remotefilelog: bump pack file formats to version 2

Drop support for format 0 and 1 packs. Version 2 will be different in
that it'll use zlib compression instead of lz4, as the latter isn't
really suitable for core.

If I can make it work, I'll make compression engines pluggable, but
the logic for handling these files is spread out enough it's a little
tricky to get that right.

Test changes are only due to pack files being renamed. The contents of
the packs are the same.

Differential Revision: https://phab.mercurial-scm.org/D4943

hgext/remotefilelog/basepack.py

@@ -1,543 +1,539 b''
 from __future__ import absolute_import
 
 import collections
 import errno
 import hashlib
 import mmap
 import os
 import struct
 import time
 
 from mercurial.i18n import _
 from mercurial import (
     policy,
     pycompat,
     util,
     vfs as vfsmod,
 )
 from . import shallowutil
 
 osutil = policy.importmod(r'osutil')
 
 # The pack version supported by this implementation. This will need to be
 # rev'd whenever the byte format changes. Ex: changing the fanout prefix,
 # changing any of the int sizes, changing the delta algorithm, etc.
 PACKVERSIONSIZE = 1
 INDEXVERSIONSIZE = 2
 
 FANOUTSTART = INDEXVERSIONSIZE
 
 # Constant that indicates a fanout table entry hasn't been filled in. (This does
 # not get serialized)
 EMPTYFANOUT = -1
 
 # The fanout prefix is the number of bytes that can be addressed by the fanout
 # table. Example: a fanout prefix of 1 means we use the first byte of a hash to
 # look in the fanout table (which will be 2^8 entries long).
 SMALLFANOUTPREFIX = 1
 LARGEFANOUTPREFIX = 2
 
 # The number of entries in the index at which point we switch to a large fanout.
 # It is chosen to balance the linear scan through a sparse fanout, with the
 # size of the bisect in actual index.
 # 2^16 / 8 was chosen because it trades off (1 step fanout scan + 5 step
 # bisect) with (8 step fanout scan + 1 step bisect)
 # 5 step bisect = log(2^16 / 8 / 255)  # fanout
 # 10 step fanout scan = 2^16 / (2^16 / 8)  # fanout space divided by entries
 SMALLFANOUTCUTOFF = 2**16 / 8
 
 # The amount of time to wait between checking for new packs. This prevents an
 # exception when data is moved to a new pack after the process has already
 # loaded the pack list.
 REFRESHRATE = 0.1
 
 if pycompat.isposix:
     # With glibc 2.7+ the 'e' flag uses O_CLOEXEC when opening.
     # The 'e' flag will be ignored on older versions of glibc.
     PACKOPENMODE = 'rbe'
 else:
     PACKOPENMODE = 'rb'
 
 class _cachebackedpacks(object):
     def __init__(self, packs, cachesize):
         self._packs = set(packs)
         self._lrucache = util.lrucachedict(cachesize)
         self._lastpack = None
 
         # Avoid cold start of the cache by populating the most recent packs
         # in the cache.
         for i in reversed(range(min(cachesize, len(packs)))):
             self._movetofront(packs[i])
 
     def _movetofront(self, pack):
         # This effectively makes pack the first entry in the cache.
         self._lrucache[pack] = True
 
     def _registerlastpackusage(self):
         if self._lastpack is not None:
             self._movetofront(self._lastpack)
             self._lastpack = None
 
     def add(self, pack):
         self._registerlastpackusage()
 
         # This method will mostly be called when packs are not in cache.
         # Therefore, adding pack to the cache.
         self._movetofront(pack)
         self._packs.add(pack)
 
     def __iter__(self):
         self._registerlastpackusage()
 
         # Cache iteration is based on LRU.
         for pack in self._lrucache:
             self._lastpack = pack
             yield pack
 
         cachedpacks = set(pack for pack in self._lrucache)
         # Yield for paths not in the cache.
         for pack in self._packs - cachedpacks:
             self._lastpack = pack
             yield pack
 
         # Data not found in any pack.
         self._lastpack = None
 
 class basepackstore(object):
     # Default cache size limit for the pack files.
     DEFAULTCACHESIZE = 100
 
     def __init__(self, ui, path):
         self.ui = ui
         self.path = path
 
         # lastrefesh is 0 so we'll immediately check for new packs on the first
         # failure.
         self.lastrefresh = 0
 
         packs = []
         for filepath, __, __ in self._getavailablepackfilessorted():
             try:
                 pack = self.getpack(filepath)
             except Exception as ex:
                 # An exception may be thrown if the pack file is corrupted
                 # somehow.  Log a warning but keep going in this case, just
                 # skipping this pack file.
                 #
                 # If this is an ENOENT error then don't even bother logging.
                 # Someone could have removed the file since we retrieved the
                 # list of paths.
                 if getattr(ex, 'errno', None) != errno.ENOENT:
                     ui.warn(_('unable to load pack %s: %s\n') % (filepath, ex))
                 continue
             packs.append(pack)
 
         self.packs = _cachebackedpacks(packs, self.DEFAULTCACHESIZE)
 
     def _getavailablepackfiles(self):
         """For each pack file (a index/data file combo), yields:
           (full path without extension, mtime, size)
 
         mtime will be the mtime of the index/data file (whichever is newer)
         size is the combined size of index/data file
         """
         indexsuffixlen = len(self.INDEXSUFFIX)
         packsuffixlen = len(self.PACKSUFFIX)
 
         ids = set()
         sizes = collections.defaultdict(lambda: 0)
         mtimes = collections.defaultdict(lambda: [])
         try:
             for filename, type, stat in osutil.listdir(self.path, stat=True):
                 id = None
                 if filename[-indexsuffixlen:] == self.INDEXSUFFIX:
                     id = filename[:-indexsuffixlen]
                 elif filename[-packsuffixlen:] == self.PACKSUFFIX:
                     id = filename[:-packsuffixlen]
 
                 # Since we expect to have two files corresponding to each ID
                 # (the index file and the pack file), we can yield once we see
                 # it twice.
                 if id:
                     sizes[id] += stat.st_size # Sum both files' sizes together
                     mtimes[id].append(stat.st_mtime)
                     if id in ids:
                         yield (os.path.join(self.path, id), max(mtimes[id]),
                             sizes[id])
                     else:
                         ids.add(id)
         except OSError as ex:
             if ex.errno != errno.ENOENT:
                 raise
 
     def _getavailablepackfilessorted(self):
         """Like `_getavailablepackfiles`, but also sorts the files by mtime,
         yielding newest files first.
 
         This is desirable, since it is more likely newer packfiles have more
         desirable data.
         """
         files = []
         for path, mtime, size in self._getavailablepackfiles():
             files.append((mtime, size, path))
         files = sorted(files, reverse=True)
         for mtime, size, path in files:
             yield path, mtime, size
 
     def gettotalsizeandcount(self):
         """Returns the total disk size (in bytes) of all the pack files in
         this store, and the count of pack files.
 
         (This might be smaller than the total size of the ``self.path``
         directory, since this only considers fuly-writen pack files, and not
         temporary files or other detritus on the directory.)
         """
         totalsize = 0
         count = 0
         for __, __, size in self._getavailablepackfiles():
             totalsize += size
             count += 1
         return totalsize, count
 
     def getmetrics(self):
         """Returns metrics on the state of this store."""
         size, count = self.gettotalsizeandcount()
         return {
             'numpacks': count,
             'totalpacksize': size,
         }
 
     def getpack(self, path):
         raise NotImplementedError()
 
     def getmissing(self, keys):
         missing = keys
         for pack in self.packs:
             missing = pack.getmissing(missing)
 
             # Ensures better performance of the cache by keeping the most
             # recently accessed pack at the beginning in subsequent iterations.
             if not missing:
                 return missing
 
         if missing:
             for pack in self.refresh():
                 missing = pack.getmissing(missing)
 
         return missing
 
     def markledger(self, ledger, options=None):
         for pack in self.packs:
             pack.markledger(ledger)
 
     def markforrefresh(self):
         """Tells the store that there may be new pack files, so the next time it
         has a lookup miss it should check for new files."""
         self.lastrefresh = 0
 
     def refresh(self):
         """Checks for any new packs on disk, adds them to the main pack list,
         and returns a list of just the new packs."""
         now = time.time()
 
         # If we experience a lot of misses (like in the case of getmissing() on
         # new objects), let's only actually check disk for new stuff every once
         # in a while. Generally this code path should only ever matter when a
         # repack is going on in the background, and that should be pretty rare
         # to have that happen twice in quick succession.
         newpacks = []
         if now > self.lastrefresh + REFRESHRATE:
             self.lastrefresh = now
             previous = set(p.path for p in self.packs)
             for filepath, __, __ in self._getavailablepackfilessorted():
                 if filepath not in previous:
                     newpack = self.getpack(filepath)
                     newpacks.append(newpack)
                     self.packs.add(newpack)
 
         return newpacks
 
 class versionmixin(object):
     # Mix-in for classes with multiple supported versions
     VERSION = None
-    SUPPORTED_VERSIONS = [0]
+    SUPPORTED_VERSIONS = [2]
 
     def _checkversion(self, version):
         if version in self.SUPPORTED_VERSIONS:
             if self.VERSION is None:
                 # only affect this instance
                 self.VERSION = version
             elif self.VERSION != version:
                 raise RuntimeError('inconsistent version: %s' % version)
         else:
             raise RuntimeError('unsupported version: %s' % version)
 
 class basepack(versionmixin):
     # The maximum amount we should read via mmap before remmaping so the old
     # pages can be released (100MB)
     MAXPAGEDIN = 100 * 1024**2
 
-    SUPPORTED_VERSIONS = [0]
+    SUPPORTED_VERSIONS = [2]
 
     def __init__(self, path):
         self.path = path
         self.packpath = path + self.PACKSUFFIX
         self.indexpath = path + self.INDEXSUFFIX
 
         self.indexsize = os.stat(self.indexpath).st_size
         self.datasize = os.stat(self.packpath).st_size
 
         self._index = None
         self._data = None
         self.freememory() # initialize the mmap
 
         version = struct.unpack('!B', self._data[:PACKVERSIONSIZE])[0]
         self._checkversion(version)
 
         version, config = struct.unpack('!BB', self._index[:INDEXVERSIONSIZE])
         self._checkversion(version)
 
         if 0b10000000 & config:
             self.params = indexparams(LARGEFANOUTPREFIX, version)
         else:
             self.params = indexparams(SMALLFANOUTPREFIX, version)
 
     @util.propertycache
     def _fanouttable(self):
         params = self.params
         rawfanout = self._index[FANOUTSTART:FANOUTSTART + params.fanoutsize]
         fanouttable = []
         for i in pycompat.xrange(0, params.fanoutcount):
             loc = i * 4
             fanoutentry = struct.unpack('!I', rawfanout[loc:loc + 4])[0]
             fanouttable.append(fanoutentry)
         return fanouttable
 
     @util.propertycache
     def _indexend(self):
-        if self.VERSION == 0:
-            return self.indexsize
-        else:
-            nodecount = struct.unpack_from('!Q', self._index,
-                                           self.params.indexstart - 8)[0]
-            return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
+        nodecount = struct.unpack_from('!Q', self._index,
+                                       self.params.indexstart - 8)[0]
+        return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
 
     def freememory(self):
         """Unmap and remap the memory to free it up after known expensive
         operations. Return True if self._data and self._index were reloaded.
         """
         if self._index:
             if self._pagedin < self.MAXPAGEDIN:
                 return False
 
             self._index.close()
             self._data.close()
 
         # TODO: use an opener/vfs to access these paths
         with open(self.indexpath, PACKOPENMODE) as indexfp:
             # memory-map the file, size 0 means whole file
             self._index = mmap.mmap(indexfp.fileno(), 0,
                                     access=mmap.ACCESS_READ)
         with open(self.packpath, PACKOPENMODE) as datafp:
             self._data = mmap.mmap(datafp.fileno(), 0, access=mmap.ACCESS_READ)
 
         self._pagedin = 0
         return True
 
     def getmissing(self, keys):
         raise NotImplementedError()
 
     def markledger(self, ledger, options=None):
         raise NotImplementedError()
 
     def cleanup(self, ledger):
         raise NotImplementedError()
 
     def __iter__(self):
         raise NotImplementedError()
 
     def iterentries(self):
         raise NotImplementedError()
 
 class mutablebasepack(versionmixin):
 
-    def __init__(self, ui, packdir, version=0):
+    def __init__(self, ui, packdir, version=2):
         self._checkversion(version)
-
+        # TODO(augie): make this configurable
+        self._compressor = 'GZ'
         opener = vfsmod.vfs(packdir)
         opener.createmode = 0o444
         self.opener = opener
 
         self.entries = {}
 
         shallowutil.mkstickygroupdir(ui, packdir)
         self.packfp, self.packpath = opener.mkstemp(
             suffix=self.PACKSUFFIX + '-tmp')
         self.idxfp, self.idxpath = opener.mkstemp(
             suffix=self.INDEXSUFFIX + '-tmp')
         self.packfp = os.fdopen(self.packfp, 'w+')
         self.idxfp = os.fdopen(self.idxfp, 'w+')
         self.sha = hashlib.sha1()
         self._closed = False
 
         # The opener provides no way of doing permission fixup on files created
         # via mkstemp, so we must fix it ourselves. We can probably fix this
         # upstream in vfs.mkstemp so we don't need to use the private method.
         opener._fixfilemode(opener.join(self.packpath))
         opener._fixfilemode(opener.join(self.idxpath))
 
         # Write header
         # TODO: make it extensible (ex: allow specifying compression algorithm,
         # a flexible key/value header, delta algorithm, fanout size, etc)
         versionbuf = struct.pack('!B', self.VERSION) # unsigned 1 byte int
         self.writeraw(versionbuf)
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         if exc_type is None:
             self.close()
         else:
             self.abort()
 
     def abort(self):
         # Unclean exit
         self._cleantemppacks()
 
     def writeraw(self, data):
         self.packfp.write(data)
         self.sha.update(data)
 
     def close(self, ledger=None):
         if self._closed:
             return
 
         try:
             sha = self.sha.hexdigest()
             self.packfp.close()
             self.writeindex()
 
             if len(self.entries) == 0:
                 # Empty pack
                 self._cleantemppacks()
                 self._closed = True
                 return None
 
             self.opener.rename(self.packpath, sha + self.PACKSUFFIX)
             try:
                 self.opener.rename(self.idxpath, sha + self.INDEXSUFFIX)
             except Exception as ex:
                 try:
                     self.opener.unlink(sha + self.PACKSUFFIX)
                 except Exception:
                     pass
                 # Throw exception 'ex' explicitly since a normal 'raise' would
                 # potentially throw an exception from the unlink cleanup.
                 raise ex
         except Exception:
             # Clean up temp packs in all exception cases
             self._cleantemppacks()
             raise
 
         self._closed = True
         result = self.opener.join(sha)
         if ledger:
             ledger.addcreated(result)
         return result
 
     def _cleantemppacks(self):
         try:
             self.opener.unlink(self.packpath)
         except Exception:
             pass
         try:
             self.opener.unlink(self.idxpath)
         except Exception:
             pass
 
     def writeindex(self):
         rawindex = ''
 
         largefanout = len(self.entries) > SMALLFANOUTCUTOFF
         if largefanout:
             params = indexparams(LARGEFANOUTPREFIX, self.VERSION)
         else:
             params = indexparams(SMALLFANOUTPREFIX, self.VERSION)
 
         fanouttable = [EMPTYFANOUT] * params.fanoutcount
 
         # Precompute the location of each entry
         locations = {}
         count = 0
         for node in sorted(self.entries.iterkeys()):
             location = count * self.INDEXENTRYLENGTH
             locations[node] = location
             count += 1
 
             # Must use [0] on the unpack result since it's always a tuple.
             fanoutkey = struct.unpack(params.fanoutstruct,
                                       node[:params.fanoutprefix])[0]
             if fanouttable[fanoutkey] == EMPTYFANOUT:
                 fanouttable[fanoutkey] = location
 
         rawfanouttable = ''
         last = 0
         for offset in fanouttable:
             offset = offset if offset != EMPTYFANOUT else last
             last = offset
             rawfanouttable += struct.pack('!I', offset)
 
         rawentrieslength = struct.pack('!Q', len(self.entries))
 
         # The index offset is the it's location in the file. So after the 2 byte
         # header and the fanouttable.
         rawindex = self.createindex(locations, 2 + len(rawfanouttable))
 
         self._writeheader(params)
         self.idxfp.write(rawfanouttable)
-        if self.VERSION == 1:
-            self.idxfp.write(rawentrieslength)
+        self.idxfp.write(rawentrieslength)
         self.idxfp.write(rawindex)
         self.idxfp.close()
 
     def createindex(self, nodelocations):
         raise NotImplementedError()
 
     def _writeheader(self, indexparams):
         # Index header
         #    <version: 1 byte>
         #    <large fanout: 1 bit> # 1 means 2^16, 0 means 2^8
         #    <unused: 7 bit> # future use (compression, delta format, etc)
         config = 0
         if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
             config = 0b10000000
         self.idxfp.write(struct.pack('!BB', self.VERSION, config))
 
 class indexparams(object):
     __slots__ = ('fanoutprefix', 'fanoutstruct', 'fanoutcount', 'fanoutsize',
                  'indexstart')
 
     def __init__(self, prefixsize, version):
         self.fanoutprefix = prefixsize
 
         # The struct pack format for fanout table location (i.e. the format that
         # converts the node prefix into an integer location in the fanout
         # table).
         if prefixsize == SMALLFANOUTPREFIX:
             self.fanoutstruct = '!B'
         elif prefixsize == LARGEFANOUTPREFIX:
             self.fanoutstruct = '!H'
         else:
             raise ValueError("invalid fanout prefix size: %s" % prefixsize)
 
         # The number of fanout table entries
         self.fanoutcount = 2**(prefixsize * 8)
 
         # The total bytes used by the fanout table
         self.fanoutsize = self.fanoutcount * 4
 
         self.indexstart = FANOUTSTART + self.fanoutsize
-        if version == 1:
-            # Skip the index length
-            self.indexstart += 8
+        # Skip the index length
+        self.indexstart += 8

hgext/remotefilelog/datapack.py

@@ -1,470 +1,456 b''
 from __future__ import absolute_import
 
 import struct
 
 from mercurial.node import hex, nullid
 from mercurial.i18n import _
 from mercurial import (
-    error,
     pycompat,
     util,
 )
 from . import (
     basepack,
     constants,
     lz4wrapper,
     shallowutil,
 )
 
 NODELENGTH = 20
 
 # The indicator value in the index for a fulltext entry.
 FULLTEXTINDEXMARK = -1
 NOBASEINDEXMARK = -2
 
 INDEXSUFFIX = '.dataidx'
 PACKSUFFIX = '.datapack'
 
 class datapackstore(basepack.basepackstore):
     INDEXSUFFIX = INDEXSUFFIX
     PACKSUFFIX = PACKSUFFIX
 
     def __init__(self, ui, path):
         super(datapackstore, self).__init__(ui, path)
 
     def getpack(self, path):
         return datapack(path)
 
     def get(self, name, node):
         raise RuntimeError("must use getdeltachain with datapackstore")
 
     def getmeta(self, name, node):
         for pack in self.packs:
             try:
                 return pack.getmeta(name, node)
             except KeyError:
                 pass
 
         for pack in self.refresh():
             try:
                 return pack.getmeta(name, node)
             except KeyError:
                 pass
 
         raise KeyError((name, hex(node)))
 
     def getdelta(self, name, node):
         for pack in self.packs:
             try:
                 return pack.getdelta(name, node)
             except KeyError:
                 pass
 
         for pack in self.refresh():
             try:
                 return pack.getdelta(name, node)
             except KeyError:
                 pass
 
         raise KeyError((name, hex(node)))
 
     def getdeltachain(self, name, node):
         for pack in self.packs:
             try:
                 return pack.getdeltachain(name, node)
             except KeyError:
                 pass
 
         for pack in self.refresh():
             try:
                 return pack.getdeltachain(name, node)
             except KeyError:
                 pass
 
         raise KeyError((name, hex(node)))
 
     def add(self, name, node, data):
         raise RuntimeError("cannot add to datapackstore")
 
 class datapack(basepack.basepack):
     INDEXSUFFIX = INDEXSUFFIX
     PACKSUFFIX = PACKSUFFIX
 
     # Format is <node><delta offset><pack data offset><pack data size>
     # See the mutabledatapack doccomment for more details.
     INDEXFORMAT = '!20siQQ'
     INDEXENTRYLENGTH = 40
 
-    SUPPORTED_VERSIONS = [0, 1]
+    SUPPORTED_VERSIONS = [2]
 
     def getmissing(self, keys):
         missing = []
         for name, node in keys:
             value = self._find(node)
             if not value:
                 missing.append((name, node))
 
         return missing
 
     def get(self, name, node):
         raise RuntimeError("must use getdeltachain with datapack (%s:%s)"
                            % (name, hex(node)))
 
     def getmeta(self, name, node):
         value = self._find(node)
         if value is None:
             raise KeyError((name, hex(node)))
 
-        # version 0 does not support metadata
-        if self.VERSION == 0:
-            return {}
-
         node, deltabaseoffset, offset, size = value
         rawentry = self._data[offset:offset + size]
 
         # see docstring of mutabledatapack for the format
         offset = 0
         offset += struct.unpack_from('!H', rawentry, offset)[0] + 2 # filename
         offset += 40 # node, deltabase node
         offset += struct.unpack_from('!Q', rawentry, offset)[0] + 8 # delta
 
         metalen = struct.unpack_from('!I', rawentry, offset)[0]
         offset += 4
 
         meta = shallowutil.parsepackmeta(rawentry[offset:offset + metalen])
 
         return meta
 
     def getdelta(self, name, node):
         value = self._find(node)
         if value is None:
             raise KeyError((name, hex(node)))
 
         node, deltabaseoffset, offset, size = value
         entry = self._readentry(offset, size, getmeta=True)
         filename, node, deltabasenode, delta, meta = entry
 
         # If we've read a lot of data from the mmap, free some memory.
         self.freememory()
 
         return delta, filename, deltabasenode, meta
 
     def getdeltachain(self, name, node):
         value = self._find(node)
         if value is None:
             raise KeyError((name, hex(node)))
 
         params = self.params
 
         # Precompute chains
         chain = [value]
         deltabaseoffset = value[1]
         entrylen = self.INDEXENTRYLENGTH
         while (deltabaseoffset != FULLTEXTINDEXMARK
                and deltabaseoffset != NOBASEINDEXMARK):
             loc = params.indexstart + deltabaseoffset
             value = struct.unpack(self.INDEXFORMAT,
                                   self._index[loc:loc + entrylen])
             deltabaseoffset = value[1]
             chain.append(value)
 
         # Read chain data
         deltachain = []
         for node, deltabaseoffset, offset, size in chain:
             filename, node, deltabasenode, delta = self._readentry(offset, size)
             deltachain.append((filename, node, filename, deltabasenode, delta))
 
         # If we've read a lot of data from the mmap, free some memory.
         self.freememory()
 
         return deltachain
 
     def _readentry(self, offset, size, getmeta=False):
         rawentry = self._data[offset:offset + size]
         self._pagedin += len(rawentry)
 
         # <2 byte len> + <filename>
         lengthsize = 2
         filenamelen = struct.unpack('!H', rawentry[:2])[0]
         filename = rawentry[lengthsize:lengthsize + filenamelen]
 
         # <20 byte node> + <20 byte deltabase>
         nodestart = lengthsize + filenamelen
         deltabasestart = nodestart + NODELENGTH
         node = rawentry[nodestart:deltabasestart]
         deltabasenode = rawentry[deltabasestart:deltabasestart + NODELENGTH]
 
         # <8 byte len> + <delta>
         deltastart = deltabasestart + NODELENGTH
         rawdeltalen = rawentry[deltastart:deltastart + 8]
         deltalen = struct.unpack('!Q', rawdeltalen)[0]
 
         delta = rawentry[deltastart + 8:deltastart + 8 + deltalen]
         delta = lz4wrapper.lz4decompress(delta)
 
         if getmeta:
-            if self.VERSION == 0:
-                meta = {}
-            else:
-                metastart = deltastart + 8 + deltalen
-                metalen = struct.unpack_from('!I', rawentry, metastart)[0]
+            metastart = deltastart + 8 + deltalen
+            metalen = struct.unpack_from('!I', rawentry, metastart)[0]
 
-                rawmeta = rawentry[metastart + 4:metastart + 4 + metalen]
-                meta = shallowutil.parsepackmeta(rawmeta)
+            rawmeta = rawentry[metastart + 4:metastart + 4 + metalen]
+            meta = shallowutil.parsepackmeta(rawmeta)
             return filename, node, deltabasenode, delta, meta
         else:
             return filename, node, deltabasenode, delta
 
     def add(self, name, node, data):
         raise RuntimeError("cannot add to datapack (%s:%s)" % (name, node))
 
     def _find(self, node):
         params = self.params
         fanoutkey = struct.unpack(params.fanoutstruct,
                                   node[:params.fanoutprefix])[0]
         fanout = self._fanouttable
 
         start = fanout[fanoutkey] + params.indexstart
         indexend = self._indexend
 
         # Scan forward to find the first non-same entry, which is the upper
         # bound.
         for i in pycompat.xrange(fanoutkey + 1, params.fanoutcount):
             end = fanout[i] + params.indexstart
             if end != start:
                 break
         else:
             end = indexend
 
         # Bisect between start and end to find node
         index = self._index
         startnode = index[start:start + NODELENGTH]
         endnode = index[end:end + NODELENGTH]
         entrylen = self.INDEXENTRYLENGTH
         if startnode == node:
             entry = index[start:start + entrylen]
         elif endnode == node:
             entry = index[end:end + entrylen]
         else:
             while start < end - entrylen:
                 mid = start  + (end - start) / 2
                 mid = mid - ((mid - params.indexstart) % entrylen)
                 midnode = index[mid:mid + NODELENGTH]
                 if midnode == node:
                     entry = index[mid:mid + entrylen]
                     break
                 if node > midnode:
                     start = mid
                     startnode = midnode
                 elif node < midnode:
                     end = mid
                     endnode = midnode
             else:
                 return None
 
         return struct.unpack(self.INDEXFORMAT, entry)
 
     def markledger(self, ledger, options=None):
         for filename, node in self:
             ledger.markdataentry(self, filename, node)
 
     def cleanup(self, ledger):
         entries = ledger.sources.get(self, [])
         allkeys = set(self)
         repackedkeys = set((e.filename, e.node) for e in entries if
                            e.datarepacked or e.gced)
 
         if len(allkeys - repackedkeys) == 0:
             if self.path not in ledger.created:
                 util.unlinkpath(self.indexpath, ignoremissing=True)
                 util.unlinkpath(self.packpath, ignoremissing=True)
 
     def __iter__(self):
         for f, n, deltabase, deltalen in self.iterentries():
             yield f, n
 
     def iterentries(self):
         # Start at 1 to skip the header
         offset = 1
         data = self._data
         while offset < self.datasize:
             oldoffset = offset
 
             # <2 byte len> + <filename>
             filenamelen = struct.unpack('!H', data[offset:offset + 2])[0]
             offset += 2
             filename = data[offset:offset + filenamelen]
             offset += filenamelen
 
             # <20 byte node>
             node = data[offset:offset + constants.NODESIZE]
             offset += constants.NODESIZE
             # <20 byte deltabase>
             deltabase = data[offset:offset + constants.NODESIZE]
             offset += constants.NODESIZE
 
             # <8 byte len> + <delta>
             rawdeltalen = data[offset:offset + 8]
             deltalen = struct.unpack('!Q', rawdeltalen)[0]
             offset += 8
 
             # it has to be at least long enough for the lz4 header.
             assert deltalen >= 4
 
             # python-lz4 stores the length of the uncompressed field as a
             # little-endian 32-bit integer at the start of the data.
             uncompressedlen = struct.unpack('<I', data[offset:offset + 4])[0]
             offset += deltalen
 
-            if self.VERSION == 1:
-                # <4 byte len> + <metadata-list>
-                metalen = struct.unpack_from('!I', data, offset)[0]
-                offset += 4 + metalen
+            # <4 byte len> + <metadata-list>
+            metalen = struct.unpack_from('!I', data, offset)[0]
+            offset += 4 + metalen
 
             yield (filename, node, deltabase, uncompressedlen)
 
             # If we've read a lot of data from the mmap, free some memory.
             self._pagedin += offset - oldoffset
             if self.freememory():
                 data = self._data
 
 class mutabledatapack(basepack.mutablebasepack):
     """A class for constructing and serializing a datapack file and index.
 
     A datapack is a pair of files that contain the revision contents for various
     file revisions in Mercurial. It contains only revision contents (like file
     contents), not any history information.
 
     It consists of two files, with the following format. All bytes are in
     network byte order (big endian).
 
     .datapack
         The pack itself is a series of revision deltas with some basic header
         information on each. A revision delta may be a fulltext, represented by
         a deltabasenode equal to the nullid.
 
         datapack = <version: 1 byte>
                    [<revision>,...]
         revision = <filename len: 2 byte unsigned int>
                    <filename>
                    <node: 20 byte>
                    <deltabasenode: 20 byte>
                    <delta len: 8 byte unsigned int>
                    <delta>
                    <metadata-list len: 4 byte unsigned int> [1]
                    <metadata-list>                          [1]
         metadata-list = [<metadata-item>, ...]
         metadata-item = <metadata-key: 1 byte>
                         <metadata-value len: 2 byte unsigned>
                         <metadata-value>
 
         metadata-key could be METAKEYFLAG or METAKEYSIZE or other single byte
         value in the future.
 
     .dataidx
         The index file consists of two parts, the fanout and the index.
 
         The index is a list of index entries, sorted by node (one per revision
         in the pack). Each entry has:
 
         - node (The 20 byte node of the entry; i.e. the commit hash, file node
                 hash, etc)
         - deltabase index offset (The location in the index of the deltabase for
                                   this entry. The deltabase is the next delta in
                                   the chain, with the chain eventually
                                   terminating in a full-text, represented by a
                                   deltabase offset of -1. This lets us compute
                                   delta chains from the index, then do
                                   sequential reads from the pack if the revision
                                   are nearby on disk.)
         - pack entry offset (The location of this entry in the datapack)
         - pack content size (The on-disk length of this entry's pack data)
 
         The fanout is a quick lookup table to reduce the number of steps for
         bisecting the index. It is a series of 4 byte pointers to positions
         within the index. It has 2^16 entries, which corresponds to hash
         prefixes [0000, 0001,..., FFFE, FFFF]. Example: the pointer in slot
         4F0A points to the index position of the first revision whose node
         starts with 4F0A. This saves log(2^16)=16 bisect steps.
 
         dataidx = <fanouttable>
                   <index>
         fanouttable = [<index offset: 4 byte unsigned int>,...] (2^16 entries)
         index = [<index entry>,...]
         indexentry = <node: 20 byte>
                      <deltabase location: 4 byte signed int>
                      <pack entry offset: 8 byte unsigned int>
                      <pack entry size: 8 byte unsigned int>
 
     [1]: new in version 1.
     """
     INDEXSUFFIX = INDEXSUFFIX
     PACKSUFFIX = PACKSUFFIX
 
     # v[01] index format: <node><delta offset><pack data offset><pack data size>
     INDEXFORMAT = datapack.INDEXFORMAT
     INDEXENTRYLENGTH = datapack.INDEXENTRYLENGTH
 
     # v1 has metadata support
-    SUPPORTED_VERSIONS = [0, 1]
+    SUPPORTED_VERSIONS = [2]
 
     def add(self, name, node, deltabasenode, delta, metadata=None):
         # metadata is a dict, ex. {METAKEYFLAG: flag}
         if len(name) > 2**16:
             raise RuntimeError(_("name too long %s") % name)
         if len(node) != 20:
             raise RuntimeError(_("node should be 20 bytes %s") % node)
 
         if node in self.entries:
             # The revision has already been added
             return
 
         # TODO: allow configurable compression
         delta = lz4wrapper.lz4compress(delta)
 
         rawdata = ''.join((
             struct.pack('!H', len(name)), # unsigned 2 byte int
             name,
             node,
             deltabasenode,
             struct.pack('!Q', len(delta)), # unsigned 8 byte int
             delta,
         ))
 
-        if self.VERSION == 1:
-            # v1 support metadata
-            rawmeta = shallowutil.buildpackmeta(metadata)
-            rawdata += struct.pack('!I', len(rawmeta)) # unsigned 4 byte
-            rawdata += rawmeta
-        else:
-            # v0 cannot store metadata, raise if metadata contains flag
-            if metadata and metadata.get(constants.METAKEYFLAG, 0) != 0:
-                raise error.ProgrammingError('v0 pack cannot store flags')
+        # v1 support metadata
+        rawmeta = shallowutil.buildpackmeta(metadata)
+        rawdata += struct.pack('!I', len(rawmeta)) # unsigned 4 byte
+        rawdata += rawmeta
 
         offset = self.packfp.tell()
 
         size = len(rawdata)
 
         self.entries[node] = (deltabasenode, offset, size)
 
         self.writeraw(rawdata)
 
     def createindex(self, nodelocations, indexoffset):
         entries = sorted((n, db, o, s) for n, (db, o, s)
                          in self.entries.iteritems())
 
         rawindex = ''
         fmt = self.INDEXFORMAT
         for node, deltabase, offset, size in entries:
             if deltabase == nullid:
                 deltabaselocation = FULLTEXTINDEXMARK
             else:
                 # Instead of storing the deltabase node in the index, let's
                 # store a pointer directly to the index entry for the deltabase.
                 deltabaselocation = nodelocations.get(deltabase,
                                                       NOBASEINDEXMARK)
 
             entry = struct.pack(fmt, node, deltabaselocation, offset, size)
             rawindex += entry
 
         return rawindex

hgext/remotefilelog/historypack.py

@@ -1,545 +1,520 b''
 from __future__ import absolute_import
 
 import hashlib
 import struct
 
 from mercurial.node import hex, nullid
 from mercurial import (
     pycompat,
     util,
 )
 from . import (
     basepack,
     constants,
     shallowutil,
 )
 
 # (filename hash, offset, size)
-INDEXFORMAT0 = '!20sQQ'
-INDEXENTRYLENGTH0 = struct.calcsize(INDEXFORMAT0)
-INDEXFORMAT1 = '!20sQQII'
-INDEXENTRYLENGTH1 = struct.calcsize(INDEXFORMAT1)
+INDEXFORMAT2 = '!20sQQII'
+INDEXENTRYLENGTH2 = struct.calcsize(INDEXFORMAT2)
 NODELENGTH = 20
 
 NODEINDEXFORMAT = '!20sQ'
 NODEINDEXENTRYLENGTH = struct.calcsize(NODEINDEXFORMAT)
 
 # (node, p1, p2, linknode)
 PACKFORMAT = "!20s20s20s20sH"
 PACKENTRYLENGTH = 82
 
 ENTRYCOUNTSIZE = 4
 
 INDEXSUFFIX = '.histidx'
 PACKSUFFIX = '.histpack'
 
 ANC_NODE = 0
 ANC_P1NODE = 1
 ANC_P2NODE = 2
 ANC_LINKNODE = 3
 ANC_COPYFROM = 4
 
 class historypackstore(basepack.basepackstore):
     INDEXSUFFIX = INDEXSUFFIX
     PACKSUFFIX = PACKSUFFIX
 
     def getpack(self, path):
         return historypack(path)
 
     def getancestors(self, name, node, known=None):
         for pack in self.packs:
             try:
                 return pack.getancestors(name, node, known=known)
             except KeyError:
                 pass
 
         for pack in self.refresh():
             try:
                 return pack.getancestors(name, node, known=known)
             except KeyError:
                 pass
 
         raise KeyError((name, node))
 
     def getnodeinfo(self, name, node):
         for pack in self.packs:
             try:
                 return pack.getnodeinfo(name, node)
             except KeyError:
                 pass
 
         for pack in self.refresh():
             try:
                 return pack.getnodeinfo(name, node)
             except KeyError:
                 pass
 
         raise KeyError((name, node))
 
     def add(self, filename, node, p1, p2, linknode, copyfrom):
         raise RuntimeError("cannot add to historypackstore (%s:%s)"
                            % (filename, hex(node)))
 
 class historypack(basepack.basepack):
     INDEXSUFFIX = INDEXSUFFIX
     PACKSUFFIX = PACKSUFFIX
 
-    SUPPORTED_VERSIONS = [0, 1]
+    SUPPORTED_VERSIONS = [2]
 
     def __init__(self, path):
         super(historypack, self).__init__(path)
-
-        if self.VERSION == 0:
-            self.INDEXFORMAT = INDEXFORMAT0
-            self.INDEXENTRYLENGTH = INDEXENTRYLENGTH0
-        else:
-            self.INDEXFORMAT = INDEXFORMAT1
-            self.INDEXENTRYLENGTH = INDEXENTRYLENGTH1
+        self.INDEXFORMAT = INDEXFORMAT2
+        self.INDEXENTRYLENGTH = INDEXENTRYLENGTH2
 
     def getmissing(self, keys):
         missing = []
         for name, node in keys:
             try:
                 self._findnode(name, node)
             except KeyError:
                 missing.append((name, node))
 
         return missing
 
     def getancestors(self, name, node, known=None):
         """Returns as many ancestors as we're aware of.
 
         return value: {
            node: (p1, p2, linknode, copyfrom),
            ...
         }
         """
         if known and node in known:
             return []
 
         ancestors = self._getancestors(name, node, known=known)
         results = {}
         for ancnode, p1, p2, linknode, copyfrom in ancestors:
             results[ancnode] = (p1, p2, linknode, copyfrom)
 
         if not results:
             raise KeyError((name, node))
         return results
 
     def getnodeinfo(self, name, node):
         # Drop the node from the tuple before returning, since the result should
         # just be (p1, p2, linknode, copyfrom)
         return self._findnode(name, node)[1:]
 
     def _getancestors(self, name, node, known=None):
         if known is None:
             known = set()
         section = self._findsection(name)
         filename, offset, size, nodeindexoffset, nodeindexsize = section
         pending = set((node,))
         o = 0
         while o < size:
             if not pending:
                 break
             entry, copyfrom = self._readentry(offset + o)
             o += PACKENTRYLENGTH
             if copyfrom:
                 o += len(copyfrom)
 
             ancnode = entry[ANC_NODE]
             if ancnode in pending:
                 pending.remove(ancnode)
                 p1node = entry[ANC_P1NODE]
                 p2node = entry[ANC_P2NODE]
                 if p1node != nullid and p1node not in known:
                     pending.add(p1node)
                 if p2node != nullid and p2node not in known:
                     pending.add(p2node)
 
                 yield (ancnode, p1node, p2node, entry[ANC_LINKNODE], copyfrom)
 
     def _readentry(self, offset):
         data = self._data
         entry = struct.unpack(PACKFORMAT, data[offset:offset + PACKENTRYLENGTH])
         copyfrom = None
         copyfromlen = entry[ANC_COPYFROM]
         if copyfromlen != 0:
             offset += PACKENTRYLENGTH
             copyfrom = data[offset:offset + copyfromlen]
         return entry, copyfrom
 
     def add(self, filename, node, p1, p2, linknode, copyfrom):
         raise RuntimeError("cannot add to historypack (%s:%s)" %
                            (filename, hex(node)))
 
     def _findnode(self, name, node):
         if self.VERSION == 0:
             ancestors = self._getancestors(name, node)
             for ancnode, p1node, p2node, linknode, copyfrom in ancestors:
                 if ancnode == node:
                     return (ancnode, p1node, p2node, linknode, copyfrom)
         else:
             section = self._findsection(name)
             nodeindexoffset, nodeindexsize = section[3:]
             entry = self._bisect(node, nodeindexoffset,
                                  nodeindexoffset + nodeindexsize,
                                  NODEINDEXENTRYLENGTH)
             if entry is not None:
                 node, offset = struct.unpack(NODEINDEXFORMAT, entry)
                 entry, copyfrom = self._readentry(offset)
                 # Drop the copyfromlen from the end of entry, and replace it
                 # with the copyfrom string.
                 return entry[:4] + (copyfrom,)
 
         raise KeyError("unable to find history for %s:%s" % (name, hex(node)))
 
     def _findsection(self, name):
         params = self.params
         namehash = hashlib.sha1(name).digest()
         fanoutkey = struct.unpack(params.fanoutstruct,
                                   namehash[:params.fanoutprefix])[0]
         fanout = self._fanouttable
 
         start = fanout[fanoutkey] + params.indexstart
         indexend = self._indexend
 
         for i in pycompat.xrange(fanoutkey + 1, params.fanoutcount):
             end = fanout[i] + params.indexstart
             if end != start:
                 break
         else:
             end = indexend
 
         entry = self._bisect(namehash, start, end, self.INDEXENTRYLENGTH)
         if not entry:
             raise KeyError(name)
 
         rawentry = struct.unpack(self.INDEXFORMAT, entry)
-        if self.VERSION == 0:
-            x, offset, size = rawentry
-            nodeindexoffset = None
-            nodeindexsize = None
-        else:
-            x, offset, size, nodeindexoffset, nodeindexsize = rawentry
-            rawnamelen = self._index[nodeindexoffset:nodeindexoffset +
-                                                     constants.FILENAMESIZE]
-            actualnamelen = struct.unpack('!H', rawnamelen)[0]
-            nodeindexoffset += constants.FILENAMESIZE
-            actualname = self._index[nodeindexoffset:nodeindexoffset +
-                                                     actualnamelen]
-            if actualname != name:
-                raise KeyError("found file name %s when looking for %s" %
-                               (actualname, name))
-            nodeindexoffset += actualnamelen
+        x, offset, size, nodeindexoffset, nodeindexsize = rawentry
+        rawnamelen = self._index[nodeindexoffset:nodeindexoffset +
+                                                 constants.FILENAMESIZE]
+        actualnamelen = struct.unpack('!H', rawnamelen)[0]
+        nodeindexoffset += constants.FILENAMESIZE
+        actualname = self._index[nodeindexoffset:nodeindexoffset +
+                                                 actualnamelen]
+        if actualname != name:
+            raise KeyError("found file name %s when looking for %s" %
+                           (actualname, name))
+        nodeindexoffset += actualnamelen
 
         filenamelength = struct.unpack('!H', self._data[offset:offset +
                                                     constants.FILENAMESIZE])[0]
         offset += constants.FILENAMESIZE
 
         actualname = self._data[offset:offset + filenamelength]
         offset += filenamelength
 
         if name != actualname:
             raise KeyError("found file name %s when looking for %s" %
                            (actualname, name))
 
         # Skip entry list size
         offset += ENTRYCOUNTSIZE
 
         nodelistoffset = offset
         nodelistsize = (size - constants.FILENAMESIZE - filenamelength -
                         ENTRYCOUNTSIZE)
         return (name, nodelistoffset, nodelistsize,
                 nodeindexoffset, nodeindexsize)
 
     def _bisect(self, node, start, end, entrylen):
         # Bisect between start and end to find node
         origstart = start
         startnode = self._index[start:start + NODELENGTH]
         endnode = self._index[end:end + NODELENGTH]
 
         if startnode == node:
             return self._index[start:start + entrylen]
         elif endnode == node:
             return self._index[end:end + entrylen]
         else:
             while start < end - entrylen:
                 mid = start + (end - start) / 2
                 mid = mid - ((mid - origstart) % entrylen)
                 midnode = self._index[mid:mid + NODELENGTH]
                 if midnode == node:
                     return self._index[mid:mid + entrylen]
                 if node > midnode:
                     start = mid
                     startnode = midnode
                 elif node < midnode:
                     end = mid
                     endnode = midnode
         return None
 
     def markledger(self, ledger, options=None):
         for filename, node in self:
             ledger.markhistoryentry(self, filename, node)
 
     def cleanup(self, ledger):
         entries = ledger.sources.get(self, [])
         allkeys = set(self)
         repackedkeys = set((e.filename, e.node) for e in entries if
                            e.historyrepacked)
 
         if len(allkeys - repackedkeys) == 0:
             if self.path not in ledger.created:
                 util.unlinkpath(self.indexpath, ignoremissing=True)
                 util.unlinkpath(self.packpath, ignoremissing=True)
 
     def __iter__(self):
         for f, n, x, x, x, x in self.iterentries():
             yield f, n
 
     def iterentries(self):
         # Start at 1 to skip the header
         offset = 1
         while offset < self.datasize:
             data = self._data
             # <2 byte len> + <filename>
             filenamelen = struct.unpack('!H', data[offset:offset +
                                                    constants.FILENAMESIZE])[0]
             offset += constants.FILENAMESIZE
             filename = data[offset:offset + filenamelen]
             offset += filenamelen
 
             revcount = struct.unpack('!I', data[offset:offset +
                                                 ENTRYCOUNTSIZE])[0]
             offset += ENTRYCOUNTSIZE
 
             for i in pycompat.xrange(revcount):
                 entry = struct.unpack(PACKFORMAT, data[offset:offset +
                                                               PACKENTRYLENGTH])
                 offset += PACKENTRYLENGTH
 
                 copyfrom = data[offset:offset + entry[ANC_COPYFROM]]
                 offset += entry[ANC_COPYFROM]
 
                 yield (filename, entry[ANC_NODE], entry[ANC_P1NODE],
                         entry[ANC_P2NODE], entry[ANC_LINKNODE], copyfrom)
 
                 self._pagedin += PACKENTRYLENGTH
 
             # If we've read a lot of data from the mmap, free some memory.
             self.freememory()
 
 class mutablehistorypack(basepack.mutablebasepack):
     """A class for constructing and serializing a histpack file and index.
 
     A history pack is a pair of files that contain the revision history for
     various file revisions in Mercurial. It contains only revision history (like
     parent pointers and linknodes), not any revision content information.
 
     It consists of two files, with the following format:
 
     .histpack
         The pack itself is a series of file revisions with some basic header
         information on each.
 
         datapack = <version: 1 byte>
                    [<filesection>,...]
         filesection = <filename len: 2 byte unsigned int>
                       <filename>
                       <revision count: 4 byte unsigned int>
                       [<revision>,...]
         revision = <node: 20 byte>
                    <p1node: 20 byte>
                    <p2node: 20 byte>
                    <linknode: 20 byte>
                    <copyfromlen: 2 byte>
                    <copyfrom>
 
         The revisions within each filesection are stored in topological order
         (newest first). If a given entry has a parent from another file (a copy)
         then p1node is the node from the other file, and copyfrom is the
         filepath of the other file.
 
     .histidx
         The index file provides a mapping from filename to the file section in
         the histpack. In V1 it also contains sub-indexes for specific nodes
         within each file. It consists of three parts, the fanout, the file index
         and the node indexes.
 
         The file index is a list of index entries, sorted by filename hash (one
         per file section in the pack). Each entry has:
 
         - node (The 20 byte hash of the filename)
         - pack entry offset (The location of this file section in the histpack)
         - pack content size (The on-disk length of this file section's pack
                              data)
         - node index offset (The location of the file's node index in the index
                              file) [1]
         - node index size (the on-disk length of this file's node index) [1]
 
         The fanout is a quick lookup table to reduce the number of steps for
         bisecting the index. It is a series of 4 byte pointers to positions
         within the index. It has 2^16 entries, which corresponds to hash
         prefixes [00, 01, 02,..., FD, FE, FF]. Example: the pointer in slot 4F
         points to the index position of the first revision whose node starts
         with 4F. This saves log(2^16) bisect steps.
 
         dataidx = <fanouttable>
                   <file count: 8 byte unsigned> [1]
                   <fileindex>
                   <node count: 8 byte unsigned> [1]
                   [<nodeindex>,...] [1]
         fanouttable = [<index offset: 4 byte unsigned int>,...] (2^16 entries)
 
         fileindex = [<file index entry>,...]
         fileindexentry = <node: 20 byte>
                          <pack file section offset: 8 byte unsigned int>
                          <pack file section size: 8 byte unsigned int>
                          <node index offset: 4 byte unsigned int> [1]
                          <node index size: 4 byte unsigned int>   [1]
         nodeindex = <filename>[<node index entry>,...] [1]
         filename = <filename len : 2 byte unsigned int><filename value> [1]
         nodeindexentry = <node: 20 byte> [1]
                          <pack file node offset: 8 byte unsigned int> [1]
 
     [1]: new in version 1.
     """
     INDEXSUFFIX = INDEXSUFFIX
     PACKSUFFIX = PACKSUFFIX
 
-    SUPPORTED_VERSIONS = [0, 1]
+    SUPPORTED_VERSIONS = [2]
 
-    def __init__(self, ui, packpath, version=0):
-        # internal config: remotefilelog.historypackv1
-        if version == 0 and ui.configbool('remotefilelog', 'historypackv1'):
-            version = 1
-
+    def __init__(self, ui, packpath, version=2):
         super(mutablehistorypack, self).__init__(ui, packpath, version=version)
         self.files = {}
         self.entrylocations = {}
         self.fileentries = {}
 
-        if version == 0:
-            self.INDEXFORMAT = INDEXFORMAT0
-            self.INDEXENTRYLENGTH = INDEXENTRYLENGTH0
-        else:
-            self.INDEXFORMAT = INDEXFORMAT1
-            self.INDEXENTRYLENGTH = INDEXENTRYLENGTH1
+        self.INDEXFORMAT = INDEXFORMAT2
+        self.INDEXENTRYLENGTH = INDEXENTRYLENGTH2
 
         self.NODEINDEXFORMAT = NODEINDEXFORMAT
         self.NODEINDEXENTRYLENGTH = NODEINDEXENTRYLENGTH
 
     def add(self, filename, node, p1, p2, linknode, copyfrom):
         copyfrom = copyfrom or ''
         copyfromlen = struct.pack('!H', len(copyfrom))
         self.fileentries.setdefault(filename, []).append((node, p1, p2,
                                                           linknode,
                                                           copyfromlen,
                                                           copyfrom))
 
     def _write(self):
         for filename in sorted(self.fileentries):
             entries = self.fileentries[filename]
             sectionstart = self.packfp.tell()
 
             # Write the file section content
             entrymap = dict((e[0], e) for e in entries)
             def parentfunc(node):
                 x, p1, p2, x, x, x = entrymap[node]
                 parents = []
                 if p1 != nullid:
                     parents.append(p1)
                 if p2 != nullid:
                     parents.append(p2)
                 return parents
 
             sortednodes = list(reversed(shallowutil.sortnodes(
                 (e[0] for e in entries),
                 parentfunc)))
 
             # Write the file section header
             self.writeraw("%s%s%s" % (
                 struct.pack('!H', len(filename)),
                 filename,
                 struct.pack('!I', len(sortednodes)),
             ))
 
             sectionlen = constants.FILENAMESIZE + len(filename) + 4
 
             rawstrings = []
 
             # Record the node locations for the index
             locations = self.entrylocations.setdefault(filename, {})
             offset = sectionstart + sectionlen
             for node in sortednodes:
                 locations[node] = offset
                 raw = '%s%s%s%s%s%s' % entrymap[node]
                 rawstrings.append(raw)
                 offset += len(raw)
 
             rawdata = ''.join(rawstrings)
             sectionlen += len(rawdata)
 
             self.writeraw(rawdata)
 
             # Record metadata for the index
             self.files[filename] = (sectionstart, sectionlen)
             node = hashlib.sha1(filename).digest()
             self.entries[node] = node
 
     def close(self, ledger=None):
         if self._closed:
             return
 
         self._write()
 
         return super(mutablehistorypack, self).close(ledger=ledger)
 
     def createindex(self, nodelocations, indexoffset):
         fileindexformat = self.INDEXFORMAT
         fileindexlength = self.INDEXENTRYLENGTH
         nodeindexformat = self.NODEINDEXFORMAT
         nodeindexlength = self.NODEINDEXENTRYLENGTH
-        version = self.VERSION
 
         files = ((hashlib.sha1(filename).digest(), filename, offset, size)
                 for filename, (offset, size) in self.files.iteritems())
         files = sorted(files)
 
         # node index is after file index size, file index, and node index size
         indexlensize = struct.calcsize('!Q')
         nodeindexoffset = (indexoffset + indexlensize +
                            (len(files) * fileindexlength) + indexlensize)
 
         fileindexentries = []
         nodeindexentries = []
         nodecount = 0
         for namehash, filename, offset, size in files:
             # File section index
-            if version == 0:
-                rawentry = struct.pack(fileindexformat, namehash, offset, size)
-            else:
-                nodelocations = self.entrylocations[filename]
+            nodelocations = self.entrylocations[filename]
 
-                nodeindexsize = len(nodelocations) * nodeindexlength
+            nodeindexsize = len(nodelocations) * nodeindexlength
 
-                rawentry = struct.pack(fileindexformat, namehash, offset, size,
-                                       nodeindexoffset, nodeindexsize)
-                # Node index
-                nodeindexentries.append(struct.pack(constants.FILENAMESTRUCT,
-                                                    len(filename)) + filename)
-                nodeindexoffset += constants.FILENAMESIZE + len(filename)
+            rawentry = struct.pack(fileindexformat, namehash, offset, size,
+                                   nodeindexoffset, nodeindexsize)
+            # Node index
+            nodeindexentries.append(struct.pack(constants.FILENAMESTRUCT,
+                                                len(filename)) + filename)
+            nodeindexoffset += constants.FILENAMESIZE + len(filename)
 
-                for node, location in sorted(nodelocations.iteritems()):
-                    nodeindexentries.append(struct.pack(nodeindexformat, node,
-                                                        location))
-                    nodecount += 1
+            for node, location in sorted(nodelocations.iteritems()):
+                nodeindexentries.append(struct.pack(nodeindexformat, node,
+                                                    location))
+                nodecount += 1
 
-                nodeindexoffset += len(nodelocations) * nodeindexlength
+            nodeindexoffset += len(nodelocations) * nodeindexlength
 
             fileindexentries.append(rawentry)
 
         nodecountraw = ''
-        if version == 1:
-            nodecountraw = struct.pack('!Q', nodecount)
+        nodecountraw = struct.pack('!Q', nodecount)
         return (''.join(fileindexentries) + nodecountraw +
                 ''.join(nodeindexentries))

hgext/remotefilelog/repack.py

@@ -1,784 +1,781 b''
 from __future__ import absolute_import
 
 import os
 import time
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     short,
 )
 from mercurial import (
     encoding,
     error,
     mdiff,
     policy,
     pycompat,
     scmutil,
     util,
     vfs,
 )
 from mercurial.utils import procutil
 from . import (
     constants,
     contentstore,
     datapack,
     extutil,
     historypack,
     metadatastore,
     shallowutil,
 )
 
 osutil = policy.importmod(r'osutil')
 
 class RepackAlreadyRunning(error.Abort):
     pass
 
 if util.safehasattr(util, '_hgexecutable'):
     # Before 5be286db
     _hgexecutable = util.hgexecutable
 else:
     from mercurial.utils import procutil
     _hgexecutable = procutil.hgexecutable
 
 def backgroundrepack(repo, incremental=True, packsonly=False):
     cmd = [_hgexecutable(), '-R', repo.origroot, 'repack']
     msg = _("(running background repack)\n")
     if incremental:
         cmd.append('--incremental')
         msg = _("(running background incremental repack)\n")
     if packsonly:
         cmd.append('--packsonly')
     repo.ui.warn(msg)
     procutil.runbgcommand(cmd, encoding.environ)
 
 def fullrepack(repo, options=None):
     """If ``packsonly`` is True, stores creating only loose objects are skipped.
     """
     if util.safehasattr(repo, 'shareddatastores'):
         datasource = contentstore.unioncontentstore(
             *repo.shareddatastores)
         historysource = metadatastore.unionmetadatastore(
             *repo.sharedhistorystores,
             allowincomplete=True)
 
         packpath = shallowutil.getcachepackpath(
             repo,
             constants.FILEPACK_CATEGORY)
         _runrepack(repo, datasource, historysource, packpath,
                    constants.FILEPACK_CATEGORY, options=options)
 
     if util.safehasattr(repo.manifestlog, 'datastore'):
         localdata, shareddata = _getmanifeststores(repo)
         lpackpath, ldstores, lhstores = localdata
         spackpath, sdstores, shstores = shareddata
 
         # Repack the shared manifest store
         datasource = contentstore.unioncontentstore(*sdstores)
         historysource = metadatastore.unionmetadatastore(
                         *shstores,
                         allowincomplete=True)
         _runrepack(repo, datasource, historysource, spackpath,
                    constants.TREEPACK_CATEGORY, options=options)
 
         # Repack the local manifest store
         datasource = contentstore.unioncontentstore(
                         *ldstores,
                         allowincomplete=True)
         historysource = metadatastore.unionmetadatastore(
                         *lhstores,
                         allowincomplete=True)
         _runrepack(repo, datasource, historysource, lpackpath,
                    constants.TREEPACK_CATEGORY, options=options)
 
 def incrementalrepack(repo, options=None):
     """This repacks the repo by looking at the distribution of pack files in the
     repo and performing the most minimal repack to keep the repo in good shape.
     """
     if util.safehasattr(repo, 'shareddatastores'):
         packpath = shallowutil.getcachepackpath(
             repo,
             constants.FILEPACK_CATEGORY)
         _incrementalrepack(repo,
                            repo.shareddatastores,
                            repo.sharedhistorystores,
                            packpath,
                            constants.FILEPACK_CATEGORY,
                            options=options)
 
     if util.safehasattr(repo.manifestlog, 'datastore'):
         localdata, shareddata = _getmanifeststores(repo)
         lpackpath, ldstores, lhstores = localdata
         spackpath, sdstores, shstores = shareddata
 
         # Repack the shared manifest store
         _incrementalrepack(repo,
                            sdstores,
                            shstores,
                            spackpath,
                            constants.TREEPACK_CATEGORY,
                            options=options)
 
         # Repack the local manifest store
         _incrementalrepack(repo,
                            ldstores,
                            lhstores,
                            lpackpath,
                            constants.TREEPACK_CATEGORY,
                            allowincompletedata=True,
                            options=options)
 
 def _getmanifeststores(repo):
     shareddatastores = repo.manifestlog.shareddatastores
     localdatastores = repo.manifestlog.localdatastores
     sharedhistorystores = repo.manifestlog.sharedhistorystores
     localhistorystores = repo.manifestlog.localhistorystores
 
     sharedpackpath = shallowutil.getcachepackpath(repo,
                                             constants.TREEPACK_CATEGORY)
     localpackpath = shallowutil.getlocalpackpath(repo.svfs.vfs.base,
                                             constants.TREEPACK_CATEGORY)
 
     return ((localpackpath, localdatastores, localhistorystores),
             (sharedpackpath, shareddatastores, sharedhistorystores))
 
 def _topacks(packpath, files, constructor):
     paths = list(os.path.join(packpath, p) for p in files)
     packs = list(constructor(p) for p in paths)
     return packs
 
 def _deletebigpacks(repo, folder, files):
     """Deletes packfiles that are bigger than ``packs.maxpacksize``.
 
     Returns ``files` with the removed files omitted."""
     maxsize = repo.ui.configbytes("packs", "maxpacksize")
     if maxsize <= 0:
         return files
 
     # This only considers datapacks today, but we could broaden it to include
     # historypacks.
     VALIDEXTS = [".datapack", ".dataidx"]
 
     # Either an oversize index or datapack will trigger cleanup of the whole
     # pack:
     oversized = set([os.path.splitext(path)[0] for path, ftype, stat in files
         if (stat.st_size > maxsize and (os.path.splitext(path)[1]
                                         in VALIDEXTS))])
 
     for rootfname in oversized:
         rootpath = os.path.join(folder, rootfname)
         for ext in VALIDEXTS:
             path = rootpath + ext
             repo.ui.debug('removing oversize packfile %s (%s)\n' %
                           (path, util.bytecount(os.stat(path).st_size)))
             os.unlink(path)
     return [row for row in files if os.path.basename(row[0]) not in oversized]
 
 def _incrementalrepack(repo, datastore, historystore, packpath, category,
         allowincompletedata=False, options=None):
     shallowutil.mkstickygroupdir(repo.ui, packpath)
 
     files = osutil.listdir(packpath, stat=True)
     files = _deletebigpacks(repo, packpath, files)
     datapacks = _topacks(packpath,
         _computeincrementaldatapack(repo.ui, files),
         datapack.datapack)
     datapacks.extend(s for s in datastore
                      if not isinstance(s, datapack.datapackstore))
 
     historypacks = _topacks(packpath,
         _computeincrementalhistorypack(repo.ui, files),
         historypack.historypack)
     historypacks.extend(s for s in historystore
                         if not isinstance(s, historypack.historypackstore))
 
     # ``allhistory{files,packs}`` contains all known history packs, even ones we
     # don't plan to repack. They are used during the datapack repack to ensure
     # good ordering of nodes.
     allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX,
                             historypack.INDEXSUFFIX)
     allhistorypacks = _topacks(packpath,
         (f for f, mode, stat in allhistoryfiles),
         historypack.historypack)
     allhistorypacks.extend(s for s in historystore
                         if not isinstance(s, historypack.historypackstore))
     _runrepack(repo,
                contentstore.unioncontentstore(
                    *datapacks,
                    allowincomplete=allowincompletedata),
                metadatastore.unionmetadatastore(
                    *historypacks,
                    allowincomplete=True),
                packpath, category,
                fullhistory=metadatastore.unionmetadatastore(
                    *allhistorypacks,
                    allowincomplete=True),
                 options=options)
 
 def _computeincrementaldatapack(ui, files):
     opts = {
         'gencountlimit' : ui.configint(
             'remotefilelog', 'data.gencountlimit'),
         'generations' : ui.configlist(
             'remotefilelog', 'data.generations'),
         'maxrepackpacks' : ui.configint(
             'remotefilelog', 'data.maxrepackpacks'),
         'repackmaxpacksize' : ui.configbytes(
             'remotefilelog', 'data.repackmaxpacksize'),
         'repacksizelimit' : ui.configbytes(
             'remotefilelog', 'data.repacksizelimit'),
     }
 
     packfiles = _allpackfileswithsuffix(
         files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX)
     return _computeincrementalpack(packfiles, opts)
 
 def _computeincrementalhistorypack(ui, files):
     opts = {
         'gencountlimit' : ui.configint(
             'remotefilelog', 'history.gencountlimit'),
         'generations' : ui.configlist(
             'remotefilelog', 'history.generations', ['100MB']),
         'maxrepackpacks' : ui.configint(
             'remotefilelog', 'history.maxrepackpacks'),
         'repackmaxpacksize' : ui.configbytes(
             'remotefilelog', 'history.repackmaxpacksize', '400MB'),
         'repacksizelimit' : ui.configbytes(
             'remotefilelog', 'history.repacksizelimit'),
     }
 
     packfiles = _allpackfileswithsuffix(
         files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX)
     return _computeincrementalpack(packfiles, opts)
 
 def _allpackfileswithsuffix(files, packsuffix, indexsuffix):
     result = []
     fileset = set(fn for fn, mode, stat in files)
     for filename, mode, stat in files:
         if not filename.endswith(packsuffix):
             continue
 
         prefix = filename[:-len(packsuffix)]
 
         # Don't process a pack if it doesn't have an index.
         if (prefix + indexsuffix) not in fileset:
             continue
         result.append((prefix, mode, stat))
 
     return result
 
 def _computeincrementalpack(files, opts):
     """Given a set of pack files along with the configuration options, this
     function computes the list of files that should be packed as part of an
     incremental repack.
 
     It tries to strike a balance between keeping incremental repacks cheap (i.e.
     packing small things when possible, and rolling the packs up to the big ones
     over time).
     """
 
     limits = list(sorted((util.sizetoint(s) for s in opts['generations']),
                                 reverse=True))
     limits.append(0)
 
     # Group the packs by generation (i.e. by size)
     generations = []
     for i in pycompat.xrange(len(limits)):
         generations.append([])
 
     sizes = {}
     for prefix, mode, stat in files:
         size = stat.st_size
         if size > opts['repackmaxpacksize']:
             continue
 
         sizes[prefix] = size
         for i, limit in enumerate(limits):
             if size > limit:
                 generations[i].append(prefix)
                 break
 
     # Steps for picking what packs to repack:
     # 1. Pick the largest generation with > gencountlimit pack files.
     # 2. Take the smallest three packs.
     # 3. While total-size-of-packs < repacksizelimit: add another pack
 
     # Find the largest generation with more than gencountlimit packs
     genpacks = []
     for i, limit in enumerate(limits):
         if len(generations[i]) > opts['gencountlimit']:
             # Sort to be smallest last, for easy popping later
             genpacks.extend(sorted(generations[i], reverse=True,
                                    key=lambda x: sizes[x]))
             break
 
     # Take as many packs from the generation as we can
     chosenpacks = genpacks[-3:]
     genpacks = genpacks[:-3]
     repacksize = sum(sizes[n] for n in chosenpacks)
     while (repacksize < opts['repacksizelimit'] and genpacks and
            len(chosenpacks) < opts['maxrepackpacks']):
         chosenpacks.append(genpacks.pop())
         repacksize += sizes[chosenpacks[-1]]
 
     return chosenpacks
 
 def _runrepack(repo, data, history, packpath, category, fullhistory=None,
                options=None):
     shallowutil.mkstickygroupdir(repo.ui, packpath)
 
     def isold(repo, filename, node):
         """Check if the file node is older than a limit.
         Unless a limit is specified in the config the default limit is taken.
         """
         filectx = repo.filectx(filename, fileid=node)
         filetime = repo[filectx.linkrev()].date()
 
         ttl = repo.ui.configint('remotefilelog', 'nodettl')
 
         limit = time.time() - ttl
         return filetime[0] < limit
 
     garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')
     if not fullhistory:
         fullhistory = history
     packer = repacker(repo, data, history, fullhistory, category,
                       gc=garbagecollect, isold=isold, options=options)
 
-    # internal config: remotefilelog.datapackversion
-    dv = repo.ui.configint('remotefilelog', 'datapackversion', 0)
-
-    with datapack.mutabledatapack(repo.ui, packpath, version=dv) as dpack:
+    with datapack.mutabledatapack(repo.ui, packpath, version=2) as dpack:
         with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
             try:
                 packer.run(dpack, hpack)
             except error.LockHeld:
                 raise RepackAlreadyRunning(_("skipping repack - another repack "
                                              "is already running"))
 
 def keepset(repo, keyfn, lastkeepkeys=None):
     """Computes a keepset which is not garbage collected.
     'keyfn' is a function that maps filename, node to a unique key.
     'lastkeepkeys' is an optional argument and if provided the keepset
     function updates lastkeepkeys with more keys and returns the result.
     """
     if not lastkeepkeys:
         keepkeys = set()
     else:
         keepkeys = lastkeepkeys
 
     # We want to keep:
     # 1. Working copy parent
     # 2. Draft commits
     # 3. Parents of draft commits
     # 4. Pullprefetch and bgprefetchrevs revsets if specified
     revs = ['.', 'draft()', 'parents(draft())']
     prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)
     if prefetchrevs:
         revs.append('(%s)' % prefetchrevs)
     prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)
     if prefetchrevs:
         revs.append('(%s)' % prefetchrevs)
     revs = '+'.join(revs)
 
     revs = ['sort((%s), "topo")' % revs]
     keep = scmutil.revrange(repo, revs)
 
     processed = set()
     lastmanifest = None
 
     # process the commits in toposorted order starting from the oldest
     for r in reversed(keep._list):
         if repo[r].p1().rev() in processed:
             # if the direct parent has already been processed
             # then we only need to process the delta
             m = repo[r].manifestctx().readdelta()
         else:
             # otherwise take the manifest and diff it
             # with the previous manifest if one exists
             if lastmanifest:
                 m = repo[r].manifest().diff(lastmanifest)
             else:
                 m = repo[r].manifest()
         lastmanifest = repo[r].manifest()
         processed.add(r)
 
         # populate keepkeys with keys from the current manifest
         if type(m) is dict:
             # m is a result of diff of two manifests and is a dictionary that
             # maps filename to ((newnode, newflag), (oldnode, oldflag)) tuple
             for filename, diff in m.iteritems():
                 if diff[0][0] is not None:
                     keepkeys.add(keyfn(filename, diff[0][0]))
         else:
             # m is a manifest object
             for filename, filenode in m.iteritems():
                 keepkeys.add(keyfn(filename, filenode))
 
     return keepkeys
 
 class repacker(object):
     """Class for orchestrating the repack of data and history information into a
     new format.
     """
     def __init__(self, repo, data, history, fullhistory, category, gc=False,
                  isold=None, options=None):
         self.repo = repo
         self.data = data
         self.history = history
         self.fullhistory = fullhistory
         self.unit = constants.getunits(category)
         self.garbagecollect = gc
         self.options = options
         if self.garbagecollect:
             if not isold:
                 raise ValueError("Function 'isold' is not properly specified")
             # use (filename, node) tuple as a keepset key
             self.keepkeys = keepset(repo, lambda f, n : (f, n))
             self.isold = isold
 
     def run(self, targetdata, targethistory):
         ledger = repackledger()
 
         with extutil.flock(repacklockvfs(self.repo).join("repacklock"),
                            _('repacking %s') % self.repo.origroot, timeout=0):
             self.repo.hook('prerepack')
 
             # Populate ledger from source
             self.data.markledger(ledger, options=self.options)
             self.history.markledger(ledger, options=self.options)
 
             # Run repack
             self.repackdata(ledger, targetdata)
             self.repackhistory(ledger, targethistory)
 
             # Call cleanup on each source
             for source in ledger.sources:
                 source.cleanup(ledger)
 
     def _chainorphans(self, ui, filename, nodes, orphans, deltabases):
         """Reorderes ``orphans`` into a single chain inside ``nodes`` and
         ``deltabases``.
 
         We often have orphan entries (nodes without a base that aren't
         referenced by other nodes -- i.e., part of a chain) due to gaps in
         history. Rather than store them as individual fulltexts, we prefer to
         insert them as one chain sorted by size.
         """
         if not orphans:
             return nodes
 
         def getsize(node, default=0):
             meta = self.data.getmeta(filename, node)
             if constants.METAKEYSIZE in meta:
                 return meta[constants.METAKEYSIZE]
             else:
                 return default
 
         # Sort orphans by size; biggest first is preferred, since it's more
         # likely to be the newest version assuming files grow over time.
         # (Sort by node first to ensure the sort is stable.)
         orphans = sorted(orphans)
         orphans = list(sorted(orphans, key=getsize, reverse=True))
         if ui.debugflag:
             ui.debug("%s: orphan chain: %s\n" % (filename,
                 ", ".join([short(s) for s in orphans])))
 
         # Create one contiguous chain and reassign deltabases.
         for i, node in enumerate(orphans):
             if i == 0:
                 deltabases[node] = (nullid, 0)
             else:
                 parent = orphans[i - 1]
                 deltabases[node] = (parent, deltabases[parent][1] + 1)
         nodes = filter(lambda node: node not in orphans, nodes)
         nodes += orphans
         return nodes
 
     def repackdata(self, ledger, target):
         ui = self.repo.ui
         maxchainlen = ui.configint('packs', 'maxchainlen', 1000)
 
         byfile = {}
         for entry in ledger.entries.itervalues():
             if entry.datasource:
                 byfile.setdefault(entry.filename, {})[entry.node] = entry
 
         count = 0
         for filename, entries in sorted(byfile.iteritems()):
             ui.progress(_("repacking data"), count, unit=self.unit,
                         total=len(byfile))
 
             ancestors = {}
             nodes = list(node for node in entries.iterkeys())
             nohistory = []
             for i, node in enumerate(nodes):
                 if node in ancestors:
                     continue
                 ui.progress(_("building history"), i, unit='nodes',
                             total=len(nodes))
                 try:
                     ancestors.update(self.fullhistory.getancestors(filename,
                         node, known=ancestors))
                 except KeyError:
                     # Since we're packing data entries, we may not have the
                     # corresponding history entries for them. It's not a big
                     # deal, but the entries won't be delta'd perfectly.
                     nohistory.append(node)
             ui.progress(_("building history"), None)
 
             # Order the nodes children first, so we can produce reverse deltas
             orderednodes = list(reversed(self._toposort(ancestors)))
             if len(nohistory) > 0:
                 ui.debug('repackdata: %d nodes without history\n' %
                          len(nohistory))
             orderednodes.extend(sorted(nohistory))
 
             # Filter orderednodes to just the nodes we want to serialize (it
             # currently also has the edge nodes' ancestors).
             orderednodes = filter(lambda node: node in nodes, orderednodes)
 
             # Garbage collect old nodes:
             if self.garbagecollect:
                 neworderednodes = []
                 for node in orderednodes:
                     # If the node is old and is not in the keepset, we skip it,
                     # and mark as garbage collected
                     if ((filename, node) not in self.keepkeys and
                         self.isold(self.repo, filename, node)):
                         entries[node].gced = True
                         continue
                     neworderednodes.append(node)
                 orderednodes = neworderednodes
 
             # Compute delta bases for nodes:
             deltabases = {}
             nobase = set()
             referenced = set()
             nodes = set(nodes)
             for i, node in enumerate(orderednodes):
                 ui.progress(_("processing nodes"), i, unit='nodes',
                             total=len(orderednodes))
                 # Find delta base
                 # TODO: allow delta'ing against most recent descendant instead
                 # of immediate child
                 deltatuple = deltabases.get(node, None)
                 if deltatuple is None:
                     deltabase, chainlen = nullid, 0
                     deltabases[node] = (nullid, 0)
                     nobase.add(node)
                 else:
                     deltabase, chainlen = deltatuple
                     referenced.add(deltabase)
 
                 # Use available ancestor information to inform our delta choices
                 ancestorinfo = ancestors.get(node)
                 if ancestorinfo:
                     p1, p2, linknode, copyfrom = ancestorinfo
 
                     # The presence of copyfrom means we're at a point where the
                     # file was copied from elsewhere. So don't attempt to do any
                     # deltas with the other file.
                     if copyfrom:
                         p1 = nullid
 
                     if chainlen < maxchainlen:
                         # Record this child as the delta base for its parents.
                         # This may be non optimal, since the parents may have
                         # many children, and this will only choose the last one.
                         # TODO: record all children and try all deltas to find
                         # best
                         if p1 != nullid:
                             deltabases[p1] = (node, chainlen + 1)
                         if p2 != nullid:
                             deltabases[p2] = (node, chainlen + 1)
 
             # experimental config: repack.chainorphansbysize
             if ui.configbool('repack', 'chainorphansbysize'):
                 orphans = nobase - referenced
                 orderednodes = self._chainorphans(ui, filename, orderednodes,
                     orphans, deltabases)
 
             # Compute deltas and write to the pack
             for i, node in enumerate(orderednodes):
                 deltabase, chainlen = deltabases[node]
                 # Compute delta
                 # TODO: Optimize the deltachain fetching. Since we're
                 # iterating over the different version of the file, we may
                 # be fetching the same deltachain over and over again.
                 meta = None
                 if deltabase != nullid:
                     deltaentry = self.data.getdelta(filename, node)
                     delta, deltabasename, origdeltabase, meta = deltaentry
                     size = meta.get(constants.METAKEYSIZE)
                     if (deltabasename != filename or origdeltabase != deltabase
                         or size is None):
                         deltabasetext = self.data.get(filename, deltabase)
                         original = self.data.get(filename, node)
                         size = len(original)
                         delta = mdiff.textdiff(deltabasetext, original)
                 else:
                     delta = self.data.get(filename, node)
                     size = len(delta)
                     meta = self.data.getmeta(filename, node)
 
                 # TODO: don't use the delta if it's larger than the fulltext
                 if constants.METAKEYSIZE not in meta:
                     meta[constants.METAKEYSIZE] = size
                 target.add(filename, node, deltabase, delta, meta)
 
                 entries[node].datarepacked = True
 
             ui.progress(_("processing nodes"), None)
             count += 1
 
         ui.progress(_("repacking data"), None)
         target.close(ledger=ledger)
 
     def repackhistory(self, ledger, target):
         ui = self.repo.ui
 
         byfile = {}
         for entry in ledger.entries.itervalues():
             if entry.historysource:
                 byfile.setdefault(entry.filename, {})[entry.node] = entry
 
         count = 0
         for filename, entries in sorted(byfile.iteritems()):
             ancestors = {}
             nodes = list(node for node in entries.iterkeys())
 
             for node in nodes:
                 if node in ancestors:
                     continue
                 ancestors.update(self.history.getancestors(filename, node,
                                                            known=ancestors))
 
             # Order the nodes children first
             orderednodes = reversed(self._toposort(ancestors))
 
             # Write to the pack
             dontprocess = set()
             for node in orderednodes:
                 p1, p2, linknode, copyfrom = ancestors[node]
 
                 # If the node is marked dontprocess, but it's also in the
                 # explicit entries set, that means the node exists both in this
                 # file and in another file that was copied to this file.
                 # Usually this happens if the file was copied to another file,
                 # then the copy was deleted, then reintroduced without copy
                 # metadata. The original add and the new add have the same hash
                 # since the content is identical and the parents are null.
                 if node in dontprocess and node not in entries:
                     # If copyfrom == filename, it means the copy history
                     # went to come other file, then came back to this one, so we
                     # should continue processing it.
                     if p1 != nullid and copyfrom != filename:
                         dontprocess.add(p1)
                     if p2 != nullid:
                         dontprocess.add(p2)
                     continue
 
                 if copyfrom:
                     dontprocess.add(p1)
 
                 target.add(filename, node, p1, p2, linknode, copyfrom)
 
                 if node in entries:
                     entries[node].historyrepacked = True
 
             count += 1
             ui.progress(_("repacking history"), count, unit=self.unit,
                         total=len(byfile))
 
         ui.progress(_("repacking history"), None)
         target.close(ledger=ledger)
 
     def _toposort(self, ancestors):
         def parentfunc(node):
             p1, p2, linknode, copyfrom = ancestors[node]
             parents = []
             if p1 != nullid:
                 parents.append(p1)
             if p2 != nullid:
                 parents.append(p2)
             return parents
 
         sortednodes = shallowutil.sortnodes(ancestors.keys(), parentfunc)
         return sortednodes
 
 class repackledger(object):
     """Storage for all the bookkeeping that happens during a repack. It contains
     the list of revisions being repacked, what happened to each revision, and
     which source store contained which revision originally (for later cleanup).
     """
     def __init__(self):
         self.entries = {}
         self.sources = {}
         self.created = set()
 
     def markdataentry(self, source, filename, node):
         """Mark the given filename+node revision as having a data rev in the
         given source.
         """
         entry = self._getorcreateentry(filename, node)
         entry.datasource = True
         entries = self.sources.get(source)
         if not entries:
             entries = set()
             self.sources[source] = entries
         entries.add(entry)
 
     def markhistoryentry(self, source, filename, node):
         """Mark the given filename+node revision as having a history rev in the
         given source.
         """
         entry = self._getorcreateentry(filename, node)
         entry.historysource = True
         entries = self.sources.get(source)
         if not entries:
             entries = set()
             self.sources[source] = entries
         entries.add(entry)
 
     def _getorcreateentry(self, filename, node):
         key = (filename, node)
         value = self.entries.get(key)
         if not value:
             value = repackentry(filename, node)
             self.entries[key] = value
 
         return value
 
     def addcreated(self, value):
         self.created.add(value)
 
 class repackentry(object):
     """Simple class representing a single revision entry in the repackledger.
     """
     __slots__ = ['filename', 'node', 'datasource', 'historysource',
                  'datarepacked', 'historyrepacked', 'gced']
     def __init__(self, filename, node):
         self.filename = filename
         self.node = node
         # If the revision has a data entry in the source
         self.datasource = False
         # If the revision has a history entry in the source
         self.historysource = False
         # If the revision's data entry was repacked into the repack target
         self.datarepacked = False
         # If the revision's history entry was repacked into the repack target
         self.historyrepacked = False
         # If garbage collected
         self.gced = False
 
 def repacklockvfs(repo):
     if util.safehasattr(repo, 'name'):
         # Lock in the shared cache so repacks across multiple copies of the same
         # repo are coordinated.
         sharedcachepath = shallowutil.getcachepackpath(
             repo,
             constants.FILEPACK_CATEGORY)
         return vfs.vfs(sharedcachepath)
     else:
         return repo.svfs

tests/test-remotefilelog-bgprefetch.t

@@ -1,370 +1,370 b''
   $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
   $ export PYTHONPATH
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ hginit master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > EOF
   $ echo x > x
   $ echo z > z
   $ hg commit -qAm x
   $ echo x2 > x
   $ echo y > y
   $ hg commit -qAm y
   $ echo w > w
   $ rm z
   $ hg commit -qAm w
   $ hg bookmark foo
 
   $ cd ..
 
 # clone the repo
 
   $ hgcloneshallow ssh://user@dummy/master shallow --noupdate
   streaming all changes
   2 files to transfer, 776 bytes of data
   transferred 776 bytes in * seconds (*/sec) (glob)
   searching for changes
   no changes found
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is. Also set prefetchdelay config
 # to zero so that there is no delay between prefetches.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > prefetchdelay=0
   > EOF
   $ cd ..
 
 # prefetch a revision
   $ cd shallow
 
   $ hg prefetch -r 0
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
 
   $ hg cat -r 0 x
   x
 
 # background prefetch on pull when configured
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > pullprefetch=bookmark()
   > backgroundprefetch=True
   > EOF
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/6b4b6f66ef8c-b4b8bdaf-backup.hg (glob)
 
   $ clearcache
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 0 changes to 0 files
   updating bookmark foo
   new changesets 6b4b6f66ef8c
   (run 'hg update' to get a working copy)
   prefetching file contents
   $ sleep 0.5
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/ef95c5376f34698742fe34f315fd82136f8f68c0
   $TESTTMP/hgcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
   $TESTTMP/hgcache/master/af/f024fe4ab0fece4091de044c58c9ae4233383a/bb6ccd5dceaa5e9dc220e0dad65e051b94f69a2c
   $TESTTMP/hgcache/repos
 
 # background prefetch with repack on pull when configured
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > backgroundrepack=True
   > EOF
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/6b4b6f66ef8c-b4b8bdaf-backup.hg (glob)
 
   $ clearcache
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 0 changes to 0 files
   updating bookmark foo
   new changesets 6b4b6f66ef8c
   (run 'hg update' to get a working copy)
   prefetching file contents
   $ sleep 0.5
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/94d53eef9e622533aec1fc6d8053cb086e785d21.histidx
-  $TESTTMP/hgcache/master/packs/94d53eef9e622533aec1fc6d8053cb086e785d21.histpack
-  $TESTTMP/hgcache/master/packs/f3644bc7773e8289deda7f765138120c838f4e6e.dataidx
-  $TESTTMP/hgcache/master/packs/f3644bc7773e8289deda7f765138120c838f4e6e.datapack
+  $TESTTMP/hgcache/master/packs/3616094d229ed39e2593f79c772676d4ec00253a.dataidx
+  $TESTTMP/hgcache/master/packs/3616094d229ed39e2593f79c772676d4ec00253a.datapack
+  $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx
+  $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # background prefetch with repack on update when wcprevset configured
 
   $ clearcache
   $ hg up -r 0
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $TESTTMP/hgcache/master/39/5df8f7c51f007019cb30201c49e884b46b92fa/69a1b67522704ec122181c0890bd16e9d3e7516a
   $TESTTMP/hgcache/repos
 
   $ hg up -r 1
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=.::
   > EOF
 
   $ clearcache
   $ hg up -r 0
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   * files fetched over * fetches - (* misses, 0.00% hit ratio) over *s (glob)
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 1
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that file 'w' was prefetched - it was not part of the update operation and therefore
 # could only be downloaded by the background prefetch
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0:
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # background prefetch with repack on commit when wcprevset configured
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=0::
   > EOF
 
   $ clearcache
   $ find $CACHEDIR -type f | sort
   $ echo b > b
   $ hg commit -qAm b
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob)
   $ hg bookmark temporary
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 1
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore
 # could only be downloaded by the background prefetch
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0:
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # background prefetch with repack on rebase when wcprevset configured
 
   $ hg up -r 2
   3 files updated, 0 files merged, 3 files removed, 0 files unresolved
   (leaving bookmark temporary)
   $ clearcache
   $ find $CACHEDIR -type f | sort
   $ hg rebase -s temporary -d foo
   rebasing 3:58147a5b5242 "b" (temporary tip)
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg (glob)
   3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 1
   $ hg debugwaitonrepack >/dev/null 2>%1
 
 # Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore
 # could only be downloaded by the background prefetch
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0:
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # Check that foregound prefetch with no arguments blocks until background prefetches finish
 
   $ hg up -r 3
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ clearcache
   $ hg prefetch --repack
   waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?)
   got lock after * seconds (glob) (?)
   (running background incremental repack)
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
 
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that files were prefetched
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0:
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # Check that foreground prefetch fetches revs specified by '. + draft() + bgprefetchrevs + pullprefetch'
 
   $ clearcache
   $ hg prefetch --repack
   waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?)
   got lock after * seconds (glob) (?)
   (running background incremental repack)
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx
-  $TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
+  $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that files were prefetched
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0:
+  $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # Test that if data was prefetched and repacked we dont need to prefetch it again
 # It ensures that Mercurial looks not only in loose files but in packs as well
 
   $ hg prefetch --repack
   (running background incremental repack)

tests/test-remotefilelog-datapack.py

@@ -1,388 +1,375 b''
 #!/usr/bin/env python
 from __future__ import absolute_import, print_function
 
 import hashlib
 import os
 import random
 import shutil
 import stat
 import struct
 import sys
 import tempfile
 import time
 import unittest
 
 import silenttestrunner
 
 # Load the local remotefilelog, not the system one
 sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')]
 from mercurial.node import nullid
 from mercurial import (
     ui as uimod,
 )
 from hgext.remotefilelog import (
     basepack,
     constants,
     datapack,
 )
 
 class datapacktestsbase(object):
     def __init__(self, datapackreader, paramsavailable):
         self.datapackreader = datapackreader
         self.paramsavailable = paramsavailable
 
     def setUp(self):
         self.tempdirs = []
 
     def tearDown(self):
         for d in self.tempdirs:
             shutil.rmtree(d)
 
     def makeTempDir(self):
         tempdir = tempfile.mkdtemp()
         self.tempdirs.append(tempdir)
         return tempdir
 
     def getHash(self, content):
         return hashlib.sha1(content).digest()
 
     def getFakeHash(self):
         return ''.join(chr(random.randint(0, 255)) for _ in range(20))
 
-    def createPack(self, revisions=None, packdir=None, version=0):
+    def createPack(self, revisions=None, packdir=None):
         if revisions is None:
             revisions = [("filename", self.getFakeHash(), nullid, "content")]
 
         if packdir is None:
             packdir = self.makeTempDir()
 
-        packer = datapack.mutabledatapack(
-            uimod.ui(), packdir, version=version)
+        packer = datapack.mutabledatapack(uimod.ui(), packdir, version=2)
 
         for args in revisions:
             filename, node, base, content = args[0:4]
             # meta is optional
             meta = None
             if len(args) > 4:
                 meta = args[4]
             packer.add(filename, node, base, content, metadata=meta)
 
         path = packer.close()
         return self.datapackreader(path)
 
     def _testAddSingle(self, content):
         """Test putting a simple blob into a pack and reading it out.
         """
         filename = "foo"
         node = self.getHash(content)
 
         revisions = [(filename, node, nullid, content)]
         pack = self.createPack(revisions)
         if self.paramsavailable:
             self.assertEquals(pack.params.fanoutprefix,
                               basepack.SMALLFANOUTPREFIX)
 
         chain = pack.getdeltachain(filename, node)
         self.assertEquals(content, chain[0][4])
 
     def testAddSingle(self):
         self._testAddSingle('')
 
     def testAddSingleEmpty(self):
         self._testAddSingle('abcdef')
 
     def testAddMultiple(self):
         """Test putting multiple unrelated blobs into a pack and reading them
         out.
         """
         revisions = []
         for i in range(10):
             filename = "foo%s" % i
             content = "abcdef%s" % i
             node = self.getHash(content)
             revisions.append((filename, node, self.getFakeHash(), content))
 
         pack = self.createPack(revisions)
 
         for filename, node, base, content in revisions:
             entry = pack.getdelta(filename, node)
             self.assertEquals((content, filename, base, {}), entry)
 
             chain = pack.getdeltachain(filename, node)
             self.assertEquals(content, chain[0][4])
 
     def testAddDeltas(self):
         """Test putting multiple delta blobs into a pack and read the chain.
         """
         revisions = []
         filename = "foo"
         lastnode = nullid
         for i in range(10):
             content = "abcdef%s" % i
             node = self.getHash(content)
             revisions.append((filename, node, lastnode, content))
             lastnode = node
 
         pack = self.createPack(revisions)
 
         entry = pack.getdelta(filename, revisions[0][1])
         realvalue = (revisions[0][3], filename, revisions[0][2], {})
         self.assertEquals(entry, realvalue)
 
         # Test that the chain for the final entry has all the others
         chain = pack.getdeltachain(filename, node)
         for i in range(10):
             content = "abcdef%s" % i
             self.assertEquals(content, chain[-i - 1][4])
 
     def testPackMany(self):
         """Pack many related and unrelated objects.
         """
         # Build a random pack file
         revisions = []
         blobs = {}
         random.seed(0)
         for i in range(100):
             filename = "filename-%s" % i
             filerevs = []
             for j in range(random.randint(1, 100)):
                 content = "content-%s" % j
                 node = self.getHash(content)
                 lastnode = nullid
                 if len(filerevs) > 0:
                     lastnode = filerevs[random.randint(0, len(filerevs) - 1)]
                 filerevs.append(node)
                 blobs[(filename, node, lastnode)] = content
                 revisions.append((filename, node, lastnode, content))
 
         pack = self.createPack(revisions)
 
         # Verify the pack contents
         for (filename, node, lastnode), content in sorted(blobs.iteritems()):
             chain = pack.getdeltachain(filename, node)
             for entry in chain:
                 expectedcontent = blobs[(entry[0], entry[1], entry[3])]
                 self.assertEquals(entry[4], expectedcontent)
 
     def testPackMetadata(self):
         revisions = []
         for i in range(100):
             filename = '%s.txt' % i
             content = 'put-something-here \n' * i
             node = self.getHash(content)
             meta = {constants.METAKEYFLAG: i ** 4,
                     constants.METAKEYSIZE: len(content),
                     'Z': 'random_string',
                     '_': '\0' * i}
             revisions.append((filename, node, nullid, content, meta))
-        pack = self.createPack(revisions, version=1)
+        pack = self.createPack(revisions)
         for name, node, x, content, origmeta in revisions:
             parsedmeta = pack.getmeta(name, node)
             # flag == 0 should be optimized out
             if origmeta[constants.METAKEYFLAG] == 0:
                 del origmeta[constants.METAKEYFLAG]
             self.assertEquals(parsedmeta, origmeta)
 
-    def testPackMetadataThrows(self):
-        filename = '1'
-        content = '2'
-        node = self.getHash(content)
-        meta = {constants.METAKEYFLAG: 3}
-        revisions = [(filename, node, nullid, content, meta)]
-        try:
-            self.createPack(revisions, version=0)
-            self.assertTrue(False, "should throw if metadata is not supported")
-        except RuntimeError:
-            pass
-
     def testGetMissing(self):
         """Test the getmissing() api.
         """
         revisions = []
         filename = "foo"
         lastnode = nullid
         for i in range(10):
             content = "abcdef%s" % i
             node = self.getHash(content)
             revisions.append((filename, node, lastnode, content))
             lastnode = node
 
         pack = self.createPack(revisions)
 
         missing = pack.getmissing([("foo", revisions[0][1])])
         self.assertFalse(missing)
 
         missing = pack.getmissing([("foo", revisions[0][1]),
                                    ("foo", revisions[1][1])])
         self.assertFalse(missing)
 
         fakenode = self.getFakeHash()
         missing = pack.getmissing([("foo", revisions[0][1]), ("foo", fakenode)])
         self.assertEquals(missing, [("foo", fakenode)])
 
     def testAddThrows(self):
         pack = self.createPack()
 
         try:
             pack.add('filename', nullid, 'contents')
             self.assertTrue(False, "datapack.add should throw")
         except RuntimeError:
             pass
 
     def testBadVersionThrows(self):
         pack = self.createPack()
         path = pack.path + '.datapack'
         with open(path) as f:
             raw = f.read()
         raw = struct.pack('!B', 255) + raw[1:]
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
         with open(path, 'w+') as f:
             f.write(raw)
 
         try:
             pack = self.datapackreader(pack.path)
             self.assertTrue(False, "bad version number should have thrown")
         except RuntimeError:
             pass
 
     def testMissingDeltabase(self):
         fakenode = self.getFakeHash()
         revisions = [("filename", fakenode, self.getFakeHash(), "content")]
         pack = self.createPack(revisions)
         chain = pack.getdeltachain("filename", fakenode)
         self.assertEquals(len(chain), 1)
 
     def testLargePack(self):
         """Test creating and reading from a large pack with over X entries.
         This causes it to use a 2^16 fanout table instead."""
         revisions = []
         blobs = {}
         total = basepack.SMALLFANOUTCUTOFF + 1
         for i in xrange(total):
             filename = "filename-%s" % i
             content = filename
             node = self.getHash(content)
             blobs[(filename, node)] = content
             revisions.append((filename, node, nullid, content))
 
         pack = self.createPack(revisions)
         if self.paramsavailable:
             self.assertEquals(pack.params.fanoutprefix,
                               basepack.LARGEFANOUTPREFIX)
 
         for (filename, node), content in blobs.iteritems():
             actualcontent = pack.getdeltachain(filename, node)[0][4]
             self.assertEquals(actualcontent, content)
 
     def testPacksCache(self):
         """Test that we remember the most recent packs while fetching the delta
         chain."""
 
         packdir = self.makeTempDir()
         deltachains = []
 
         numpacks = 10
         revisionsperpack = 100
 
         for i in range(numpacks):
             chain = []
             revision = (str(i), self.getFakeHash(), nullid, "content")
 
             for _ in range(revisionsperpack):
                 chain.append(revision)
                 revision = (
                     str(i),
                     self.getFakeHash(),
                     revision[1],
                     self.getFakeHash()
                 )
 
             self.createPack(chain, packdir)
             deltachains.append(chain)
 
         class testdatapackstore(datapack.datapackstore):
             # Ensures that we are not keeping everything in the cache.
             DEFAULTCACHESIZE = numpacks / 2
 
         store = testdatapackstore(uimod.ui(), packdir)
 
         random.shuffle(deltachains)
         for randomchain in deltachains:
             revision = random.choice(randomchain)
             chain = store.getdeltachain(revision[0], revision[1])
 
             mostrecentpack = next(iter(store.packs), None)
             self.assertEquals(
                 mostrecentpack.getdeltachain(revision[0], revision[1]),
                 chain
             )
 
             self.assertEquals(randomchain.index(revision) + 1, len(chain))
 
     # perf test off by default since it's slow
     def _testIndexPerf(self):
         random.seed(0)
         print("Multi-get perf test")
         packsizes = [
             100,
             10000,
             100000,
             500000,
             1000000,
             3000000,
         ]
         lookupsizes = [
             10,
             100,
             1000,
             10000,
             100000,
             1000000,
         ]
         for packsize in packsizes:
             revisions = []
             for i in xrange(packsize):
                 filename = "filename-%s" % i
                 content = "content-%s" % i
                 node = self.getHash(content)
                 revisions.append((filename, node, nullid, content))
 
             path = self.createPack(revisions).path
 
             # Perf of large multi-get
             import gc
             gc.disable()
             pack = self.datapackreader(path)
             for lookupsize in lookupsizes:
                 if lookupsize > packsize:
                     continue
                 random.shuffle(revisions)
                 findnodes = [(rev[0], rev[1]) for rev in revisions]
 
                 start = time.time()
                 pack.getmissing(findnodes[:lookupsize])
                 elapsed = time.time() - start
                 print ("%s pack %s lookups = %0.04f" %
                        (('%s' % packsize).rjust(7),
                         ('%s' % lookupsize).rjust(7),
                         elapsed))
 
             print("")
             gc.enable()
 
         # The perf test is meant to produce output, so we always fail the test
         # so the user sees the output.
         raise RuntimeError("perf test always fails")
 
 class datapacktests(datapacktestsbase, unittest.TestCase):
     def __init__(self, *args, **kwargs):
         datapacktestsbase.__init__(self, datapack.datapack, True)
         unittest.TestCase.__init__(self, *args, **kwargs)
 
 # TODO:
 # datapack store:
 # - getmissing
 # - GC two packs into one
 
 if __name__ == '__main__':
     silenttestrunner.main(__name__)

tests/test-remotefilelog-gc.t

@@ -1,113 +1,113 b''
   $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
   $ export PYTHONPATH
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ hginit master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > serverexpiration=-1
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # commit a new version of x so we can gc the old one
 
   $ cd master
   $ echo y > x
   $ hg commit -qAm y
   $ cd ..
 
   $ cd shallow
   $ hg pull -q
   $ hg update -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
   $ cd ..
 
 # gc client cache
 
   $ lastweek=`$PYTHON -c 'import datetime,time; print(datetime.datetime.fromtimestamp(time.time() - (86400 * 7)).strftime("%y%m%d%H%M"))'`
   $ find $CACHEDIR -type f -exec touch -t $lastweek {} \;
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0 (glob)
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $TESTTMP/hgcache/repos (glob)
   $ hg gc
   finished: removed 1 of 2 files (0.00 GB to 0.00 GB)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $TESTTMP/hgcache/repos
 
 # gc server cache
 
   $ find master/.hg/remotefilelogcache -type f | sort
   master/.hg/remotefilelogcache/x/1406e74118627694268417491f018a4a883152f0 (glob)
   master/.hg/remotefilelogcache/x/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $ hg gc master
   finished: removed 0 of 1 files (0.00 GB to 0.00 GB)
   $ find master/.hg/remotefilelogcache -type f | sort
   master/.hg/remotefilelogcache/x/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
 
 # Test that GC keepset includes pullprefetch revset if it is configured
 
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > pullprefetch=all()
   > EOF
   $ hg prefetch
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
   $ cd ..
   $ hg gc
   finished: removed 0 of 2 files (0.00 GB to 0.00 GB)
 
 # Ensure that there are 2 versions of the file in cache
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0 (glob)
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $TESTTMP/hgcache/repos (glob)
 
 # Test that if garbage collection on repack and repack on hg gc flags are set then incremental repack with garbage collector is run
 
   $ hg gc --config remotefilelog.gcrepack=True --config remotefilelog.repackonhggc=True
 
 # Ensure that loose files are repacked
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/8d3499c65d926e4f107cf03c6b0df833222025b4.histidx
-  $TESTTMP/hgcache/master/packs/8d3499c65d926e4f107cf03c6b0df833222025b4.histpack
-  $TESTTMP/hgcache/master/packs/9c7046f8cad0417c39aa7c03ce13e0ba991306c2.dataidx
-  $TESTTMP/hgcache/master/packs/9c7046f8cad0417c39aa7c03ce13e0ba991306c2.datapack
+  $TESTTMP/hgcache/master/packs/173691d550fabb9d33db8da192f1c9bc62dd11a4.dataidx
+  $TESTTMP/hgcache/master/packs/173691d550fabb9d33db8da192f1c9bc62dd11a4.datapack
+  $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histidx
+  $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that warning is displayed when there are no valid repos in repofile
 
   $ cp $CACHEDIR/repos $CACHEDIR/repos.bak
   $ echo " " > $CACHEDIR/repos
   $ hg gc
   warning: no valid repos in repofile
   $ mv $CACHEDIR/repos.bak $CACHEDIR/repos
 
 # Test that warning is displayed when the repo path is malformed
 
   $ printf "asdas\0das" >> $CACHEDIR/repos
   $ hg gc 2>&1 | head -n2
   warning: malformed path: * (glob)
   Traceback (most recent call last):

tests/test-remotefilelog-gcrepack.t

@@ -1,160 +1,160 b''
   $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
   $ export PYTHONPATH
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ hginit master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ echo y > y
   $ rm x
   $ hg commit -qAm DxAy
   $ echo yy > y
   $ hg commit -qAm y
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # Prefetch all data and repack
 
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=all()
   > EOF
 
   $ hg prefetch
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ hg repack
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
-  $TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack
-  $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack
+  $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
+  $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1.datapack
 
 # Ensure that all file versions were prefetched
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4:
+  $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1406e7411862  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   50dbc4572b8e  000000000000  3             3
   076f5e2225b3  50dbc4572b8e  14            2
   
   Total:                      17            5         (240.0% bigger)
 
 # Test garbage collection during repack
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=tip
   > gcrepack=True
   > nodettl=86400
   > EOF
 
   $ hg repack
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
-  $TESTTMP/hgcache/master/packs/05baa499c6b07f2bf0ea3d2c8151da1cb86f5e33.datapack
-  $TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack
+  $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
+  $TESTTMP/hgcache/master/packs/b868298fad9bf477b4e9d9455226c440b0135fe6.datapack
 
 # Ensure that file 'x' was garbage collected. It should be GCed because it is not in the keepset
 # and is old (commit date is 0.0 in tests). Ensure that file 'y' is present as it is in the keepset.
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/05baa499c6b07f2bf0ea3d2c8151da1cb86f5e33:
+  $TESTTMP/hgcache/master/packs/b868298fad9bf477b4e9d9455226c440b0135fe6:
   y:
   Node          Delta Base    Delta Length  Blob Size
   50dbc4572b8e  000000000000  3             3
   
   Total:                      3             3         (0.0% bigger)
 
 # Prefetch all data again and repack for later garbage collection
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=all()
   > EOF
 
   $ hg prefetch
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ hg repack
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
-  $TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack
-  $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack
+  $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
+  $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1.datapack
 
 # Ensure that all file versions were prefetched
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4:
+  $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1406e7411862  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   50dbc4572b8e  000000000000  3             3
   076f5e2225b3  50dbc4572b8e  14            2
   
   Total:                      17            5         (240.0% bigger)
 
 # Test garbage collection during repack. Ensure that new files are not removed even though they are not in the keepset
 # For the purposes of the test the TTL of a file is set to current time + 100 seconds. i.e. all commits in tests have
 # a date of 1970 and therefore to prevent garbage collection we have to set nodettl to be farther from 1970 than we are now.
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=
   > nodettl=$(($(date +%s) + 100))
   > EOF
 
   $ hg repack
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
-  $TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack
-  $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack
+  $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
+  $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1.datapack
 
 # Ensure that all file versions were prefetched
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
-  $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4:
+  $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1406e7411862  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   50dbc4572b8e  000000000000  3             3
   076f5e2225b3  50dbc4572b8e  14            2
   
   Total:                      17            5         (240.0% bigger)

tests/test-remotefilelog-histpack.py

@@ -1,274 +1,274 b''
 #!/usr/bin/env python
 from __future__ import absolute_import
 
 import hashlib
 import os
 import random
 import shutil
 import stat
 import struct
 import sys
 import tempfile
 import unittest
 
 import silenttestrunner
 
 from mercurial.node import nullid
 from mercurial import (
     ui as uimod,
 )
 # Load the local remotefilelog, not the system one
 sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')]
 from hgext.remotefilelog import (
     basepack,
     historypack,
 )
 
 class histpacktests(unittest.TestCase):
     def setUp(self):
         self.tempdirs = []
 
     def tearDown(self):
         for d in self.tempdirs:
             shutil.rmtree(d)
 
     def makeTempDir(self):
         tempdir = tempfile.mkdtemp()
         self.tempdirs.append(tempdir)
         return tempdir
 
     def getHash(self, content):
         return hashlib.sha1(content).digest()
 
     def getFakeHash(self):
         return ''.join(chr(random.randint(0, 255)) for _ in range(20))
 
     def createPack(self, revisions=None):
         """Creates and returns a historypack containing the specified revisions.
 
         `revisions` is a list of tuples, where each tuple contains a filanem,
         node, p1node, p2node, and linknode.
         """
         if revisions is None:
             revisions = [("filename", self.getFakeHash(), nullid, nullid,
                           self.getFakeHash(), None)]
 
         packdir = self.makeTempDir()
         packer = historypack.mutablehistorypack(uimod.ui(), packdir,
-                                                version=1)
+                                                version=2)
 
         for filename, node, p1, p2, linknode, copyfrom in revisions:
             packer.add(filename, node, p1, p2, linknode, copyfrom)
 
         path = packer.close()
         return historypack.historypack(path)
 
     def testAddSingle(self):
         """Test putting a single entry into a pack and reading it out.
         """
         filename = "foo"
         node = self.getFakeHash()
         p1 = self.getFakeHash()
         p2 = self.getFakeHash()
         linknode = self.getFakeHash()
 
         revisions = [(filename, node, p1, p2, linknode, None)]
         pack = self.createPack(revisions)
 
         actual = pack.getancestors(filename, node)[node]
         self.assertEquals(p1, actual[0])
         self.assertEquals(p2, actual[1])
         self.assertEquals(linknode, actual[2])
 
     def testAddMultiple(self):
         """Test putting multiple unrelated revisions into a pack and reading
         them out.
         """
         revisions = []
         for i in range(10):
             filename = "foo-%s" % i
             node = self.getFakeHash()
             p1 = self.getFakeHash()
             p2 = self.getFakeHash()
             linknode = self.getFakeHash()
             revisions.append((filename, node, p1, p2, linknode, None))
 
         pack = self.createPack(revisions)
 
         for filename, node, p1, p2, linknode, copyfrom in revisions:
             actual = pack.getancestors(filename, node)[node]
             self.assertEquals(p1, actual[0])
             self.assertEquals(p2, actual[1])
             self.assertEquals(linknode, actual[2])
             self.assertEquals(copyfrom, actual[3])
 
     def testAddAncestorChain(self):
         """Test putting multiple revisions in into a pack and read the ancestor
         chain.
         """
         revisions = []
         filename = "foo"
         lastnode = nullid
         for i in range(10):
             node = self.getFakeHash()
             revisions.append((filename, node, lastnode, nullid, nullid, None))
             lastnode = node
 
         # revisions must be added in topological order, newest first
         revisions = list(reversed(revisions))
         pack = self.createPack(revisions)
 
         # Test that the chain has all the entries
         ancestors = pack.getancestors(revisions[0][0], revisions[0][1])
         for filename, node, p1, p2, linknode, copyfrom in revisions:
             ap1, ap2, alinknode, acopyfrom = ancestors[node]
             self.assertEquals(ap1, p1)
             self.assertEquals(ap2, p2)
             self.assertEquals(alinknode, linknode)
             self.assertEquals(acopyfrom, copyfrom)
 
     def testPackMany(self):
         """Pack many related and unrelated ancestors.
         """
         # Build a random pack file
         allentries = {}
         ancestorcounts = {}
         revisions = []
         random.seed(0)
         for i in range(100):
             filename = "filename-%s" % i
             entries = []
             p2 = nullid
             linknode = nullid
             for j in range(random.randint(1, 100)):
                 node = self.getFakeHash()
                 p1 = nullid
                 if len(entries) > 0:
                     p1 = entries[random.randint(0, len(entries) - 1)]
                 entries.append(node)
                 revisions.append((filename, node, p1, p2, linknode, None))
                 allentries[(filename, node)] = (p1, p2, linknode)
                 if p1 == nullid:
                     ancestorcounts[(filename, node)] = 1
                 else:
                     newcount = ancestorcounts[(filename, p1)] + 1
                     ancestorcounts[(filename, node)] = newcount
 
         # Must add file entries in reverse topological order
         revisions = list(reversed(revisions))
         pack = self.createPack(revisions)
 
         # Verify the pack contents
         for (filename, node), (p1, p2, lastnode) in allentries.iteritems():
             ancestors = pack.getancestors(filename, node)
             self.assertEquals(ancestorcounts[(filename, node)],
                               len(ancestors))
             for anode, (ap1, ap2, alinknode, copyfrom) in ancestors.iteritems():
                 ep1, ep2, elinknode = allentries[(filename, anode)]
                 self.assertEquals(ap1, ep1)
                 self.assertEquals(ap2, ep2)
                 self.assertEquals(alinknode, elinknode)
                 self.assertEquals(copyfrom, None)
 
     def testGetNodeInfo(self):
         revisions = []
         filename = "foo"
         lastnode = nullid
         for i in range(10):
             node = self.getFakeHash()
             revisions.append((filename, node, lastnode, nullid, nullid, None))
             lastnode = node
 
         pack = self.createPack(revisions)
 
         # Test that getnodeinfo returns the expected results
         for filename, node, p1, p2, linknode, copyfrom in revisions:
             ap1, ap2, alinknode, acopyfrom = pack.getnodeinfo(filename, node)
             self.assertEquals(ap1, p1)
             self.assertEquals(ap2, p2)
             self.assertEquals(alinknode, linknode)
             self.assertEquals(acopyfrom, copyfrom)
 
     def testGetMissing(self):
         """Test the getmissing() api.
         """
         revisions = []
         filename = "foo"
         for i in range(10):
             node = self.getFakeHash()
             p1 = self.getFakeHash()
             p2 = self.getFakeHash()
             linknode = self.getFakeHash()
             revisions.append((filename, node, p1, p2, linknode, None))
 
         pack = self.createPack(revisions)
 
         missing = pack.getmissing([(filename, revisions[0][1])])
         self.assertFalse(missing)
 
         missing = pack.getmissing([(filename, revisions[0][1]),
                                    (filename, revisions[1][1])])
         self.assertFalse(missing)
 
         fakenode = self.getFakeHash()
         missing = pack.getmissing([(filename, revisions[0][1]),
                                    (filename, fakenode)])
         self.assertEquals(missing, [(filename, fakenode)])
 
         # Test getmissing on a non-existant filename
         missing = pack.getmissing([("bar", fakenode)])
         self.assertEquals(missing, [("bar", fakenode)])
 
     def testAddThrows(self):
         pack = self.createPack()
 
         try:
             pack.add('filename', nullid, nullid, nullid, nullid, None)
             self.assertTrue(False, "historypack.add should throw")
         except RuntimeError:
             pass
 
     def testBadVersionThrows(self):
         pack = self.createPack()
         path = pack.path + '.histpack'
         with open(path) as f:
             raw = f.read()
         raw = struct.pack('!B', 255) + raw[1:]
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
         with open(path, 'w+') as f:
             f.write(raw)
 
         try:
             pack = historypack.historypack(pack.path)
             self.assertTrue(False, "bad version number should have thrown")
         except RuntimeError:
             pass
 
     def testLargePack(self):
         """Test creating and reading from a large pack with over X entries.
         This causes it to use a 2^16 fanout table instead."""
         total = basepack.SMALLFANOUTCUTOFF + 1
         revisions = []
         for i in xrange(total):
             filename = "foo-%s" % i
             node = self.getFakeHash()
             p1 = self.getFakeHash()
             p2 = self.getFakeHash()
             linknode = self.getFakeHash()
             revisions.append((filename, node, p1, p2, linknode, None))
 
         pack = self.createPack(revisions)
         self.assertEquals(pack.params.fanoutprefix, basepack.LARGEFANOUTPREFIX)
 
         for filename, node, p1, p2, linknode, copyfrom in revisions:
             actual = pack.getancestors(filename, node)[node]
             self.assertEquals(p1, actual[0])
             self.assertEquals(p2, actual[1])
             self.assertEquals(linknode, actual[2])
             self.assertEquals(copyfrom, actual[3])
 # TODO:
 # histpack store:
 # - repack two packs into one
 
 if __name__ == '__main__':
     silenttestrunner.main(__name__)

tests/test-remotefilelog-repack-fast.t

@@ -1,385 +1,385 b''
   $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
   $ export PYTHONPATH
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ cat >> $HGRCPATH <<EOF
   > [remotefilelog]
   > fastdatapack=True
   > EOF
 
   $ hginit master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > serverexpiration=-1
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ echo x >> x
   $ hg commit -qAm x2
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # Test that repack cleans up the old files and creates new packs
 
   $ cd shallow
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/11
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
   $TESTTMP/hgcache/repos
 
   $ hg repack
 
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/packs
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that the packs are readonly
   $ ls_l $CACHEDIR/master/packs
-  -r--r--r--    1145 276d308429d0303762befa376788300f0310f90e.histidx
-  -r--r--r--     172 276d308429d0303762befa376788300f0310f90e.histpack
-  -r--r--r--    1074 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  -r--r--r--      69 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  -r--r--r--    1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  -r--r--r--     172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  -r--r--r--    1074 add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  -r--r--r--      69 add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   -rw-r--r--       0 repacklock
 
 # Test that the data in the new packs is accessible
   $ hg cat -r . x
   x
   x
 
 # Test that adding new data and repacking it results in the loose data and the
 # old packs being combined.
 
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x3
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --traceback
 
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Verify all the file data is still available
   $ hg cat -r . x
   x
   x
   x
   $ hg cat -r '.^' x
   x
   x
 
 # Test that repacking again without new data does not delete the pack files
 # and did not change the pack names
   $ hg repack
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Run two repacks at once
   $ hg repack --config "hooks.prerepack=sleep 3" &
   $ sleep 1
   $ hg repack
   skipping repack - another repack is already running
   $ hg debugwaitonrepack >/dev/null 2>&1
 
 # Run repack in the background
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x4
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --background
   (running background repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histidx
-  $TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histpack
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.dataidx
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.dataidx
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
+  $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
+  $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test debug commands
 
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15:
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15:
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
   x:
   Node                                      Delta Base                                Delta Length  Blob Size
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  8             8
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  12            6
   aee31534993a501858fb6dd96a065671922e7d51  d4a3ed9310e5bd9887e3bf779da5077efab28216  12            4
   
   Total:                                                                              32            18        (77.8% bigger)
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15:
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
   
   x
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  77029ab56e83ea2115dd53ff87483682abe5d7ca  12
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5  8
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
 
 # Test copy tracing from a pack
   $ cd ../master
   $ hg mv x y
   $ hg commit -m 'move x to y'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg log -f y -T '{desc}\n'
   move x to y
   x4
   x3
   x2
   x
 
 # Test copy trace across rename and back
   $ cp -R $TESTTMP/hgcache/master/packs $TESTTMP/backuppacks
   $ cd ../master
   $ hg mv y x
   $ hg commit -m 'move y back to x'
   $ hg revert -r 0 x
   $ mv x y
   $ hg add y
   $ echo >> y
   $ hg revert x
   $ hg commit -m 'add y back without metadata'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ ls $TESTTMP/hgcache/master/packs
-  e8fdf7ae22b772dcc291f905b9c6e5f381d28739.dataidx
-  e8fdf7ae22b772dcc291f905b9c6e5f381d28739.datapack
-  ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histidx
-  ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histpack
+  308a7aba9c54a0b71ae5adbbccd00c0aff20876e.dataidx
+  308a7aba9c54a0b71ae5adbbccd00c0aff20876e.datapack
+  bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
+  bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
   repacklock
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   cd410a44d584  577959738234  000000000000  609547eda446  y
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
   
   y
   Node          P1 Node       P2 Node       Link Node     Copy From
   577959738234  1bb2e6237e03  000000000000  c7faf2fc439a  x
   21f46f2721e7  000000000000  000000000000  d6868642b790  
   $ hg strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
   $ hg -R ../master strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/master/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
 
   $ rm -rf $TESTTMP/hgcache/master/packs
   $ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs
 
 # Test repacking datapack without history
   $ rm -rf $CACHEDIR/master/packs/*hist*
   $ hg repack
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/a8d86ff8e1a11a77a85f5fea567f56a757583eda:
+  $TESTTMP/hgcache/master/packs/ba4649b56263282b0699f9a6e7e34a4a2bac1638:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   577959738234  000000000000  70            8
   
   Total:                      70            8         (775.0% bigger)
 
   $ hg cat -r ".^" x
   x
   x
   x
   x
 
 Incremental repack
   $ rm -rf $CACHEDIR/master/packs/*
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > data.generations=60
   >   150
   > EOF
 
 Single pack - repack does nothing
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   [1]
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   [1]
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 3
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--     225 06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 1 gen3 pack, 1 gen0 pack - does nothing
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--     225 06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 Pull should run background repack
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > backgroundrepack=True
   > EOF
   $ clearcache
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 3
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   no changes found
   (running background incremental repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     301 09b8bf49256b3fc2175977ba97d6402e91a9a604.datapack
+  -r--r--r--     301 671913bebdb7b95aae52a546662753eac7606e40.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack
+  -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
 
 Test environment variable resolution
   $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $TESTTMP/envcache | sort
   $TESTTMP/envcache
   $TESTTMP/envcache/master
   $TESTTMP/envcache/master/95
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/577959738234a1eb241ed3ed4b22a575833f56e0
   $TESTTMP/envcache/repos
 
 Test local remotefilelog blob is correct when based on a pack
   $ hg prefetch -r .
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ echo >> y
   $ hg commit -m y2
   $ hg debugremotefilelog .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808
   size: 9 bytes
   path: .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808 
   key: b70860edba4f 
   
           node =>           p1            p2      linknode     copyfrom
   b70860edba4f => 577959738234  000000000000  08d3fbc98c48  
   577959738234 => 1bb2e6237e03  000000000000  c7faf2fc439a  x
   1bb2e6237e03 => d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5 => aee31534993a  000000000000  421535db10b6  
   aee31534993a => 1406e7411862  000000000000  a89d614e2364  
   1406e7411862 => 000000000000  000000000000  b292c1e3311f  

tests/test-remotefilelog-repack.t

@@ -1,463 +1,463 b''
   $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
   $ export PYTHONPATH
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ hginit master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > serverexpiration=-1
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ echo x >> x
   $ hg commit -qAm x2
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # Test that repack cleans up the old files and creates new packs
 
   $ cd shallow
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/11
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
   $TESTTMP/hgcache/repos
 
   $ hg repack
 
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/packs
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that the packs are readonly
   $ ls_l $CACHEDIR/master/packs
-  -r--r--r--    1145 276d308429d0303762befa376788300f0310f90e.histidx
-  -r--r--r--     172 276d308429d0303762befa376788300f0310f90e.histpack
-  -r--r--r--    1074 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  -r--r--r--      69 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  -r--r--r--    1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  -r--r--r--     172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  -r--r--r--    1074 add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  -r--r--r--      69 add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   -rw-r--r--       0 repacklock
 
 # Test that the data in the new packs is accessible
   $ hg cat -r . x
   x
   x
 
 # Test that adding new data and repacking it results in the loose data and the
 # old packs being combined.
 
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x3
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # First assert that with --packsonly, the loose object will be ignored:
 
   $ hg repack --packsonly
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
-  $TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-  $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
+  $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
+  $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --traceback
 
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Verify all the file data is still available
   $ hg cat -r . x
   x
   x
   x
   $ hg cat -r '.^' x
   x
   x
 
 # Test that repacking again without new data does not delete the pack files
 # and did not change the pack names
   $ hg repack
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Run two repacks at once
   $ hg repack --config "hooks.prerepack=sleep 3" &
   $ sleep 1
   $ hg repack
   skipping repack - another repack is already running
   $ hg debugwaitonrepack >/dev/null 2>&1
 
 # Run repack in the background
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x4
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx
-  $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx
-  $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
+  $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
+  $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --background
   (running background repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ find $CACHEDIR -type f | sort
-  $TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histidx
-  $TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histpack
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.dataidx
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.dataidx
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
+  $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
+  $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
   $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test debug commands
 
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15:
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15:
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
   x:
   Node                                      Delta Base                                Delta Length  Blob Size
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  8             8
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  12            6
   aee31534993a501858fb6dd96a065671922e7d51  d4a3ed9310e5bd9887e3bf779da5077efab28216  12            4
   
   Total:                                                                              32            18        (77.8% bigger)
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216
-  $TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15:
+  $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
   
   x
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  77029ab56e83ea2115dd53ff87483682abe5d7ca  12
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5  8
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
 
 # Test copy tracing from a pack
   $ cd ../master
   $ hg mv x y
   $ hg commit -m 'move x to y'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg log -f y -T '{desc}\n'
   move x to y
   x4
   x3
   x2
   x
 
 # Test copy trace across rename and back
   $ cp -R $TESTTMP/hgcache/master/packs $TESTTMP/backuppacks
   $ cd ../master
   $ hg mv y x
   $ hg commit -m 'move y back to x'
   $ hg revert -r 0 x
   $ mv x y
   $ hg add y
   $ echo >> y
   $ hg revert x
   $ hg commit -m 'add y back without metadata'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ ls $TESTTMP/hgcache/master/packs
-  e8fdf7ae22b772dcc291f905b9c6e5f381d28739.dataidx
-  e8fdf7ae22b772dcc291f905b9c6e5f381d28739.datapack
-  ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histidx
-  ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histpack
+  308a7aba9c54a0b71ae5adbbccd00c0aff20876e.dataidx
+  308a7aba9c54a0b71ae5adbbccd00c0aff20876e.datapack
+  bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
+  bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
   repacklock
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   cd410a44d584  577959738234  000000000000  609547eda446  y
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
   
   y
   Node          P1 Node       P2 Node       Link Node     Copy From
   577959738234  1bb2e6237e03  000000000000  c7faf2fc439a  x
   21f46f2721e7  000000000000  000000000000  d6868642b790  
   $ hg strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
   $ hg -R ../master strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/master/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
 
   $ rm -rf $TESTTMP/hgcache/master/packs
   $ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs
 
 # Test repacking datapack without history
   $ rm -rf $CACHEDIR/master/packs/*hist*
   $ hg repack
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/a8d86ff8e1a11a77a85f5fea567f56a757583eda:
+  $TESTTMP/hgcache/master/packs/ba4649b56263282b0699f9a6e7e34a4a2bac1638:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   577959738234  000000000000  70            8
   
   Total:                      70            8         (775.0% bigger)
 
   $ hg cat -r ".^" x
   x
   x
   x
   x
 
 Incremental repack
   $ rm -rf $CACHEDIR/master/packs/*
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > data.generations=60
   >   150
   > EOF
 
 Single pack - repack does nothing
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   [1]
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   [1]
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 38
   abort: unknown revision '38'!
   [255]
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 For the data packs, setting the limit for the repackmaxpacksize to be 64 such
 that data pack with size 65 is more than the limit. This effectively ensures
 that no generation has 3 packs and therefore, no packs are chosen for the
 incremental repacking. As for the history packs, setting repackmaxpacksize to be
 0 which should always result in no repacking.
   $ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=64 \
   > --config remotefilelog.history.repackmaxpacksize=0
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     147 935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--     147 1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--     254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 Setting limit for the repackmaxpacksize to be the size of the biggest pack file
 which ensures that it is effectively ignored in the incremental repacking.
   $ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=65 \
   > --config remotefilelog.history.repackmaxpacksize=336
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     147 935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--     147 1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--     254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 1 gen3 pack, 1 gen0 pack - does nothing
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     147 935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack
-  -r--r--r--      67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack
+  -r--r--r--     147 1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
+  -r--r--r--      67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack
-  -r--r--r--      90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  -r--r--r--     254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
+  -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 Pull should run background repack
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > backgroundrepack=True
   > EOF
   $ clearcache
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 3
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   no changes found
   (running background incremental repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-  -r--r--r--     301 09b8bf49256b3fc2175977ba97d6402e91a9a604.datapack
+  -r--r--r--     301 671913bebdb7b95aae52a546662753eac7606e40.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-  -r--r--r--     336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack
+  -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
 
 Test environment variable resolution
   $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $TESTTMP/envcache | sort
   $TESTTMP/envcache
   $TESTTMP/envcache/master
   $TESTTMP/envcache/master/95
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/577959738234a1eb241ed3ed4b22a575833f56e0
   $TESTTMP/envcache/repos
 
 Test local remotefilelog blob is correct when based on a pack
   $ hg prefetch -r .
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ echo >> y
   $ hg commit -m y2
   $ hg debugremotefilelog .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808
   size: 9 bytes
   path: .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808 
   key: b70860edba4f 
   
           node =>           p1            p2      linknode     copyfrom
   b70860edba4f => 577959738234  000000000000  08d3fbc98c48  
   577959738234 => 1bb2e6237e03  000000000000  c7faf2fc439a  x
   1bb2e6237e03 => d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5 => aee31534993a  000000000000  421535db10b6  
   aee31534993a => 1406e7411862  000000000000  a89d614e2364  
   1406e7411862 => 000000000000  000000000000  b292c1e3311f  
 
 Test limiting the max delta chain length
   $ hg repack --config packs.maxchainlen=1
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.dataidx
-  $TESTTMP/hgcache/master/packs/a8378d888fe4325250720e51311a65d2509be742:
+  $TESTTMP/hgcache/master/packs/80f7c299aeb22849ebc28fbd11bb76078aa55918:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  000000000000  4             4
   1406e7411862  aee31534993a  12            2
   
   Total:                      36            20        (80.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   577959738234  000000000000  70            8
   
   Total:                      70            8         (775.0% bigger)
 
 Test huge pack cleanup using different values of packs.maxpacksize:
   $ hg repack --incremental --debug
   $ hg repack --incremental --debug --config packs.maxpacksize=512
-  removing oversize packfile $TESTTMP/hgcache/master/packs/a8378d888fe4325250720e51311a65d2509be742.datapack (426 bytes)
-  removing oversize packfile $TESTTMP/hgcache/master/packs/a8378d888fe4325250720e51311a65d2509be742.dataidx (1.21 KB)
+  removing oversize packfile $TESTTMP/hgcache/master/packs/80f7c299aeb22849ebc28fbd11bb76078aa55918.datapack (426 bytes)
+  removing oversize packfile $TESTTMP/hgcache/master/packs/80f7c299aeb22849ebc28fbd11bb76078aa55918.dataidx (1.21 KB)
 
 Do a repack where the new pack reuses a delta from the old pack
   $ clearcache
   $ hg prefetch -r '2::3'
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg debugdatapack $CACHEDIR/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/abf210f6c3aa4dd0ecc7033633ad73591be16c95:
+  $TESTTMP/hgcache/master/packs/90cfb1a45e2549500caace30add04b58a6b243af:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   
   Total:                      20            14        (42.9% bigger)
   $ hg prefetch -r '0::1'
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg debugdatapack $CACHEDIR/master/packs/*.datapack
-  $TESTTMP/hgcache/master/packs/09b8bf49256b3fc2175977ba97d6402e91a9a604:
+  $TESTTMP/hgcache/master/packs/671913bebdb7b95aae52a546662753eac7606e40:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   1406e7411862  aee31534993a  12            2
   
   Total:                      44            20        (120.0% bigger)