remotefilelog: replace repack lock to solve race condition

2c74337e6483 reduced the probability of race-conditions when starting
background repack and prefetch and we saw the difference in our CI instance
with all failures disappearing except one where one call to waitonrepack seems
to returns too early.

I'm not sure what exactly goes wrong but I realized that while the prefetch
operation uses a standard Mercurial lock, the repack operation is using a
custom lock based on `fcntl.flock` on available platforms. As `extutil.flock`
fallback on traditional Mercurial locks on other platforms and the tests are
stable on my laptop, our CI environment and GCC112, I'm sending this patch to
standardize the behavior across environments.

Differential Revision: https://phab.mercurial-scm.org/D6844

hgext/remotefilelog/debugcommands.py

@@ -1,378 +1,378 b''
 # debugcommands.py - debug logic for remotefilelog
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import hashlib
 import os
 import zlib
 
 from mercurial.node import bin, hex, nullid, short
 from mercurial.i18n import _
 from mercurial import (
     error,
     filelog,
+    lock as lockmod,
     node as nodemod,
     pycompat,
     revlog,
 )
 from . import (
     constants,
     datapack,
-    extutil,
     fileserverclient,
     historypack,
     repack,
     shallowutil,
 )
 
 def debugremotefilelog(ui, path, **opts):
     decompress = opts.get(r'decompress')
 
     size, firstnode, mapping = parsefileblob(path, decompress)
 
     ui.status(_("size: %d bytes\n") % (size))
     ui.status(_("path: %s \n") % (path))
     ui.status(_("key: %s \n") % (short(firstnode)))
     ui.status(_("\n"))
     ui.status(_("%12s => %12s %13s %13s %12s\n") %
               ("node", "p1", "p2", "linknode", "copyfrom"))
 
     queue = [firstnode]
     while queue:
         node = queue.pop(0)
         p1, p2, linknode, copyfrom = mapping[node]
         ui.status(_("%s => %s  %s  %s  %s\n") %
             (short(node), short(p1), short(p2), short(linknode), copyfrom))
         if p1 != nullid:
             queue.append(p1)
         if p2 != nullid:
             queue.append(p2)
 
 def buildtemprevlog(repo, file):
     # get filename key
     filekey = nodemod.hex(hashlib.sha1(file).digest())
     filedir = os.path.join(repo.path, 'store/data', filekey)
 
     # sort all entries based on linkrev
     fctxs = []
     for filenode in os.listdir(filedir):
         if '_old' not in filenode:
             fctxs.append(repo.filectx(file, fileid=bin(filenode)))
 
     fctxs = sorted(fctxs, key=lambda x: x.linkrev())
 
     # add to revlog
     temppath = repo.sjoin('data/temprevlog.i')
     if os.path.exists(temppath):
         os.remove(temppath)
     r = filelog.filelog(repo.svfs, 'temprevlog')
 
     class faket(object):
         def add(self, a, b, c):
             pass
     t = faket()
     for fctx in fctxs:
         if fctx.node() not in repo:
             continue
 
         p = fctx.filelog().parents(fctx.filenode())
         meta = {}
         if fctx.renamed():
             meta['copy'] = fctx.renamed()[0]
             meta['copyrev'] = hex(fctx.renamed()[1])
 
         r.add(fctx.data(), meta, t, fctx.linkrev(), p[0], p[1])
 
     return r
 
 def debugindex(orig, ui, repo, file_=None, **opts):
     """dump the contents of an index file"""
     if (opts.get(r'changelog') or
         opts.get(r'manifest') or
         opts.get(r'dir') or
         not shallowutil.isenabled(repo) or
         not repo.shallowmatch(file_)):
         return orig(ui, repo, file_, **opts)
 
     r = buildtemprevlog(repo, file_)
 
     # debugindex like normal
     format = opts.get('format', 0)
     if format not in (0, 1):
         raise error.Abort(_("unknown format %d") % format)
 
     generaldelta = r.version & revlog.FLAG_GENERALDELTA
     if generaldelta:
         basehdr = ' delta'
     else:
         basehdr = '  base'
 
     if format == 0:
         ui.write(("   rev    offset  length " + basehdr + " linkrev"
                   " nodeid       p1           p2\n"))
     elif format == 1:
         ui.write(("   rev flag   offset   length"
                   "     size " + basehdr + "   link     p1     p2"
                   "       nodeid\n"))
 
     for i in r:
         node = r.node(i)
         if generaldelta:
             base = r.deltaparent(i)
         else:
             base = r.chainbase(i)
         if format == 0:
             try:
                 pp = r.parents(node)
             except Exception:
                 pp = [nullid, nullid]
             ui.write("% 6d % 9d % 7d % 6d % 7d %s %s %s\n" % (
                     i, r.start(i), r.length(i), base, r.linkrev(i),
                     short(node), short(pp[0]), short(pp[1])))
         elif format == 1:
             pr = r.parentrevs(i)
             ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % (
                     i, r.flags(i), r.start(i), r.length(i), r.rawsize(i),
                     base, r.linkrev(i), pr[0], pr[1], short(node)))
 
 def debugindexdot(orig, ui, repo, file_):
     """dump an index DAG as a graphviz dot file"""
     if not shallowutil.isenabled(repo):
         return orig(ui, repo, file_)
 
     r = buildtemprevlog(repo, os.path.basename(file_)[:-2])
 
     ui.write(("digraph G {\n"))
     for i in r:
         node = r.node(i)
         pp = r.parents(node)
         ui.write("\t%d -> %d\n" % (r.rev(pp[0]), i))
         if pp[1] != nullid:
             ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i))
     ui.write("}\n")
 
 def verifyremotefilelog(ui, path, **opts):
     decompress = opts.get(r'decompress')
 
     for root, dirs, files in os.walk(path):
         for file in files:
             if file == "repos":
                 continue
             filepath = os.path.join(root, file)
             size, firstnode, mapping = parsefileblob(filepath, decompress)
             for p1, p2, linknode, copyfrom in mapping.itervalues():
                 if linknode == nullid:
                     actualpath = os.path.relpath(root, path)
                     key = fileserverclient.getcachekey("reponame", actualpath,
                                                        file)
                     ui.status("%s %s\n" % (key, os.path.relpath(filepath,
                                                                 path)))
 
 def _decompressblob(raw):
     return zlib.decompress(raw)
 
 def parsefileblob(path, decompress):
     f = open(path, "rb")
     try:
         raw = f.read()
     finally:
         f.close()
 
     if decompress:
         raw = _decompressblob(raw)
 
     offset, size, flags = shallowutil.parsesizeflags(raw)
     start = offset + size
 
     firstnode = None
 
     mapping = {}
     while start < len(raw):
         divider = raw.index('\0', start + 80)
 
         currentnode = raw[start:(start + 20)]
         if not firstnode:
             firstnode = currentnode
 
         p1 = raw[(start + 20):(start + 40)]
         p2 = raw[(start + 40):(start + 60)]
         linknode = raw[(start + 60):(start + 80)]
         copyfrom = raw[(start + 80):divider]
 
         mapping[currentnode] = (p1, p2, linknode, copyfrom)
         start = divider + 1
 
     return size, firstnode, mapping
 
 def debugdatapack(ui, *paths, **opts):
     for path in paths:
         if '.data' in path:
             path = path[:path.index('.data')]
         ui.write("%s:\n" % path)
         dpack = datapack.datapack(path)
         node = opts.get(r'node')
         if node:
             deltachain = dpack.getdeltachain('', bin(node))
             dumpdeltachain(ui, deltachain, **opts)
             return
 
         if opts.get(r'long'):
             hashformatter = hex
             hashlen = 42
         else:
             hashformatter = short
             hashlen = 14
 
         lastfilename = None
         totaldeltasize = 0
         totalblobsize = 0
         def printtotals():
             if lastfilename is not None:
                 ui.write("\n")
             if not totaldeltasize or not totalblobsize:
                 return
             difference = totalblobsize - totaldeltasize
             deltastr = "%0.1f%% %s" % (
                 (100.0 * abs(difference) / totalblobsize),
                 ("smaller" if difference > 0 else "bigger"))
 
             ui.write(("Total:%s%s  %s (%s)\n") % (
                 "".ljust(2 * hashlen - len("Total:")),
                 ('%d' % totaldeltasize).ljust(12),
                 ('%d' % totalblobsize).ljust(9),
                 deltastr
             ))
 
         bases = {}
         nodes = set()
         failures = 0
         for filename, node, deltabase, deltalen in dpack.iterentries():
             bases[node] = deltabase
             if node in nodes:
                 ui.write(("Bad entry: %s appears twice\n" % short(node)))
                 failures += 1
             nodes.add(node)
             if filename != lastfilename:
                 printtotals()
                 name = '(empty name)' if filename == '' else filename
                 ui.write("%s:\n" % name)
                 ui.write("%s%s%s%s\n" % (
                     "Node".ljust(hashlen),
                     "Delta Base".ljust(hashlen),
                     "Delta Length".ljust(14),
                     "Blob Size".ljust(9)))
                 lastfilename = filename
                 totalblobsize = 0
                 totaldeltasize = 0
 
             # Metadata could be missing, in which case it will be an empty dict.
             meta = dpack.getmeta(filename, node)
             if constants.METAKEYSIZE in meta:
                 blobsize = meta[constants.METAKEYSIZE]
                 totaldeltasize += deltalen
                 totalblobsize += blobsize
             else:
                 blobsize = "(missing)"
             ui.write("%s  %s  %s%s\n" % (
                 hashformatter(node),
                 hashformatter(deltabase),
                 ('%d' % deltalen).ljust(14),
                 pycompat.bytestr(blobsize)))
 
         if filename is not None:
             printtotals()
 
         failures += _sanitycheck(ui, set(nodes), bases)
         if failures > 1:
             ui.warn(("%d failures\n" % failures))
             return 1
 
 def _sanitycheck(ui, nodes, bases):
     """
     Does some basic sanity checking on a packfiles with ``nodes`` ``bases`` (a
     mapping of node->base):
 
     - Each deltabase must itself be a node elsewhere in the pack
     - There must be no cycles
     """
     failures = 0
     for node in nodes:
         seen = set()
         current = node
         deltabase = bases[current]
 
         while deltabase != nullid:
             if deltabase not in nodes:
                 ui.warn(("Bad entry: %s has an unknown deltabase (%s)\n" %
                         (short(node), short(deltabase))))
                 failures += 1
                 break
 
             if deltabase in seen:
                 ui.warn(("Bad entry: %s has a cycle (at %s)\n" %
                         (short(node), short(deltabase))))
                 failures += 1
                 break
 
             current = deltabase
             seen.add(current)
             deltabase = bases[current]
         # Since ``node`` begins a valid chain, reset/memoize its base to nullid
         # so we don't traverse it again.
         bases[node] = nullid
     return failures
 
 def dumpdeltachain(ui, deltachain, **opts):
     hashformatter = hex
     hashlen = 40
 
     lastfilename = None
     for filename, node, filename, deltabasenode, delta in deltachain:
         if filename != lastfilename:
             ui.write("\n%s\n" % filename)
             lastfilename = filename
         ui.write("%s  %s  %s  %s\n" % (
             "Node".ljust(hashlen),
             "Delta Base".ljust(hashlen),
             "Delta SHA1".ljust(hashlen),
             "Delta Length".ljust(6),
         ))
 
         ui.write("%s  %s  %s  %d\n" % (
             hashformatter(node),
             hashformatter(deltabasenode),
             nodemod.hex(hashlib.sha1(delta).digest()),
             len(delta)))
 
 def debughistorypack(ui, path):
     if '.hist' in path:
         path = path[:path.index('.hist')]
     hpack = historypack.historypack(path)
 
     lastfilename = None
     for entry in hpack.iterentries():
         filename, node, p1node, p2node, linknode, copyfrom = entry
         if filename != lastfilename:
             ui.write("\n%s\n" % filename)
             ui.write("%s%s%s%s%s\n" % (
                 "Node".ljust(14),
                 "P1 Node".ljust(14),
                 "P2 Node".ljust(14),
                 "Link Node".ljust(14),
                 "Copy From"))
             lastfilename = filename
         ui.write("%s  %s  %s  %s  %s\n" % (short(node), short(p1node),
             short(p2node), short(linknode), copyfrom))
 
 def debugwaitonrepack(repo):
-    with extutil.flock(repack.repacklockvfs(repo).join('repacklock'), ''):
+    with lockmod.lock(repack.repacklockvfs(repo), "repacklock", timeout=-1):
         return
 
 def debugwaitonprefetch(repo):
     with repo._lock(repo.svfs, "prefetchlock", True, None,
                          None, _('prefetching in %s') % repo.origroot):
         pass

hgext/remotefilelog/repack.py

@@ -1,779 +1,779 b''
 from __future__ import absolute_import
 
 import os
 import time
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     short,
 )
 from mercurial import (
     encoding,
     error,
+    lock as lockmod,
     mdiff,
     policy,
     pycompat,
     scmutil,
     util,
     vfs,
 )
 from mercurial.utils import procutil
 from . import (
     constants,
     contentstore,
     datapack,
-    extutil,
     historypack,
     metadatastore,
     shallowutil,
 )
 
 osutil = policy.importmod(r'osutil')
 
 class RepackAlreadyRunning(error.Abort):
     pass
 
 def backgroundrepack(repo, incremental=True, packsonly=False,
                      ensurestart=False):
     cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'repack']
     msg = _("(running background repack)\n")
     if incremental:
         cmd.append('--incremental')
         msg = _("(running background incremental repack)\n")
     if packsonly:
         cmd.append('--packsonly')
     repo.ui.warn(msg)
     # We know this command will find a binary, so don't block on it starting.
     procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart)
 
 def fullrepack(repo, options=None):
     """If ``packsonly`` is True, stores creating only loose objects are skipped.
     """
     if util.safehasattr(repo, 'shareddatastores'):
         datasource = contentstore.unioncontentstore(
             *repo.shareddatastores)
         historysource = metadatastore.unionmetadatastore(
             *repo.sharedhistorystores,
             allowincomplete=True)
 
         packpath = shallowutil.getcachepackpath(
             repo,
             constants.FILEPACK_CATEGORY)
         _runrepack(repo, datasource, historysource, packpath,
                    constants.FILEPACK_CATEGORY, options=options)
 
     if util.safehasattr(repo.manifestlog, 'datastore'):
         localdata, shareddata = _getmanifeststores(repo)
         lpackpath, ldstores, lhstores = localdata
         spackpath, sdstores, shstores = shareddata
 
         # Repack the shared manifest store
         datasource = contentstore.unioncontentstore(*sdstores)
         historysource = metadatastore.unionmetadatastore(
                         *shstores,
                         allowincomplete=True)
         _runrepack(repo, datasource, historysource, spackpath,
                    constants.TREEPACK_CATEGORY, options=options)
 
         # Repack the local manifest store
         datasource = contentstore.unioncontentstore(
                         *ldstores,
                         allowincomplete=True)
         historysource = metadatastore.unionmetadatastore(
                         *lhstores,
                         allowincomplete=True)
         _runrepack(repo, datasource, historysource, lpackpath,
                    constants.TREEPACK_CATEGORY, options=options)
 
 def incrementalrepack(repo, options=None):
     """This repacks the repo by looking at the distribution of pack files in the
     repo and performing the most minimal repack to keep the repo in good shape.
     """
     if util.safehasattr(repo, 'shareddatastores'):
         packpath = shallowutil.getcachepackpath(
             repo,
             constants.FILEPACK_CATEGORY)
         _incrementalrepack(repo,
                            repo.shareddatastores,
                            repo.sharedhistorystores,
                            packpath,
                            constants.FILEPACK_CATEGORY,
                            options=options)
 
     if util.safehasattr(repo.manifestlog, 'datastore'):
         localdata, shareddata = _getmanifeststores(repo)
         lpackpath, ldstores, lhstores = localdata
         spackpath, sdstores, shstores = shareddata
 
         # Repack the shared manifest store
         _incrementalrepack(repo,
                            sdstores,
                            shstores,
                            spackpath,
                            constants.TREEPACK_CATEGORY,
                            options=options)
 
         # Repack the local manifest store
         _incrementalrepack(repo,
                            ldstores,
                            lhstores,
                            lpackpath,
                            constants.TREEPACK_CATEGORY,
                            allowincompletedata=True,
                            options=options)
 
 def _getmanifeststores(repo):
     shareddatastores = repo.manifestlog.shareddatastores
     localdatastores = repo.manifestlog.localdatastores
     sharedhistorystores = repo.manifestlog.sharedhistorystores
     localhistorystores = repo.manifestlog.localhistorystores
 
     sharedpackpath = shallowutil.getcachepackpath(repo,
                                             constants.TREEPACK_CATEGORY)
     localpackpath = shallowutil.getlocalpackpath(repo.svfs.vfs.base,
                                             constants.TREEPACK_CATEGORY)
 
     return ((localpackpath, localdatastores, localhistorystores),
             (sharedpackpath, shareddatastores, sharedhistorystores))
 
 def _topacks(packpath, files, constructor):
     paths = list(os.path.join(packpath, p) for p in files)
     packs = list(constructor(p) for p in paths)
     return packs
 
 def _deletebigpacks(repo, folder, files):
     """Deletes packfiles that are bigger than ``packs.maxpacksize``.
 
     Returns ``files` with the removed files omitted."""
     maxsize = repo.ui.configbytes("packs", "maxpacksize")
     if maxsize <= 0:
         return files
 
     # This only considers datapacks today, but we could broaden it to include
     # historypacks.
     VALIDEXTS = [".datapack", ".dataidx"]
 
     # Either an oversize index or datapack will trigger cleanup of the whole
     # pack:
     oversized = {os.path.splitext(path)[0] for path, ftype, stat in files
         if (stat.st_size > maxsize and (os.path.splitext(path)[1]
                                         in VALIDEXTS))}
 
     for rootfname in oversized:
         rootpath = os.path.join(folder, rootfname)
         for ext in VALIDEXTS:
             path = rootpath + ext
             repo.ui.debug('removing oversize packfile %s (%s)\n' %
                           (path, util.bytecount(os.stat(path).st_size)))
             os.unlink(path)
     return [row for row in files if os.path.basename(row[0]) not in oversized]
 
 def _incrementalrepack(repo, datastore, historystore, packpath, category,
         allowincompletedata=False, options=None):
     shallowutil.mkstickygroupdir(repo.ui, packpath)
 
     files = osutil.listdir(packpath, stat=True)
     files = _deletebigpacks(repo, packpath, files)
     datapacks = _topacks(packpath,
         _computeincrementaldatapack(repo.ui, files),
         datapack.datapack)
     datapacks.extend(s for s in datastore
                      if not isinstance(s, datapack.datapackstore))
 
     historypacks = _topacks(packpath,
         _computeincrementalhistorypack(repo.ui, files),
         historypack.historypack)
     historypacks.extend(s for s in historystore
                         if not isinstance(s, historypack.historypackstore))
 
     # ``allhistory{files,packs}`` contains all known history packs, even ones we
     # don't plan to repack. They are used during the datapack repack to ensure
     # good ordering of nodes.
     allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX,
                             historypack.INDEXSUFFIX)
     allhistorypacks = _topacks(packpath,
         (f for f, mode, stat in allhistoryfiles),
         historypack.historypack)
     allhistorypacks.extend(s for s in historystore
                         if not isinstance(s, historypack.historypackstore))
     _runrepack(repo,
                contentstore.unioncontentstore(
                    *datapacks,
                    allowincomplete=allowincompletedata),
                metadatastore.unionmetadatastore(
                    *historypacks,
                    allowincomplete=True),
                packpath, category,
                fullhistory=metadatastore.unionmetadatastore(
                    *allhistorypacks,
                    allowincomplete=True),
                 options=options)
 
 def _computeincrementaldatapack(ui, files):
     opts = {
         'gencountlimit' : ui.configint(
             'remotefilelog', 'data.gencountlimit'),
         'generations' : ui.configlist(
             'remotefilelog', 'data.generations'),
         'maxrepackpacks' : ui.configint(
             'remotefilelog', 'data.maxrepackpacks'),
         'repackmaxpacksize' : ui.configbytes(
             'remotefilelog', 'data.repackmaxpacksize'),
         'repacksizelimit' : ui.configbytes(
             'remotefilelog', 'data.repacksizelimit'),
     }
 
     packfiles = _allpackfileswithsuffix(
         files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX)
     return _computeincrementalpack(packfiles, opts)
 
 def _computeincrementalhistorypack(ui, files):
     opts = {
         'gencountlimit' : ui.configint(
             'remotefilelog', 'history.gencountlimit'),
         'generations' : ui.configlist(
             'remotefilelog', 'history.generations', ['100MB']),
         'maxrepackpacks' : ui.configint(
             'remotefilelog', 'history.maxrepackpacks'),
         'repackmaxpacksize' : ui.configbytes(
             'remotefilelog', 'history.repackmaxpacksize', '400MB'),
         'repacksizelimit' : ui.configbytes(
             'remotefilelog', 'history.repacksizelimit'),
     }
 
     packfiles = _allpackfileswithsuffix(
         files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX)
     return _computeincrementalpack(packfiles, opts)
 
 def _allpackfileswithsuffix(files, packsuffix, indexsuffix):
     result = []
     fileset = set(fn for fn, mode, stat in files)
     for filename, mode, stat in files:
         if not filename.endswith(packsuffix):
             continue
 
         prefix = filename[:-len(packsuffix)]
 
         # Don't process a pack if it doesn't have an index.
         if (prefix + indexsuffix) not in fileset:
             continue
         result.append((prefix, mode, stat))
 
     return result
 
 def _computeincrementalpack(files, opts):
     """Given a set of pack files along with the configuration options, this
     function computes the list of files that should be packed as part of an
     incremental repack.
 
     It tries to strike a balance between keeping incremental repacks cheap (i.e.
     packing small things when possible, and rolling the packs up to the big ones
     over time).
     """
 
     limits = list(sorted((util.sizetoint(s) for s in opts['generations']),
                                 reverse=True))
     limits.append(0)
 
     # Group the packs by generation (i.e. by size)
     generations = []
     for i in pycompat.xrange(len(limits)):
         generations.append([])
 
     sizes = {}
     for prefix, mode, stat in files:
         size = stat.st_size
         if size > opts['repackmaxpacksize']:
             continue
 
         sizes[prefix] = size
         for i, limit in enumerate(limits):
             if size > limit:
                 generations[i].append(prefix)
                 break
 
     # Steps for picking what packs to repack:
     # 1. Pick the largest generation with > gencountlimit pack files.
     # 2. Take the smallest three packs.
     # 3. While total-size-of-packs < repacksizelimit: add another pack
 
     # Find the largest generation with more than gencountlimit packs
     genpacks = []
     for i, limit in enumerate(limits):
         if len(generations[i]) > opts['gencountlimit']:
             # Sort to be smallest last, for easy popping later
             genpacks.extend(sorted(generations[i], reverse=True,
                                    key=lambda x: sizes[x]))
             break
 
     # Take as many packs from the generation as we can
     chosenpacks = genpacks[-3:]
     genpacks = genpacks[:-3]
     repacksize = sum(sizes[n] for n in chosenpacks)
     while (repacksize < opts['repacksizelimit'] and genpacks and
            len(chosenpacks) < opts['maxrepackpacks']):
         chosenpacks.append(genpacks.pop())
         repacksize += sizes[chosenpacks[-1]]
 
     return chosenpacks
 
 def _runrepack(repo, data, history, packpath, category, fullhistory=None,
                options=None):
     shallowutil.mkstickygroupdir(repo.ui, packpath)
 
     def isold(repo, filename, node):
         """Check if the file node is older than a limit.
         Unless a limit is specified in the config the default limit is taken.
         """
         filectx = repo.filectx(filename, fileid=node)
         filetime = repo[filectx.linkrev()].date()
 
         ttl = repo.ui.configint('remotefilelog', 'nodettl')
 
         limit = time.time() - ttl
         return filetime[0] < limit
 
     garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')
     if not fullhistory:
         fullhistory = history
     packer = repacker(repo, data, history, fullhistory, category,
                       gc=garbagecollect, isold=isold, options=options)
 
     with datapack.mutabledatapack(repo.ui, packpath) as dpack:
         with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
             try:
                 packer.run(dpack, hpack)
             except error.LockHeld:
                 raise RepackAlreadyRunning(_("skipping repack - another repack "
                                              "is already running"))
 
 def keepset(repo, keyfn, lastkeepkeys=None):
     """Computes a keepset which is not garbage collected.
     'keyfn' is a function that maps filename, node to a unique key.
     'lastkeepkeys' is an optional argument and if provided the keepset
     function updates lastkeepkeys with more keys and returns the result.
     """
     if not lastkeepkeys:
         keepkeys = set()
     else:
         keepkeys = lastkeepkeys
 
     # We want to keep:
     # 1. Working copy parent
     # 2. Draft commits
     # 3. Parents of draft commits
     # 4. Pullprefetch and bgprefetchrevs revsets if specified
     revs = ['.', 'draft()', 'parents(draft())']
     prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)
     if prefetchrevs:
         revs.append('(%s)' % prefetchrevs)
     prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)
     if prefetchrevs:
         revs.append('(%s)' % prefetchrevs)
     revs = '+'.join(revs)
 
     revs = ['sort((%s), "topo")' % revs]
     keep = scmutil.revrange(repo, revs)
 
     processed = set()
     lastmanifest = None
 
     # process the commits in toposorted order starting from the oldest
     for r in reversed(keep._list):
         if repo[r].p1().rev() in processed:
             # if the direct parent has already been processed
             # then we only need to process the delta
             m = repo[r].manifestctx().readdelta()
         else:
             # otherwise take the manifest and diff it
             # with the previous manifest if one exists
             if lastmanifest:
                 m = repo[r].manifest().diff(lastmanifest)
             else:
                 m = repo[r].manifest()
         lastmanifest = repo[r].manifest()
         processed.add(r)
 
         # populate keepkeys with keys from the current manifest
         if type(m) is dict:
             # m is a result of diff of two manifests and is a dictionary that
             # maps filename to ((newnode, newflag), (oldnode, oldflag)) tuple
             for filename, diff in m.iteritems():
                 if diff[0][0] is not None:
                     keepkeys.add(keyfn(filename, diff[0][0]))
         else:
             # m is a manifest object
             for filename, filenode in m.iteritems():
                 keepkeys.add(keyfn(filename, filenode))
 
     return keepkeys
 
 class repacker(object):
     """Class for orchestrating the repack of data and history information into a
     new format.
     """
     def __init__(self, repo, data, history, fullhistory, category, gc=False,
                  isold=None, options=None):
         self.repo = repo
         self.data = data
         self.history = history
         self.fullhistory = fullhistory
         self.unit = constants.getunits(category)
         self.garbagecollect = gc
         self.options = options
         if self.garbagecollect:
             if not isold:
                 raise ValueError("Function 'isold' is not properly specified")
             # use (filename, node) tuple as a keepset key
             self.keepkeys = keepset(repo, lambda f, n : (f, n))
             self.isold = isold
 
     def run(self, targetdata, targethistory):
         ledger = repackledger()
 
-        with extutil.flock(repacklockvfs(self.repo).join("repacklock"),
-                           _('repacking %s') % self.repo.origroot, timeout=0):
+        with lockmod.lock(repacklockvfs(self.repo), "repacklock", desc=None,
+                          timeout=0):
             self.repo.hook('prerepack')
 
             # Populate ledger from source
             self.data.markledger(ledger, options=self.options)
             self.history.markledger(ledger, options=self.options)
 
             # Run repack
             self.repackdata(ledger, targetdata)
             self.repackhistory(ledger, targethistory)
 
             # Call cleanup on each source
             for source in ledger.sources:
                 source.cleanup(ledger)
 
     def _chainorphans(self, ui, filename, nodes, orphans, deltabases):
         """Reorderes ``orphans`` into a single chain inside ``nodes`` and
         ``deltabases``.
 
         We often have orphan entries (nodes without a base that aren't
         referenced by other nodes -- i.e., part of a chain) due to gaps in
         history. Rather than store them as individual fulltexts, we prefer to
         insert them as one chain sorted by size.
         """
         if not orphans:
             return nodes
 
         def getsize(node, default=0):
             meta = self.data.getmeta(filename, node)
             if constants.METAKEYSIZE in meta:
                 return meta[constants.METAKEYSIZE]
             else:
                 return default
 
         # Sort orphans by size; biggest first is preferred, since it's more
         # likely to be the newest version assuming files grow over time.
         # (Sort by node first to ensure the sort is stable.)
         orphans = sorted(orphans)
         orphans = list(sorted(orphans, key=getsize, reverse=True))
         if ui.debugflag:
             ui.debug("%s: orphan chain: %s\n" % (filename,
                 ", ".join([short(s) for s in orphans])))
 
         # Create one contiguous chain and reassign deltabases.
         for i, node in enumerate(orphans):
             if i == 0:
                 deltabases[node] = (nullid, 0)
             else:
                 parent = orphans[i - 1]
                 deltabases[node] = (parent, deltabases[parent][1] + 1)
         nodes = [n for n in nodes if n not in orphans]
         nodes += orphans
         return nodes
 
     def repackdata(self, ledger, target):
         ui = self.repo.ui
         maxchainlen = ui.configint('packs', 'maxchainlen', 1000)
 
         byfile = {}
         for entry in ledger.entries.itervalues():
             if entry.datasource:
                 byfile.setdefault(entry.filename, {})[entry.node] = entry
 
         count = 0
         repackprogress = ui.makeprogress(_("repacking data"), unit=self.unit,
                                             total=len(byfile))
         for filename, entries in sorted(byfile.iteritems()):
             repackprogress.update(count)
 
             ancestors = {}
             nodes = list(node for node in entries)
             nohistory = []
             buildprogress = ui.makeprogress(_("building history"), unit='nodes',
                                             total=len(nodes))
             for i, node in enumerate(nodes):
                 if node in ancestors:
                     continue
                 buildprogress.update(i)
                 try:
                     ancestors.update(self.fullhistory.getancestors(filename,
                         node, known=ancestors))
                 except KeyError:
                     # Since we're packing data entries, we may not have the
                     # corresponding history entries for them. It's not a big
                     # deal, but the entries won't be delta'd perfectly.
                     nohistory.append(node)
             buildprogress.complete()
 
             # Order the nodes children first, so we can produce reverse deltas
             orderednodes = list(reversed(self._toposort(ancestors)))
             if len(nohistory) > 0:
                 ui.debug('repackdata: %d nodes without history\n' %
                          len(nohistory))
             orderednodes.extend(sorted(nohistory))
 
             # Filter orderednodes to just the nodes we want to serialize (it
             # currently also has the edge nodes' ancestors).
             orderednodes = list(filter(lambda node: node in nodes,
                                 orderednodes))
 
             # Garbage collect old nodes:
             if self.garbagecollect:
                 neworderednodes = []
                 for node in orderednodes:
                     # If the node is old and is not in the keepset, we skip it,
                     # and mark as garbage collected
                     if ((filename, node) not in self.keepkeys and
                         self.isold(self.repo, filename, node)):
                         entries[node].gced = True
                         continue
                     neworderednodes.append(node)
                 orderednodes = neworderednodes
 
             # Compute delta bases for nodes:
             deltabases = {}
             nobase = set()
             referenced = set()
             nodes = set(nodes)
             processprogress = ui.makeprogress(_("processing nodes"),
                                               unit='nodes',
                                               total=len(orderednodes))
             for i, node in enumerate(orderednodes):
                 processprogress.update(i)
                 # Find delta base
                 # TODO: allow delta'ing against most recent descendant instead
                 # of immediate child
                 deltatuple = deltabases.get(node, None)
                 if deltatuple is None:
                     deltabase, chainlen = nullid, 0
                     deltabases[node] = (nullid, 0)
                     nobase.add(node)
                 else:
                     deltabase, chainlen = deltatuple
                     referenced.add(deltabase)
 
                 # Use available ancestor information to inform our delta choices
                 ancestorinfo = ancestors.get(node)
                 if ancestorinfo:
                     p1, p2, linknode, copyfrom = ancestorinfo
 
                     # The presence of copyfrom means we're at a point where the
                     # file was copied from elsewhere. So don't attempt to do any
                     # deltas with the other file.
                     if copyfrom:
                         p1 = nullid
 
                     if chainlen < maxchainlen:
                         # Record this child as the delta base for its parents.
                         # This may be non optimal, since the parents may have
                         # many children, and this will only choose the last one.
                         # TODO: record all children and try all deltas to find
                         # best
                         if p1 != nullid:
                             deltabases[p1] = (node, chainlen + 1)
                         if p2 != nullid:
                             deltabases[p2] = (node, chainlen + 1)
 
             # experimental config: repack.chainorphansbysize
             if ui.configbool('repack', 'chainorphansbysize'):
                 orphans = nobase - referenced
                 orderednodes = self._chainorphans(ui, filename, orderednodes,
                     orphans, deltabases)
 
             # Compute deltas and write to the pack
             for i, node in enumerate(orderednodes):
                 deltabase, chainlen = deltabases[node]
                 # Compute delta
                 # TODO: Optimize the deltachain fetching. Since we're
                 # iterating over the different version of the file, we may
                 # be fetching the same deltachain over and over again.
                 if deltabase != nullid:
                     deltaentry = self.data.getdelta(filename, node)
                     delta, deltabasename, origdeltabase, meta = deltaentry
                     size = meta.get(constants.METAKEYSIZE)
                     if (deltabasename != filename or origdeltabase != deltabase
                         or size is None):
                         deltabasetext = self.data.get(filename, deltabase)
                         original = self.data.get(filename, node)
                         size = len(original)
                         delta = mdiff.textdiff(deltabasetext, original)
                 else:
                     delta = self.data.get(filename, node)
                     size = len(delta)
                     meta = self.data.getmeta(filename, node)
 
                 # TODO: don't use the delta if it's larger than the fulltext
                 if constants.METAKEYSIZE not in meta:
                     meta[constants.METAKEYSIZE] = size
                 target.add(filename, node, deltabase, delta, meta)
 
                 entries[node].datarepacked = True
 
             processprogress.complete()
             count += 1
 
         repackprogress.complete()
         target.close(ledger=ledger)
 
     def repackhistory(self, ledger, target):
         ui = self.repo.ui
 
         byfile = {}
         for entry in ledger.entries.itervalues():
             if entry.historysource:
                 byfile.setdefault(entry.filename, {})[entry.node] = entry
 
         progress = ui.makeprogress(_("repacking history"), unit=self.unit,
                                    total=len(byfile))
         for filename, entries in sorted(byfile.iteritems()):
             ancestors = {}
             nodes = list(node for node in entries)
 
             for node in nodes:
                 if node in ancestors:
                     continue
                 ancestors.update(self.history.getancestors(filename, node,
                                                            known=ancestors))
 
             # Order the nodes children first
             orderednodes = reversed(self._toposort(ancestors))
 
             # Write to the pack
             dontprocess = set()
             for node in orderednodes:
                 p1, p2, linknode, copyfrom = ancestors[node]
 
                 # If the node is marked dontprocess, but it's also in the
                 # explicit entries set, that means the node exists both in this
                 # file and in another file that was copied to this file.
                 # Usually this happens if the file was copied to another file,
                 # then the copy was deleted, then reintroduced without copy
                 # metadata. The original add and the new add have the same hash
                 # since the content is identical and the parents are null.
                 if node in dontprocess and node not in entries:
                     # If copyfrom == filename, it means the copy history
                     # went to come other file, then came back to this one, so we
                     # should continue processing it.
                     if p1 != nullid and copyfrom != filename:
                         dontprocess.add(p1)
                     if p2 != nullid:
                         dontprocess.add(p2)
                     continue
 
                 if copyfrom:
                     dontprocess.add(p1)
 
                 target.add(filename, node, p1, p2, linknode, copyfrom)
 
                 if node in entries:
                     entries[node].historyrepacked = True
 
             progress.increment()
 
         progress.complete()
         target.close(ledger=ledger)
 
     def _toposort(self, ancestors):
         def parentfunc(node):
             p1, p2, linknode, copyfrom = ancestors[node]
             parents = []
             if p1 != nullid:
                 parents.append(p1)
             if p2 != nullid:
                 parents.append(p2)
             return parents
 
         sortednodes = shallowutil.sortnodes(ancestors.keys(), parentfunc)
         return sortednodes
 
 class repackledger(object):
     """Storage for all the bookkeeping that happens during a repack. It contains
     the list of revisions being repacked, what happened to each revision, and
     which source store contained which revision originally (for later cleanup).
     """
     def __init__(self):
         self.entries = {}
         self.sources = {}
         self.created = set()
 
     def markdataentry(self, source, filename, node):
         """Mark the given filename+node revision as having a data rev in the
         given source.
         """
         entry = self._getorcreateentry(filename, node)
         entry.datasource = True
         entries = self.sources.get(source)
         if not entries:
             entries = set()
             self.sources[source] = entries
         entries.add(entry)
 
     def markhistoryentry(self, source, filename, node):
         """Mark the given filename+node revision as having a history rev in the
         given source.
         """
         entry = self._getorcreateentry(filename, node)
         entry.historysource = True
         entries = self.sources.get(source)
         if not entries:
             entries = set()
             self.sources[source] = entries
         entries.add(entry)
 
     def _getorcreateentry(self, filename, node):
         key = (filename, node)
         value = self.entries.get(key)
         if not value:
             value = repackentry(filename, node)
             self.entries[key] = value
 
         return value
 
     def addcreated(self, value):
         self.created.add(value)
 
 class repackentry(object):
     """Simple class representing a single revision entry in the repackledger.
     """
     __slots__ = (r'filename', r'node', r'datasource', r'historysource',
                  r'datarepacked', r'historyrepacked', r'gced')
     def __init__(self, filename, node):
         self.filename = filename
         self.node = node
         # If the revision has a data entry in the source
         self.datasource = False
         # If the revision has a history entry in the source
         self.historysource = False
         # If the revision's data entry was repacked into the repack target
         self.datarepacked = False
         # If the revision's history entry was repacked into the repack target
         self.historyrepacked = False
         # If garbage collected
         self.gced = False
 
 def repacklockvfs(repo):
     if util.safehasattr(repo, 'name'):
         # Lock in the shared cache so repacks across multiple copies of the same
         # repo are coordinated.
         sharedcachepath = shallowutil.getcachepackpath(
             repo,
             constants.FILEPACK_CATEGORY)
         return vfs.vfs(sharedcachepath)
     else:
         return repo.svfs

tests/test-remotefilelog-bgprefetch.t

@@ -1,381 +1,376 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 # devel.remotefilelog.ensurestart: reduce race condition with
 # waiton{repack/prefetch}
   $ cat >> $HGRCPATH <<EOF
   > [devel]
   > remotefilelog.ensurestart=True
   > EOF
 
   $ hg init master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > EOF
   $ echo x > x
   $ echo z > z
   $ hg commit -qAm x
   $ echo x2 > x
   $ echo y > y
   $ hg commit -qAm y
   $ echo w > w
   $ rm z
   $ hg commit -qAm w
   $ hg bookmark foo
 
   $ cd ..
 
 # clone the repo
 
   $ hgcloneshallow ssh://user@dummy/master shallow --noupdate
   streaming all changes
   2 files to transfer, 776 bytes of data
   transferred 776 bytes in * seconds (*/sec) (glob)
   searching for changes
   no changes found
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is. Also set prefetchdelay config
 # to zero so that there is no delay between prefetches.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > prefetchdelay=0
   > EOF
   $ cd ..
 
 # prefetch a revision
   $ cd shallow
 
   $ hg prefetch -r 0
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
 
   $ hg cat -r 0 x
   x
 
 # background prefetch on pull when configured
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > pullprefetch=bookmark()
   > backgroundprefetch=True
   > EOF
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/6b4b6f66ef8c-b4b8bdaf-backup.hg (glob)
 
   $ clearcache
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   updating bookmark foo
   added 1 changesets with 0 changes to 0 files
   new changesets 6b4b6f66ef8c
   (run 'hg update' to get a working copy)
   prefetching file contents
   $ sleep 0.5
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/ef95c5376f34698742fe34f315fd82136f8f68c0
   $TESTTMP/hgcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
   $TESTTMP/hgcache/master/af/f024fe4ab0fece4091de044c58c9ae4233383a/bb6ccd5dceaa5e9dc220e0dad65e051b94f69a2c
   $TESTTMP/hgcache/repos
 
 # background prefetch with repack on pull when configured
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > backgroundrepack=True
   > EOF
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/6b4b6f66ef8c-b4b8bdaf-backup.hg (glob)
 
   $ clearcache
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   updating bookmark foo
   added 1 changesets with 0 changes to 0 files
   new changesets 6b4b6f66ef8c
   (run 'hg update' to get a working copy)
   prefetching file contents
   $ sleep 0.5
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ sleep 0.5
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx
   $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack
   $TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.dataidx
   $TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # background prefetch with repack on update when wcprevset configured
 
   $ clearcache
   $ hg up -r 0
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $TESTTMP/hgcache/master/39/5df8f7c51f007019cb30201c49e884b46b92fa/69a1b67522704ec122181c0890bd16e9d3e7516a
   $TESTTMP/hgcache/repos
 
   $ hg up -r 1
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=.::
   > EOF
 
   $ clearcache
   $ hg up -r 0
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   * files fetched over * fetches - (* misses, 0.00% hit ratio) over *s (glob)
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 1
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ sleep 1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that file 'w' was prefetched - it was not part of the update operation and therefore
 # could only be downloaded by the background prefetch
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # background prefetch with repack on commit when wcprevset configured
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > bgprefetchrevs=0::
   > EOF
 
   $ clearcache
   $ find $CACHEDIR -type f | sort
   $ echo b > b
   $ hg commit -qAm b
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob)
   $ hg bookmark temporary
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 1
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ sleep 1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore
 # could only be downloaded by the background prefetch
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # background prefetch with repack on rebase when wcprevset configured
 
   $ hg up -r 2
   3 files updated, 0 files merged, 3 files removed, 0 files unresolved
   (leaving bookmark temporary)
   $ clearcache
   $ find $CACHEDIR -type f | sort
   $ hg rebase -s temporary -d foo
   rebasing 3:58147a5b5242 "b" (temporary tip)
-  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg (glob)
+  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg
   3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
   $ sleep 1
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ sleep 1
 
 # Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore
 # could only be downloaded by the background prefetch
 
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # Check that foregound prefetch with no arguments blocks until background prefetches finish
 
   $ hg up -r 3
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ clearcache
   $ hg prefetch --repack
   waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?)
   got lock after * seconds (glob) (?)
   (running background incremental repack)
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
 
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ sleep 0.5
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that files were prefetched
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # Check that foreground prefetch fetches revs specified by '. + draft() + bgprefetchrevs + pullprefetch'
 
   $ clearcache
   $ hg prefetch --repack
   waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?)
   got lock after * seconds (glob) (?)
   (running background incremental repack)
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>%1
   $ sleep 0.5
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that files were prefetched
   $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
   w:
   Node          Delta Base    Delta Length  Blob Size
   bb6ccd5dceaa  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   x:
   Node          Delta Base    Delta Length  Blob Size
   ef95c5376f34  000000000000  3             3
   1406e7411862  ef95c5376f34  14            2
   
   Total:                      17            5         (240.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   076f5e2225b3  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
   z:
   Node          Delta Base    Delta Length  Blob Size
   69a1b6752270  000000000000  2             2
   
   Total:                      2             2         (0.0% bigger)
 
 # Test that if data was prefetched and repacked we dont need to prefetch it again
 # It ensures that Mercurial looks not only in loose files but in packs as well
 
   $ hg prefetch --repack
   (running background incremental repack)

tests/test-remotefilelog-gc.t

@@ -1,112 +1,111 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ hg init master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > serverexpiration=-1
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # commit a new version of x so we can gc the old one
 
   $ cd master
   $ echo y > x
   $ hg commit -qAm y
   $ cd ..
 
   $ cd shallow
   $ hg pull -q
   $ hg update -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
   $ cd ..
 
 # gc client cache
 
   $ lastweek=`$PYTHON -c 'import datetime,time; print(datetime.datetime.fromtimestamp(time.time() - (86400 * 7)).strftime("%y%m%d%H%M"))'`
   $ find $CACHEDIR -type f -exec touch -t $lastweek {} \;
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0 (glob)
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $TESTTMP/hgcache/repos (glob)
   $ hg gc
   finished: removed 1 of 2 files (0.00 GB to 0.00 GB)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $TESTTMP/hgcache/repos
 
 # gc server cache
 
   $ find master/.hg/remotefilelogcache -type f | sort
   master/.hg/remotefilelogcache/x/1406e74118627694268417491f018a4a883152f0 (glob)
   master/.hg/remotefilelogcache/x/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $ hg gc master
   finished: removed 0 of 1 files (0.00 GB to 0.00 GB)
   $ find master/.hg/remotefilelogcache -type f | sort
   master/.hg/remotefilelogcache/x/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
 
 # Test that GC keepset includes pullprefetch revset if it is configured
 
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > pullprefetch=all()
   > EOF
   $ hg prefetch
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
   $ cd ..
   $ hg gc
   finished: removed 0 of 2 files (0.00 GB to 0.00 GB)
 
 # Ensure that there are 2 versions of the file in cache
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0 (glob)
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/48023ec064c1d522f0d792a5a912bb1bf7859a4a (glob)
   $TESTTMP/hgcache/repos (glob)
 
 # Test that if garbage collection on repack and repack on hg gc flags are set then incremental repack with garbage collector is run
 
   $ hg gc --config remotefilelog.gcrepack=True --config remotefilelog.repackonhggc=True
 
 # Ensure that loose files are repacked
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.dataidx
   $TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.datapack
   $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histidx
   $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that warning is displayed when there are no valid repos in repofile
 
   $ cp $CACHEDIR/repos $CACHEDIR/repos.bak
   $ echo " " > $CACHEDIR/repos
   $ hg gc
   warning: no valid repos in repofile
   $ mv $CACHEDIR/repos.bak $CACHEDIR/repos
 
 # Test that warning is displayed when the repo path is malformed
 
   $ printf "asdas\0das" >> $CACHEDIR/repos
   $ hg gc
   abort: invalid path asdas\x00da: .*(null|NULL).* (re)
   [255]

tests/test-remotefilelog-repack-fast.t

@@ -1,387 +1,379 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 # devel.remotefilelog.ensurestart: reduce race condition with
 # waiton{repack/prefetch}
   $ cat >> $HGRCPATH <<EOF
   > [remotefilelog]
   > fastdatapack=True
   > [devel]
   > remotefilelog.ensurestart=True
   > EOF
 
   $ hg init master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > serverexpiration=-1
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ echo x >> x
   $ hg commit -qAm x2
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # Test that repack cleans up the old files and creates new packs
 
   $ cd shallow
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/11
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
   $TESTTMP/hgcache/repos
 
   $ hg repack
 
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/packs
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that the packs are readonly
   $ ls_l $CACHEDIR/master/packs
   -r--r--r--    1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   -r--r--r--     172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   -r--r--r--    1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   -r--r--r--      72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  -rw-r--r--       0 repacklock
 
 # Test that the data in the new packs is accessible
   $ hg cat -r . x
   x
   x
 
 # Test that adding new data and repacking it results in the loose data and the
 # old packs being combined.
 
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x3
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --traceback
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Verify all the file data is still available
   $ hg cat -r . x
   x
   x
   x
   $ hg cat -r '.^' x
   x
   x
 
 # Test that repacking again without new data does not delete the pack files
 # and did not change the pack names
   $ hg repack
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Run two repacks at once
   $ hg repack --config "hooks.prerepack=sleep 3" &
   $ sleep 1
   $ hg repack
   skipping repack - another repack is already running
   $ hg debugwaitonrepack >/dev/null 2>&1
 
 # Run repack in the background
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x4
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --background
   (running background repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test debug commands
 
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
   x:
   Node                                      Delta Base                                Delta Length  Blob Size
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  8             8
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  12            6
   aee31534993a501858fb6dd96a065671922e7d51  d4a3ed9310e5bd9887e3bf779da5077efab28216  12            4
   
   Total:                                                                              32            18        (77.8% bigger)
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
   
   x
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  77029ab56e83ea2115dd53ff87483682abe5d7ca  12
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5  8
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
 
 # Test copy tracing from a pack
   $ cd ../master
   $ hg mv x y
   $ hg commit -m 'move x to y'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg log -f y -T '{desc}\n'
   move x to y
   x4
   x3
   x2
   x
 
 # Test copy trace across rename and back
   $ cp -R $TESTTMP/hgcache/master/packs $TESTTMP/backuppacks
   $ cd ../master
   $ hg mv y x
   $ hg commit -m 'move y back to x'
   $ hg revert -r 0 x
   $ mv x y
   $ hg add y
   $ echo >> y
   $ hg revert x
   $ hg commit -m 'add y back without metadata'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ ls $TESTTMP/hgcache/master/packs
   bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
   bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
   fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx
   fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack
-  repacklock
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   cd410a44d584  577959738234  000000000000  609547eda446  y
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
   
   y
   Node          P1 Node       P2 Node       Link Node     Copy From
   577959738234  1bb2e6237e03  000000000000  c7faf2fc439a  x
   21f46f2721e7  000000000000  000000000000  d6868642b790  
   $ hg strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
   $ hg -R ../master strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/master/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
 
   $ rm -rf $TESTTMP/hgcache/master/packs
   $ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs
 
 # Test repacking datapack without history
   $ rm -rf $CACHEDIR/master/packs/*hist*
   $ hg repack
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/922aca43dbbeda4d250565372e8892ec7b08da6a:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   577959738234  000000000000  70            8
   
   Total:                      70            8         (775.0% bigger)
 
   $ hg cat -r ".^" x
   x
   x
   x
   x
 
 Incremental repack
   $ rm -rf $CACHEDIR/master/packs/*
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > data.generations=60
   >   150
   > EOF
 
 Single pack - repack does nothing
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   [1]
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   [1]
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 3
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   -r--r--r--     226 39443fa1064182e93d968b5cba292eb5283260d0.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 1 gen3 pack, 1 gen0 pack - does nothing
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   -r--r--r--     226 39443fa1064182e93d968b5cba292eb5283260d0.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 Pull should run background repack
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > backgroundrepack=True
   > EOF
   $ clearcache
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 3
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   no changes found
   (running background incremental repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--     303 156a6c1c83aeb69422d7936e0a46ba9bc06a71c0.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
 
 Test environment variable resolution
   $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $TESTTMP/envcache | sort
   $TESTTMP/envcache
   $TESTTMP/envcache/master
   $TESTTMP/envcache/master/95
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/577959738234a1eb241ed3ed4b22a575833f56e0
   $TESTTMP/envcache/repos
 
 Test local remotefilelog blob is correct when based on a pack
   $ hg prefetch -r .
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ echo >> y
   $ hg commit -m y2
   $ hg debugremotefilelog .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808
   size: 9 bytes
   path: .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808 
   key: b70860edba4f 
   
           node =>           p1            p2      linknode     copyfrom
   b70860edba4f => 577959738234  000000000000  08d3fbc98c48  
   577959738234 => 1bb2e6237e03  000000000000  c7faf2fc439a  x
   1bb2e6237e03 => d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5 => aee31534993a  000000000000  421535db10b6  
   aee31534993a => 1406e7411862  000000000000  a89d614e2364  
   1406e7411862 => 000000000000  000000000000  b292c1e3311f  

tests/test-remotefilelog-repack.t

@@ -1,468 +1,459 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 # devel.remotefilelog.ensurestart: reduce race condition with
 # waiton{repack/prefetch}
   $ cat >> $HGRCPATH <<EOF
   > [devel]
   > remotefilelog.ensurestart=True
   > EOF
 
   $ hg init master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > serverexpiration=-1
   > EOF
   $ echo x > x
   $ hg commit -qAm x
   $ echo x >> x
   $ hg commit -qAm x2
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 # Set the prefetchdays config to zero so that all commits are prefetched
 # no matter what their creation date is.
   $ cd shallow
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > prefetchdays=0
   > EOF
   $ cd ..
 
 # Test that repack cleans up the old files and creates new packs
 
   $ cd shallow
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/11
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
   $TESTTMP/hgcache/repos
 
   $ hg repack
 
   $ find $CACHEDIR | sort
   $TESTTMP/hgcache
   $TESTTMP/hgcache/master
   $TESTTMP/hgcache/master/packs
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that the packs are readonly
   $ ls_l $CACHEDIR/master/packs
   -r--r--r--    1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   -r--r--r--     172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   -r--r--r--    1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   -r--r--r--      72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  -rw-r--r--       0 repacklock
 
 # Test that the data in the new packs is accessible
   $ hg cat -r . x
   x
   x
 
 # Test that adding new data and repacking it results in the loose data and the
 # old packs being combined.
 
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x3
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # First assert that with --packsonly, the loose object will be ignored:
 
   $ hg repack --packsonly
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --traceback
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Verify all the file data is still available
   $ hg cat -r . x
   x
   x
   x
   $ hg cat -r '.^' x
   x
   x
 
 # Test that repacking again without new data does not delete the pack files
 # and did not change the pack names
   $ hg repack
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Run two repacks at once
   $ hg repack --config "hooks.prerepack=sleep 3" &
   $ sleep 1
   $ hg repack
   skipping repack - another repack is already running
   $ hg debugwaitonrepack >/dev/null 2>&1
 
 # Run repack in the background
   $ cd ../master
   $ echo x >> x
   $ hg commit -m x4
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --background
   (running background repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test debug commands
 
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
   x:
   Node                                      Delta Base                                Delta Length  Blob Size
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  8             8
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  12            6
   aee31534993a501858fb6dd96a065671922e7d51  d4a3ed9310e5bd9887e3bf779da5077efab28216  12            4
   
   Total:                                                                              32            18        (77.8% bigger)
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
   
   x
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   d4a3ed9310e5bd9887e3bf779da5077efab28216  1bb2e6237e035c8f8ef508e281f1ce075bc6db72  77029ab56e83ea2115dd53ff87483682abe5d7ca  12
   Node                                      Delta Base                                Delta SHA1                                Delta Length
   1bb2e6237e035c8f8ef508e281f1ce075bc6db72  0000000000000000000000000000000000000000  7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5  8
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
 
 # Test copy tracing from a pack
   $ cd ../master
   $ hg mv x y
   $ hg commit -m 'move x to y'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg log -f y -T '{desc}\n'
   move x to y
   x4
   x3
   x2
   x
 
 # Test copy trace across rename and back
   $ cp -R $TESTTMP/hgcache/master/packs $TESTTMP/backuppacks
   $ cd ../master
   $ hg mv y x
   $ hg commit -m 'move y back to x'
   $ hg revert -r 0 x
   $ mv x y
   $ hg add y
   $ echo >> y
   $ hg revert x
   $ hg commit -m 'add y back without metadata'
   $ cd ../shallow
   $ hg pull -q
   $ hg up -q tip
   2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ ls $TESTTMP/hgcache/master/packs
   bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
   bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
   fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx
   fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack
-  repacklock
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
   Node          P1 Node       P2 Node       Link Node     Copy From
   cd410a44d584  577959738234  000000000000  609547eda446  y
   1bb2e6237e03  d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5  aee31534993a  000000000000  421535db10b6  
   aee31534993a  1406e7411862  000000000000  a89d614e2364  
   1406e7411862  000000000000  000000000000  b292c1e3311f  
   
   y
   Node          P1 Node       P2 Node       Link Node     Copy From
   577959738234  1bb2e6237e03  000000000000  c7faf2fc439a  x
   21f46f2721e7  000000000000  000000000000  d6868642b790  
   $ hg strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
   $ hg -R ../master strip -r '.^'
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/master/.hg/strip-backup/609547eda446-b26b56a8-backup.hg (glob)
 
   $ rm -rf $TESTTMP/hgcache/master/packs
   $ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs
 
 # Test repacking datapack without history
   $ rm -rf $CACHEDIR/master/packs/*hist*
   $ hg repack
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/922aca43dbbeda4d250565372e8892ec7b08da6a:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   
   Total:                      32            18        (77.8% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   577959738234  000000000000  70            8
   
   Total:                      70            8         (775.0% bigger)
 
   $ hg cat -r ".^" x
   x
   x
   x
   x
 
 Incremental repack
   $ rm -rf $CACHEDIR/master/packs/*
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > data.generations=60
   >   150
   > EOF
 
 Single pack - repack does nothing
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   [1]
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   [1]
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 38
   abort: unknown revision '38'!
   [255]
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 For the data packs, setting the limit for the repackmaxpacksize to be 64 such
 that data pack with size 65 is more than the limit. This effectively ensures
 that no generation has 3 packs and therefore, no packs are chosen for the
 incremental repacking. As for the history packs, setting repackmaxpacksize to be
 0 which should always result in no repacking.
   $ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=64 \
   > --config remotefilelog.history.repackmaxpacksize=0
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   -r--r--r--     149 78840d69389c7404327f7477e3931c89945c37d1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 Setting limit for the repackmaxpacksize to be the size of the biggest pack file
 which ensures that it is effectively ignored in the incremental repacking.
   $ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=65 \
   > --config remotefilelog.history.repackmaxpacksize=336
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   -r--r--r--     149 78840d69389c7404327f7477e3931c89945c37d1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 1 gen3 pack, 1 gen0 pack - does nothing
   $ hg repack --incremental
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--      70 052643fdcdebbd42d7c180a651a30d46098e6fe1.datapack
   -r--r--r--     149 78840d69389c7404327f7477e3931c89945c37d1.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
   -r--r--r--      90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
 
 Pull should run background repack
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > backgroundrepack=True
   > EOF
   $ clearcache
   $ hg prefetch -r 0
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 1
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 2
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ hg prefetch -r 3
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
 
   $ hg pull
   pulling from ssh://user@dummy/master
   searching for changes
   no changes found
   (running background incremental repack)
   $ sleep 0.5
   $ hg debugwaitonrepack >/dev/null 2>&1
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--     303 156a6c1c83aeb69422d7936e0a46ba9bc06a71c0.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
   -r--r--r--     336 604552d403a1381749faf656feca0ca265a6d52c.histpack
 
 Test environment variable resolution
   $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ find $TESTTMP/envcache | sort
   $TESTTMP/envcache
   $TESTTMP/envcache/master
   $TESTTMP/envcache/master/95
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a
   $TESTTMP/envcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/577959738234a1eb241ed3ed4b22a575833f56e0
   $TESTTMP/envcache/repos
 
 Test local remotefilelog blob is correct when based on a pack
   $ hg prefetch -r .
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
   $ echo >> y
   $ hg commit -m y2
   $ hg debugremotefilelog .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808
   size: 9 bytes
   path: .hg/store/data/95cb0bfd2977c761298d9624e4b4d4c72a39974a/b70860edba4f8242a1d52f2a94679dd23cb76808 
   key: b70860edba4f 
   
           node =>           p1            p2      linknode     copyfrom
   b70860edba4f => 577959738234  000000000000  08d3fbc98c48  
   577959738234 => 1bb2e6237e03  000000000000  c7faf2fc439a  x
   1bb2e6237e03 => d4a3ed9310e5  000000000000  0b03bbc9e1e7  
   d4a3ed9310e5 => aee31534993a  000000000000  421535db10b6  
   aee31534993a => 1406e7411862  000000000000  a89d614e2364  
   1406e7411862 => 000000000000  000000000000  b292c1e3311f  
 
 Test limiting the max delta chain length
   $ hg repack --config packs.maxchainlen=1
   $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.dataidx
   $TESTTMP/hgcache/master/packs/f258af4c033dd5cd32b4dbc42a1efcd8e4c7d909:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  000000000000  4             4
   1406e7411862  aee31534993a  12            2
   
   Total:                      36            20        (80.0% bigger)
   y:
   Node          Delta Base    Delta Length  Blob Size
   577959738234  000000000000  70            8
   
   Total:                      70            8         (775.0% bigger)
 
 Test huge pack cleanup using different values of packs.maxpacksize:
   $ hg repack --incremental --debug
   $ hg repack --incremental --debug --config packs.maxpacksize=512
   removing oversize packfile $TESTTMP/hgcache/master/packs/f258af4c033dd5cd32b4dbc42a1efcd8e4c7d909.datapack (425 bytes)
   removing oversize packfile $TESTTMP/hgcache/master/packs/f258af4c033dd5cd32b4dbc42a1efcd8e4c7d909.dataidx (1.21 KB)
 
 Do a repack where the new pack reuses a delta from the old pack
   $ clearcache
   $ hg prefetch -r '2::3'
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg debugdatapack $CACHEDIR/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/9ec6b30891bd851320acb7c66b69a2bdf41c8df3:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   
   Total:                      20            14        (42.9% bigger)
   $ hg prefetch -r '0::1'
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
   $ hg repack
   $ hg debugdatapack $CACHEDIR/master/packs/*.datapack
   $TESTTMP/hgcache/master/packs/156a6c1c83aeb69422d7936e0a46ba9bc06a71c0:
   x:
   Node          Delta Base    Delta Length  Blob Size
   1bb2e6237e03  000000000000  8             8
   d4a3ed9310e5  1bb2e6237e03  12            6
   aee31534993a  d4a3ed9310e5  12            4
   1406e7411862  aee31534993a  12            2
   
   Total:                      44            20        (120.0% bigger)