interfaces: create a new folder for interfaces and move repository.py in it

I was trying to understand current interfaces and write new ones and I realized
we need to improve how current interfaces are organised. This creates a
dedicated folder for defining interfaces and move `repository.py` which defines
all the current interfaces inside it.

Differential Revision: https://phab.mercurial-scm.org/D6741

contrib/import-checker.py

@@ -1,748 +1,749
 #!/usr/bin/env python
 
 from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import os
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
 if True: # disable lexical sorting checks
     try:
         import BaseHTTPServer as basehttpserver
     except ImportError:
         basehttpserver = None
     import zlib
 
 import testparseutil
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
     'bzrlib',
     'hgclient',
     'mercurial',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
+    'mercurial.interfaces',
     'mercurial.node',
     # for revlog to re-export constant to extensions
     'mercurial.revlogutils.constants',
     'mercurial.revlogutils.flagutil',
     # for cffi modules to re-export pure functions
     'mercurial.pure.base85',
     'mercurial.pure.bdiff',
     'mercurial.pure.mpatch',
     'mercurial.pure.osutil',
     'mercurial.pure.parsers',
     # third-party imports should be directly imported
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
 )
 
 # Whitelist of symbols that can be directly imported.
 directsymbols = (
     'demandimport',
 )
 
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 def usingabsolute(root):
     """Whether absolute imports are being used."""
     if sys.version_info[0] >= 3:
         return True
 
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 for n in node.names:
                     if n.name == 'absolute_import':
                         return True
 
     return False
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 def dotted_name_of_path(path):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     return '.'.join(parts)
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
     paths of locally defined (= Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__', 'foo.foo1',
     ...              'foo.bar.__init__', 'foo.bar.bar1',
     ...              'baz.__init__', 'baz.baz1'}
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
     >>> fromlocal('foo1', 1)
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
     >>> fromlocal2('bar2', 1)
     False
     >>> fromlocal2('bar', 2)
     ('foo.bar', 'foo.bar.__init__', True)
     """
     if not isinstance(modulename, str):
         modulename = modulename.decode('ascii')
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
     def fromlocal(name, level=0):
         # name is false value when relative imports are used.
         if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
             if not level:
                 # Check relative name first.
                 candidates = [prefix + name, name]
             else:
                 candidates = ['.'.join(modulename.split('.')[:-level]) +
                               '.' + name]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
     return fromlocal
 
 def populateextmods(localmods):
     """Populate C extension modules based on pure modules"""
     newlocalmods = set(localmods)
     for n in localmods:
         if n.startswith('mercurial.pure.'):
             m = n[len('mercurial.pure.'):]
             newlocalmods.add('mercurial.cext.' + m)
             newlocalmods.add('mercurial.cffi._' + m)
     return newlocalmods
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> py3 = sys.version_info[0] >= 3
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods or py3
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods or py3
     True
 
     >>> 'cffi' in mods
     True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield '__builtin__'
     yield 'builtins' # python3 only
     yield 'importlib.abc' # python3 only
     yield 'importlib.machinery' # python3 only
     yield 'importlib.util' # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
         yield m
     for m in ['cffi']:
         yield m
     stdlib_prefixes = {sys.prefix, sys.exec_prefix}
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (basehttpserver, zlib):
         if mod is None:
             continue
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
     sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
     for libpath in sys.path:
         # We want to walk everything in sys.path that starts with something in
         # stdlib_prefixes, but not directories from the hg sources.
         if (os.path.abspath(libpath).startswith(sourceroot)
             or not any(libpath.startswith(p) for p in stdlib_prefixes)):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
                 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath and d in ('hgdemandimport', 'hgext',
                                                 'mercurial')):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1:]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 stdlib_modules = set(list_stdlib_modules())
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: set of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
             if modnotfound:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
                 yield dottedpath
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention.
 
     We have 2 conventions: legacy and modern. The modern convention is in
     effect when using absolute imports.
 
     The legacy convention only looks for mixed imports. The modern convention
     is much more thorough.
     """
     root = ast.parse(source)
     absolute = usingabsolute(root)
 
     if absolute:
         return verify_modern_convention(module, root, localmods)
     else:
         return verify_stdlib_on_own_line(root)
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
       must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed, unless that
       library is also a local module.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     if not isinstance(module, str):
         module = module.decode('ascii')
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
     # Whether a local/non-stdlib, non-symbol import has been seen.
     seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     laststdlib = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(module, node, localmods,
                                               node.col_offset + 4):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg('multiple imported names: %s',
                           ', '.join(n.name for n in node.names))
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             stdlib = name in stdlib_modules
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname and laststdlib == stdlib:
                     yield msg('imports not lexically sorted: %s < %s',
                               name, lastname)
 
             lastname = name
             laststdlib = stdlib
 
             # stdlib imports should be before local imports.
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg('stdlib import "%s" follows local import: %s',
                           name, seenlocal)
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg('%s module must be "as" aliased to %s',
                           name, requirealias[name])
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[:-node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or (
                     fullname in stdlib_modules
                     and fullname not in localmods
                     and fullname + '.__init__' not in localmods):
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
             found = fromlocal(node.module, node.level)
             if found and found[2]:  # node.module is a package
                 prefix = found[0] + '.'
                 symbols = (n.name for n in node.names
                            if not fromlocal(prefix + n.name))
             else:
                 symbols = (n.name for n in node.names)
             symbols = [sym for sym in symbols if sym not in directsymbols]
             if node.module and node.col_offset == root_col_offset:
                 if symbols and fullname not in allowsymbolimports:
                     yield msg('direct symbol import %s from %s',
                               ', '.join(symbols), fullname)
 
                 if symbols and seennonsymbollocal:
                     yield msg('symbol import follows non-symbol import: %s',
                               fullname)
             if not symbols and fullname not in stdlib_modules:
                 seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
 
                 # Only allow 1 group per level.
                 if (node.level in seenlevels
                     and node.col_offset == root_col_offset):
                     yield msg('multiple "from %s import" statements',
                               '.' * node.level)
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg('higher-level import should come first: %s',
                               fullname)
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg('imports from %s not lexically sorted: %s < %s',
                               fullname, n.name, lastentryname)
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg('%s from %s must be "as" aliased to %s',
                               n.name, fullname, requirealias[n.name])
 
 def verify_stdlib_on_own_line(root):
     """Given some python source, verify that stdlib imports are done
     in separate statements from relative local module imports.
 
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
     [('mixed imports\\n   stdlib:    sys\\n   relative:  foo', 1)]
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
     []
     >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
     []
     """
     for node in ast.walk(root):
         if isinstance(node, ast.Import):
             from_stdlib = {False: [], True: []}
             for n in node.names:
                 from_stdlib[n.name in stdlib_modules].append(n.name)
             if from_stdlib[True] and from_stdlib[False]:
                 yield ('mixed imports\n   stdlib:    %s\n   relative:  %s' %
                        (', '.join(sorted(from_stdlib[True])),
                         ', '.join(sorted(from_stdlib[False]))), node.lineno)
 
 class CircularImport(Exception):
     pass
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 def _cycle_sortkey(c):
     return len(c), c
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def _forcestr(thing):
     ...     if not isinstance(thing, str):
     ...         return thing.decode('ascii')
     ...     return thing
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, b"example", lines):
     ...         print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
     ...         print(repr(_forcestr(s)))
     >>> lines = [
     ...   'comment',
     ...   '  >>> from __future__ import print_function',
     ...   "  >>> ' multiline",
     ...   "  ... string'",
     ...   '  ',
     ...   'comment',
     ...   '  $ cat > foo.py <<EOF',
     ...   '  > from __future__ import print_function',
     ...   '  > EOF',
     ... ]
     >>> test(b"example.t", lines)
     example[2] doctest.py 1
     "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
     example[8] foo.py 7
     'from __future__ import print_function\\n'
     """
     errors = []
     for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
         if not name:
             # use 'doctest.py', in order to make already existing
             # doctest above pass instantly
             name = 'doctest.py'
         # "starts" is "line number" (1-origin), but embedded() is
         # expected to return "line offset" (0-origin). Therefore, this
         # yields "starts - 1".
         if not isinstance(modname, str):
             modname = modname.decode('utf8')
         yield code, "%s[%d]" % (modname, starts), name, starts - 1
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
     if not f.endswith('.t'):
         with open(f, 'rb') as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         with open(f, 'r') as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname.encode('utf8'), t, line
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmodpaths = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
         modname = dotted_name_of_path(source_path)
         localmodpaths[modname] = source_path
     localmods = populateextmods(localmodpaths)
     for localmodname, source_path in sorted(localmodpaths.items()):
         if not isinstance(localmodname, bytes):
             # This is only safe because all hg's files are ascii
             localmodname = localmodname.encode('ascii')
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(src, modname, name, localmods,
                                      ignore_nested=True))
                 for error, lineno in verify_import_convention(modname, src,
                                                               localmods):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print('%s:%d: SyntaxError: %s' %
                       (source_path, e.lineno + line, e))
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

hgext/lfs/__init__.py

@@ -1,383 +1,386
 # lfs - hash-preserving large file support using Git-LFS protocol
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """lfs - large file support (EXPERIMENTAL)
 
 This extension allows large files to be tracked outside of the normal
 repository storage and stored on a centralized server, similar to the
 ``largefiles`` extension.  The ``git-lfs`` protocol is used when
 communicating with the server, so existing git infrastructure can be
 harnessed.  Even though the files are stored outside of the repository,
 they are still integrity checked in the same manner as normal files.
 
 The files stored outside of the repository are downloaded on demand,
 which reduces the time to clone, and possibly the local disk usage.
 This changes fundamental workflows in a DVCS, so careful thought
 should be given before deploying it.  :hg:`convert` can be used to
 convert LFS repositories to normal repositories that no longer
 require this extension, and do so without changing the commit hashes.
 This allows the extension to be disabled if the centralized workflow
 becomes burdensome.  However, the pre and post convert clones will
 not be able to communicate with each other unless the extension is
 enabled on both.
 
 To start a new repository, or to add LFS files to an existing one, just
 create an ``.hglfs`` file as described below in the root directory of
 the repository.  Typically, this file should be put under version
 control, so that the settings will propagate to other repositories with
 push and pull.  During any commit, Mercurial will consult this file to
 determine if an added or modified file should be stored externally.  The
 type of storage depends on the characteristics of the file at each
 commit.  A file that is near a size threshold may switch back and forth
 between LFS and normal storage, as needed.
 
 Alternately, both normal repositories and largefile controlled
 repositories can be converted to LFS by using :hg:`convert` and the
 ``lfs.track`` config option described below.  The ``.hglfs`` file
 should then be created and added, to control subsequent LFS selection.
 The hashes are also unchanged in this case.  The LFS and non-LFS
 repositories can be distinguished because the LFS repository will
 abort any command if this extension is disabled.
 
 Committed LFS files are held locally, until the repository is pushed.
 Prior to pushing the normal repository data, the LFS files that are
 tracked by the outgoing commits are automatically uploaded to the
 configured central server.  No LFS files are transferred on
 :hg:`pull` or :hg:`clone`.  Instead, the files are downloaded on
 demand as they need to be read, if a cached copy cannot be found
 locally.  Both committing and downloading an LFS file will link the
 file to a usercache, to speed up future access.  See the `usercache`
 config setting described below.
 
 .hglfs::
 
     The extension reads its configuration from a versioned ``.hglfs``
     configuration file found in the root of the working directory. The
     ``.hglfs`` file uses the same syntax as all other Mercurial
     configuration files. It uses a single section, ``[track]``.
 
     The ``[track]`` section specifies which files are stored as LFS (or
     not). Each line is keyed by a file pattern, with a predicate value.
     The first file pattern match is used, so put more specific patterns
     first.  The available predicates are ``all()``, ``none()``, and
     ``size()``. See "hg help filesets.size" for the latter.
 
     Example versioned ``.hglfs`` file::
 
       [track]
       # No Makefile or python file, anywhere, will be LFS
       **Makefile = none()
       **.py = none()
 
       **.zip = all()
       **.exe = size(">1MB")
 
       # Catchall for everything not matched above
       ** = size(">10MB")
 
 Configs::
 
     [lfs]
     # Remote endpoint. Multiple protocols are supported:
     # - http(s)://user:[e-mail]/path
     #   git-lfs endpoint
     # - file:///tmp/path
     #   local filesystem, usually for testing
     # if unset, lfs will assume the remote repository also handles blob storage
     # for http(s) URLs.  Otherwise, lfs will prompt to set this when it must
     # use this value.
     # (default: unset)
     url = https://example.com/repo.git/info/lfs
 
     # Which files to track in LFS.  Path tests are "**.extname" for file
     # extensions, and "path:under/some/directory" for path prefix.  Both
     # are relative to the repository root.
     # File size can be tested with the "size()" fileset, and tests can be
     # joined with fileset operators.  (See "hg help filesets.operators".)
     #
     # Some examples:
     # - all()                       # everything
     # - none()                      # nothing
     # - size(">20MB")               # larger than 20MB
     # - !**.txt                     # anything not a *.txt file
     # - **.zip | **.tar.gz | **.7z  # some types of compressed files
     # - path:bin                    # files under "bin" in the project root
     # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz
     #     | (path:bin & !path:/bin/README) | size(">1GB")
     # (default: none())
     #
     # This is ignored if there is a tracked '.hglfs' file, and this setting
     # will eventually be deprecated and removed.
     track = size(">10M")
 
     # how many times to retry before giving up on transferring an object
     retry = 5
 
     # the local directory to store lfs files for sharing across local clones.
     # If not set, the cache is located in an OS specific cache location.
     usercache = /path/to/global/cache
 """
 
 from __future__ import absolute_import
 
 import sys
 
 from mercurial.i18n import _
 
 from mercurial import (
     config,
     context,
     error,
     exchange,
     extensions,
     exthelper,
     filelog,
     filesetlang,
     localrepo,
     minifileset,
     node,
     pycompat,
-    repository,
     revlog,
     scmutil,
     templateutil,
     util,
 )
 
+from mercurial.interfaces import (
+    repository,
+)
+
 from . import (
     blobstore,
     wireprotolfsserver,
     wrapper,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 eh = exthelper.exthelper()
 eh.merge(wrapper.eh)
 eh.merge(wireprotolfsserver.eh)
 
 cmdtable = eh.cmdtable
 configtable = eh.configtable
 extsetup = eh.finalextsetup
 uisetup = eh.finaluisetup
 filesetpredicate = eh.filesetpredicate
 reposetup = eh.finalreposetup
 templatekeyword = eh.templatekeyword
 
 eh.configitem('experimental', 'lfs.serve',
     default=True,
 )
 eh.configitem('experimental', 'lfs.user-agent',
     default=None,
 )
 eh.configitem('experimental', 'lfs.disableusercache',
     default=False,
 )
 eh.configitem('experimental', 'lfs.worker-enable',
     default=False,
 )
 
 eh.configitem('lfs', 'url',
     default=None,
 )
 eh.configitem('lfs', 'usercache',
     default=None,
 )
 # Deprecated
 eh.configitem('lfs', 'threshold',
     default=None,
 )
 eh.configitem('lfs', 'track',
     default='none()',
 )
 eh.configitem('lfs', 'retry',
     default=5,
 )
 
 lfsprocessor = (
     wrapper.readfromstore,
     wrapper.writetostore,
     wrapper.bypasscheckhash,
 )
 
 def featuresetup(ui, supported):
     # don't die on seeing a repo with the lfs requirement
     supported |= {'lfs'}
 
 @eh.uisetup
 def _uisetup(ui):
     localrepo.featuresetupfuncs.add(featuresetup)
 
 @eh.reposetup
 def _reposetup(ui, repo):
     # Nothing to do with a remote repo
     if not repo.local():
         return
 
     repo.svfs.lfslocalblobstore = blobstore.local(repo)
     repo.svfs.lfsremoteblobstore = blobstore.remote(repo)
 
     class lfsrepo(repo.__class__):
         @localrepo.unfilteredmethod
         def commitctx(self, ctx, error=False, origctx=None):
             repo.svfs.options['lfstrack'] = _trackedmatcher(self)
             return super(lfsrepo, self).commitctx(ctx, error, origctx=origctx)
 
     repo.__class__ = lfsrepo
 
     if 'lfs' not in repo.requirements:
         def checkrequireslfs(ui, repo, **kwargs):
             if 'lfs' in repo.requirements:
                 return 0
 
             last = kwargs.get(r'node_last')
             _bin = node.bin
             if last:
                 s = repo.set('%n:%n', _bin(kwargs[r'node']), _bin(last))
             else:
                 s = repo.set('%n', _bin(kwargs[r'node']))
             match = repo._storenarrowmatch
             for ctx in s:
                 # TODO: is there a way to just walk the files in the commit?
                 if any(ctx[f].islfs() for f in ctx.files()
                        if f in ctx and match(f)):
                     repo.requirements.add('lfs')
                     repo.features.add(repository.REPO_FEATURE_LFS)
                     repo._writerequirements()
                     repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
                     break
 
         ui.setconfig('hooks', 'commit.lfs', checkrequireslfs, 'lfs')
         ui.setconfig('hooks', 'pretxnchangegroup.lfs', checkrequireslfs, 'lfs')
     else:
         repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
 
 def _trackedmatcher(repo):
     """Return a function (path, size) -> bool indicating whether or not to
     track a given file with lfs."""
     if not repo.wvfs.exists('.hglfs'):
         # No '.hglfs' in wdir.  Fallback to config for now.
         trackspec = repo.ui.config('lfs', 'track')
 
         # deprecated config: lfs.threshold
         threshold = repo.ui.configbytes('lfs', 'threshold')
         if threshold:
             filesetlang.parse(trackspec)  # make sure syntax errors are confined
             trackspec = "(%s) | size('>%d')" % (trackspec, threshold)
 
         return minifileset.compile(trackspec)
 
     data = repo.wvfs.tryread('.hglfs')
     if not data:
         return lambda p, s: False
 
     # Parse errors here will abort with a message that points to the .hglfs file
     # and line number.
     cfg = config.config()
     cfg.parse('.hglfs', data)
 
     try:
         rules = [(minifileset.compile(pattern), minifileset.compile(rule))
                  for pattern, rule in cfg.items('track')]
     except error.ParseError as e:
         # The original exception gives no indicator that the error is in the
         # .hglfs file, so add that.
 
         # TODO: See if the line number of the file can be made available.
         raise error.Abort(_('parse error in .hglfs: %s') % e)
 
     def _match(path, size):
         for pat, rule in rules:
             if pat(path, size):
                 return rule(path, size)
 
         return False
 
     return _match
 
 # Called by remotefilelog
 def wrapfilelog(filelog):
     wrapfunction = extensions.wrapfunction
 
     wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision)
     wrapfunction(filelog, 'renamed', wrapper.filelogrenamed)
     wrapfunction(filelog, 'size', wrapper.filelogsize)
 
 @eh.wrapfunction(localrepo, 'resolverevlogstorevfsoptions')
 def _resolverevlogstorevfsoptions(orig, ui, requirements, features):
     opts = orig(ui, requirements, features)
     for name, module in extensions.extensions(ui):
         if module is sys.modules[__name__]:
             if revlog.REVIDX_EXTSTORED in opts[b'flagprocessors']:
                 msg = (_(b"cannot register multiple processors on flag '%#x'.")
                        % revlog.REVIDX_EXTSTORED)
                 raise error.Abort(msg)
 
             opts[b'flagprocessors'][revlog.REVIDX_EXTSTORED] = lfsprocessor
             break
 
     return opts
 
 @eh.extsetup
 def _extsetup(ui):
     wrapfilelog(filelog.filelog)
 
     context.basefilectx.islfs = wrapper.filectxislfs
 
     scmutil.fileprefetchhooks.add('lfs', wrapper._prefetchfiles)
 
     # Make bundle choose changegroup3 instead of changegroup2. This affects
     # "hg bundle" command. Note: it does not cover all bundle formats like
     # "packed1". Using "packed1" with lfs will likely cause trouble.
     exchange._bundlespeccontentopts["v2"]["cg.version"] = "03"
 
 @eh.filesetpredicate('lfs()')
 def lfsfileset(mctx, x):
     """File that uses LFS storage."""
     # i18n: "lfs" is a keyword
     filesetlang.getargs(x, 0, 0, _("lfs takes no arguments"))
     ctx = mctx.ctx
     def lfsfilep(f):
         return wrapper.pointerfromctx(ctx, f, removed=True) is not None
     return mctx.predicate(lfsfilep, predrepr='<lfs>')
 
 @eh.templatekeyword('lfs_files', requires={'ctx'})
 def lfsfiles(context, mapping):
     """List of strings. All files modified, added, or removed by this
     changeset."""
     ctx = context.resource(mapping, 'ctx')
 
     pointers = wrapper.pointersfromctx(ctx, removed=True) # {path: pointer}
     files = sorted(pointers.keys())
 
     def pointer(v):
         # In the file spec, version is first and the other keys are sorted.
         sortkeyfunc = lambda x: (x[0] != 'version', x)
         items = sorted(pointers[v].iteritems(), key=sortkeyfunc)
         return util.sortdict(items)
 
     makemap = lambda v: {
         'file': v,
         'lfsoid': pointers[v].oid() if pointers[v] else None,
         'lfspointer': templateutil.hybriddict(pointer(v)),
     }
 
     # TODO: make the separator ', '?
     f = templateutil._showcompatlist(context, mapping, 'lfs_file', files)
     return templateutil.hybrid(f, files, makemap, pycompat.identity)
 
 @eh.command('debuglfsupload',
             [('r', 'rev', [], _('upload large files introduced by REV'))])
 def debuglfsupload(ui, repo, **opts):
     """upload lfs blobs added by the working copy parent or given revisions"""
     revs = opts.get(r'rev', [])
     pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs))
     wrapper.uploadblobs(repo, pointers)

hgext/lfs/wrapper.py

@@ -1,446 +1,449
 # wrapper.py - methods wrapping core mercurial logic
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import hashlib
 
 from mercurial.i18n import _
 from mercurial.node import bin, hex, nullid, short
 
 from mercurial import (
     bundle2,
     changegroup,
     cmdutil,
     context,
     error,
     exchange,
     exthelper,
     localrepo,
-    repository,
     revlog,
     scmutil,
     upgrade,
     util,
     vfs as vfsmod,
     wireprotov1server,
 )
 
+from mercurial.interfaces import (
+    repository,
+)
+
 from mercurial.utils import (
     storageutil,
     stringutil,
 )
 
 from ..largefiles import lfutil
 
 from . import (
     blobstore,
     pointer,
 )
 
 eh = exthelper.exthelper()
 
 @eh.wrapfunction(localrepo, 'makefilestorage')
 def localrepomakefilestorage(orig, requirements, features, **kwargs):
     if b'lfs' in requirements:
         features.add(repository.REPO_FEATURE_LFS)
 
     return orig(requirements=requirements, features=features, **kwargs)
 
 @eh.wrapfunction(changegroup, 'allsupportedversions')
 def allsupportedversions(orig, ui):
     versions = orig(ui)
     versions.add('03')
     return versions
 
 @eh.wrapfunction(wireprotov1server, '_capabilities')
 def _capabilities(orig, repo, proto):
     '''Wrap server command to announce lfs server capability'''
     caps = orig(repo, proto)
     if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
         # Advertise a slightly different capability when lfs is *required*, so
         # that the client knows it MUST load the extension.  If lfs is not
         # required on the server, there's no reason to autoload the extension
         # on the client.
         if b'lfs' in repo.requirements:
             caps.append('lfs-serve')
 
         caps.append('lfs')
     return caps
 
 def bypasscheckhash(self, text):
     return False
 
 def readfromstore(self, text):
     """Read filelog content from local blobstore transform for flagprocessor.
 
     Default tranform for flagprocessor, returning contents from blobstore.
     Returns a 2-typle (text, validatehash) where validatehash is True as the
     contents of the blobstore should be checked using checkhash.
     """
     p = pointer.deserialize(text)
     oid = p.oid()
     store = self.opener.lfslocalblobstore
     if not store.has(oid):
         p.filename = self.filename
         self.opener.lfsremoteblobstore.readbatch([p], store)
 
     # The caller will validate the content
     text = store.read(oid, verify=False)
 
     # pack hg filelog metadata
     hgmeta = {}
     for k in p.keys():
         if k.startswith('x-hg-'):
             name = k[len('x-hg-'):]
             hgmeta[name] = p[k]
     if hgmeta or text.startswith('\1\n'):
         text = storageutil.packmeta(hgmeta, text)
 
     return (text, True)
 
 def writetostore(self, text):
     # hg filelog metadata (includes rename, etc)
     hgmeta, offset = storageutil.parsemeta(text)
     if offset and offset > 0:
         # lfs blob does not contain hg filelog metadata
         text = text[offset:]
 
     # git-lfs only supports sha256
     oid = hex(hashlib.sha256(text).digest())
     self.opener.lfslocalblobstore.write(oid, text)
 
     # replace contents with metadata
     longoid = 'sha256:%s' % oid
     metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
 
     # by default, we expect the content to be binary. however, LFS could also
     # be used for non-binary content. add a special entry for non-binary data.
     # this will be used by filectx.isbinary().
     if not stringutil.binary(text):
         # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
         metadata['x-is-binary'] = '0'
 
     # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
     if hgmeta is not None:
         for k, v in hgmeta.iteritems():
             metadata['x-hg-%s' % k] = v
 
     rawtext = metadata.serialize()
     return (rawtext, False)
 
 def _islfs(rlog, node=None, rev=None):
     if rev is None:
         if node is None:
             # both None - likely working copy content where node is not ready
             return False
         rev = rlog._revlog.rev(node)
     else:
         node = rlog._revlog.node(rev)
     if node == nullid:
         return False
     flags = rlog._revlog.flags(rev)
     return bool(flags & revlog.REVIDX_EXTSTORED)
 
 # Wrapping may also be applied by remotefilelog
 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
                        cachedelta=None, node=None,
                        flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
     # The matcher isn't available if reposetup() wasn't called.
     lfstrack = self._revlog.opener.options.get('lfstrack')
 
     if lfstrack:
         textlen = len(text)
         # exclude hg rename meta from file size
         meta, offset = storageutil.parsemeta(text)
         if offset:
             textlen -= offset
 
         if lfstrack(self._revlog.filename, textlen):
             flags |= revlog.REVIDX_EXTSTORED
 
     return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
                 node=node, flags=flags, **kwds)
 
 # Wrapping may also be applied by remotefilelog
 def filelogrenamed(orig, self, node):
     if _islfs(self, node):
         rawtext = self._revlog.rawdata(node)
         if not rawtext:
             return False
         metadata = pointer.deserialize(rawtext)
         if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
             return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
         else:
             return False
     return orig(self, node)
 
 # Wrapping may also be applied by remotefilelog
 def filelogsize(orig, self, rev):
     if _islfs(self, rev=rev):
         # fast path: use lfs metadata to answer size
         rawtext = self._revlog.rawdata(rev)
         metadata = pointer.deserialize(rawtext)
         return int(metadata['size'])
     return orig(self, rev)
 
 @eh.wrapfunction(context.basefilectx, 'cmp')
 def filectxcmp(orig, self, fctx):
     """returns True if text is different than fctx"""
     # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
     if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
         # fast path: check LFS oid
         p1 = pointer.deserialize(self.rawdata())
         p2 = pointer.deserialize(fctx.rawdata())
         return p1.oid() != p2.oid()
     return orig(self, fctx)
 
 @eh.wrapfunction(context.basefilectx, 'isbinary')
 def filectxisbinary(orig, self):
     if self.islfs():
         # fast path: use lfs metadata to answer isbinary
         metadata = pointer.deserialize(self.rawdata())
         # if lfs metadata says nothing, assume it's binary by default
         return bool(int(metadata.get('x-is-binary', 1)))
     return orig(self)
 
 def filectxislfs(self):
     return _islfs(self.filelog(), self.filenode())
 
 @eh.wrapfunction(cmdutil, '_updatecatformatter')
 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
     orig(fm, ctx, matcher, path, decode)
     fm.data(rawdata=ctx[path].rawdata())
 
 @eh.wrapfunction(scmutil, 'wrapconvertsink')
 def convertsink(orig, sink):
     sink = orig(sink)
     if sink.repotype == 'hg':
         class lfssink(sink.__class__):
             def putcommit(self, files, copies, parents, commit, source, revmap,
                           full, cleanp2):
                 pc = super(lfssink, self).putcommit
                 node = pc(files, copies, parents, commit, source, revmap, full,
                           cleanp2)
 
                 if 'lfs' not in self.repo.requirements:
                     ctx = self.repo[node]
 
                     # The file list may contain removed files, so check for
                     # membership before assuming it is in the context.
                     if any(f in ctx and ctx[f].islfs() for f, n in files):
                         self.repo.requirements.add('lfs')
                         self.repo._writerequirements()
 
                 return node
 
         sink.__class__ = lfssink
 
     return sink
 
 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
 # options and blob stores are passed from othervfs to the new readonlyvfs.
 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
 def vfsinit(orig, self, othervfs):
     orig(self, othervfs)
     # copy lfs related options
     for k, v in othervfs.options.items():
         if k.startswith('lfs'):
             self.options[k] = v
     # also copy lfs blobstores. note: this can run before reposetup, so lfs
     # blobstore attributes are not always ready at this time.
     for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
         if util.safehasattr(othervfs, name):
             setattr(self, name, getattr(othervfs, name))
 
 def _prefetchfiles(repo, revs, match):
     """Ensure that required LFS blobs are present, fetching them as a group if
     needed."""
     if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
         return
 
     pointers = []
     oids = set()
     localstore = repo.svfs.lfslocalblobstore
 
     for rev in revs:
         ctx = repo[rev]
         for f in ctx.walk(match):
             p = pointerfromctx(ctx, f)
             if p and p.oid() not in oids and not localstore.has(p.oid()):
                 p.filename = f
                 pointers.append(p)
                 oids.add(p.oid())
 
     if pointers:
         # Recalculating the repo store here allows 'paths.default' that is set
         # on the repo by a clone command to be used for the update.
         blobstore.remote(repo).readbatch(pointers, localstore)
 
 def _canskipupload(repo):
     # Skip if this hasn't been passed to reposetup()
     if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
         return True
 
     # if remotestore is a null store, upload is a no-op and can be skipped
     return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 def candownload(repo):
     # Skip if this hasn't been passed to reposetup()
     if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
         return False
 
     # if remotestore is a null store, downloads will lead to nothing
     return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 def uploadblobsfromrevs(repo, revs):
     '''upload lfs blobs introduced by revs
 
     Note: also used by other extensions e. g. infinitepush. avoid renaming.
     '''
     if _canskipupload(repo):
         return
     pointers = extractpointers(repo, revs)
     uploadblobs(repo, pointers)
 
 def prepush(pushop):
     """Prepush hook.
 
     Read through the revisions to push, looking for filelog entries that can be
     deserialized into metadata so that we can block the push on their upload to
     the remote blobstore.
     """
     return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
 
 @eh.wrapfunction(exchange, 'push')
 def push(orig, repo, remote, *args, **kwargs):
     """bail on push if the extension isn't enabled on remote when needed, and
     update the remote store based on the destination path."""
     if 'lfs' in repo.requirements:
         # If the remote peer is for a local repo, the requirement tests in the
         # base class method enforce lfs support.  Otherwise, some revisions in
         # this repo use lfs, and the remote repo needs the extension loaded.
         if not remote.local() and not remote.capable('lfs'):
             # This is a copy of the message in exchange.push() when requirements
             # are missing between local repos.
             m = _("required features are not supported in the destination: %s")
             raise error.Abort(m % 'lfs',
                               hint=_('enable the lfs extension on the server'))
 
         # Repositories where this extension is disabled won't have the field.
         # But if there's a requirement, then the extension must be loaded AND
         # there may be blobs to push.
         remotestore = repo.svfs.lfsremoteblobstore
         try:
             repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
             return orig(repo, remote, *args, **kwargs)
         finally:
             repo.svfs.lfsremoteblobstore = remotestore
     else:
         return orig(repo, remote, *args, **kwargs)
 
 # when writing a bundle via "hg bundle" command, upload related LFS blobs
 @eh.wrapfunction(bundle2, 'writenewbundle')
 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
                    *args, **kwargs):
     """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
     uploadblobsfromrevs(repo, outgoing.missing)
     return orig(ui, repo, source, filename, bundletype, outgoing, *args,
                 **kwargs)
 
 def extractpointers(repo, revs):
     """return a list of lfs pointers added by given revs"""
     repo.ui.debug('lfs: computing set of blobs to upload\n')
     pointers = {}
 
     makeprogress = repo.ui.makeprogress
     with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
         for r in revs:
             ctx = repo[r]
             for p in pointersfromctx(ctx).values():
                 pointers[p.oid()] = p
             progress.increment()
         return sorted(pointers.values(), key=lambda p: p.oid())
 
 def pointerfromctx(ctx, f, removed=False):
     """return a pointer for the named file from the given changectx, or None if
     the file isn't LFS.
 
     Optionally, the pointer for a file deleted from the context can be returned.
     Since no such pointer is actually stored, and to distinguish from a non LFS
     file, this pointer is represented by an empty dict.
     """
     _ctx = ctx
     if f not in ctx:
         if not removed:
             return None
         if f in ctx.p1():
             _ctx = ctx.p1()
         elif f in ctx.p2():
             _ctx = ctx.p2()
         else:
             return None
     fctx = _ctx[f]
     if not _islfs(fctx.filelog(), fctx.filenode()):
         return None
     try:
         p = pointer.deserialize(fctx.rawdata())
         if ctx == _ctx:
             return p
         return {}
     except pointer.InvalidPointer as ex:
         raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
                           % (f, short(_ctx.node()), ex))
 
 def pointersfromctx(ctx, removed=False):
     """return a dict {path: pointer} for given single changectx.
 
     If ``removed`` == True and the LFS file was removed from ``ctx``, the value
     stored for the path is an empty dict.
     """
     result = {}
     m = ctx.repo().narrowmatch()
 
     # TODO: consider manifest.fastread() instead
     for f in ctx.files():
         if not m(f):
             continue
         p = pointerfromctx(ctx, f, removed=removed)
         if p is not None:
             result[f] = p
     return result
 
 def uploadblobs(repo, pointers):
     """upload given pointers from local blobstore"""
     if not pointers:
         return
 
     remoteblob = repo.svfs.lfsremoteblobstore
     remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
 
 @eh.wrapfunction(upgrade, '_finishdatamigration')
 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
     orig(ui, srcrepo, dstrepo, requirements)
 
     # Skip if this hasn't been passed to reposetup()
     if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
         util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
         srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
         dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
 
         for dirpath, dirs, files in srclfsvfs.walk():
             for oid in files:
                 ui.write(_('copying lfs blob %s\n') % oid)
                 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
 
 @eh.wrapfunction(upgrade, 'preservedrequirements')
 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if 'lfs' in repo.requirements:
         reqs.add('lfs')
     return reqs

hgext/narrow/__init__.py

@@ -1,71 +1,74
 # __init__.py - narrowhg extension
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 '''create clones which fetch history data for subset of files (EXPERIMENTAL)'''
 
 from __future__ import absolute_import
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 from mercurial import (
     localrepo,
     registrar,
+)
+
+from mercurial.interfaces import (
     repository,
 )
 
 from . import (
     narrowbundle2,
     narrowcommands,
     narrowrepo,
     narrowtemplates,
     narrowwirepeer,
 )
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 # Narrowhg *has* support for serving ellipsis nodes (which are used at
 # least by Google's internal server), but that support is pretty
 # fragile and has a lot of problems on real-world repositories that
 # have complex graph topologies. This could probably be corrected, but
 # absent someone needing the full support for ellipsis nodes in
 # repositories with merges, it's unlikely this work will get done. As
 # of this writining in late 2017, all repositories large enough for
 # ellipsis nodes to be a hard requirement also enforce strictly linear
 # history for other scaling reasons.
 configitem('experimental', 'narrowservebrokenellipses',
            default=False,
            alias=[('narrow', 'serveellipses')],
 )
 
 # Export the commands table for Mercurial to see.
 cmdtable = narrowcommands.table
 
 def featuresetup(ui, features):
     features.add(repository.NARROW_REQUIREMENT)
 
 def uisetup(ui):
     """Wraps user-facing mercurial commands with narrow-aware versions."""
     localrepo.featuresetupfuncs.add(featuresetup)
     narrowbundle2.setup()
     narrowcommands.setup()
     narrowwirepeer.uisetup()
 
 def reposetup(ui, repo):
     """Wraps local repositories with narrow repo support."""
     if not repo.local():
         return
 
     repo.ui.setconfig('experimental', 'narrow', True, 'narrow-ext')
     if repository.NARROW_REQUIREMENT in repo.requirements:
         narrowrepo.wraprepo(repo)
         narrowwirepeer.reposetup(repo)
 
 templatekeyword = narrowtemplates.templatekeyword
 revsetpredicate = narrowtemplates.revsetpredicate

hgext/narrow/narrowbundle2.py

@@ -1,301 +1,303
 # narrowbundle2.py - bundle2 extensions for narrow repository support
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import struct
 
 from mercurial.i18n import _
 from mercurial.node import (
     bin,
     nullid,
 )
 from mercurial import (
     bundle2,
     changegroup,
     error,
     exchange,
     localrepo,
     narrowspec,
     repair,
-    repository,
     util,
     wireprototypes,
 )
+from mercurial.interfaces import (
+    repository,
+)
 from mercurial.utils import (
     stringutil,
 )
 
 _NARROWACL_SECTION = 'narrowacl'
 _CHANGESPECPART = 'narrow:changespec'
 _RESSPECS = 'narrow:responsespec'
 _SPECPART = 'narrow:spec'
 _SPECPART_INCLUDE = 'include'
 _SPECPART_EXCLUDE = 'exclude'
 _KILLNODESIGNAL = 'KILL'
 _DONESIGNAL = 'DONE'
 _ELIDEDCSHEADER = '>20s20s20sl' # cset id, p1, p2, len(text)
 _ELIDEDMFHEADER = '>20s20s20s20sl' # manifest id, p1, p2, link id, len(text)
 _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
 _MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
 
 # Serve a changegroup for a client with a narrow clone.
 def getbundlechangegrouppart_narrow(bundler, repo, source,
                                     bundlecaps=None, b2caps=None, heads=None,
                                     common=None, **kwargs):
     assert repo.ui.configbool('experimental', 'narrowservebrokenellipses')
 
     cgversions = b2caps.get('changegroup')
     cgversions = [v for v in cgversions
                   if v in changegroup.supportedoutgoingversions(repo)]
     if not cgversions:
         raise ValueError(_('no common changegroup version'))
     version = max(cgversions)
 
     oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
     oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
     newinclude = sorted(filter(bool, kwargs.get(r'includepats', [])))
     newexclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
     known = {bin(n) for n in kwargs.get(r'known', [])}
     generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
                             newexclude, version, common, heads, known,
                             kwargs.get(r'depth', None))
 
 def generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
                             newexclude, version, common, heads, known, depth):
     newmatch = narrowspec.match(repo.root, include=newinclude,
                                 exclude=newexclude)
     if depth is not None:
         depth = int(depth)
         if depth < 1:
             raise error.Abort(_('depth must be positive, got %d') % depth)
 
     heads = set(heads or repo.heads())
     common = set(common or [nullid])
     if known and (oldinclude != newinclude or oldexclude != newexclude):
         # Steps:
         # 1. Send kill for "$known & ::common"
         #
         # 2. Send changegroup for ::common
         #
         # 3. Proceed.
         #
         # In the future, we can send kills for only the specific
         # nodes we know should go away or change shape, and then
         # send a data stream that tells the client something like this:
         #
         # a) apply this changegroup
         # b) apply nodes XXX, YYY, ZZZ that you already have
         # c) goto a
         #
         # until they've built up the full new state.
         # Convert to revnums and intersect with "common". The client should
         # have made it a subset of "common" already, but let's be safe.
         known = set(repo.revs("%ln & ::%ln", known, common))
         # TODO: we could send only roots() of this set, and the
         # list of nodes in common, and the client could work out
         # what to strip, instead of us explicitly sending every
         # single node.
         deadrevs = known
         def genkills():
             for r in deadrevs:
                 yield _KILLNODESIGNAL
                 yield repo.changelog.node(r)
             yield _DONESIGNAL
         bundler.newpart(_CHANGESPECPART, data=genkills())
         newvisit, newfull, newellipsis = exchange._computeellipsis(
             repo, set(), common, known, newmatch)
         if newvisit:
             packer = changegroup.getbundler(version, repo,
                                             matcher=newmatch,
                                             ellipses=True,
                                             shallow=depth is not None,
                                             ellipsisroots=newellipsis,
                                             fullnodes=newfull)
             cgdata = packer.generate(common, newvisit, False, 'narrow_widen')
 
             part = bundler.newpart('changegroup', data=cgdata)
             part.addparam('version', version)
             if 'treemanifest' in repo.requirements:
                 part.addparam('treemanifest', '1')
 
     visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
         repo, common, heads, set(), newmatch, depth=depth)
 
     repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
     if visitnodes:
         packer = changegroup.getbundler(version, repo,
                                         matcher=newmatch,
                                         ellipses=True,
                                         shallow=depth is not None,
                                         ellipsisroots=ellipsisroots,
                                         fullnodes=relevant_nodes)
         cgdata = packer.generate(common, visitnodes, False, 'narrow_widen')
 
         part = bundler.newpart('changegroup', data=cgdata)
         part.addparam('version', version)
         if 'treemanifest' in repo.requirements:
             part.addparam('treemanifest', '1')
 
 @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
 def _handlechangespec_2(op, inpart):
     # XXX: This bundle2 handling is buggy and should be removed after hg5.2 is
     # released. New servers will send a mandatory bundle2 part named
     # 'Narrowspec' and will send specs as data instead of params.
     # Refer to issue5952 and 6019
     includepats = set(inpart.params.get(_SPECPART_INCLUDE, '').splitlines())
     excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, '').splitlines())
     narrowspec.validatepatterns(includepats)
     narrowspec.validatepatterns(excludepats)
 
     if not repository.NARROW_REQUIREMENT in op.repo.requirements:
         op.repo.requirements.add(repository.NARROW_REQUIREMENT)
         op.repo._writerequirements()
     op.repo.setnarrowpats(includepats, excludepats)
     narrowspec.copytoworkingcopy(op.repo)
 
 @bundle2.parthandler(_RESSPECS)
 def _handlenarrowspecs(op, inpart):
     data = inpart.read()
     inc, exc = data.split('\0')
     includepats = set(inc.splitlines())
     excludepats = set(exc.splitlines())
     narrowspec.validatepatterns(includepats)
     narrowspec.validatepatterns(excludepats)
 
     if repository.NARROW_REQUIREMENT not in op.repo.requirements:
         op.repo.requirements.add(repository.NARROW_REQUIREMENT)
         op.repo._writerequirements()
     op.repo.setnarrowpats(includepats, excludepats)
     narrowspec.copytoworkingcopy(op.repo)
 
 @bundle2.parthandler(_CHANGESPECPART)
 def _handlechangespec(op, inpart):
     repo = op.repo
     cl = repo.changelog
 
     # changesets which need to be stripped entirely. either they're no longer
     # needed in the new narrow spec, or the server is sending a replacement
     # in the changegroup part.
     clkills = set()
 
     # A changespec part contains all the updates to ellipsis nodes
     # that will happen as a result of widening or narrowing a
     # repo. All the changes that this block encounters are ellipsis
     # nodes or flags to kill an existing ellipsis.
     chunksignal = changegroup.readexactly(inpart, 4)
     while chunksignal != _DONESIGNAL:
         if chunksignal == _KILLNODESIGNAL:
             # a node used to be an ellipsis but isn't anymore
             ck = changegroup.readexactly(inpart, 20)
             if cl.hasnode(ck):
                 clkills.add(ck)
         else:
             raise error.Abort(
                 _('unexpected changespec node chunk type: %s') % chunksignal)
         chunksignal = changegroup.readexactly(inpart, 4)
 
     if clkills:
         # preserve bookmarks that repair.strip() would otherwise strip
         op._bookmarksbackup = repo._bookmarks
         class dummybmstore(dict):
             def applychanges(self, repo, tr, changes):
                 pass
         localrepo.localrepository._bookmarks.set(repo, dummybmstore())
         chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
                                  topic='widen')
         if chgrpfile:
             op._widen_uninterr = repo.ui.uninterruptible()
             op._widen_uninterr.__enter__()
             # presence of _widen_bundle attribute activates widen handler later
             op._widen_bundle = chgrpfile
     # Set the new narrowspec if we're widening. The setnewnarrowpats() method
     # will currently always be there when using the core+narrowhg server, but
     # other servers may include a changespec part even when not widening (e.g.
     # because we're deepening a shallow repo).
     if util.safehasattr(repo, 'setnewnarrowpats'):
         repo.setnewnarrowpats()
 
 def handlechangegroup_widen(op, inpart):
     """Changegroup exchange handler which restores temporarily-stripped nodes"""
     # We saved a bundle with stripped node data we must now restore.
     # This approach is based on mercurial/repair.py@6ee26a53c111.
     repo = op.repo
     ui = op.ui
 
     chgrpfile = op._widen_bundle
     del op._widen_bundle
     vfs = repo.vfs
 
     ui.note(_("adding branch\n"))
     f = vfs.open(chgrpfile, "rb")
     try:
         gen = exchange.readbundle(ui, f, chgrpfile, vfs)
         if not ui.verbose:
             # silence internal shuffling chatter
             ui.pushbuffer()
         if isinstance(gen, bundle2.unbundle20):
             with repo.transaction('strip') as tr:
                 bundle2.processbundle(repo, gen, lambda: tr)
         else:
             gen.apply(repo, 'strip', 'bundle:' + vfs.join(chgrpfile), True)
         if not ui.verbose:
             ui.popbuffer()
     finally:
         f.close()
 
     # remove undo files
     for undovfs, undofile in repo.undofiles():
         try:
             undovfs.unlink(undofile)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 ui.warn(_('error removing %s: %s\n') %
                         (undovfs.join(undofile), stringutil.forcebytestr(e)))
 
     # Remove partial backup only if there were no exceptions
     op._widen_uninterr.__exit__(None, None, None)
     vfs.unlink(chgrpfile)
 
 def setup():
     """Enable narrow repo support in bundle2-related extension points."""
     getbundleargs = wireprototypes.GETBUNDLE_ARGUMENTS
 
     getbundleargs['narrow'] = 'boolean'
     getbundleargs['depth'] = 'plain'
     getbundleargs['oldincludepats'] = 'csv'
     getbundleargs['oldexcludepats'] = 'csv'
     getbundleargs['known'] = 'csv'
 
     # Extend changegroup serving to handle requests from narrow clients.
     origcgfn = exchange.getbundle2partsmapping['changegroup']
     def wrappedcgfn(*args, **kwargs):
         repo = args[1]
         if repo.ui.has_section(_NARROWACL_SECTION):
             kwargs = exchange.applynarrowacl(repo, kwargs)
 
         if (kwargs.get(r'narrow', False) and
             repo.ui.configbool('experimental', 'narrowservebrokenellipses')):
             getbundlechangegrouppart_narrow(*args, **kwargs)
         else:
             origcgfn(*args, **kwargs)
     exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
 
     # Extend changegroup receiver so client can fixup after widen requests.
     origcghandler = bundle2.parthandlermapping['changegroup']
     def wrappedcghandler(op, inpart):
         origcghandler(op, inpart)
         if util.safehasattr(op, '_widen_bundle'):
             handlechangegroup_widen(op, inpart)
         if util.safehasattr(op, '_bookmarksbackup'):
             localrepo.localrepository._bookmarks.set(op.repo,
                                                      op._bookmarksbackup)
             del op._bookmarksbackup
 
     wrappedcghandler.params = origcghandler.params
     bundle2.parthandlermapping['changegroup'] = wrappedcghandler

hgext/narrow/narrowcommands.py

@@ -1,478 +1,480
 # narrowcommands.py - command modifications for narrowhg extension
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import itertools
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     bundle2,
     cmdutil,
     commands,
     discovery,
     encoding,
     error,
     exchange,
     extensions,
     hg,
     narrowspec,
     node,
     pycompat,
     registrar,
     repair,
-    repository,
     repoview,
     sparse,
     util,
     wireprototypes,
 )
+from mercurial.interfaces import (
+    repository,
+)
 
 table = {}
 command = registrar.command(table)
 
 def setup():
     """Wraps user-facing mercurial commands with narrow-aware versions."""
 
     entry = extensions.wrapcommand(commands.table, 'clone', clonenarrowcmd)
     entry[1].append(('', 'narrow', None,
                      _("create a narrow clone of select files")))
     entry[1].append(('', 'depth', '',
                      _("limit the history fetched by distance from heads")))
     entry[1].append(('', 'narrowspec', '',
                      _("read narrowspecs from file")))
     # TODO(durin42): unify sparse/narrow --include/--exclude logic a bit
     if 'sparse' not in extensions.enabled():
         entry[1].append(('', 'include', [],
                          _("specifically fetch this file/directory")))
         entry[1].append(
             ('', 'exclude', [],
              _("do not fetch this file/directory, even if included")))
 
     entry = extensions.wrapcommand(commands.table, 'pull', pullnarrowcmd)
     entry[1].append(('', 'depth', '',
                      _("limit the history fetched by distance from heads")))
 
     extensions.wrapcommand(commands.table, 'archive', archivenarrowcmd)
 
 def clonenarrowcmd(orig, ui, repo, *args, **opts):
     """Wraps clone command, so 'hg clone' first wraps localrepo.clone()."""
     opts = pycompat.byteskwargs(opts)
     wrappedextraprepare = util.nullcontextmanager()
     narrowspecfile = opts['narrowspec']
 
     if narrowspecfile:
         filepath = os.path.join(encoding.getcwd(), narrowspecfile)
         ui.status(_("reading narrowspec from '%s'\n") % filepath)
         try:
             fdata = util.readfile(filepath)
         except IOError as inst:
             raise error.Abort(_("cannot read narrowspecs from '%s': %s") %
                               (filepath, encoding.strtolocal(inst.strerror)))
 
         includes, excludes, profiles = sparse.parseconfig(ui, fdata, 'narrow')
         if profiles:
             raise error.Abort(_("cannot specify other files using '%include' in"
                                 " narrowspec"))
 
         narrowspec.validatepatterns(includes)
         narrowspec.validatepatterns(excludes)
 
         # narrowspec is passed so we should assume that user wants narrow clone
         opts['narrow'] = True
         opts['include'].extend(includes)
         opts['exclude'].extend(excludes)
 
     if opts['narrow']:
         def pullbundle2extraprepare_widen(orig, pullop, kwargs):
             orig(pullop, kwargs)
 
             if opts.get('depth'):
                 kwargs['depth'] = opts['depth']
         wrappedextraprepare = extensions.wrappedfunction(exchange,
             '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
 
     with wrappedextraprepare:
         return orig(ui, repo, *args, **pycompat.strkwargs(opts))
 
 def pullnarrowcmd(orig, ui, repo, *args, **opts):
     """Wraps pull command to allow modifying narrow spec."""
     wrappedextraprepare = util.nullcontextmanager()
     if repository.NARROW_REQUIREMENT in repo.requirements:
 
         def pullbundle2extraprepare_widen(orig, pullop, kwargs):
             orig(pullop, kwargs)
             if opts.get(r'depth'):
                 kwargs['depth'] = opts[r'depth']
         wrappedextraprepare = extensions.wrappedfunction(exchange,
             '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
 
     with wrappedextraprepare:
         return orig(ui, repo, *args, **opts)
 
 def archivenarrowcmd(orig, ui, repo, *args, **opts):
     """Wraps archive command to narrow the default includes."""
     if repository.NARROW_REQUIREMENT in repo.requirements:
         repo_includes, repo_excludes = repo.narrowpats
         includes = set(opts.get(r'include', []))
         excludes = set(opts.get(r'exclude', []))
         includes, excludes, unused_invalid = narrowspec.restrictpatterns(
             includes, excludes, repo_includes, repo_excludes)
         if includes:
             opts[r'include'] = includes
         if excludes:
             opts[r'exclude'] = excludes
     return orig(ui, repo, *args, **opts)
 
 def pullbundle2extraprepare(orig, pullop, kwargs):
     repo = pullop.repo
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return orig(pullop, kwargs)
 
     if wireprototypes.NARROWCAP not in pullop.remote.capabilities():
         raise error.Abort(_("server does not support narrow clones"))
     orig(pullop, kwargs)
     kwargs['narrow'] = True
     include, exclude = repo.narrowpats
     kwargs['oldincludepats'] = include
     kwargs['oldexcludepats'] = exclude
     if include:
         kwargs['includepats'] = include
     if exclude:
         kwargs['excludepats'] = exclude
     # calculate known nodes only in ellipses cases because in non-ellipses cases
     # we have all the nodes
     if wireprototypes.ELLIPSESCAP1 in pullop.remote.capabilities():
         kwargs['known'] = [node.hex(ctx.node()) for ctx in
                            repo.set('::%ln', pullop.common)
                            if ctx.node() != node.nullid]
         if not kwargs['known']:
             # Mercurial serializes an empty list as '' and deserializes it as
             # [''], so delete it instead to avoid handling the empty string on
             # the server.
             del kwargs['known']
 
 extensions.wrapfunction(exchange,'_pullbundle2extraprepare',
                         pullbundle2extraprepare)
 
 def _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
             newincludes, newexcludes, force):
     oldmatch = narrowspec.match(repo.root, oldincludes, oldexcludes)
     newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
 
     # This is essentially doing "hg outgoing" to find all local-only
     # commits. We will then check that the local-only commits don't
     # have any changes to files that will be untracked.
     unfi = repo.unfiltered()
     outgoing = discovery.findcommonoutgoing(unfi, remote,
                                             commoninc=commoninc)
     ui.status(_('looking for local changes to affected paths\n'))
     localnodes = []
     for n in itertools.chain(outgoing.missing, outgoing.excluded):
         if any(oldmatch(f) and not newmatch(f) for f in unfi[n].files()):
             localnodes.append(n)
     revstostrip = unfi.revs('descendants(%ln)', localnodes)
     hiddenrevs = repoview.filterrevs(repo, 'visible')
     visibletostrip = list(repo.changelog.node(r)
                           for r in (revstostrip - hiddenrevs))
     if visibletostrip:
         ui.status(_('The following changeset(s) or their ancestors have '
                     'local changes not on the remote:\n'))
         maxnodes = 10
         if ui.verbose or len(visibletostrip) <= maxnodes:
             for n in visibletostrip:
                 ui.status('%s\n' % node.short(n))
         else:
             for n in visibletostrip[:maxnodes]:
                 ui.status('%s\n' % node.short(n))
             ui.status(_('...and %d more, use --verbose to list all\n') %
                       (len(visibletostrip) - maxnodes))
         if not force:
             raise error.Abort(_('local changes found'),
                               hint=_('use --force-delete-local-changes to '
                                      'ignore'))
 
     with ui.uninterruptible():
         if revstostrip:
             tostrip = [unfi.changelog.node(r) for r in revstostrip]
             if repo['.'].node() in tostrip:
                 # stripping working copy, so move to a different commit first
                 urev = max(repo.revs('(::%n) - %ln + null',
                                      repo['.'].node(), visibletostrip))
                 hg.clean(repo, urev)
             overrides = {('devel', 'strip-obsmarkers'): False}
             with ui.configoverride(overrides, 'narrow'):
                 repair.strip(ui, unfi, tostrip, topic='narrow')
 
         todelete = []
         for f, f2, size in repo.store.datafiles():
             if f.startswith('data/'):
                 file = f[5:-2]
                 if not newmatch(file):
                     todelete.append(f)
             elif f.startswith('meta/'):
                 dir = f[5:-13]
                 dirs = sorted(util.dirs({dir})) + [dir]
                 include = True
                 for d in dirs:
                     visit = newmatch.visitdir(d)
                     if not visit:
                         include = False
                         break
                     if visit == 'all':
                         break
                 if not include:
                     todelete.append(f)
 
         repo.destroying()
 
         with repo.transaction('narrowing'):
             # Update narrowspec before removing revlogs, so repo won't be
             # corrupt in case of crash
             repo.setnarrowpats(newincludes, newexcludes)
 
             for f in todelete:
                 ui.status(_('deleting %s\n') % f)
                 util.unlinkpath(repo.svfs.join(f))
                 repo.store.markremoved(f)
 
             narrowspec.updateworkingcopy(repo, assumeclean=True)
             narrowspec.copytoworkingcopy(repo)
 
         repo.destroyed()
 
 def _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
            newincludes, newexcludes):
     # for now we assume that if a server has ellipses enabled, we will be
     # exchanging ellipses nodes. In future we should add ellipses as a client
     # side requirement (maybe) to distinguish a client is shallow or not and
     # then send that information to server whether we want ellipses or not.
     # Theoretically a non-ellipses repo should be able to use narrow
     # functionality from an ellipses enabled server
     remotecap = remote.capabilities()
     ellipsesremote = any(cap in remotecap
                          for cap in wireprototypes.SUPPORTED_ELLIPSESCAP)
 
     # check whether we are talking to a server which supports old version of
     # ellipses capabilities
     isoldellipses = (ellipsesremote and wireprototypes.ELLIPSESCAP1 in
                      remotecap and wireprototypes.ELLIPSESCAP not in remotecap)
 
     def pullbundle2extraprepare_widen(orig, pullop, kwargs):
         orig(pullop, kwargs)
         # The old{in,ex}cludepats have already been set by orig()
         kwargs['includepats'] = newincludes
         kwargs['excludepats'] = newexcludes
     wrappedextraprepare = extensions.wrappedfunction(exchange,
         '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
 
     # define a function that narrowbundle2 can call after creating the
     # backup bundle, but before applying the bundle from the server
     def setnewnarrowpats():
         repo.setnarrowpats(newincludes, newexcludes)
     repo.setnewnarrowpats = setnewnarrowpats
     # silence the devel-warning of applying an empty changegroup
     overrides = {('devel', 'all-warnings'): False}
 
     common = commoninc[0]
     with ui.uninterruptible():
         if ellipsesremote:
             ds = repo.dirstate
             p1, p2 = ds.p1(), ds.p2()
             with ds.parentchange():
                 ds.setparents(node.nullid, node.nullid)
         if isoldellipses:
             with wrappedextraprepare:
                 exchange.pull(repo, remote, heads=common)
         else:
             known = []
             if ellipsesremote:
                 known = [node.hex(ctx.node()) for ctx in
                          repo.set('::%ln', common)
                          if ctx.node() != node.nullid]
             with remote.commandexecutor() as e:
                 bundle = e.callcommand('narrow_widen', {
                     'oldincludes': oldincludes,
                     'oldexcludes': oldexcludes,
                     'newincludes': newincludes,
                     'newexcludes': newexcludes,
                     'cgversion': '03',
                     'commonheads': common,
                     'known': known,
                     'ellipses': ellipsesremote,
                 }).result()
 
             trmanager = exchange.transactionmanager(repo, 'widen', remote.url())
             with trmanager, repo.ui.configoverride(overrides, 'widen'):
                 op = bundle2.bundleoperation(repo, trmanager.transaction,
                                              source='widen')
                 # TODO: we should catch error.Abort here
                 bundle2.processbundle(repo, bundle, op=op)
 
         if ellipsesremote:
             with ds.parentchange():
                 ds.setparents(p1, p2)
 
         with repo.transaction('widening'):
             repo.setnewnarrowpats()
             narrowspec.updateworkingcopy(repo)
             narrowspec.copytoworkingcopy(repo)
 
 # TODO(rdamazio): Make new matcher format and update description
 @command('tracked',
     [('', 'addinclude', [], _('new paths to include')),
      ('', 'removeinclude', [], _('old paths to no longer include')),
      ('', 'addexclude', [], _('new paths to exclude')),
      ('', 'import-rules', '', _('import narrowspecs from a file')),
      ('', 'removeexclude', [], _('old paths to no longer exclude')),
      ('', 'clear', False, _('whether to replace the existing narrowspec')),
      ('', 'force-delete-local-changes', False,
        _('forces deletion of local changes when narrowing')),
      ('', 'update-working-copy', False,
       _('update working copy when the store has changed')),
     ] + commands.remoteopts,
     _('[OPTIONS]... [REMOTE]'),
     inferrepo=True)
 def trackedcmd(ui, repo, remotepath=None, *pats, **opts):
     """show or change the current narrowspec
 
     With no argument, shows the current narrowspec entries, one per line. Each
     line will be prefixed with 'I' or 'X' for included or excluded patterns,
     respectively.
 
     The narrowspec is comprised of expressions to match remote files and/or
     directories that should be pulled into your client.
     The narrowspec has *include* and *exclude* expressions, with excludes always
     trumping includes: that is, if a file matches an exclude expression, it will
     be excluded even if it also matches an include expression.
     Excluding files that were never included has no effect.
 
     Each included or excluded entry is in the format described by
     'hg help patterns'.
 
     The options allow you to add or remove included and excluded expressions.
 
     If --clear is specified, then all previous includes and excludes are DROPPED
     and replaced by the new ones specified to --addinclude and --addexclude.
     If --clear is specified without any further options, the narrowspec will be
     empty and will not match any files.
 
     --import-rules accepts a path to a file containing rules, allowing you to
     add --addinclude, --addexclude rules in bulk. Like the other include and
     exclude switches, the changes are applied immediately.
     """
     opts = pycompat.byteskwargs(opts)
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         raise error.Abort(_('the tracked command is only supported on '
                             'repositories cloned with --narrow'))
 
     # Before supporting, decide whether it "hg tracked --clear" should mean
     # tracking no paths or all paths.
     if opts['clear']:
         raise error.Abort(_('the --clear option is not yet supported'))
 
     # import rules from a file
     newrules = opts.get('import_rules')
     if newrules:
         try:
             filepath = os.path.join(encoding.getcwd(), newrules)
             fdata = util.readfile(filepath)
         except IOError as inst:
             raise error.Abort(_("cannot read narrowspecs from '%s': %s") %
                               (filepath, encoding.strtolocal(inst.strerror)))
         includepats, excludepats, profiles = sparse.parseconfig(ui, fdata,
                                                                 'narrow')
         if profiles:
             raise error.Abort(_("including other spec files using '%include' "
                                 "is not supported in narrowspec"))
         opts['addinclude'].extend(includepats)
         opts['addexclude'].extend(excludepats)
 
     addedincludes = narrowspec.parsepatterns(opts['addinclude'])
     removedincludes = narrowspec.parsepatterns(opts['removeinclude'])
     addedexcludes = narrowspec.parsepatterns(opts['addexclude'])
     removedexcludes = narrowspec.parsepatterns(opts['removeexclude'])
 
     update_working_copy = opts['update_working_copy']
     only_show = not (addedincludes or removedincludes or addedexcludes or
                      removedexcludes or newrules or update_working_copy)
 
     oldincludes, oldexcludes = repo.narrowpats
 
     # filter the user passed additions and deletions into actual additions and
     # deletions of excludes and includes
     addedincludes -= oldincludes
     removedincludes &= oldincludes
     addedexcludes -= oldexcludes
     removedexcludes &= oldexcludes
 
     widening = addedincludes or removedexcludes
     narrowing = removedincludes or addedexcludes
 
     # Only print the current narrowspec.
     if only_show:
         ui.pager('tracked')
         fm = ui.formatter('narrow', opts)
         for i in sorted(oldincludes):
             fm.startitem()
             fm.write('status', '%s ', 'I', label='narrow.included')
             fm.write('pat', '%s\n', i, label='narrow.included')
         for i in sorted(oldexcludes):
             fm.startitem()
             fm.write('status', '%s ', 'X', label='narrow.excluded')
             fm.write('pat', '%s\n', i, label='narrow.excluded')
         fm.end()
         return 0
 
     if update_working_copy:
         with repo.wlock(), repo.lock(), repo.transaction('narrow-wc'):
             narrowspec.updateworkingcopy(repo)
             narrowspec.copytoworkingcopy(repo)
         return 0
 
     if not widening and not narrowing:
         ui.status(_("nothing to widen or narrow\n"))
         return 0
 
     with repo.wlock(), repo.lock():
         cmdutil.bailifchanged(repo)
 
         # Find the revisions we have in common with the remote. These will
         # be used for finding local-only changes for narrowing. They will
         # also define the set of revisions to update for widening.
         remotepath = ui.expandpath(remotepath or 'default')
         url, branches = hg.parseurl(remotepath)
         ui.status(_('comparing with %s\n') % util.hidepassword(url))
         remote = hg.peer(repo, opts, url)
 
         # check narrow support before doing anything if widening needs to be
         # performed. In future we should also abort if client is ellipses and
         # server does not support ellipses
         if widening and wireprototypes.NARROWCAP not in remote.capabilities():
             raise error.Abort(_("server does not support narrow clones"))
 
         commoninc = discovery.findcommonincoming(repo, remote)
 
         if narrowing:
             newincludes = oldincludes - removedincludes
             newexcludes = oldexcludes | addedexcludes
             _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
                     newincludes, newexcludes,
                     opts['force_delete_local_changes'])
             # _narrow() updated the narrowspec and _widen() below needs to
             # use the updated values as its base (otherwise removed includes
             # and addedexcludes will be lost in the resulting narrowspec)
             oldincludes = newincludes
             oldexcludes = newexcludes
 
         if widening:
             newincludes = oldincludes | addedincludes
             newexcludes = oldexcludes - removedexcludes
             _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
                     newincludes, newexcludes)
 
     return 0

hgext/sqlitestore.py

@@ -1,1174 +1,1176
 # sqlitestore.py - Storage backend that uses SQLite
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """store repository data in SQLite (EXPERIMENTAL)
 
 The sqlitestore extension enables the storage of repository data in SQLite.
 
 This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY
 GUARANTEES. This means that repositories created with this extension may
 only be usable with the exact version of this extension/Mercurial that was
 used. The extension attempts to enforce this in order to prevent repository
 corruption.
 
 In addition, several features are not yet supported or have known bugs:
 
 * Only some data is stored in SQLite. Changeset, manifest, and other repository
   data is not yet stored in SQLite.
 * Transactions are not robust. If the process is aborted at the right time
   during transaction close/rollback, the repository could be in an inconsistent
   state. This problem will diminish once all repository data is tracked by
   SQLite.
 * Bundle repositories do not work (the ability to use e.g.
   `hg -R <bundle-file> log` to automatically overlay a bundle on top of the
   existing repository).
 * Various other features don't work.
 
 This extension should work for basic clone/pull, update, and commit workflows.
 Some history rewriting operations may fail due to lack of support for bundle
 repositories.
 
 To use, activate the extension and set the ``storage.new-repo-backend`` config
 option to ``sqlite`` to enable new repositories to use SQLite for storage.
 """
 
 # To run the test suite with repos using SQLite by default, execute the
 # following:
 #
 # HGREPOFEATURES="sqlitestore" run-tests.py \
 #     --extra-config-opt extensions.sqlitestore= \
 #     --extra-config-opt storage.new-repo-backend=sqlite
 
 from __future__ import absolute_import
 
 import hashlib
 import sqlite3
 import struct
 import threading
 import zlib
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     nullrev,
     short,
 )
 from mercurial.thirdparty import (
     attr,
 )
 from mercurial import (
     ancestor,
     dagop,
     encoding,
     error,
     extensions,
     localrepo,
     mdiff,
     pycompat,
     registrar,
-    repository,
     util,
     verify,
 )
+from mercurial.interfaces import (
+    repository,
+)
 from mercurial.utils import (
     interfaceutil,
     storageutil,
 )
 
 try:
     from mercurial import zstd
     zstd.__version__
 except ImportError:
     zstd = None
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # experimental config: storage.sqlite.compression
 configitem('storage', 'sqlite.compression',
            default='zstd' if zstd else 'zlib',
            experimental=True)
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 REQUIREMENT = b'exp-sqlite-001'
 REQUIREMENT_ZSTD = b'exp-sqlite-comp-001=zstd'
 REQUIREMENT_ZLIB = b'exp-sqlite-comp-001=zlib'
 REQUIREMENT_NONE = b'exp-sqlite-comp-001=none'
 REQUIREMENT_SHALLOW_FILES = b'exp-sqlite-shallow-files'
 
 CURRENT_SCHEMA_VERSION = 1
 
 COMPRESSION_NONE = 1
 COMPRESSION_ZSTD = 2
 COMPRESSION_ZLIB = 3
 
 FLAG_CENSORED = 1
 FLAG_MISSING_P1 = 2
 FLAG_MISSING_P2 = 4
 
 CREATE_SCHEMA = [
     # Deltas are stored as content-indexed blobs.
     # compression column holds COMPRESSION_* constant for how the
     # delta is encoded.
 
     r'CREATE TABLE delta ('
     r'    id INTEGER PRIMARY KEY, '
     r'    compression INTEGER NOT NULL, '
     r'    hash BLOB UNIQUE ON CONFLICT ABORT, '
     r'    delta BLOB NOT NULL '
     r')',
 
     # Tracked paths are denormalized to integers to avoid redundant
     # storage of the path name.
     r'CREATE TABLE filepath ('
     r'    id INTEGER PRIMARY KEY, '
     r'    path BLOB NOT NULL '
     r')',
 
     r'CREATE UNIQUE INDEX filepath_path '
     r'    ON filepath (path)',
 
     # We have a single table for all file revision data.
     # Each file revision is uniquely described by a (path, rev) and
     # (path, node).
     #
     # Revision data is stored as a pointer to the delta producing this
     # revision and the file revision whose delta should be applied before
     # that one. One can reconstruct the delta chain by recursively following
     # the delta base revision pointers until one encounters NULL.
     #
     # flags column holds bitwise integer flags controlling storage options.
     # These flags are defined by the FLAG_* constants.
     r'CREATE TABLE fileindex ('
     r'    id INTEGER PRIMARY KEY, '
     r'    pathid INTEGER REFERENCES filepath(id), '
     r'    revnum INTEGER NOT NULL, '
     r'    p1rev INTEGER NOT NULL, '
     r'    p2rev INTEGER NOT NULL, '
     r'    linkrev INTEGER NOT NULL, '
     r'    flags INTEGER NOT NULL, '
     r'    deltaid INTEGER REFERENCES delta(id), '
     r'    deltabaseid INTEGER REFERENCES fileindex(id), '
     r'    node BLOB NOT NULL '
     r')',
 
     r'CREATE UNIQUE INDEX fileindex_pathrevnum '
     r'    ON fileindex (pathid, revnum)',
 
     r'CREATE UNIQUE INDEX fileindex_pathnode '
     r'    ON fileindex (pathid, node)',
 
     # Provide a view over all file data for convenience.
     r'CREATE VIEW filedata AS '
     r'SELECT '
     r'    fileindex.id AS id, '
     r'    filepath.id AS pathid, '
     r'    filepath.path AS path, '
     r'    fileindex.revnum AS revnum, '
     r'    fileindex.node AS node, '
     r'    fileindex.p1rev AS p1rev, '
     r'    fileindex.p2rev AS p2rev, '
     r'    fileindex.linkrev AS linkrev, '
     r'    fileindex.flags AS flags, '
     r'    fileindex.deltaid AS deltaid, '
     r'    fileindex.deltabaseid AS deltabaseid '
     r'FROM filepath, fileindex '
     r'WHERE fileindex.pathid=filepath.id',
 
     r'PRAGMA user_version=%d' % CURRENT_SCHEMA_VERSION,
 ]
 
 def resolvedeltachain(db, pathid, node, revisioncache,
                       stoprids, zstddctx=None):
     """Resolve a delta chain for a file node."""
 
     # TODO the "not in ({stops})" here is possibly slowing down the query
     # because it needs to perform the lookup on every recursive invocation.
     # This could possibly be faster if we created a temporary query with
     # baseid "poisoned" to null and limited the recursive filter to
     # "is not null".
     res = db.execute(
         r'WITH RECURSIVE '
         r'    deltachain(deltaid, baseid) AS ('
         r'        SELECT deltaid, deltabaseid FROM fileindex '
         r'            WHERE pathid=? AND node=? '
         r'        UNION ALL '
         r'        SELECT fileindex.deltaid, deltabaseid '
         r'            FROM fileindex, deltachain '
         r'            WHERE '
         r'                fileindex.id=deltachain.baseid '
         r'                AND deltachain.baseid IS NOT NULL '
         r'                AND fileindex.id NOT IN ({stops}) '
         r'    ) '
         r'SELECT deltachain.baseid, compression, delta '
         r'FROM deltachain, delta '
         r'WHERE delta.id=deltachain.deltaid'.format(
             stops=r','.join([r'?'] * len(stoprids))),
         tuple([pathid, node] + list(stoprids.keys())))
 
     deltas = []
     lastdeltabaseid = None
 
     for deltabaseid, compression, delta in res:
         lastdeltabaseid = deltabaseid
 
         if compression == COMPRESSION_ZSTD:
             delta = zstddctx.decompress(delta)
         elif compression == COMPRESSION_NONE:
             delta = delta
         elif compression == COMPRESSION_ZLIB:
             delta = zlib.decompress(delta)
         else:
             raise SQLiteStoreError('unhandled compression type: %d' %
                                    compression)
 
         deltas.append(delta)
 
     if lastdeltabaseid in stoprids:
         basetext = revisioncache[stoprids[lastdeltabaseid]]
     else:
         basetext = deltas.pop()
 
     deltas.reverse()
     fulltext = mdiff.patches(basetext, deltas)
 
     # SQLite returns buffer instances for blob columns on Python 2. This
     # type can propagate through the delta application layer. Because
     # downstream callers assume revisions are bytes, cast as needed.
     if not isinstance(fulltext, bytes):
         fulltext = bytes(delta)
 
     return fulltext
 
 def insertdelta(db, compression, hash, delta):
     try:
         return db.execute(
             r'INSERT INTO delta (compression, hash, delta) '
             r'VALUES (?, ?, ?)',
             (compression, hash, delta)).lastrowid
     except sqlite3.IntegrityError:
         return db.execute(
             r'SELECT id FROM delta WHERE hash=?',
             (hash,)).fetchone()[0]
 
 class SQLiteStoreError(error.StorageError):
     pass
 
 @attr.s
 class revisionentry(object):
     rid = attr.ib()
     rev = attr.ib()
     node = attr.ib()
     p1rev = attr.ib()
     p2rev = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     linkrev = attr.ib()
     flags = attr.ib()
 
 @interfaceutil.implementer(repository.irevisiondelta)
 @attr.s(slots=True)
 class sqliterevisiondelta(object):
     node = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     basenode = attr.ib()
     flags = attr.ib()
     baserevisionsize = attr.ib()
     revision = attr.ib()
     delta = attr.ib()
     linknode = attr.ib(default=None)
 
 @interfaceutil.implementer(repository.iverifyproblem)
 @attr.s(frozen=True)
 class sqliteproblem(object):
     warning = attr.ib(default=None)
     error = attr.ib(default=None)
     node = attr.ib(default=None)
 
 @interfaceutil.implementer(repository.ifilestorage)
 class sqlitefilestore(object):
     """Implements storage for an individual tracked path."""
 
     def __init__(self, db, path, compression):
         self._db = db
         self._path = path
 
         self._pathid = None
 
         # revnum -> node
         self._revtonode = {}
         # node -> revnum
         self._nodetorev = {}
         # node -> data structure
         self._revisions = {}
 
         self._revisioncache = util.lrucachedict(10)
 
         self._compengine = compression
 
         if compression == 'zstd':
             self._cctx = zstd.ZstdCompressor(level=3)
             self._dctx = zstd.ZstdDecompressor()
         else:
             self._cctx = None
             self._dctx = None
 
         self._refreshindex()
 
     def _refreshindex(self):
         self._revtonode = {}
         self._nodetorev = {}
         self._revisions = {}
 
         res = list(self._db.execute(
             r'SELECT id FROM filepath WHERE path=?', (self._path,)))
 
         if not res:
             self._pathid = None
             return
 
         self._pathid = res[0][0]
 
         res = self._db.execute(
             r'SELECT id, revnum, node, p1rev, p2rev, linkrev, flags '
             r'FROM fileindex '
             r'WHERE pathid=? '
             r'ORDER BY revnum ASC',
             (self._pathid,))
 
         for i, row in enumerate(res):
             rid, rev, node, p1rev, p2rev, linkrev, flags = row
 
             if i != rev:
                 raise SQLiteStoreError(_('sqlite database has inconsistent '
                                          'revision numbers'))
 
             if p1rev == nullrev:
                 p1node = nullid
             else:
                 p1node = self._revtonode[p1rev]
 
             if p2rev == nullrev:
                 p2node = nullid
             else:
                 p2node = self._revtonode[p2rev]
 
             entry = revisionentry(
                 rid=rid,
                 rev=rev,
                 node=node,
                 p1rev=p1rev,
                 p2rev=p2rev,
                 p1node=p1node,
                 p2node=p2node,
                 linkrev=linkrev,
                 flags=flags)
 
             self._revtonode[rev] = node
             self._nodetorev[node] = rev
             self._revisions[node] = entry
 
     # Start of ifileindex interface.
 
     def __len__(self):
         return len(self._revisions)
 
     def __iter__(self):
         return iter(pycompat.xrange(len(self._revisions)))
 
     def hasnode(self, node):
         if node == nullid:
             return False
 
         return node in self._nodetorev
 
     def revs(self, start=0, stop=None):
         return storageutil.iterrevs(len(self._revisions), start=start,
                                     stop=stop)
 
     def parents(self, node):
         if node == nullid:
             return nullid, nullid
 
         if node not in self._revisions:
             raise error.LookupError(node, self._path, _('no node'))
 
         entry = self._revisions[node]
         return entry.p1node, entry.p2node
 
     def parentrevs(self, rev):
         if rev == nullrev:
             return nullrev, nullrev
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         entry = self._revisions[self._revtonode[rev]]
         return entry.p1rev, entry.p2rev
 
     def rev(self, node):
         if node == nullid:
             return nullrev
 
         if node not in self._nodetorev:
             raise error.LookupError(node, self._path, _('no node'))
 
         return self._nodetorev[node]
 
     def node(self, rev):
         if rev == nullrev:
             return nullid
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         return self._revtonode[rev]
 
     def lookup(self, node):
         return storageutil.fileidlookup(self, node, self._path)
 
     def linkrev(self, rev):
         if rev == nullrev:
             return nullrev
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         entry = self._revisions[self._revtonode[rev]]
         return entry.linkrev
 
     def iscensored(self, rev):
         if rev == nullrev:
             return False
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         return self._revisions[self._revtonode[rev]].flags & FLAG_CENSORED
 
     def commonancestorsheads(self, node1, node2):
         rev1 = self.rev(node1)
         rev2 = self.rev(node2)
 
         ancestors = ancestor.commonancestorsheads(self.parentrevs, rev1, rev2)
         return pycompat.maplist(self.node, ancestors)
 
     def descendants(self, revs):
         # TODO we could implement this using a recursive SQL query, which
         # might be faster.
         return dagop.descendantrevs(revs, self.revs, self.parentrevs)
 
     def heads(self, start=None, stop=None):
         if start is None and stop is None:
             if not len(self):
                 return [nullid]
 
         startrev = self.rev(start) if start is not None else nullrev
         stoprevs = {self.rev(n) for n in stop or []}
 
         revs = dagop.headrevssubset(self.revs, self.parentrevs,
                                     startrev=startrev, stoprevs=stoprevs)
 
         return [self.node(rev) for rev in revs]
 
     def children(self, node):
         rev = self.rev(node)
 
         res = self._db.execute(
             r'SELECT'
             r'  node '
             r'  FROM filedata '
             r'  WHERE path=? AND (p1rev=? OR p2rev=?) '
             r'  ORDER BY revnum ASC',
             (self._path, rev, rev))
 
         return [row[0] for row in res]
 
     # End of ifileindex interface.
 
     # Start of ifiledata interface.
 
     def size(self, rev):
         if rev == nullrev:
             return 0
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         node = self._revtonode[rev]
 
         if self.renamed(node):
             return len(self.read(node))
 
         return len(self.revision(node))
 
     def revision(self, node, raw=False, _verifyhash=True):
         if node in (nullid, nullrev):
             return b''
 
         if isinstance(node, int):
             node = self.node(node)
 
         if node not in self._nodetorev:
             raise error.LookupError(node, self._path, _('no node'))
 
         if node in self._revisioncache:
             return self._revisioncache[node]
 
         # Because we have a fulltext revision cache, we are able to
         # short-circuit delta chain traversal and decompression as soon as
         # we encounter a revision in the cache.
 
         stoprids = {self._revisions[n].rid: n
                     for n in self._revisioncache}
 
         if not stoprids:
             stoprids[-1] = None
 
         fulltext = resolvedeltachain(self._db, self._pathid, node,
                                      self._revisioncache, stoprids,
                                      zstddctx=self._dctx)
 
         # Don't verify hashes if parent nodes were rewritten, as the hash
         # wouldn't verify.
         if self._revisions[node].flags & (FLAG_MISSING_P1 | FLAG_MISSING_P2):
             _verifyhash = False
 
         if _verifyhash:
             self._checkhash(fulltext, node)
             self._revisioncache[node] = fulltext
 
         return fulltext
 
     def rawdata(self, *args, **kwargs):
         return self.revision(*args, **kwargs)
 
     def read(self, node):
         return storageutil.filtermetadata(self.revision(node))
 
     def renamed(self, node):
         return storageutil.filerevisioncopied(self, node)
 
     def cmp(self, node, fulltext):
         return not storageutil.filedataequivalent(self, node, fulltext)
 
     def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
                       assumehaveparentrevisions=False,
                       deltamode=repository.CG_DELTAMODE_STD):
         if nodesorder not in ('nodes', 'storage', 'linear', None):
             raise error.ProgrammingError('unhandled value for nodesorder: %s' %
                                          nodesorder)
 
         nodes = [n for n in nodes if n != nullid]
 
         if not nodes:
             return
 
         # TODO perform in a single query.
         res = self._db.execute(
             r'SELECT revnum, deltaid FROM fileindex '
             r'WHERE pathid=? '
             r'    AND node in (%s)' % (r','.join([r'?'] * len(nodes))),
             tuple([self._pathid] + nodes))
 
         deltabases = {}
 
         for rev, deltaid in res:
             res = self._db.execute(
                 r'SELECT revnum from fileindex WHERE pathid=? AND deltaid=?',
                 (self._pathid, deltaid))
             deltabases[rev] = res.fetchone()[0]
 
         # TODO define revdifffn so we can use delta from storage.
         for delta in storageutil.emitrevisions(
             self, nodes, nodesorder, sqliterevisiondelta,
             deltaparentfn=deltabases.__getitem__,
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode):
 
             yield delta
 
     # End of ifiledata interface.
 
     # Start of ifilemutation interface.
 
     def add(self, filedata, meta, transaction, linkrev, p1, p2):
         if meta or filedata.startswith(b'\x01\n'):
             filedata = storageutil.packmeta(meta, filedata)
 
         return self.addrevision(filedata, transaction, linkrev, p1, p2)
 
     def addrevision(self, revisiondata, transaction, linkrev, p1, p2, node=None,
                     flags=0, cachedelta=None):
         if flags:
             raise SQLiteStoreError(_('flags not supported on revisions'))
 
         validatehash = node is not None
         node = node or storageutil.hashrevisionsha1(revisiondata, p1, p2)
 
         if validatehash:
             self._checkhash(revisiondata, node, p1, p2)
 
         if node in self._nodetorev:
             return node
 
         node = self._addrawrevision(node, revisiondata, transaction, linkrev,
                                     p1, p2)
 
         self._revisioncache[node] = revisiondata
         return node
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None,
                  maybemissingparents=False):
         nodes = []
 
         for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
             storeflags = 0
 
             if wireflags & repository.REVISION_FLAG_CENSORED:
                 storeflags |= FLAG_CENSORED
 
             if wireflags & ~repository.REVISION_FLAG_CENSORED:
                 raise SQLiteStoreError('unhandled revision flag')
 
             if maybemissingparents:
                 if p1 != nullid and not self.hasnode(p1):
                     p1 = nullid
                     storeflags |= FLAG_MISSING_P1
 
                 if p2 != nullid and not self.hasnode(p2):
                     p2 = nullid
                     storeflags |= FLAG_MISSING_P2
 
             baserev = self.rev(deltabase)
 
             # If base is censored, delta must be full replacement in a single
             # patch operation.
             if baserev != nullrev and self.iscensored(baserev):
                 hlen = struct.calcsize('>lll')
                 oldlen = len(self.rawdata(deltabase, _verifyhash=False))
                 newlen = len(delta) - hlen
 
                 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
                     raise error.CensoredBaseError(self._path,
                                                   deltabase)
 
             if (not (storeflags & FLAG_CENSORED)
                 and storageutil.deltaiscensored(
                     delta, baserev, lambda x: len(self.rawdata(x)))):
                 storeflags |= FLAG_CENSORED
 
             linkrev = linkmapper(linknode)
 
             nodes.append(node)
 
             if node in self._revisions:
                 # Possibly reset parents to make them proper.
                 entry = self._revisions[node]
 
                 if entry.flags & FLAG_MISSING_P1 and p1 != nullid:
                     entry.p1node = p1
                     entry.p1rev = self._nodetorev[p1]
                     entry.flags &= ~FLAG_MISSING_P1
 
                     self._db.execute(
                         r'UPDATE fileindex SET p1rev=?, flags=? '
                         r'WHERE id=?',
                         (self._nodetorev[p1], entry.flags, entry.rid))
 
                 if entry.flags & FLAG_MISSING_P2 and p2 != nullid:
                     entry.p2node = p2
                     entry.p2rev = self._nodetorev[p2]
                     entry.flags &= ~FLAG_MISSING_P2
 
                     self._db.execute(
                         r'UPDATE fileindex SET p2rev=?, flags=? '
                         r'WHERE id=?',
                         (self._nodetorev[p1], entry.flags, entry.rid))
 
                 continue
 
             if deltabase == nullid:
                 text = mdiff.patch(b'', delta)
                 storedelta = None
             else:
                 text = None
                 storedelta = (deltabase, delta)
 
             self._addrawrevision(node, text, transaction, linkrev, p1, p2,
                                  storedelta=storedelta, flags=storeflags)
 
             if addrevisioncb:
                 addrevisioncb(self, node)
 
         return nodes
 
     def censorrevision(self, tr, censornode, tombstone=b''):
         tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
 
         # This restriction is cargo culted from revlogs and makes no sense for
         # SQLite, since columns can be resized at will.
         if len(tombstone) > len(self.rawdata(censornode)):
             raise error.Abort(_('censor tombstone must be no longer than '
                                 'censored data'))
 
         # We need to replace the censored revision's data with the tombstone.
         # But replacing that data will have implications for delta chains that
         # reference it.
         #
         # While "better," more complex strategies are possible, we do something
         # simple: we find delta chain children of the censored revision and we
         # replace those incremental deltas with fulltexts of their corresponding
         # revision. Then we delete the now-unreferenced delta and original
         # revision and insert a replacement.
 
         # Find the delta to be censored.
         censoreddeltaid = self._db.execute(
             r'SELECT deltaid FROM fileindex WHERE id=?',
             (self._revisions[censornode].rid,)).fetchone()[0]
 
         # Find all its delta chain children.
         # TODO once we support storing deltas for !files, we'll need to look
         # for those delta chains too.
         rows = list(self._db.execute(
             r'SELECT id, pathid, node FROM fileindex '
             r'WHERE deltabaseid=? OR deltaid=?',
             (censoreddeltaid, censoreddeltaid)))
 
         for row in rows:
             rid, pathid, node = row
 
             fulltext = resolvedeltachain(self._db, pathid, node, {}, {-1: None},
                                          zstddctx=self._dctx)
 
             deltahash = hashlib.sha1(fulltext).digest()
 
             if self._compengine == 'zstd':
                 deltablob = self._cctx.compress(fulltext)
                 compression = COMPRESSION_ZSTD
             elif self._compengine == 'zlib':
                 deltablob = zlib.compress(fulltext)
                 compression = COMPRESSION_ZLIB
             elif self._compengine == 'none':
                 deltablob = fulltext
                 compression = COMPRESSION_NONE
             else:
                 raise error.ProgrammingError('unhandled compression engine: %s'
                                              % self._compengine)
 
             if len(deltablob) >= len(fulltext):
                 deltablob = fulltext
                 compression = COMPRESSION_NONE
 
             deltaid = insertdelta(self._db, compression, deltahash, deltablob)
 
             self._db.execute(
                 r'UPDATE fileindex SET deltaid=?, deltabaseid=NULL '
                 r'WHERE id=?', (deltaid, rid))
 
         # Now create the tombstone delta and replace the delta on the censored
         # node.
         deltahash = hashlib.sha1(tombstone).digest()
         tombstonedeltaid = insertdelta(self._db, COMPRESSION_NONE,
                                        deltahash, tombstone)
 
         flags = self._revisions[censornode].flags
         flags |= FLAG_CENSORED
 
         self._db.execute(
             r'UPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL '
             r'WHERE pathid=? AND node=?',
             (flags, tombstonedeltaid, self._pathid, censornode))
 
         self._db.execute(
             r'DELETE FROM delta WHERE id=?', (censoreddeltaid,))
 
         self._refreshindex()
         self._revisioncache.clear()
 
     def getstrippoint(self, minlink):
         return storageutil.resolvestripinfo(minlink, len(self) - 1,
                                             [self.rev(n) for n in self.heads()],
                                             self.linkrev,
                                             self.parentrevs)
 
     def strip(self, minlink, transaction):
         if not len(self):
             return
 
         rev, _ignored = self.getstrippoint(minlink)
 
         if rev == len(self):
             return
 
         for rev in self.revs(rev):
             self._db.execute(
                 r'DELETE FROM fileindex WHERE pathid=? AND node=?',
                 (self._pathid, self.node(rev)))
 
         # TODO how should we garbage collect data in delta table?
 
         self._refreshindex()
 
     # End of ifilemutation interface.
 
     # Start of ifilestorage interface.
 
     def files(self):
         return []
 
     def storageinfo(self, exclusivefiles=False, sharedfiles=False,
                     revisionscount=False, trackedsize=False,
                     storedsize=False):
         d = {}
 
         if exclusivefiles:
             d['exclusivefiles'] = []
 
         if sharedfiles:
             # TODO list sqlite file(s) here.
             d['sharedfiles'] = []
 
         if revisionscount:
             d['revisionscount'] = len(self)
 
         if trackedsize:
             d['trackedsize'] = sum(len(self.revision(node))
                                        for node in self._nodetorev)
 
         if storedsize:
             # TODO implement this?
             d['storedsize'] = None
 
         return d
 
     def verifyintegrity(self, state):
         state['skipread'] = set()
 
         for rev in self:
             node = self.node(rev)
 
             try:
                 self.revision(node)
             except Exception as e:
                 yield sqliteproblem(
                     error=_('unpacking %s: %s') % (short(node), e),
                     node=node)
 
                 state['skipread'].add(node)
 
     # End of ifilestorage interface.
 
     def _checkhash(self, fulltext, node, p1=None, p2=None):
         if p1 is None and p2 is None:
             p1, p2 = self.parents(node)
 
         if node == storageutil.hashrevisionsha1(fulltext, p1, p2):
             return
 
         try:
             del self._revisioncache[node]
         except KeyError:
             pass
 
         if storageutil.iscensoredtext(fulltext):
             raise error.CensoredNodeError(self._path, node, fulltext)
 
         raise SQLiteStoreError(_('integrity check failed on %s') %
                                self._path)
 
     def _addrawrevision(self, node, revisiondata, transaction, linkrev,
                         p1, p2, storedelta=None, flags=0):
         if self._pathid is None:
             res = self._db.execute(
                 r'INSERT INTO filepath (path) VALUES (?)', (self._path,))
             self._pathid = res.lastrowid
 
         # For simplicity, always store a delta against p1.
         # TODO we need a lot more logic here to make behavior reasonable.
 
         if storedelta:
             deltabase, delta = storedelta
 
             if isinstance(deltabase, int):
                 deltabase = self.node(deltabase)
 
         else:
             assert revisiondata is not None
             deltabase = p1
 
             if deltabase == nullid:
                 delta = revisiondata
             else:
                 delta = mdiff.textdiff(self.revision(self.rev(deltabase)),
                                        revisiondata)
 
         # File index stores a pointer to its delta and the parent delta.
         # The parent delta is stored via a pointer to the fileindex PK.
         if deltabase == nullid:
             baseid = None
         else:
             baseid = self._revisions[deltabase].rid
 
         # Deltas are stored with a hash of their content. This allows
         # us to de-duplicate. The table is configured to ignore conflicts
         # and it is faster to just insert and silently noop than to look
         # first.
         deltahash = hashlib.sha1(delta).digest()
 
         if self._compengine == 'zstd':
             deltablob = self._cctx.compress(delta)
             compression = COMPRESSION_ZSTD
         elif self._compengine == 'zlib':
             deltablob = zlib.compress(delta)
             compression = COMPRESSION_ZLIB
         elif self._compengine == 'none':
             deltablob = delta
             compression = COMPRESSION_NONE
         else:
             raise error.ProgrammingError('unhandled compression engine: %s' %
                                          self._compengine)
 
         # Don't store compressed data if it isn't practical.
         if len(deltablob) >= len(delta):
             deltablob = delta
             compression = COMPRESSION_NONE
 
         deltaid = insertdelta(self._db, compression, deltahash, deltablob)
 
         rev = len(self)
 
         if p1 == nullid:
             p1rev = nullrev
         else:
             p1rev = self._nodetorev[p1]
 
         if p2 == nullid:
             p2rev = nullrev
         else:
             p2rev = self._nodetorev[p2]
 
         rid = self._db.execute(
             r'INSERT INTO fileindex ('
             r'    pathid, revnum, node, p1rev, p2rev, linkrev, flags, '
             r'    deltaid, deltabaseid) '
             r'    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
             (self._pathid, rev, node, p1rev, p2rev, linkrev, flags,
              deltaid, baseid)
         ).lastrowid
 
         entry = revisionentry(
             rid=rid,
             rev=rev,
             node=node,
             p1rev=p1rev,
             p2rev=p2rev,
             p1node=p1,
             p2node=p2,
             linkrev=linkrev,
             flags=flags)
 
         self._nodetorev[node] = rev
         self._revtonode[rev] = node
         self._revisions[node] = entry
 
         return node
 
 class sqliterepository(localrepo.localrepository):
     def cancopy(self):
         return False
 
     def transaction(self, *args, **kwargs):
         current = self.currenttransaction()
 
         tr = super(sqliterepository, self).transaction(*args, **kwargs)
 
         if current:
             return tr
 
         self._dbconn.execute(r'BEGIN TRANSACTION')
 
         def committransaction(_):
             self._dbconn.commit()
 
         tr.addfinalize('sqlitestore', committransaction)
 
         return tr
 
     @property
     def _dbconn(self):
         # SQLite connections can only be used on the thread that created
         # them. In most cases, this "just works." However, hgweb uses
         # multiple threads.
         tid = threading.current_thread().ident
 
         if self._db:
             if self._db[0] == tid:
                 return self._db[1]
 
         db = makedb(self.svfs.join('db.sqlite'))
         self._db = (tid, db)
 
         return db
 
 def makedb(path):
     """Construct a database handle for a database at path."""
 
     db = sqlite3.connect(encoding.strfromlocal(path))
     db.text_factory = bytes
 
     res = db.execute(r'PRAGMA user_version').fetchone()[0]
 
     # New database.
     if res == 0:
         for statement in CREATE_SCHEMA:
             db.execute(statement)
 
         db.commit()
 
     elif res == CURRENT_SCHEMA_VERSION:
         pass
 
     else:
         raise error.Abort(_('sqlite database has unrecognized version'))
 
     db.execute(r'PRAGMA journal_mode=WAL')
 
     return db
 
 def featuresetup(ui, supported):
     supported.add(REQUIREMENT)
 
     if zstd:
         supported.add(REQUIREMENT_ZSTD)
 
     supported.add(REQUIREMENT_ZLIB)
     supported.add(REQUIREMENT_NONE)
     supported.add(REQUIREMENT_SHALLOW_FILES)
     supported.add(repository.NARROW_REQUIREMENT)
 
 def newreporequirements(orig, ui, createopts):
     if createopts['backend'] != 'sqlite':
         return orig(ui, createopts)
 
     # This restriction can be lifted once we have more confidence.
     if 'sharedrepo' in createopts:
         raise error.Abort(_('shared repositories not supported with SQLite '
                             'store'))
 
     # This filtering is out of an abundance of caution: we want to ensure
     # we honor creation options and we do that by annotating exactly the
     # creation options we recognize.
     known = {
         'narrowfiles',
         'backend',
         'shallowfilestore',
     }
 
     unsupported = set(createopts) - known
     if unsupported:
         raise error.Abort(_('SQLite store does not support repo creation '
                             'option: %s') % ', '.join(sorted(unsupported)))
 
     # Since we're a hybrid store that still relies on revlogs, we fall back
     # to using the revlogv1 backend's storage requirements then adding our
     # own requirement.
     createopts['backend'] = 'revlogv1'
     requirements = orig(ui, createopts)
     requirements.add(REQUIREMENT)
 
     compression = ui.config('storage', 'sqlite.compression')
 
     if compression == 'zstd' and not zstd:
         raise error.Abort(_('storage.sqlite.compression set to "zstd" but '
                             'zstandard compression not available to this '
                             'Mercurial install'))
 
     if compression == 'zstd':
         requirements.add(REQUIREMENT_ZSTD)
     elif compression == 'zlib':
         requirements.add(REQUIREMENT_ZLIB)
     elif compression == 'none':
         requirements.add(REQUIREMENT_NONE)
     else:
         raise error.Abort(_('unknown compression engine defined in '
                             'storage.sqlite.compression: %s') % compression)
 
     if createopts.get('shallowfilestore'):
         requirements.add(REQUIREMENT_SHALLOW_FILES)
 
     return requirements
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
 class sqlitefilestorage(object):
     """Repository file storage backed by SQLite."""
     def file(self, path):
         if path[0] == b'/':
             path = path[1:]
 
         if REQUIREMENT_ZSTD in self.requirements:
             compression = 'zstd'
         elif REQUIREMENT_ZLIB in self.requirements:
             compression = 'zlib'
         elif REQUIREMENT_NONE in self.requirements:
             compression = 'none'
         else:
             raise error.Abort(_('unable to determine what compression engine '
                                 'to use for SQLite storage'))
 
         return sqlitefilestore(self._dbconn, path, compression)
 
 def makefilestorage(orig, requirements, features, **kwargs):
     """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
     if REQUIREMENT in requirements:
         if REQUIREMENT_SHALLOW_FILES in requirements:
             features.add(repository.REPO_FEATURE_SHALLOW_FILE_STORAGE)
 
         return sqlitefilestorage
     else:
         return orig(requirements=requirements, features=features, **kwargs)
 
 def makemain(orig, ui, requirements, **kwargs):
     if REQUIREMENT in requirements:
         if REQUIREMENT_ZSTD in requirements and not zstd:
             raise error.Abort(_('repository uses zstandard compression, which '
                                 'is not available to this Mercurial install'))
 
         return sqliterepository
 
     return orig(requirements=requirements, **kwargs)
 
 def verifierinit(orig, self, *args, **kwargs):
     orig(self, *args, **kwargs)
 
     # We don't care that files in the store don't align with what is
     # advertised. So suppress these warnings.
     self.warnorphanstorefiles = False
 
 def extsetup(ui):
     localrepo.featuresetupfuncs.add(featuresetup)
     extensions.wrapfunction(localrepo, 'newreporequirements',
                             newreporequirements)
     extensions.wrapfunction(localrepo, 'makefilestorage',
                             makefilestorage)
     extensions.wrapfunction(localrepo, 'makemain',
                             makemain)
     extensions.wrapfunction(verify.verifier, '__init__',
                             verifierinit)
 
 def reposetup(ui, repo):
     if isinstance(repo, sqliterepository):
         repo._db = None
 
     # TODO check for bundlerepository?

mercurial/changegroup.py

@@ -1,1423 +1,1426
 # changegroup.py - Mercurial changegroup manipulation functions
 #
 #  Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import os
 import struct
 import weakref
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
     nullrev,
     short,
 )
 
 from . import (
     error,
     match as matchmod,
     mdiff,
     phases,
     pycompat,
+    util,
+)
+
+from .interfaces import (
     repository,
-    util,
 )
 
 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
 
 LFS_REQUIREMENT = 'lfs'
 
 readexactly = util.readexactly
 
 def getchunk(stream):
     """return the next chunk from stream as a string"""
     d = readexactly(stream, 4)
     l = struct.unpack(">l", d)[0]
     if l <= 4:
         if l:
             raise error.Abort(_("invalid chunk length %d") % l)
         return ""
     return readexactly(stream, l - 4)
 
 def chunkheader(length):
     """return a changegroup chunk header (string)"""
     return struct.pack(">l", length + 4)
 
 def closechunk():
     """return a changegroup chunk header (string) for a zero-length chunk"""
     return struct.pack(">l", 0)
 
 def _fileheader(path):
     """Obtain a changegroup chunk header for a named path."""
     return chunkheader(len(path)) + path
 
 def writechunks(ui, chunks, filename, vfs=None):
     """Write chunks to a file and return its filename.
 
     The stream is assumed to be a bundle file.
     Existing files will not be overwritten.
     If no filename is specified, a temporary file is created.
     """
     fh = None
     cleanup = None
     try:
         if filename:
             if vfs:
                 fh = vfs.open(filename, "wb")
             else:
                 # Increase default buffer size because default is usually
                 # small (4k is common on Linux).
                 fh = open(filename, "wb", 131072)
         else:
             fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
             fh = os.fdopen(fd, r"wb")
         cleanup = filename
         for c in chunks:
             fh.write(c)
         cleanup = None
         return filename
     finally:
         if fh is not None:
             fh.close()
         if cleanup is not None:
             if filename and vfs:
                 vfs.unlink(cleanup)
             else:
                 os.unlink(cleanup)
 
 class cg1unpacker(object):
     """Unpacker for cg1 changegroup streams.
 
     A changegroup unpacker handles the framing of the revision data in
     the wire format. Most consumers will want to use the apply()
     method to add the changes from the changegroup to a repository.
 
     If you're forwarding a changegroup unmodified to another consumer,
     use getchunks(), which returns an iterator of changegroup
     chunks. This is mostly useful for cases where you need to know the
     data stream has ended by observing the end of the changegroup.
 
     deltachunk() is useful only if you're applying delta data. Most
     consumers should prefer apply() instead.
 
     A few other public methods exist. Those are used only for
     bundlerepo and some debug commands - their use is discouraged.
     """
     deltaheader = _CHANGEGROUPV1_DELTA_HEADER
     deltaheadersize = deltaheader.size
     version = '01'
     _grouplistcount = 1 # One list of files after the manifests
 
     def __init__(self, fh, alg, extras=None):
         if alg is None:
             alg = 'UN'
         if alg not in util.compengines.supportedbundletypes:
             raise error.Abort(_('unknown stream compression type: %s')
                              % alg)
         if alg == 'BZ':
             alg = '_truncatedBZ'
 
         compengine = util.compengines.forbundletype(alg)
         self._stream = compengine.decompressorreader(fh)
         self._type = alg
         self.extras = extras or {}
         self.callback = None
 
     # These methods (compressed, read, seek, tell) all appear to only
     # be used by bundlerepo, but it's a little hard to tell.
     def compressed(self):
         return self._type is not None and self._type != 'UN'
     def read(self, l):
         return self._stream.read(l)
     def seek(self, pos):
         return self._stream.seek(pos)
     def tell(self):
         return self._stream.tell()
     def close(self):
         return self._stream.close()
 
     def _chunklength(self):
         d = readexactly(self._stream, 4)
         l = struct.unpack(">l", d)[0]
         if l <= 4:
             if l:
                 raise error.Abort(_("invalid chunk length %d") % l)
             return 0
         if self.callback:
             self.callback()
         return l - 4
 
     def changelogheader(self):
         """v10 does not have a changelog header chunk"""
         return {}
 
     def manifestheader(self):
         """v10 does not have a manifest header chunk"""
         return {}
 
     def filelogheader(self):
         """return the header of the filelogs chunk, v10 only has the filename"""
         l = self._chunklength()
         if not l:
             return {}
         fname = readexactly(self._stream, l)
         return {'filename': fname}
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, cs = headertuple
         if prevnode is None:
             deltabase = p1
         else:
             deltabase = prevnode
         flags = 0
         return node, p1, p2, deltabase, cs, flags
 
     def deltachunk(self, prevnode):
         l = self._chunklength()
         if not l:
             return {}
         headerdata = readexactly(self._stream, self.deltaheadersize)
         header = self.deltaheader.unpack(headerdata)
         delta = readexactly(self._stream, l - self.deltaheadersize)
         node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
         return (node, p1, p2, cs, deltabase, delta, flags)
 
     def getchunks(self):
         """returns all the chunks contains in the bundle
 
         Used when you need to forward the binary stream to a file or another
         network API. To do so, it parse the changegroup data, otherwise it will
         block in case of sshrepo because it don't know the end of the stream.
         """
         # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
         # and a list of filelogs. For changegroup 3, we expect 4 parts:
         # changelog, manifestlog, a list of tree manifestlogs, and a list of
         # filelogs.
         #
         # Changelog and manifestlog parts are terminated with empty chunks. The
         # tree and file parts are a list of entry sections. Each entry section
         # is a series of chunks terminating in an empty chunk. The list of these
         # entry sections is terminated in yet another empty chunk, so we know
         # we've reached the end of the tree/file list when we reach an empty
         # chunk that was proceeded by no non-empty chunks.
 
         parts = 0
         while parts < 2 + self._grouplistcount:
             noentries = True
             while True:
                 chunk = getchunk(self)
                 if not chunk:
                     # The first two empty chunks represent the end of the
                     # changelog and the manifestlog portions. The remaining
                     # empty chunks represent either A) the end of individual
                     # tree or file entries in the file list, or B) the end of
                     # the entire list. It's the end of the entire list if there
                     # were no entries (i.e. noentries is True).
                     if parts < 2:
                         parts += 1
                     elif noentries:
                         parts += 1
                     break
                 noentries = False
                 yield chunkheader(len(chunk))
                 pos = 0
                 while pos < len(chunk):
                     next = pos + 2**20
                     yield chunk[pos:next]
                     pos = next
             yield closechunk()
 
     def _unpackmanifests(self, repo, revmap, trp, prog):
         self.callback = prog.increment
         # no need to check for empty manifest group here:
         # if the result of the merge of 1 and 2 is the same in 3 and 4,
         # no new manifest will be created and the manifest group will
         # be empty during the pull
         self.manifestheader()
         deltas = self.deltaiter()
         repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp)
         prog.complete()
         self.callback = None
 
     def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
               expectedtotal=None):
         """Add the changegroup returned by source.read() to this repo.
         srctype is a string like 'push', 'pull', or 'unbundle'.  url is
         the URL of the repo where this changegroup is coming from.
 
         Return an integer summarizing the change to this repo:
         - nothing changed or no source: 0
         - more heads than before: 1+added heads (2..n)
         - fewer heads than before: -1-removed heads (-2..-n)
         - number of heads stays the same: 1
         """
         repo = repo.unfiltered()
         def csmap(x):
             repo.ui.debug("add changeset %s\n" % short(x))
             return len(cl)
 
         def revmap(x):
             return cl.rev(x)
 
         changesets = files = revisions = 0
 
         try:
             # The transaction may already carry source information. In this
             # case we use the top level data. We overwrite the argument
             # because we need to use the top level value (if they exist)
             # in this function.
             srctype = tr.hookargs.setdefault('source', srctype)
             tr.hookargs.setdefault('url', url)
             repo.hook('prechangegroup',
                       throw=True, **pycompat.strkwargs(tr.hookargs))
 
             # write changelog data to temp files so concurrent readers
             # will not see an inconsistent view
             cl = repo.changelog
             cl.delayupdate(tr)
             oldheads = set(cl.heads())
 
             trp = weakref.proxy(tr)
             # pull off the changeset group
             repo.ui.status(_("adding changesets\n"))
             clstart = len(cl)
             progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
                                             total=expectedtotal)
             self.callback = progress.increment
 
             efiles = set()
             def onchangelog(cl, node):
                 efiles.update(cl.readfiles(node))
 
             self.changelogheader()
             deltas = self.deltaiter()
             cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
             efiles = len(efiles)
 
             if not cgnodes:
                 repo.ui.develwarn('applied empty changelog from changegroup',
                                   config='warn-empty-changegroup')
             clend = len(cl)
             changesets = clend - clstart
             progress.complete()
             self.callback = None
 
             # pull off the manifest group
             repo.ui.status(_("adding manifests\n"))
             # We know that we'll never have more manifests than we had
             # changesets.
             progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
                                             total=changesets)
             self._unpackmanifests(repo, revmap, trp, progress)
 
             needfiles = {}
             if repo.ui.configbool('server', 'validate'):
                 cl = repo.changelog
                 ml = repo.manifestlog
                 # validate incoming csets have their manifests
                 for cset in pycompat.xrange(clstart, clend):
                     mfnode = cl.changelogrevision(cset).manifest
                     mfest = ml[mfnode].readdelta()
                     # store file cgnodes we must see
                     for f, n in mfest.iteritems():
                         needfiles.setdefault(f, set()).add(n)
 
             # process the files
             repo.ui.status(_("adding file changes\n"))
             newrevs, newfiles = _addchangegroupfiles(
                 repo, self, revmap, trp, efiles, needfiles)
             revisions += newrevs
             files += newfiles
 
             deltaheads = 0
             if oldheads:
                 heads = cl.heads()
                 deltaheads = len(heads) - len(oldheads)
                 for h in heads:
                     if h not in oldheads and repo[h].closesbranch():
                         deltaheads -= 1
             htext = ""
             if deltaheads:
                 htext = _(" (%+d heads)") % deltaheads
 
             repo.ui.status(_("added %d changesets"
                              " with %d changes to %d files%s\n")
                              % (changesets, revisions, files, htext))
             repo.invalidatevolatilesets()
 
             if changesets > 0:
                 if 'node' not in tr.hookargs:
                     tr.hookargs['node'] = hex(cl.node(clstart))
                     tr.hookargs['node_last'] = hex(cl.node(clend - 1))
                     hookargs = dict(tr.hookargs)
                 else:
                     hookargs = dict(tr.hookargs)
                     hookargs['node'] = hex(cl.node(clstart))
                     hookargs['node_last'] = hex(cl.node(clend - 1))
                 repo.hook('pretxnchangegroup',
                           throw=True, **pycompat.strkwargs(hookargs))
 
             added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
             phaseall = None
             if srctype in ('push', 'serve'):
                 # Old servers can not push the boundary themselves.
                 # New servers won't push the boundary if changeset already
                 # exists locally as secret
                 #
                 # We should not use added here but the list of all change in
                 # the bundle
                 if repo.publishing():
                     targetphase = phaseall = phases.public
                 else:
                     # closer target phase computation
 
                     # Those changesets have been pushed from the
                     # outside, their phases are going to be pushed
                     # alongside. Therefor `targetphase` is
                     # ignored.
                     targetphase = phaseall = phases.draft
             if added:
                 phases.registernew(repo, tr, targetphase, added)
             if phaseall is not None:
                 phases.advanceboundary(repo, tr, phaseall, cgnodes)
 
             if changesets > 0:
 
                 def runhooks():
                     # These hooks run when the lock releases, not when the
                     # transaction closes. So it's possible for the changelog
                     # to have changed since we last saw it.
                     if clstart >= len(repo):
                         return
 
                     repo.hook("changegroup", **pycompat.strkwargs(hookargs))
 
                     for n in added:
                         args = hookargs.copy()
                         args['node'] = hex(n)
                         del args['node_last']
                         repo.hook("incoming", **pycompat.strkwargs(args))
 
                     newheads = [h for h in repo.heads()
                                 if h not in oldheads]
                     repo.ui.log("incoming",
                                 "%d incoming changes - new heads: %s\n",
                                 len(added),
                                 ', '.join([hex(c[:6]) for c in newheads]))
 
                 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
                                 lambda tr: repo._afterlock(runhooks))
         finally:
             repo.ui.flush()
         # never return 0 here:
         if deltaheads < 0:
             ret = deltaheads - 1
         else:
             ret = deltaheads + 1
         return ret
 
     def deltaiter(self):
         """
         returns an iterator of the deltas in this changegroup
 
         Useful for passing to the underlying storage system to be stored.
         """
         chain = None
         for chunkdata in iter(lambda: self.deltachunk(chain), {}):
             # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
             yield chunkdata
             chain = chunkdata[0]
 
 class cg2unpacker(cg1unpacker):
     """Unpacker for cg2 streams.
 
     cg2 streams add support for generaldelta, so the delta header
     format is slightly different. All other features about the data
     remain the same.
     """
     deltaheader = _CHANGEGROUPV2_DELTA_HEADER
     deltaheadersize = deltaheader.size
     version = '02'
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, deltabase, cs = headertuple
         flags = 0
         return node, p1, p2, deltabase, cs, flags
 
 class cg3unpacker(cg2unpacker):
     """Unpacker for cg3 streams.
 
     cg3 streams add support for exchanging treemanifests and revlog
     flags. It adds the revlog flags to the delta header and an empty chunk
     separating manifests and files.
     """
     deltaheader = _CHANGEGROUPV3_DELTA_HEADER
     deltaheadersize = deltaheader.size
     version = '03'
     _grouplistcount = 2 # One list of manifests and one list of files
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, deltabase, cs, flags = headertuple
         return node, p1, p2, deltabase, cs, flags
 
     def _unpackmanifests(self, repo, revmap, trp, prog):
         super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
         for chunkdata in iter(self.filelogheader, {}):
             # If we get here, there are directory manifests in the changegroup
             d = chunkdata["filename"]
             repo.ui.debug("adding %s revisions\n" % d)
             deltas = self.deltaiter()
             if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
                 raise error.Abort(_("received dir revlog group is empty"))
 
 class headerlessfixup(object):
     def __init__(self, fh, h):
         self._h = h
         self._fh = fh
     def read(self, n):
         if self._h:
             d, self._h = self._h[:n], self._h[n:]
             if len(d) < n:
                 d += readexactly(self._fh, n - len(d))
             return d
         return readexactly(self._fh, n)
 
 def _revisiondeltatochunks(delta, headerfn):
     """Serialize a revisiondelta to changegroup chunks."""
 
     # The captured revision delta may be encoded as a delta against
     # a base revision or as a full revision. The changegroup format
     # requires that everything on the wire be deltas. So for full
     # revisions, we need to invent a header that says to rewrite
     # data.
 
     if delta.delta is not None:
         prefix, data = b'', delta.delta
     elif delta.basenode == nullid:
         data = delta.revision
         prefix = mdiff.trivialdiffheader(len(data))
     else:
         data = delta.revision
         prefix = mdiff.replacediffheader(delta.baserevisionsize,
                                          len(data))
 
     meta = headerfn(delta)
 
     yield chunkheader(len(meta) + len(prefix) + len(data))
     yield meta
     if prefix:
         yield prefix
     yield data
 
 def _sortnodesellipsis(store, nodes, cl, lookup):
     """Sort nodes for changegroup generation."""
     # Ellipses serving mode.
     #
     # In a perfect world, we'd generate better ellipsis-ified graphs
     # for non-changelog revlogs. In practice, we haven't started doing
     # that yet, so the resulting DAGs for the manifestlog and filelogs
     # are actually full of bogus parentage on all the ellipsis
     # nodes. This has the side effect that, while the contents are
     # correct, the individual DAGs might be completely out of whack in
     # a case like 882681bc3166 and its ancestors (back about 10
     # revisions or so) in the main hg repo.
     #
     # The one invariant we *know* holds is that the new (potentially
     # bogus) DAG shape will be valid if we order the nodes in the
     # order that they're introduced in dramatis personae by the
     # changelog, so what we do is we sort the non-changelog histories
     # by the order in which they are used by the changelog.
     key = lambda n: cl.rev(lookup(n))
     return sorted(nodes, key=key)
 
 def _resolvenarrowrevisioninfo(cl, store, ischangelog, rev, linkrev,
                                linknode, clrevtolocalrev, fullclnodes,
                                precomputedellipsis):
     linkparents = precomputedellipsis[linkrev]
     def local(clrev):
         """Turn a changelog revnum into a local revnum.
 
         The ellipsis dag is stored as revnums on the changelog,
         but when we're producing ellipsis entries for
         non-changelog revlogs, we need to turn those numbers into
         something local. This does that for us, and during the
         changelog sending phase will also expand the stored
         mappings as needed.
         """
         if clrev == nullrev:
             return nullrev
 
         if ischangelog:
             return clrev
 
         # Walk the ellipsis-ized changelog breadth-first looking for a
         # change that has been linked from the current revlog.
         #
         # For a flat manifest revlog only a single step should be necessary
         # as all relevant changelog entries are relevant to the flat
         # manifest.
         #
         # For a filelog or tree manifest dirlog however not every changelog
         # entry will have been relevant, so we need to skip some changelog
         # nodes even after ellipsis-izing.
         walk = [clrev]
         while walk:
             p = walk[0]
             walk = walk[1:]
             if p in clrevtolocalrev:
                 return clrevtolocalrev[p]
             elif p in fullclnodes:
                 walk.extend([pp for pp in cl.parentrevs(p)
                                 if pp != nullrev])
             elif p in precomputedellipsis:
                 walk.extend([pp for pp in precomputedellipsis[p]
                                 if pp != nullrev])
             else:
                 # In this case, we've got an ellipsis with parents
                 # outside the current bundle (likely an
                 # incremental pull). We "know" that we can use the
                 # value of this same revlog at whatever revision
                 # is pointed to by linknode. "Know" is in scare
                 # quotes because I haven't done enough examination
                 # of edge cases to convince myself this is really
                 # a fact - it works for all the (admittedly
                 # thorough) cases in our testsuite, but I would be
                 # somewhat unsurprised to find a case in the wild
                 # where this breaks down a bit. That said, I don't
                 # know if it would hurt anything.
                 for i in pycompat.xrange(rev, 0, -1):
                     if store.linkrev(i) == clrev:
                         return i
                 # We failed to resolve a parent for this node, so
                 # we crash the changegroup construction.
                 raise error.Abort(
                     'unable to resolve parent while packing %r %r'
                     ' for changeset %r' % (store.indexfile, rev, clrev))
 
         return nullrev
 
     if not linkparents or (
         store.parentrevs(rev) == (nullrev, nullrev)):
         p1, p2 = nullrev, nullrev
     elif len(linkparents) == 1:
         p1, = sorted(local(p) for p in linkparents)
         p2 = nullrev
     else:
         p1, p2 = sorted(local(p) for p in linkparents)
 
     p1node, p2node = store.node(p1), store.node(p2)
 
     return p1node, p2node, linknode
 
 def deltagroup(repo, store, nodes, ischangelog, lookup, forcedeltaparentprev,
                topic=None,
                ellipses=False, clrevtolocalrev=None, fullclnodes=None,
                precomputedellipsis=None):
     """Calculate deltas for a set of revisions.
 
     Is a generator of ``revisiondelta`` instances.
 
     If topic is not None, progress detail will be generated using this
     topic name (e.g. changesets, manifests, etc).
     """
     if not nodes:
         return
 
     cl = repo.changelog
 
     if ischangelog:
         # `hg log` shows changesets in storage order. To preserve order
         # across clones, send out changesets in storage order.
         nodesorder = 'storage'
     elif ellipses:
         nodes = _sortnodesellipsis(store, nodes, cl, lookup)
         nodesorder = 'nodes'
     else:
         nodesorder = None
 
     # Perform ellipses filtering and revision massaging. We do this before
     # emitrevisions() because a) filtering out revisions creates less work
     # for emitrevisions() b) dropping revisions would break emitrevisions()'s
     # assumptions about delta choices and we would possibly send a delta
     # referencing a missing base revision.
     #
     # Also, calling lookup() has side-effects with regards to populating
     # data structures. If we don't call lookup() for each node or if we call
     # lookup() after the first pass through each node, things can break -
     # possibly intermittently depending on the python hash seed! For that
     # reason, we store a mapping of all linknodes during the initial node
     # pass rather than use lookup() on the output side.
     if ellipses:
         filtered = []
         adjustedparents = {}
         linknodes = {}
 
         for node in nodes:
             rev = store.rev(node)
             linknode = lookup(node)
             linkrev = cl.rev(linknode)
             clrevtolocalrev[linkrev] = rev
 
             # If linknode is in fullclnodes, it means the corresponding
             # changeset was a full changeset and is being sent unaltered.
             if linknode in fullclnodes:
                 linknodes[node] = linknode
 
             # If the corresponding changeset wasn't in the set computed
             # as relevant to us, it should be dropped outright.
             elif linkrev not in precomputedellipsis:
                 continue
 
             else:
                 # We could probably do this later and avoid the dict
                 # holding state. But it likely doesn't matter.
                 p1node, p2node, linknode = _resolvenarrowrevisioninfo(
                     cl, store, ischangelog, rev, linkrev, linknode,
                     clrevtolocalrev, fullclnodes, precomputedellipsis)
 
                 adjustedparents[node] = (p1node, p2node)
                 linknodes[node] = linknode
 
             filtered.append(node)
 
         nodes = filtered
 
     # We expect the first pass to be fast, so we only engage the progress
     # meter for constructing the revision deltas.
     progress = None
     if topic is not None:
         progress = repo.ui.makeprogress(topic, unit=_('chunks'),
                                         total=len(nodes))
 
     configtarget = repo.ui.config('devel', 'bundle.delta')
     if configtarget not in ('', 'p1', 'full'):
         msg = _("""config "devel.bundle.delta" as unknown value: %s""")
         repo.ui.warn(msg % configtarget)
 
     deltamode = repository.CG_DELTAMODE_STD
     if forcedeltaparentprev:
         deltamode = repository.CG_DELTAMODE_PREV
     elif configtarget == 'p1':
         deltamode = repository.CG_DELTAMODE_P1
     elif configtarget == 'full':
         deltamode = repository.CG_DELTAMODE_FULL
 
     revisions = store.emitrevisions(
         nodes,
         nodesorder=nodesorder,
         revisiondata=True,
         assumehaveparentrevisions=not ellipses,
         deltamode=deltamode)
 
     for i, revision in enumerate(revisions):
         if progress:
             progress.update(i + 1)
 
         if ellipses:
             linknode = linknodes[revision.node]
 
             if revision.node in adjustedparents:
                 p1node, p2node = adjustedparents[revision.node]
                 revision.p1node = p1node
                 revision.p2node = p2node
                 revision.flags |= repository.REVISION_FLAG_ELLIPSIS
 
         else:
             linknode = lookup(revision.node)
 
         revision.linknode = linknode
         yield revision
 
     if progress:
         progress.complete()
 
 class cgpacker(object):
     def __init__(self, repo, oldmatcher, matcher, version,
                  builddeltaheader, manifestsend,
                  forcedeltaparentprev=False,
                  bundlecaps=None, ellipses=False,
                  shallow=False, ellipsisroots=None, fullnodes=None):
         """Given a source repo, construct a bundler.
 
         oldmatcher is a matcher that matches on files the client already has.
         These will not be included in the changegroup.
 
         matcher is a matcher that matches on files to include in the
         changegroup. Used to facilitate sparse changegroups.
 
         forcedeltaparentprev indicates whether delta parents must be against
         the previous revision in a delta group. This should only be used for
         compatibility with changegroup version 1.
 
         builddeltaheader is a callable that constructs the header for a group
         delta.
 
         manifestsend is a chunk to send after manifests have been fully emitted.
 
         ellipses indicates whether ellipsis serving mode is enabled.
 
         bundlecaps is optional and can be used to specify the set of
         capabilities which can be used to build the bundle. While bundlecaps is
         unused in core Mercurial, extensions rely on this feature to communicate
         capabilities to customize the changegroup packer.
 
         shallow indicates whether shallow data might be sent. The packer may
         need to pack file contents not introduced by the changes being packed.
 
         fullnodes is the set of changelog nodes which should not be ellipsis
         nodes. We store this rather than the set of nodes that should be
         ellipsis because for very large histories we expect this to be
         significantly smaller.
         """
         assert oldmatcher
         assert matcher
         self._oldmatcher = oldmatcher
         self._matcher = matcher
 
         self.version = version
         self._forcedeltaparentprev = forcedeltaparentprev
         self._builddeltaheader = builddeltaheader
         self._manifestsend = manifestsend
         self._ellipses = ellipses
 
         # Set of capabilities we can use to build the bundle.
         if bundlecaps is None:
             bundlecaps = set()
         self._bundlecaps = bundlecaps
         self._isshallow = shallow
         self._fullclnodes = fullnodes
 
         # Maps ellipsis revs to their roots at the changelog level.
         self._precomputedellipsis = ellipsisroots
 
         self._repo = repo
 
         if self._repo.ui.verbose and not self._repo.ui.debugflag:
             self._verbosenote = self._repo.ui.note
         else:
             self._verbosenote = lambda s: None
 
     def generate(self, commonrevs, clnodes, fastpathlinkrev, source,
                  changelog=True):
         """Yield a sequence of changegroup byte chunks.
         If changelog is False, changelog data won't be added to changegroup
         """
 
         repo = self._repo
         cl = repo.changelog
 
         self._verbosenote(_('uncompressed size of bundle content:\n'))
         size = 0
 
         clstate, deltas = self._generatechangelog(cl, clnodes,
                                                   generate=changelog)
         for delta in deltas:
             for chunk in _revisiondeltatochunks(delta,
                                                 self._builddeltaheader):
                 size += len(chunk)
                 yield chunk
 
         close = closechunk()
         size += len(close)
         yield closechunk()
 
         self._verbosenote(_('%8.i (changelog)\n') % size)
 
         clrevorder = clstate['clrevorder']
         manifests = clstate['manifests']
         changedfiles = clstate['changedfiles']
 
         # We need to make sure that the linkrev in the changegroup refers to
         # the first changeset that introduced the manifest or file revision.
         # The fastpath is usually safer than the slowpath, because the filelogs
         # are walked in revlog order.
         #
         # When taking the slowpath when the manifest revlog uses generaldelta,
         # the manifest may be walked in the "wrong" order. Without 'clrevorder',
         # we would get an incorrect linkrev (see fix in cc0ff93d0c0c).
         #
         # When taking the fastpath, we are only vulnerable to reordering
         # of the changelog itself. The changelog never uses generaldelta and is
         # never reordered. To handle this case, we simply take the slowpath,
         # which already has the 'clrevorder' logic. This was also fixed in
         # cc0ff93d0c0c.
 
         # Treemanifests don't work correctly with fastpathlinkrev
         # either, because we don't discover which directory nodes to
         # send along with files. This could probably be fixed.
         fastpathlinkrev = fastpathlinkrev and (
             'treemanifest' not in repo.requirements)
 
         fnodes = {}  # needed file nodes
 
         size = 0
         it = self.generatemanifests(
             commonrevs, clrevorder, fastpathlinkrev, manifests, fnodes, source,
             clstate['clrevtomanifestrev'])
 
         for tree, deltas in it:
             if tree:
                 assert self.version == b'03'
                 chunk = _fileheader(tree)
                 size += len(chunk)
                 yield chunk
 
             for delta in deltas:
                 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
                 for chunk in chunks:
                     size += len(chunk)
                     yield chunk
 
             close = closechunk()
             size += len(close)
             yield close
 
         self._verbosenote(_('%8.i (manifests)\n') % size)
         yield self._manifestsend
 
         mfdicts = None
         if self._ellipses and self._isshallow:
             mfdicts = [(self._repo.manifestlog[n].read(), lr)
                        for (n, lr) in manifests.iteritems()]
 
         manifests.clear()
         clrevs = set(cl.rev(x) for x in clnodes)
 
         it = self.generatefiles(changedfiles, commonrevs,
                                 source, mfdicts, fastpathlinkrev,
                                 fnodes, clrevs)
 
         for path, deltas in it:
             h = _fileheader(path)
             size = len(h)
             yield h
 
             for delta in deltas:
                 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
                 for chunk in chunks:
                     size += len(chunk)
                     yield chunk
 
             close = closechunk()
             size += len(close)
             yield close
 
             self._verbosenote(_('%8.i  %s\n') % (size, path))
 
         yield closechunk()
 
         if clnodes:
             repo.hook('outgoing', node=hex(clnodes[0]), source=source)
 
     def _generatechangelog(self, cl, nodes, generate=True):
         """Generate data for changelog chunks.
 
         Returns a 2-tuple of a dict containing state and an iterable of
         byte chunks. The state will not be fully populated until the
         chunk stream has been fully consumed.
 
         if generate is False, the state will be fully populated and no chunk
         stream will be yielded
         """
         clrevorder = {}
         manifests = {}
         mfl = self._repo.manifestlog
         changedfiles = set()
         clrevtomanifestrev = {}
 
         state = {
             'clrevorder': clrevorder,
             'manifests': manifests,
             'changedfiles': changedfiles,
             'clrevtomanifestrev': clrevtomanifestrev,
         }
 
         if not (generate or self._ellipses):
             # sort the nodes in storage order
             nodes = sorted(nodes, key=cl.rev)
             for node in nodes:
                 c = cl.changelogrevision(node)
                 clrevorder[node] = len(clrevorder)
                 # record the first changeset introducing this manifest version
                 manifests.setdefault(c.manifest, node)
                 # Record a complete list of potentially-changed files in
                 # this manifest.
                 changedfiles.update(c.files)
 
             return state, ()
 
         # Callback for the changelog, used to collect changed files and
         # manifest nodes.
         # Returns the linkrev node (identity in the changelog case).
         def lookupcl(x):
             c = cl.changelogrevision(x)
             clrevorder[x] = len(clrevorder)
 
             if self._ellipses:
                 # Only update manifests if x is going to be sent. Otherwise we
                 # end up with bogus linkrevs specified for manifests and
                 # we skip some manifest nodes that we should otherwise
                 # have sent.
                 if (x in self._fullclnodes
                     or cl.rev(x) in self._precomputedellipsis):
 
                     manifestnode = c.manifest
                     # Record the first changeset introducing this manifest
                     # version.
                     manifests.setdefault(manifestnode, x)
                     # Set this narrow-specific dict so we have the lowest
                     # manifest revnum to look up for this cl revnum. (Part of
                     # mapping changelog ellipsis parents to manifest ellipsis
                     # parents)
                     clrevtomanifestrev.setdefault(
                         cl.rev(x), mfl.rev(manifestnode))
                 # We can't trust the changed files list in the changeset if the
                 # client requested a shallow clone.
                 if self._isshallow:
                     changedfiles.update(mfl[c.manifest].read().keys())
                 else:
                     changedfiles.update(c.files)
             else:
                 # record the first changeset introducing this manifest version
                 manifests.setdefault(c.manifest, x)
                 # Record a complete list of potentially-changed files in
                 # this manifest.
                 changedfiles.update(c.files)
 
             return x
 
         gen = deltagroup(
             self._repo, cl, nodes, True, lookupcl,
             self._forcedeltaparentprev,
             ellipses=self._ellipses,
             topic=_('changesets'),
             clrevtolocalrev={},
             fullclnodes=self._fullclnodes,
             precomputedellipsis=self._precomputedellipsis)
 
         return state, gen
 
     def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev,
                           manifests, fnodes, source, clrevtolocalrev):
         """Returns an iterator of changegroup chunks containing manifests.
 
         `source` is unused here, but is used by extensions like remotefilelog to
         change what is sent based in pulls vs pushes, etc.
         """
         repo = self._repo
         mfl = repo.manifestlog
         tmfnodes = {'': manifests}
 
         # Callback for the manifest, used to collect linkrevs for filelog
         # revisions.
         # Returns the linkrev node (collected in lookupcl).
         def makelookupmflinknode(tree, nodes):
             if fastpathlinkrev:
                 assert not tree
                 return manifests.__getitem__
 
             def lookupmflinknode(x):
                 """Callback for looking up the linknode for manifests.
 
                 Returns the linkrev node for the specified manifest.
 
                 SIDE EFFECT:
 
                 1) fclnodes gets populated with the list of relevant
                    file nodes if we're not using fastpathlinkrev
                 2) When treemanifests are in use, collects treemanifest nodes
                    to send
 
                 Note that this means manifests must be completely sent to
                 the client before you can trust the list of files and
                 treemanifests to send.
                 """
                 clnode = nodes[x]
                 mdata = mfl.get(tree, x).readfast(shallow=True)
                 for p, n, fl in mdata.iterentries():
                     if fl == 't': # subdirectory manifest
                         subtree = tree + p + '/'
                         tmfclnodes = tmfnodes.setdefault(subtree, {})
                         tmfclnode = tmfclnodes.setdefault(n, clnode)
                         if clrevorder[clnode] < clrevorder[tmfclnode]:
                             tmfclnodes[n] = clnode
                     else:
                         f = tree + p
                         fclnodes = fnodes.setdefault(f, {})
                         fclnode = fclnodes.setdefault(n, clnode)
                         if clrevorder[clnode] < clrevorder[fclnode]:
                             fclnodes[n] = clnode
                 return clnode
             return lookupmflinknode
 
         while tmfnodes:
             tree, nodes = tmfnodes.popitem()
 
             should_visit = self._matcher.visitdir(tree[:-1])
             if tree and not should_visit:
                 continue
 
             store = mfl.getstorage(tree)
 
             if not should_visit:
                 # No nodes to send because this directory is out of
                 # the client's view of the repository (probably
                 # because of narrow clones). Do this even for the root
                 # directory (tree=='')
                 prunednodes = []
             else:
                 # Avoid sending any manifest nodes we can prove the
                 # client already has by checking linkrevs. See the
                 # related comment in generatefiles().
                 prunednodes = self._prunemanifests(store, nodes, commonrevs)
 
             if tree and not prunednodes:
                 continue
 
             lookupfn = makelookupmflinknode(tree, nodes)
 
             deltas = deltagroup(
                 self._repo, store, prunednodes, False, lookupfn,
                 self._forcedeltaparentprev,
                 ellipses=self._ellipses,
                 topic=_('manifests'),
                 clrevtolocalrev=clrevtolocalrev,
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis)
 
             if not self._oldmatcher.visitdir(store.tree[:-1]):
                 yield tree, deltas
             else:
                 # 'deltas' is a generator and we need to consume it even if
                 # we are not going to send it because a side-effect is that
                 # it updates tmdnodes (via lookupfn)
                 for d in deltas:
                     pass
                 if not tree:
                     yield tree, []
 
     def _prunemanifests(self, store, nodes, commonrevs):
         if not self._ellipses:
             # In non-ellipses case and large repositories, it is better to
             # prevent calling of store.rev and store.linkrev on a lot of
             # nodes as compared to sending some extra data
             return nodes.copy()
         # This is split out as a separate method to allow filtering
         # commonrevs in extension code.
         #
         # TODO(augie): this shouldn't be required, instead we should
         # make filtering of revisions to send delegated to the store
         # layer.
         frev, flr = store.rev, store.linkrev
         return [n for n in nodes if flr(frev(n)) not in commonrevs]
 
     # The 'source' parameter is useful for extensions
     def generatefiles(self, changedfiles, commonrevs, source,
                       mfdicts, fastpathlinkrev, fnodes, clrevs):
         changedfiles = [f for f in changedfiles
                         if self._matcher(f) and not self._oldmatcher(f)]
 
         if not fastpathlinkrev:
             def normallinknodes(unused, fname):
                 return fnodes.get(fname, {})
         else:
             cln = self._repo.changelog.node
 
             def normallinknodes(store, fname):
                 flinkrev = store.linkrev
                 fnode = store.node
                 revs = ((r, flinkrev(r)) for r in store)
                 return dict((fnode(r), cln(lr))
                             for r, lr in revs if lr in clrevs)
 
         clrevtolocalrev = {}
 
         if self._isshallow:
             # In a shallow clone, the linknodes callback needs to also include
             # those file nodes that are in the manifests we sent but weren't
             # introduced by those manifests.
             commonctxs = [self._repo[c] for c in commonrevs]
             clrev = self._repo.changelog.rev
 
             def linknodes(flog, fname):
                 for c in commonctxs:
                     try:
                         fnode = c.filenode(fname)
                         clrevtolocalrev[c.rev()] = flog.rev(fnode)
                     except error.ManifestLookupError:
                         pass
                 links = normallinknodes(flog, fname)
                 if len(links) != len(mfdicts):
                     for mf, lr in mfdicts:
                         fnode = mf.get(fname, None)
                         if fnode in links:
                             links[fnode] = min(links[fnode], lr, key=clrev)
                         elif fnode:
                             links[fnode] = lr
                 return links
         else:
             linknodes = normallinknodes
 
         repo = self._repo
         progress = repo.ui.makeprogress(_('files'), unit=_('files'),
                                         total=len(changedfiles))
         for i, fname in enumerate(sorted(changedfiles)):
             filerevlog = repo.file(fname)
             if not filerevlog:
                 raise error.Abort(_("empty or missing file data for %s") %
                                   fname)
 
             clrevtolocalrev.clear()
 
             linkrevnodes = linknodes(filerevlog, fname)
             # Lookup for filenodes, we collected the linkrev nodes above in the
             # fastpath case and with lookupmf in the slowpath case.
             def lookupfilelog(x):
                 return linkrevnodes[x]
 
             frev, flr = filerevlog.rev, filerevlog.linkrev
             # Skip sending any filenode we know the client already
             # has. This avoids over-sending files relatively
             # inexpensively, so it's not a problem if we under-filter
             # here.
             filenodes = [n for n in linkrevnodes
                          if flr(frev(n)) not in commonrevs]
 
             if not filenodes:
                 continue
 
             progress.update(i + 1, item=fname)
 
             deltas = deltagroup(
                 self._repo, filerevlog, filenodes, False, lookupfilelog,
                 self._forcedeltaparentprev,
                 ellipses=self._ellipses,
                 clrevtolocalrev=clrevtolocalrev,
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis)
 
             yield fname, deltas
 
         progress.complete()
 
 def _makecg1packer(repo, oldmatcher, matcher, bundlecaps,
                    ellipses=False, shallow=False, ellipsisroots=None,
                    fullnodes=None):
     builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.linknode)
 
     return cgpacker(repo, oldmatcher, matcher, b'01',
                     builddeltaheader=builddeltaheader,
                     manifestsend=b'',
                     forcedeltaparentprev=True,
                     bundlecaps=bundlecaps,
                     ellipses=ellipses,
                     shallow=shallow,
                     ellipsisroots=ellipsisroots,
                     fullnodes=fullnodes)
 
 def _makecg2packer(repo, oldmatcher, matcher, bundlecaps,
                    ellipses=False, shallow=False, ellipsisroots=None,
                    fullnodes=None):
     builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode)
 
     return cgpacker(repo, oldmatcher, matcher, b'02',
                     builddeltaheader=builddeltaheader,
                     manifestsend=b'',
                     bundlecaps=bundlecaps,
                     ellipses=ellipses,
                     shallow=shallow,
                     ellipsisroots=ellipsisroots,
                     fullnodes=fullnodes)
 
 def _makecg3packer(repo, oldmatcher, matcher, bundlecaps,
                    ellipses=False, shallow=False, ellipsisroots=None,
                    fullnodes=None):
     builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
 
     return cgpacker(repo, oldmatcher, matcher, b'03',
                     builddeltaheader=builddeltaheader,
                     manifestsend=closechunk(),
                     bundlecaps=bundlecaps,
                     ellipses=ellipses,
                     shallow=shallow,
                     ellipsisroots=ellipsisroots,
                     fullnodes=fullnodes)
 
 _packermap = {'01': (_makecg1packer, cg1unpacker),
              # cg2 adds support for exchanging generaldelta
              '02': (_makecg2packer, cg2unpacker),
              # cg3 adds support for exchanging revlog flags and treemanifests
              '03': (_makecg3packer, cg3unpacker),
 }
 
 def allsupportedversions(repo):
     versions = set(_packermap.keys())
     if not (repo.ui.configbool('experimental', 'changegroup3') or
             repo.ui.configbool('experimental', 'treemanifest') or
             'treemanifest' in repo.requirements):
         versions.discard('03')
     return versions
 
 # Changegroup versions that can be applied to the repo
 def supportedincomingversions(repo):
     return allsupportedversions(repo)
 
 # Changegroup versions that can be created from the repo
 def supportedoutgoingversions(repo):
     versions = allsupportedversions(repo)
     if 'treemanifest' in repo.requirements:
         # Versions 01 and 02 support only flat manifests and it's just too
         # expensive to convert between the flat manifest and tree manifest on
         # the fly. Since tree manifests are hashed differently, all of history
         # would have to be converted. Instead, we simply don't even pretend to
         # support versions 01 and 02.
         versions.discard('01')
         versions.discard('02')
     if repository.NARROW_REQUIREMENT in repo.requirements:
         # Versions 01 and 02 don't support revlog flags, and we need to
         # support that for stripping and unbundling to work.
         versions.discard('01')
         versions.discard('02')
     if LFS_REQUIREMENT in repo.requirements:
         # Versions 01 and 02 don't support revlog flags, and we need to
         # mark LFS entries with REVIDX_EXTSTORED.
         versions.discard('01')
         versions.discard('02')
 
     return versions
 
 def localversion(repo):
     # Finds the best version to use for bundles that are meant to be used
     # locally, such as those from strip and shelve, and temporary bundles.
     return max(supportedoutgoingversions(repo))
 
 def safeversion(repo):
     # Finds the smallest version that it's safe to assume clients of the repo
     # will support. For example, all hg versions that support generaldelta also
     # support changegroup 02.
     versions = supportedoutgoingversions(repo)
     if 'generaldelta' in repo.requirements:
         versions.discard('01')
     assert versions
     return min(versions)
 
 def getbundler(version, repo, bundlecaps=None, oldmatcher=None,
                matcher=None, ellipses=False, shallow=False,
                ellipsisroots=None, fullnodes=None):
     assert version in supportedoutgoingversions(repo)
 
     if matcher is None:
         matcher = matchmod.always()
     if oldmatcher is None:
         oldmatcher = matchmod.never()
 
     if version == '01' and not matcher.always():
         raise error.ProgrammingError('version 01 changegroups do not support '
                                      'sparse file matchers')
 
     if ellipses and version in (b'01', b'02'):
         raise error.Abort(
             _('ellipsis nodes require at least cg3 on client and server, '
               'but negotiated version %s') % version)
 
     # Requested files could include files not in the local store. So
     # filter those out.
     matcher = repo.narrowmatch(matcher)
 
     fn = _packermap[version][0]
     return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses,
               shallow=shallow, ellipsisroots=ellipsisroots,
               fullnodes=fullnodes)
 
 def getunbundler(version, fh, alg, extras=None):
     return _packermap[version][1](fh, alg, extras=extras)
 
 def _changegroupinfo(repo, nodes, source):
     if repo.ui.verbose or source == 'bundle':
         repo.ui.status(_("%d changesets found\n") % len(nodes))
     if repo.ui.debugflag:
         repo.ui.debug("list of changesets:\n")
         for node in nodes:
             repo.ui.debug("%s\n" % hex(node))
 
 def makechangegroup(repo, outgoing, version, source, fastpath=False,
                     bundlecaps=None):
     cgstream = makestream(repo, outgoing, version, source,
                           fastpath=fastpath, bundlecaps=bundlecaps)
     return getunbundler(version, util.chunkbuffer(cgstream), None,
                         {'clcount': len(outgoing.missing) })
 
 def makestream(repo, outgoing, version, source, fastpath=False,
                bundlecaps=None, matcher=None):
     bundler = getbundler(version, repo, bundlecaps=bundlecaps,
                          matcher=matcher)
 
     repo = repo.unfiltered()
     commonrevs = outgoing.common
     csets = outgoing.missing
     heads = outgoing.missingheads
     # We go through the fast path if we get told to, or if all (unfiltered
     # heads have been requested (since we then know there all linkrevs will
     # be pulled by the client).
     heads.sort()
     fastpathlinkrev = fastpath or (
             repo.filtername is None and heads == sorted(repo.heads()))
 
     repo.hook('preoutgoing', throw=True, source=source)
     _changegroupinfo(repo, csets, source)
     return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
 
 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
     revisions = 0
     files = 0
     progress = repo.ui.makeprogress(_('files'), unit=_('files'),
                                     total=expectedfiles)
     for chunkdata in iter(source.filelogheader, {}):
         files += 1
         f = chunkdata["filename"]
         repo.ui.debug("adding %s revisions\n" % f)
         progress.increment()
         fl = repo.file(f)
         o = len(fl)
         try:
             deltas = source.deltaiter()
             if not fl.addgroup(deltas, revmap, trp):
                 raise error.Abort(_("received file revlog group is empty"))
         except error.CensoredBaseError as e:
             raise error.Abort(_("received delta base is censored: %s") % e)
         revisions += len(fl) - o
         if f in needfiles:
             needs = needfiles[f]
             for new in pycompat.xrange(o, len(fl)):
                 n = fl.node(new)
                 if n in needs:
                     needs.remove(n)
                 else:
                     raise error.Abort(
                         _("received spurious file revlog entry"))
             if not needs:
                 del needfiles[f]
     progress.complete()
 
     for f, needs in needfiles.iteritems():
         fl = repo.file(f)
         for n in needs:
             try:
                 fl.rev(n)
             except error.LookupError:
                 raise error.Abort(
                     _('missing file data for %s:%s - run hg verify') %
                     (f, hex(n)))
 
     return revisions, files

mercurial/exchange.py

@@ -1,2701 +1,2703
 # exchange.py - utility to exchange data between repos.
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import hashlib
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
     nullrev,
 )
 from .thirdparty import (
     attr,
 )
 from . import (
     bookmarks as bookmod,
     bundle2,
     changegroup,
     discovery,
     error,
     exchangev2,
     lock as lockmod,
     logexchange,
     narrowspec,
     obsolete,
     phases,
     pushkey,
     pycompat,
-    repository,
     scmutil,
     sslutil,
     streamclone,
     url as urlmod,
     util,
     wireprototypes,
 )
+from .interfaces import (
+    repository,
+)
 from .utils import (
     stringutil,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 _NARROWACL_SECTION = 'narrowacl'
 
 # Maps bundle version human names to changegroup versions.
 _bundlespeccgversions = {'v1': '01',
                          'v2': '02',
                          'packed1': 's1',
                          'bundle2': '02', #legacy
                         }
 
 # Maps bundle version with content opts to choose which part to bundle
 _bundlespeccontentopts = {
     'v1': {
         'changegroup': True,
         'cg.version': '01',
         'obsolescence': False,
         'phases': False,
         'tagsfnodescache': False,
         'revbranchcache': False
     },
     'v2': {
         'changegroup': True,
         'cg.version': '02',
         'obsolescence': False,
         'phases': False,
         'tagsfnodescache': True,
         'revbranchcache': True
     },
     'packed1' : {
         'cg.version': 's1'
     }
 }
 _bundlespeccontentopts['bundle2'] = _bundlespeccontentopts['v2']
 
 _bundlespecvariants = {"streamv2": {"changegroup": False, "streamv2": True,
                                     "tagsfnodescache": False,
                                     "revbranchcache": False}}
 
 # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
 _bundlespecv1compengines = {'gzip', 'bzip2', 'none'}
 
 @attr.s
 class bundlespec(object):
     compression = attr.ib()
     wirecompression = attr.ib()
     version = attr.ib()
     wireversion = attr.ib()
     params = attr.ib()
     contentopts = attr.ib()
 
 def parsebundlespec(repo, spec, strict=True):
     """Parse a bundle string specification into parts.
 
     Bundle specifications denote a well-defined bundle/exchange format.
     The content of a given specification should not change over time in
     order to ensure that bundles produced by a newer version of Mercurial are
     readable from an older version.
 
     The string currently has the form:
 
        <compression>-<type>[;<parameter0>[;<parameter1>]]
 
     Where <compression> is one of the supported compression formats
     and <type> is (currently) a version string. A ";" can follow the type and
     all text afterwards is interpreted as URI encoded, ";" delimited key=value
     pairs.
 
     If ``strict`` is True (the default) <compression> is required. Otherwise,
     it is optional.
 
     Returns a bundlespec object of (compression, version, parameters).
     Compression will be ``None`` if not in strict mode and a compression isn't
     defined.
 
     An ``InvalidBundleSpecification`` is raised when the specification is
     not syntactically well formed.
 
     An ``UnsupportedBundleSpecification`` is raised when the compression or
     bundle type/version is not recognized.
 
     Note: this function will likely eventually return a more complex data
     structure, including bundle2 part information.
     """
     def parseparams(s):
         if ';' not in s:
             return s, {}
 
         params = {}
         version, paramstr = s.split(';', 1)
 
         for p in paramstr.split(';'):
             if '=' not in p:
                 raise error.InvalidBundleSpecification(
                     _('invalid bundle specification: '
                       'missing "=" in parameter: %s') % p)
 
             key, value = p.split('=', 1)
             key = urlreq.unquote(key)
             value = urlreq.unquote(value)
             params[key] = value
 
         return version, params
 
 
     if strict and '-' not in spec:
         raise error.InvalidBundleSpecification(
                 _('invalid bundle specification; '
                   'must be prefixed with compression: %s') % spec)
 
     if '-' in spec:
         compression, version = spec.split('-', 1)
 
         if compression not in util.compengines.supportedbundlenames:
             raise error.UnsupportedBundleSpecification(
                     _('%s compression is not supported') % compression)
 
         version, params = parseparams(version)
 
         if version not in _bundlespeccgversions:
             raise error.UnsupportedBundleSpecification(
                     _('%s is not a recognized bundle version') % version)
     else:
         # Value could be just the compression or just the version, in which
         # case some defaults are assumed (but only when not in strict mode).
         assert not strict
 
         spec, params = parseparams(spec)
 
         if spec in util.compengines.supportedbundlenames:
             compression = spec
             version = 'v1'
             # Generaldelta repos require v2.
             if 'generaldelta' in repo.requirements:
                 version = 'v2'
             # Modern compression engines require v2.
             if compression not in _bundlespecv1compengines:
                 version = 'v2'
         elif spec in _bundlespeccgversions:
             if spec == 'packed1':
                 compression = 'none'
             else:
                 compression = 'bzip2'
             version = spec
         else:
             raise error.UnsupportedBundleSpecification(
                     _('%s is not a recognized bundle specification') % spec)
 
     # Bundle version 1 only supports a known set of compression engines.
     if version == 'v1' and compression not in _bundlespecv1compengines:
         raise error.UnsupportedBundleSpecification(
             _('compression engine %s is not supported on v1 bundles') %
             compression)
 
     # The specification for packed1 can optionally declare the data formats
     # required to apply it. If we see this metadata, compare against what the
     # repo supports and error if the bundle isn't compatible.
     if version == 'packed1' and 'requirements' in params:
         requirements = set(params['requirements'].split(','))
         missingreqs = requirements - repo.supportedformats
         if missingreqs:
             raise error.UnsupportedBundleSpecification(
                     _('missing support for repository features: %s') %
                       ', '.join(sorted(missingreqs)))
 
     # Compute contentopts based on the version
     contentopts = _bundlespeccontentopts.get(version, {}).copy()
 
     # Process the variants
     if "stream" in params and params["stream"] == "v2":
         variant = _bundlespecvariants["streamv2"]
         contentopts.update(variant)
 
     engine = util.compengines.forbundlename(compression)
     compression, wirecompression = engine.bundletype()
     wireversion = _bundlespeccgversions[version]
 
     return bundlespec(compression, wirecompression, version, wireversion,
                       params, contentopts)
 
 def readbundle(ui, fh, fname, vfs=None):
     header = changegroup.readexactly(fh, 4)
 
     alg = None
     if not fname:
         fname = "stream"
         if not header.startswith('HG') and header.startswith('\0'):
             fh = changegroup.headerlessfixup(fh, header)
             header = "HG10"
             alg = 'UN'
     elif vfs:
         fname = vfs.join(fname)
 
     magic, version = header[0:2], header[2:4]
 
     if magic != 'HG':
         raise error.Abort(_('%s: not a Mercurial bundle') % fname)
     if version == '10':
         if alg is None:
             alg = changegroup.readexactly(fh, 2)
         return changegroup.cg1unpacker(fh, alg)
     elif version.startswith('2'):
         return bundle2.getunbundler(ui, fh, magicstring=magic + version)
     elif version == 'S1':
         return streamclone.streamcloneapplier(fh)
     else:
         raise error.Abort(_('%s: unknown bundle version %s') % (fname, version))
 
 def getbundlespec(ui, fh):
     """Infer the bundlespec from a bundle file handle.
 
     The input file handle is seeked and the original seek position is not
     restored.
     """
     def speccompression(alg):
         try:
             return util.compengines.forbundletype(alg).bundletype()[0]
         except KeyError:
             return None
 
     b = readbundle(ui, fh, None)
     if isinstance(b, changegroup.cg1unpacker):
         alg = b._type
         if alg == '_truncatedBZ':
             alg = 'BZ'
         comp = speccompression(alg)
         if not comp:
             raise error.Abort(_('unknown compression algorithm: %s') % alg)
         return '%s-v1' % comp
     elif isinstance(b, bundle2.unbundle20):
         if 'Compression' in b.params:
             comp = speccompression(b.params['Compression'])
             if not comp:
                 raise error.Abort(_('unknown compression algorithm: %s') % comp)
         else:
             comp = 'none'
 
         version = None
         for part in b.iterparts():
             if part.type == 'changegroup':
                 version = part.params['version']
                 if version in ('01', '02'):
                     version = 'v2'
                 else:
                     raise error.Abort(_('changegroup version %s does not have '
                                         'a known bundlespec') % version,
                                       hint=_('try upgrading your Mercurial '
                                               'client'))
             elif part.type == 'stream2' and version is None:
                 # A stream2 part requires to be part of a v2 bundle
                 requirements = urlreq.unquote(part.params['requirements'])
                 splitted = requirements.split()
                 params = bundle2._formatrequirementsparams(splitted)
                 return 'none-v2;stream=v2;%s' % params
 
         if not version:
             raise error.Abort(_('could not identify changegroup version in '
                                 'bundle'))
 
         return '%s-%s' % (comp, version)
     elif isinstance(b, streamclone.streamcloneapplier):
         requirements = streamclone.readbundle1header(fh)[2]
         formatted = bundle2._formatrequirementsparams(requirements)
         return 'none-packed1;%s' % formatted
     else:
         raise error.Abort(_('unknown bundle type: %s') % b)
 
 def _computeoutgoing(repo, heads, common):
     """Computes which revs are outgoing given a set of common
     and a set of heads.
 
     This is a separate function so extensions can have access to
     the logic.
 
     Returns a discovery.outgoing object.
     """
     cl = repo.changelog
     if common:
         hasnode = cl.hasnode
         common = [n for n in common if hasnode(n)]
     else:
         common = [nullid]
     if not heads:
         heads = cl.heads()
     return discovery.outgoing(repo, common, heads)
 
 def _checkpublish(pushop):
     repo = pushop.repo
     ui = repo.ui
     behavior = ui.config('experimental', 'auto-publish')
     if pushop.publish or behavior not in ('warn', 'confirm', 'abort'):
         return
     remotephases = listkeys(pushop.remote, 'phases')
     if not remotephases.get('publishing', False):
         return
 
     if pushop.revs is None:
         published = repo.filtered('served').revs('not public()')
     else:
         published = repo.revs('::%ln - public()', pushop.revs)
     if published:
         if behavior == 'warn':
             ui.warn(_('%i changesets about to be published\n')
                     % len(published))
         elif behavior == 'confirm':
             if ui.promptchoice(_('push and publish %i changesets (yn)?'
                                  '$$ &Yes $$ &No') % len(published)):
                 raise error.Abort(_('user quit'))
         elif behavior == 'abort':
             msg = _('push would publish %i changesets') % len(published)
             hint = _("use --publish or adjust 'experimental.auto-publish'"
                      " config")
             raise error.Abort(msg, hint=hint)
 
 def _forcebundle1(op):
     """return true if a pull/push must use bundle1
 
     This function is used to allow testing of the older bundle version"""
     ui = op.repo.ui
     # The goal is this config is to allow developer to choose the bundle
     # version used during exchanged. This is especially handy during test.
     # Value is a list of bundle version to be picked from, highest version
     # should be used.
     #
     # developer config: devel.legacy.exchange
     exchange = ui.configlist('devel', 'legacy.exchange')
     forcebundle1 = 'bundle2' not in exchange and 'bundle1' in exchange
     return forcebundle1 or not op.remote.capable('bundle2')
 
 class pushoperation(object):
     """A object that represent a single push operation
 
     Its purpose is to carry push related state and very common operations.
 
     A new pushoperation should be created at the beginning of each push and
     discarded afterward.
     """
 
     def __init__(self, repo, remote, force=False, revs=None, newbranch=False,
                  bookmarks=(), publish=False, pushvars=None):
         # repo we push from
         self.repo = repo
         self.ui = repo.ui
         # repo we push to
         self.remote = remote
         # force option provided
         self.force = force
         # revs to be pushed (None is "all")
         self.revs = revs
         # bookmark explicitly pushed
         self.bookmarks = bookmarks
         # allow push of new branch
         self.newbranch = newbranch
         # step already performed
         # (used to check what steps have been already performed through bundle2)
         self.stepsdone = set()
         # Integer version of the changegroup push result
         # - None means nothing to push
         # - 0 means HTTP error
         # - 1 means we pushed and remote head count is unchanged *or*
         #   we have outgoing changesets but refused to push
         # - other values as described by addchangegroup()
         self.cgresult = None
         # Boolean value for the bookmark push
         self.bkresult = None
         # discover.outgoing object (contains common and outgoing data)
         self.outgoing = None
         # all remote topological heads before the push
         self.remoteheads = None
         # Details of the remote branch pre and post push
         #
         # mapping: {'branch': ([remoteheads],
         #                      [newheads],
         #                      [unsyncedheads],
         #                      [discardedheads])}
         # - branch: the branch name
         # - remoteheads: the list of remote heads known locally
         #                None if the branch is new
         # - newheads: the new remote heads (known locally) with outgoing pushed
         # - unsyncedheads: the list of remote heads unknown locally.
         # - discardedheads: the list of remote heads made obsolete by the push
         self.pushbranchmap = None
         # testable as a boolean indicating if any nodes are missing locally.
         self.incoming = None
         # summary of the remote phase situation
         self.remotephases = None
         # phases changes that must be pushed along side the changesets
         self.outdatedphases = None
         # phases changes that must be pushed if changeset push fails
         self.fallbackoutdatedphases = None
         # outgoing obsmarkers
         self.outobsmarkers = set()
         # outgoing bookmarks
         self.outbookmarks = []
         # transaction manager
         self.trmanager = None
         # map { pushkey partid -> callback handling failure}
         # used to handle exception from mandatory pushkey part failure
         self.pkfailcb = {}
         # an iterable of pushvars or None
         self.pushvars = pushvars
         # publish pushed changesets
         self.publish = publish
 
     @util.propertycache
     def futureheads(self):
         """future remote heads if the changeset push succeeds"""
         return self.outgoing.missingheads
 
     @util.propertycache
     def fallbackheads(self):
         """future remote heads if the changeset push fails"""
         if self.revs is None:
             # not target to push, all common are relevant
             return self.outgoing.commonheads
         unfi = self.repo.unfiltered()
         # I want cheads = heads(::missingheads and ::commonheads)
         # (missingheads is revs with secret changeset filtered out)
         #
         # This can be expressed as:
         #     cheads = ( (missingheads and ::commonheads)
         #              + (commonheads and ::missingheads))"
         #              )
         #
         # while trying to push we already computed the following:
         #     common = (::commonheads)
         #     missing = ((commonheads::missingheads) - commonheads)
         #
         # We can pick:
         # * missingheads part of common (::commonheads)
         common = self.outgoing.common
         nm = self.repo.changelog.nodemap
         cheads = [node for node in self.revs if nm[node] in common]
         # and
         # * commonheads parents on missing
         revset = unfi.set('%ln and parents(roots(%ln))',
                          self.outgoing.commonheads,
                          self.outgoing.missing)
         cheads.extend(c.node() for c in revset)
         return cheads
 
     @property
     def commonheads(self):
         """set of all common heads after changeset bundle push"""
         if self.cgresult:
             return self.futureheads
         else:
             return self.fallbackheads
 
 # mapping of message used when pushing bookmark
 bookmsgmap = {'update': (_("updating bookmark %s\n"),
                          _('updating bookmark %s failed!\n')),
               'export': (_("exporting bookmark %s\n"),
                          _('exporting bookmark %s failed!\n')),
               'delete': (_("deleting remote bookmark %s\n"),
                          _('deleting remote bookmark %s failed!\n')),
               }
 
 
 def push(repo, remote, force=False, revs=None, newbranch=False, bookmarks=(),
          publish=False, opargs=None):
     '''Push outgoing changesets (limited by revs) from a local
     repository to remote. Return an integer:
       - None means nothing to push
       - 0 means HTTP error
       - 1 means we pushed and remote head count is unchanged *or*
         we have outgoing changesets but refused to push
       - other values as described by addchangegroup()
     '''
     if opargs is None:
         opargs = {}
     pushop = pushoperation(repo, remote, force, revs, newbranch, bookmarks,
                            publish, **pycompat.strkwargs(opargs))
     if pushop.remote.local():
         missing = (set(pushop.repo.requirements)
                    - pushop.remote.local().supported)
         if missing:
             msg = _("required features are not"
                     " supported in the destination:"
                     " %s") % (', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     if not pushop.remote.canpush():
         raise error.Abort(_("destination does not support push"))
 
     if not pushop.remote.capable('unbundle'):
         raise error.Abort(_('cannot push: destination does not support the '
                             'unbundle wire protocol command'))
 
     # get lock as we might write phase data
     wlock = lock = None
     try:
         # bundle2 push may receive a reply bundle touching bookmarks
         # requiring the wlock. Take it now to ensure proper ordering.
         maypushback = pushop.ui.configbool('experimental', 'bundle2.pushback')
         if ((not _forcebundle1(pushop)) and
             maypushback and
             not bookmod.bookmarksinstore(repo)):
             wlock = pushop.repo.wlock()
         lock = pushop.repo.lock()
         pushop.trmanager = transactionmanager(pushop.repo,
                                               'push-response',
                                               pushop.remote.url())
     except error.LockUnavailable as err:
         # source repo cannot be locked.
         # We do not abort the push, but just disable the local phase
         # synchronisation.
         msg = ('cannot lock source repository: %s\n'
                % stringutil.forcebytestr(err))
         pushop.ui.debug(msg)
 
     with wlock or util.nullcontextmanager():
         with lock or util.nullcontextmanager():
             with pushop.trmanager or util.nullcontextmanager():
                 pushop.repo.checkpush(pushop)
                 _checkpublish(pushop)
                 _pushdiscovery(pushop)
                 if not _forcebundle1(pushop):
                     _pushbundle2(pushop)
                 _pushchangeset(pushop)
                 _pushsyncphase(pushop)
                 _pushobsolete(pushop)
                 _pushbookmark(pushop)
 
     if repo.ui.configbool('experimental', 'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pushop
 
 # list of steps to perform discovery before push
 pushdiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pushdiscoverymapping = {}
 
 def pushdiscovery(stepname):
     """decorator for function performing discovery before push
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pushdiscovery dictionary directly."""
     def dec(func):
         assert stepname not in pushdiscoverymapping
         pushdiscoverymapping[stepname] = func
         pushdiscoveryorder.append(stepname)
         return func
     return dec
 
 def _pushdiscovery(pushop):
     """Run all discovery steps"""
     for stepname in pushdiscoveryorder:
         step = pushdiscoverymapping[stepname]
         step(pushop)
 
 @pushdiscovery('changeset')
 def _pushdiscoverychangeset(pushop):
     """discover the changeset that need to be pushed"""
     fci = discovery.findcommonincoming
     if pushop.revs:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force,
                         ancestorsof=pushop.revs)
     else:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
     common, inc, remoteheads = commoninc
     fco = discovery.findcommonoutgoing
     outgoing = fco(pushop.repo, pushop.remote, onlyheads=pushop.revs,
                    commoninc=commoninc, force=pushop.force)
     pushop.outgoing = outgoing
     pushop.remoteheads = remoteheads
     pushop.incoming = inc
 
 @pushdiscovery('phase')
 def _pushdiscoveryphase(pushop):
     """discover the phase that needs to be pushed
 
     (computed for both success and failure case for changesets push)"""
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     remotephases = listkeys(pushop.remote, 'phases')
 
     if (pushop.ui.configbool('ui', '_usedassubrepo')
         and remotephases    # server supports phases
         and not pushop.outgoing.missing # no changesets to be pushed
         and remotephases.get('publishing', False)):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset are to be pushed
         # - and remote is publishing
         # We may be in issue 3781 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         pushop.outdatedphases = []
         pushop.fallbackoutdatedphases = []
         return
 
     pushop.remotephases = phases.remotephasessummary(pushop.repo,
                                                      pushop.fallbackheads,
                                                      remotephases)
     droots = pushop.remotephases.draftroots
 
     extracond = ''
     if not pushop.remotephases.publishing:
         extracond = ' and public()'
     revset = 'heads((%%ln::%%ln) %s)' % extracond
     # Get the list of all revs draft on remote by public here.
     # XXX Beware that revset break if droots is not strictly
     # XXX root we may want to ensure it is but it is costly
     fallback = list(unfi.set(revset, droots, pushop.fallbackheads))
     if not pushop.remotephases.publishing and pushop.publish:
         future = list(unfi.set('%ln and (not public() or %ln::)',
                                pushop.futureheads, droots))
     elif not outgoing.missing:
         future = fallback
     else:
         # adds changeset we are going to push as draft
         #
         # should not be necessary for publishing server, but because of an
         # issue fixed in xxxxx we have to do it anyway.
         fdroots = list(unfi.set('roots(%ln  + %ln::)',
                        outgoing.missing, droots))
         fdroots = [f.node() for f in fdroots]
         future = list(unfi.set(revset, fdroots, pushop.futureheads))
     pushop.outdatedphases = future
     pushop.fallbackoutdatedphases = fallback
 
 @pushdiscovery('obsmarker')
 def _pushdiscoveryobsmarkers(pushop):
     if not obsolete.isenabled(pushop.repo, obsolete.exchangeopt):
         return
 
     if not pushop.repo.obsstore:
         return
 
     if 'obsolete' not in listkeys(pushop.remote, 'namespaces'):
         return
 
     repo = pushop.repo
     # very naive computation, that can be quite expensive on big repo.
     # However: evolution is currently slow on them anyway.
     nodes = (c.node() for c in repo.set('::%ln', pushop.futureheads))
     pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
 
 @pushdiscovery('bookmarks')
 def _pushdiscoverybookmarks(pushop):
     ui = pushop.ui
     repo = pushop.repo.unfiltered()
     remote = pushop.remote
     ui.debug("checking for updated bookmarks\n")
     ancestors = ()
     if pushop.revs:
         revnums = pycompat.maplist(repo.changelog.rev, pushop.revs)
         ancestors = repo.changelog.ancestors(revnums, inclusive=True)
 
     remotebookmark = listkeys(remote, 'bookmarks')
 
     explicit = {repo._bookmarks.expandname(bookmark)
                 for bookmark in pushop.bookmarks}
 
     remotebookmark = bookmod.unhexlifybookmarks(remotebookmark)
     comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
 
     def safehex(x):
         if x is None:
             return x
         return hex(x)
 
     def hexifycompbookmarks(bookmarks):
         return [(b, safehex(scid), safehex(dcid))
                 for (b, scid, dcid) in bookmarks]
 
     comp = [hexifycompbookmarks(marks) for marks in comp]
     return _processcompared(pushop, ancestors, explicit, remotebookmark, comp)
 
 def _processcompared(pushop, pushed, explicit, remotebms, comp):
     """take decision on bookmark to pull from the remote bookmark
 
     Exist to help extensions who want to alter this behavior.
     """
     addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
 
     repo = pushop.repo
 
     for b, scid, dcid in advsrc:
         if b in explicit:
             explicit.remove(b)
         if not pushed or repo[scid].rev() in pushed:
             pushop.outbookmarks.append((b, dcid, scid))
     # search added bookmark
     for b, scid, dcid in addsrc:
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, '', scid))
     # search for overwritten bookmark
     for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, dcid, scid))
     # search for bookmark to delete
     for b, scid, dcid in adddst:
         if b in explicit:
             explicit.remove(b)
             # treat as "deleted locally"
             pushop.outbookmarks.append((b, dcid, ''))
     # identical bookmarks shouldn't get reported
     for b, scid, dcid in same:
         if b in explicit:
             explicit.remove(b)
 
     if explicit:
         explicit = sorted(explicit)
         # we should probably list all of them
         pushop.ui.warn(_('bookmark %s does not exist on the local '
                          'or remote repository!\n') % explicit[0])
         pushop.bkresult = 2
 
     pushop.outbookmarks.sort()
 
 def _pushcheckoutgoing(pushop):
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     if not outgoing.missing:
         # nothing to push
         scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
         return False
     # something to push
     if not pushop.force:
         # if repo.obsstore == False --> no obsolete
         # then, save the iteration
         if unfi.obsstore:
             # this message are here for 80 char limit reason
             mso = _("push includes obsolete changeset: %s!")
             mspd = _("push includes phase-divergent changeset: %s!")
             mscd = _("push includes content-divergent changeset: %s!")
             mst = {"orphan": _("push includes orphan changeset: %s!"),
                    "phase-divergent": mspd,
                    "content-divergent": mscd}
             # If we are to push if there is at least one
             # obsolete or unstable changeset in missing, at
             # least one of the missinghead will be obsolete or
             # unstable. So checking heads only is ok
             for node in outgoing.missingheads:
                 ctx = unfi[node]
                 if ctx.obsolete():
                     raise error.Abort(mso % ctx)
                 elif ctx.isunstable():
                     # TODO print more than one instability in the abort
                     # message
                     raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
 
         discovery.checkheads(pushop)
     return True
 
 # List of names of steps to perform for an outgoing bundle2, order matters.
 b2partsgenorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 b2partsgenmapping = {}
 
 def b2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the b2partsgenmapping dictionary directly."""
     def dec(func):
         assert stepname not in b2partsgenmapping
         b2partsgenmapping[stepname] = func
         if idx is None:
             b2partsgenorder.append(stepname)
         else:
             b2partsgenorder.insert(idx, stepname)
         return func
     return dec
 
 def _pushb2ctxcheckheads(pushop, bundler):
     """Generate race condition checking parts
 
     Exists as an independent function to aid extensions
     """
     # * 'force' do not check for push race,
     # * if we don't push anything, there are nothing to check.
     if not pushop.force and pushop.outgoing.missingheads:
         allowunrelated = 'related' in bundler.capabilities.get('checkheads', ())
         emptyremote = pushop.pushbranchmap is None
         if not allowunrelated or emptyremote:
             bundler.newpart('check:heads', data=iter(pushop.remoteheads))
         else:
             affected = set()
             for branch, heads in pushop.pushbranchmap.iteritems():
                 remoteheads, newheads, unsyncedheads, discardedheads = heads
                 if remoteheads is not None:
                     remote = set(remoteheads)
                     affected |= set(discardedheads) & remote
                     affected |= remote - set(newheads)
             if affected:
                 data = iter(sorted(affected))
                 bundler.newpart('check:updated-heads', data=data)
 
 def _pushing(pushop):
     """return True if we are pushing anything"""
     return bool(pushop.outgoing.missing
                 or pushop.outdatedphases
                 or pushop.outobsmarkers
                 or pushop.outbookmarks)
 
 @b2partsgenerator('check-bookmarks')
 def _pushb2checkbookmarks(pushop, bundler):
     """insert bookmark move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasbookmarkcheck = 'bookmarks' in b2caps
     if not (pushop.outbookmarks and hasbookmarkcheck):
         return
     data = []
     for book, old, new in pushop.outbookmarks:
         old = bin(old)
         data.append((book, old))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart('check:bookmarks', data=checkdata)
 
 @b2partsgenerator('check-phases')
 def _pushb2checkphases(pushop, bundler):
     """insert phase move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasphaseheads = 'heads' in b2caps.get('phases', ())
     if pushop.remotephases is not None and hasphaseheads:
         # check that the remote phase has not changed
         checks = [[] for p in phases.allphases]
         checks[phases.public].extend(pushop.remotephases.publicheads)
         checks[phases.draft].extend(pushop.remotephases.draftroots)
         if any(checks):
             for nodes in checks:
                 nodes.sort()
             checkdata = phases.binaryencode(checks)
             bundler.newpart('check:phases', data=checkdata)
 
 @b2partsgenerator('changeset')
 def _pushb2ctx(pushop, bundler):
     """handle changegroup push through bundle2
 
     addchangegroup result is stored in the ``pushop.cgresult`` attribute.
     """
     if 'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add('changesets')
     # Send known heads to the server for race detection.
     if not _pushcheckoutgoing(pushop):
         return
     pushop.repo.prepushoutgoinghooks(pushop)
 
     _pushb2ctxcheckheads(pushop, bundler)
 
     b2caps = bundle2.bundle2caps(pushop.remote)
     version = '01'
     cgversions = b2caps.get('changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [v for v in cgversions
                       if v in changegroup.supportedoutgoingversions(
                           pushop.repo)]
         if not cgversions:
             raise error.Abort(_('no common changegroup version'))
         version = max(cgversions)
     cgstream = changegroup.makestream(pushop.repo, pushop.outgoing, version,
                                       'push')
     cgpart = bundler.newpart('changegroup', data=cgstream)
     if cgversions:
         cgpart.addparam('version', version)
     if 'treemanifest' in pushop.repo.requirements:
         cgpart.addparam('treemanifest', '1')
     def handlereply(op):
         """extract addchangegroup returns from server reply"""
         cgreplies = op.records.getreplies(cgpart.id)
         assert len(cgreplies['changegroup']) == 1
         pushop.cgresult = cgreplies['changegroup'][0]['return']
     return handlereply
 
 @b2partsgenerator('phase')
 def _pushb2phases(pushop, bundler):
     """handle phase push through bundle2"""
     if 'phases' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     ui = pushop.repo.ui
 
     legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
     haspushkey = 'pushkey' in b2caps
     hasphaseheads = 'heads' in b2caps.get('phases', ())
 
     if hasphaseheads and not legacyphase:
         return _pushb2phaseheads(pushop, bundler)
     elif haspushkey:
         return _pushb2phasespushkey(pushop, bundler)
 
 def _pushb2phaseheads(pushop, bundler):
     """push phase information through a bundle2 - binary part"""
     pushop.stepsdone.add('phases')
     if pushop.outdatedphases:
         updates = [[] for p in phases.allphases]
         updates[0].extend(h.node() for h in pushop.outdatedphases)
         phasedata = phases.binaryencode(updates)
         bundler.newpart('phase-heads', data=phasedata)
 
 def _pushb2phasespushkey(pushop, bundler):
     """push phase information through a bundle2 - pushkey part"""
     pushop.stepsdone.add('phases')
     part2node = []
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, node in part2node:
             if partid == targetid:
                 raise error.Abort(_('updating %s to public failed') % node)
 
     enc = pushkey.encode
     for newremotehead in pushop.outdatedphases:
         part = bundler.newpart('pushkey')
         part.addparam('namespace', enc('phases'))
         part.addparam('key', enc(newremotehead.hex()))
         part.addparam('old', enc('%d' % phases.draft))
         part.addparam('new', enc('%d' % phases.public))
         part2node.append((part.id, newremotehead))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         for partid, node in part2node:
             partrep = op.records.getreplies(partid)
             results = partrep['pushkey']
             assert len(results) <= 1
             msg = None
             if not results:
                 msg = _('server ignored update of %s to public!\n') % node
             elif not int(results[0]['return']):
                 msg = _('updating %s to public failed!\n') % node
             if msg is not None:
                 pushop.ui.warn(msg)
     return handlereply
 
 @b2partsgenerator('obsmarkers')
 def _pushb2obsmarkers(pushop, bundler):
     if 'obsmarkers' in pushop.stepsdone:
         return
     remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
     if obsolete.commonversion(remoteversions) is None:
         return
     pushop.stepsdone.add('obsmarkers')
     if pushop.outobsmarkers:
         markers = sorted(pushop.outobsmarkers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 @b2partsgenerator('bookmarks')
 def _pushb2bookmarks(pushop, bundler):
     """handle bookmark push through bundle2"""
     if 'bookmarks' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
 
     legacy = pushop.repo.ui.configlist('devel', 'legacy.exchange')
     legacybooks = 'bookmarks' in legacy
 
     if not legacybooks and 'bookmarks' in b2caps:
         return _pushb2bookmarkspart(pushop, bundler)
     elif 'pushkey' in b2caps:
         return _pushb2bookmarkspushkey(pushop, bundler)
 
 def _bmaction(old, new):
     """small utility for bookmark pushing"""
     if not old:
         return 'export'
     elif not new:
         return 'delete'
     return 'update'
 
 def _pushb2bookmarkspart(pushop, bundler):
     pushop.stepsdone.add('bookmarks')
     if not pushop.outbookmarks:
         return
 
     allactions = []
     data = []
     for book, old, new in pushop.outbookmarks:
         new = bin(new)
         data.append((book, new))
         allactions.append((book, _bmaction(old, new)))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart('bookmarks', data=checkdata)
 
     def handlereply(op):
         ui = pushop.ui
         # if success
         for book, action in allactions:
             ui.status(bookmsgmap[action][0] % book)
 
     return handlereply
 
 def _pushb2bookmarkspushkey(pushop, bundler):
     pushop.stepsdone.add('bookmarks')
     part2book = []
     enc = pushkey.encode
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, book, action in part2book:
             if partid == targetid:
                 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
         # we should not be called for part we did not generated
         assert False
 
     for book, old, new in pushop.outbookmarks:
         part = bundler.newpart('pushkey')
         part.addparam('namespace', enc('bookmarks'))
         part.addparam('key', enc(book))
         part.addparam('old', enc(old))
         part.addparam('new', enc(new))
         action = 'update'
         if not old:
             action = 'export'
         elif not new:
             action = 'delete'
         part2book.append((part.id, book, action))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         ui = pushop.ui
         for partid, book, action in part2book:
             partrep = op.records.getreplies(partid)
             results = partrep['pushkey']
             assert len(results) <= 1
             if not results:
                 pushop.ui.warn(_('server ignored bookmark %s update\n') % book)
             else:
                 ret = int(results[0]['return'])
                 if ret:
                     ui.status(bookmsgmap[action][0] % book)
                 else:
                     ui.warn(bookmsgmap[action][1] % book)
                     if pushop.bkresult is not None:
                         pushop.bkresult = 1
     return handlereply
 
 @b2partsgenerator('pushvars', idx=0)
 def _getbundlesendvars(pushop, bundler):
     '''send shellvars via bundle2'''
     pushvars = pushop.pushvars
     if pushvars:
         shellvars = {}
         for raw in pushvars:
             if '=' not in raw:
                 msg = ("unable to parse variable '%s', should follow "
                         "'KEY=VALUE' or 'KEY=' format")
                 raise error.Abort(msg % raw)
             k, v = raw.split('=', 1)
             shellvars[k] = v
 
         part = bundler.newpart('pushvars')
 
         for key, value in shellvars.iteritems():
             part.addparam(key, value, mandatory=False)
 
 def _pushbundle2(pushop):
     """push data to the remote using bundle2
 
     The only currently supported type of data is changegroup but this will
     evolve in the future."""
     bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
     pushback = (pushop.trmanager
                 and pushop.ui.configbool('experimental', 'bundle2.pushback'))
 
     # create reply capability
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(pushop.repo,
                                                       allowpushback=pushback,
                                                       role='client'))
     bundler.newpart('replycaps', data=capsblob)
     replyhandlers = []
     for partgenname in b2partsgenorder:
         partgen = b2partsgenmapping[partgenname]
         ret = partgen(pushop, bundler)
         if callable(ret):
             replyhandlers.append(ret)
     # do not push if nothing to push
     if bundler.nbparts <= 1:
         return
     stream = util.chunkbuffer(bundler.getchunks())
     try:
         try:
             with pushop.remote.commandexecutor() as e:
                 reply = e.callcommand('unbundle', {
                     'bundle': stream,
                     'heads': ['force'],
                     'url': pushop.remote.url(),
                 }).result()
         except error.BundleValueError as exc:
             raise error.Abort(_('missing support for %s') % exc)
         try:
             trgetter = None
             if pushback:
                 trgetter = pushop.trmanager.transaction
             op = bundle2.processbundle(pushop.repo, reply, trgetter)
         except error.BundleValueError as exc:
             raise error.Abort(_('missing support for %s') % exc)
         except bundle2.AbortFromPart as exc:
             pushop.ui.status(_('remote: %s\n') % exc)
             if exc.hint is not None:
                 pushop.ui.status(_('remote: %s\n') % ('(%s)' % exc.hint))
             raise error.Abort(_('push failed on remote'))
     except error.PushkeyFailed as exc:
         partid = int(exc.partid)
         if partid not in pushop.pkfailcb:
             raise
         pushop.pkfailcb[partid](pushop, exc)
     for rephand in replyhandlers:
         rephand(op)
 
 def _pushchangeset(pushop):
     """Make the actual push of changeset bundle to remote repo"""
     if 'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add('changesets')
     if not _pushcheckoutgoing(pushop):
         return
 
     # Should have verified this in push().
     assert pushop.remote.capable('unbundle')
 
     pushop.repo.prepushoutgoinghooks(pushop)
     outgoing = pushop.outgoing
     # TODO: get bundlecaps from remote
     bundlecaps = None
     # create a changegroup from local
     if pushop.revs is None and not (outgoing.excluded
                             or pushop.repo.changelog.filteredrevs):
         # push everything,
         # use the fast path, no race possible on push
         cg = changegroup.makechangegroup(pushop.repo, outgoing, '01', 'push',
                 fastpath=True, bundlecaps=bundlecaps)
     else:
         cg = changegroup.makechangegroup(pushop.repo, outgoing, '01',
                                         'push', bundlecaps=bundlecaps)
 
     # apply changegroup to remote
     # local repo finds heads on server, finds out what
     # revs it must push. once revs transferred, if server
     # finds it has different heads (someone else won
     # commit/push race), server aborts.
     if pushop.force:
         remoteheads = ['force']
     else:
         remoteheads = pushop.remoteheads
     # ssh: return remote's addchangegroup()
     # http: return remote's addchangegroup() or 0 for error
     pushop.cgresult = pushop.remote.unbundle(cg, remoteheads,
                                         pushop.repo.url())
 
 def _pushsyncphase(pushop):
     """synchronise phase information locally and remotely"""
     cheads = pushop.commonheads
     # even when we don't push, exchanging phase data is useful
     remotephases = listkeys(pushop.remote, 'phases')
     if (pushop.ui.configbool('ui', '_usedassubrepo')
         and remotephases    # server supports phases
         and pushop.cgresult is None # nothing was pushed
         and remotephases.get('publishing', False)):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset was pushed
         # - and remote is publishing
         # We may be in issue 3871 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         remotephases = {'publishing': 'True'}
     if not remotephases: # old server or public only reply from non-publishing
         _localphasemove(pushop, cheads)
         # don't push any phase data as there is nothing to push
     else:
         ana = phases.analyzeremotephases(pushop.repo, cheads,
                                          remotephases)
         pheads, droots = ana
         ### Apply remote phase on local
         if remotephases.get('publishing', False):
             _localphasemove(pushop, cheads)
         else: # publish = False
             _localphasemove(pushop, pheads)
             _localphasemove(pushop, cheads, phases.draft)
         ### Apply local phase on remote
 
         if pushop.cgresult:
             if 'phases' in pushop.stepsdone:
                 # phases already pushed though bundle2
                 return
             outdated = pushop.outdatedphases
         else:
             outdated = pushop.fallbackoutdatedphases
 
         pushop.stepsdone.add('phases')
 
         # filter heads already turned public by the push
         outdated = [c for c in outdated if c.node() not in pheads]
         # fallback to independent pushkey command
         for newremotehead in outdated:
             with pushop.remote.commandexecutor() as e:
                 r = e.callcommand('pushkey', {
                     'namespace': 'phases',
                     'key': newremotehead.hex(),
                     'old': '%d' % phases.draft,
                     'new': '%d' % phases.public
                 }).result()
 
             if not r:
                 pushop.ui.warn(_('updating %s to public failed!\n')
                                % newremotehead)
 
 def _localphasemove(pushop, nodes, phase=phases.public):
     """move <nodes> to <phase> in the local source repo"""
     if pushop.trmanager:
         phases.advanceboundary(pushop.repo,
                                pushop.trmanager.transaction(),
                                phase,
                                nodes)
     else:
         # repo is not locked, do not change any phases!
         # Informs the user that phases should have been moved when
         # applicable.
         actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
         phasestr = phases.phasenames[phase]
         if actualmoves:
             pushop.ui.status(_('cannot lock source repo, skipping '
                                'local %s phase update\n') % phasestr)
 
 def _pushobsolete(pushop):
     """utility function to push obsolete markers to a remote"""
     if 'obsmarkers' in pushop.stepsdone:
         return
     repo = pushop.repo
     remote = pushop.remote
     pushop.stepsdone.add('obsmarkers')
     if pushop.outobsmarkers:
         pushop.ui.debug('try to push obsolete markers to remote\n')
         rslts = []
         remotedata = obsolete._pushkeyescape(sorted(pushop.outobsmarkers))
         for key in sorted(remotedata, reverse=True):
             # reverse sort to ensure we end with dump0
             data = remotedata[key]
             rslts.append(remote.pushkey('obsolete', key, '', data))
         if [r for r in rslts if not r]:
             msg = _('failed to push some obsolete markers!\n')
             repo.ui.warn(msg)
 
 def _pushbookmark(pushop):
     """Update bookmark position on remote"""
     if pushop.cgresult == 0 or 'bookmarks' in pushop.stepsdone:
         return
     pushop.stepsdone.add('bookmarks')
     ui = pushop.ui
     remote = pushop.remote
 
     for b, old, new in pushop.outbookmarks:
         action = 'update'
         if not old:
             action = 'export'
         elif not new:
             action = 'delete'
 
         with remote.commandexecutor() as e:
             r = e.callcommand('pushkey', {
                 'namespace': 'bookmarks',
                 'key': b,
                 'old': old,
                 'new': new,
             }).result()
 
         if r:
             ui.status(bookmsgmap[action][0] % b)
         else:
             ui.warn(bookmsgmap[action][1] % b)
             # discovery can have set the value form invalid entry
             if pushop.bkresult is not None:
                 pushop.bkresult = 1
 
 class pulloperation(object):
     """A object that represent a single pull operation
 
     It purpose is to carry pull related state and very common operation.
 
     A new should be created at the beginning of each pull and discarded
     afterward.
     """
 
     def __init__(self, repo, remote, heads=None, force=False, bookmarks=(),
                  remotebookmarks=None, streamclonerequested=None,
                  includepats=None, excludepats=None, depth=None):
         # repo we pull into
         self.repo = repo
         # repo we pull from
         self.remote = remote
         # revision we try to pull (None is "all")
         self.heads = heads
         # bookmark pulled explicitly
         self.explicitbookmarks = [repo._bookmarks.expandname(bookmark)
                                   for bookmark in bookmarks]
         # do we force pull?
         self.force = force
         # whether a streaming clone was requested
         self.streamclonerequested = streamclonerequested
         # transaction manager
         self.trmanager = None
         # set of common changeset between local and remote before pull
         self.common = None
         # set of pulled head
         self.rheads = None
         # list of missing changeset to fetch remotely
         self.fetch = None
         # remote bookmarks data
         self.remotebookmarks = remotebookmarks
         # result of changegroup pulling (used as return code by pull)
         self.cgresult = None
         # list of step already done
         self.stepsdone = set()
         # Whether we attempted a clone from pre-generated bundles.
         self.clonebundleattempted = False
         # Set of file patterns to include.
         self.includepats = includepats
         # Set of file patterns to exclude.
         self.excludepats = excludepats
         # Number of ancestor changesets to pull from each pulled head.
         self.depth = depth
 
     @util.propertycache
     def pulledsubset(self):
         """heads of the set of changeset target by the pull"""
         # compute target subset
         if self.heads is None:
             # We pulled every thing possible
             # sync on everything common
             c = set(self.common)
             ret = list(self.common)
             for n in self.rheads:
                 if n not in c:
                     ret.append(n)
             return ret
         else:
             # We pulled a specific subset
             # sync on this subset
             return self.heads
 
     @util.propertycache
     def canusebundle2(self):
         return not _forcebundle1(self)
 
     @util.propertycache
     def remotebundle2caps(self):
         return bundle2.bundle2caps(self.remote)
 
     def gettransaction(self):
         # deprecated; talk to trmanager directly
         return self.trmanager.transaction()
 
 class transactionmanager(util.transactional):
     """An object to manage the life cycle of a transaction
 
     It creates the transaction on demand and calls the appropriate hooks when
     closing the transaction."""
     def __init__(self, repo, source, url):
         self.repo = repo
         self.source = source
         self.url = url
         self._tr = None
 
     def transaction(self):
         """Return an open transaction object, constructing if necessary"""
         if not self._tr:
             trname = '%s\n%s' % (self.source, util.hidepassword(self.url))
             self._tr = self.repo.transaction(trname)
             self._tr.hookargs['source'] = self.source
             self._tr.hookargs['url'] = self.url
         return self._tr
 
     def close(self):
         """close transaction if created"""
         if self._tr is not None:
             self._tr.close()
 
     def release(self):
         """release transaction if created"""
         if self._tr is not None:
             self._tr.release()
 
 def listkeys(remote, namespace):
     with remote.commandexecutor() as e:
         return e.callcommand('listkeys', {'namespace': namespace}).result()
 
 def _fullpullbundle2(repo, pullop):
     # The server may send a partial reply, i.e. when inlining
     # pre-computed bundles. In that case, update the common
     # set based on the results and pull another bundle.
     #
     # There are two indicators that the process is finished:
     # - no changeset has been added, or
     # - all remote heads are known locally.
     # The head check must use the unfiltered view as obsoletion
     # markers can hide heads.
     unfi = repo.unfiltered()
     unficl = unfi.changelog
     def headsofdiff(h1, h2):
         """Returns heads(h1 % h2)"""
         res = unfi.set('heads(%ln %% %ln)', h1, h2)
         return set(ctx.node() for ctx in res)
     def headsofunion(h1, h2):
         """Returns heads((h1 + h2) - null)"""
         res = unfi.set('heads((%ln + %ln - null))', h1, h2)
         return set(ctx.node() for ctx in res)
     while True:
         old_heads = unficl.heads()
         clstart = len(unficl)
         _pullbundle2(pullop)
         if repository.NARROW_REQUIREMENT in repo.requirements:
             # XXX narrow clones filter the heads on the server side during
             # XXX getbundle and result in partial replies as well.
             # XXX Disable pull bundles in this case as band aid to avoid
             # XXX extra round trips.
             break
         if clstart == len(unficl):
             break
         if all(unficl.hasnode(n) for n in pullop.rheads):
             break
         new_heads = headsofdiff(unficl.heads(), old_heads)
         pullop.common = headsofunion(new_heads, pullop.common)
         pullop.rheads = set(pullop.rheads) - pullop.common
 
 def pull(repo, remote, heads=None, force=False, bookmarks=(), opargs=None,
          streamclonerequested=None, includepats=None, excludepats=None,
          depth=None):
     """Fetch repository data from a remote.
 
     This is the main function used to retrieve data from a remote repository.
 
     ``repo`` is the local repository to clone into.
     ``remote`` is a peer instance.
     ``heads`` is an iterable of revisions we want to pull. ``None`` (the
     default) means to pull everything from the remote.
     ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
     default, all remote bookmarks are pulled.
     ``opargs`` are additional keyword arguments to pass to ``pulloperation``
     initialization.
     ``streamclonerequested`` is a boolean indicating whether a "streaming
     clone" is requested. A "streaming clone" is essentially a raw file copy
     of revlogs from the server. This only works when the local repository is
     empty. The default value of ``None`` means to respect the server
     configuration for preferring stream clones.
     ``includepats`` and ``excludepats`` define explicit file patterns to
     include and exclude in storage, respectively. If not defined, narrow
     patterns from the repo instance are used, if available.
     ``depth`` is an integer indicating the DAG depth of history we're
     interested in. If defined, for each revision specified in ``heads``, we
     will fetch up to this many of its ancestors and data associated with them.
 
     Returns the ``pulloperation`` created for this pull.
     """
     if opargs is None:
         opargs = {}
 
     # We allow the narrow patterns to be passed in explicitly to provide more
     # flexibility for API consumers.
     if includepats or excludepats:
         includepats = includepats or set()
         excludepats = excludepats or set()
     else:
         includepats, excludepats = repo.narrowpats
 
     narrowspec.validatepatterns(includepats)
     narrowspec.validatepatterns(excludepats)
 
     pullop = pulloperation(repo, remote, heads, force, bookmarks=bookmarks,
                            streamclonerequested=streamclonerequested,
                            includepats=includepats, excludepats=excludepats,
                            depth=depth,
                            **pycompat.strkwargs(opargs))
 
     peerlocal = pullop.remote.local()
     if peerlocal:
         missing = set(peerlocal.requirements) - pullop.repo.supported
         if missing:
             msg = _("required features are not"
                     " supported in the destination:"
                     " %s") % (', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
         wlock = repo.wlock()
     with wlock, repo.lock(), pullop.trmanager:
         # Use the modern wire protocol, if available.
         if remote.capable('command-changesetdata'):
             exchangev2.pull(pullop)
         else:
             # This should ideally be in _pullbundle2(). However, it needs to run
             # before discovery to avoid extra work.
             _maybeapplyclonebundle(pullop)
             streamclone.maybeperformlegacystreamclone(pullop)
             _pulldiscovery(pullop)
             if pullop.canusebundle2:
                 _fullpullbundle2(repo, pullop)
             _pullchangeset(pullop)
             _pullphase(pullop)
             _pullbookmarks(pullop)
             _pullobsolete(pullop)
 
     # storing remotenames
     if repo.ui.configbool('experimental', 'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pullop
 
 # list of steps to perform discovery before pull
 pulldiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pulldiscoverymapping = {}
 
 def pulldiscovery(stepname):
     """decorator for function performing discovery before pull
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pulldiscovery dictionary directly."""
     def dec(func):
         assert stepname not in pulldiscoverymapping
         pulldiscoverymapping[stepname] = func
         pulldiscoveryorder.append(stepname)
         return func
     return dec
 
 def _pulldiscovery(pullop):
     """Run all discovery steps"""
     for stepname in pulldiscoveryorder:
         step = pulldiscoverymapping[stepname]
         step(pullop)
 
 @pulldiscovery('b1:bookmarks')
 def _pullbookmarkbundle1(pullop):
     """fetch bookmark data in bundle1 case
 
     If not using bundle2, we have to fetch bookmarks before changeset
     discovery to reduce the chance and impact of race conditions."""
     if pullop.remotebookmarks is not None:
         return
     if pullop.canusebundle2 and 'listkeys' in pullop.remotebundle2caps:
         # all known bundle2 servers now support listkeys, but lets be nice with
         # new implementation.
         return
     books = listkeys(pullop.remote, 'bookmarks')
     pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
 
 
 @pulldiscovery('changegroup')
 def _pulldiscoverychangegroup(pullop):
     """discovery phase for the pull
 
     Current handle changeset discovery only, will change handle all discovery
     at some point."""
     tmp = discovery.findcommonincoming(pullop.repo,
                                        pullop.remote,
                                        heads=pullop.heads,
                                        force=pullop.force)
     common, fetch, rheads = tmp
     nm = pullop.repo.unfiltered().changelog.nodemap
     if fetch and rheads:
         # If a remote heads is filtered locally, put in back in common.
         #
         # This is a hackish solution to catch most of "common but locally
         # hidden situation".  We do not performs discovery on unfiltered
         # repository because it end up doing a pathological amount of round
         # trip for w huge amount of changeset we do not care about.
         #
         # If a set of such "common but filtered" changeset exist on the server
         # but are not including a remote heads, we'll not be able to detect it,
         scommon = set(common)
         for n in rheads:
             if n in nm:
                 if n not in scommon:
                     common.append(n)
         if set(rheads).issubset(set(common)):
             fetch = []
     pullop.common = common
     pullop.fetch = fetch
     pullop.rheads = rheads
 
 def _pullbundle2(pullop):
     """pull data using bundle2
 
     For now, the only supported data are changegroup."""
     kwargs = {'bundlecaps': caps20to10(pullop.repo, role='client')}
 
     # make ui easier to access
     ui = pullop.repo.ui
 
     # At the moment we don't do stream clones over bundle2. If that is
     # implemented then here's where the check for that will go.
     streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
 
     # declare pull perimeters
     kwargs['common'] = pullop.common
     kwargs['heads'] = pullop.heads or pullop.rheads
 
     # check server supports narrow and then adding includepats and excludepats
     servernarrow = pullop.remote.capable(wireprototypes.NARROWCAP)
     if servernarrow and pullop.includepats:
         kwargs['includepats'] = pullop.includepats
     if servernarrow and pullop.excludepats:
         kwargs['excludepats'] = pullop.excludepats
 
     if streaming:
         kwargs['cg'] = False
         kwargs['stream'] = True
         pullop.stepsdone.add('changegroup')
         pullop.stepsdone.add('phases')
 
     else:
         # pulling changegroup
         pullop.stepsdone.add('changegroup')
 
         kwargs['cg'] = pullop.fetch
 
         legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
         hasbinaryphase = 'heads' in pullop.remotebundle2caps.get('phases', ())
         if (not legacyphase and hasbinaryphase):
             kwargs['phases'] = True
             pullop.stepsdone.add('phases')
 
         if 'listkeys' in pullop.remotebundle2caps:
             if 'phases' not in pullop.stepsdone:
                 kwargs['listkeys'] = ['phases']
 
     bookmarksrequested = False
     legacybookmark = 'bookmarks' in ui.configlist('devel', 'legacy.exchange')
     hasbinarybook = 'bookmarks' in pullop.remotebundle2caps
 
     if pullop.remotebookmarks is not None:
         pullop.stepsdone.add('request-bookmarks')
 
     if ('request-bookmarks' not in pullop.stepsdone
         and pullop.remotebookmarks is None
         and not legacybookmark and hasbinarybook):
         kwargs['bookmarks'] = True
         bookmarksrequested = True
 
     if 'listkeys' in pullop.remotebundle2caps:
         if 'request-bookmarks' not in pullop.stepsdone:
             # make sure to always includes bookmark data when migrating
             # `hg incoming --bundle` to using this function.
             pullop.stepsdone.add('request-bookmarks')
             kwargs.setdefault('listkeys', []).append('bookmarks')
 
     # If this is a full pull / clone and the server supports the clone bundles
     # feature, tell the server whether we attempted a clone bundle. The
     # presence of this flag indicates the client supports clone bundles. This
     # will enable the server to treat clients that support clone bundles
     # differently from those that don't.
     if (pullop.remote.capable('clonebundles')
         and pullop.heads is None and list(pullop.common) == [nullid]):
         kwargs['cbattempted'] = pullop.clonebundleattempted
 
     if streaming:
         pullop.repo.ui.status(_('streaming all changes\n'))
     elif not pullop.fetch:
         pullop.repo.ui.status(_("no changes found\n"))
         pullop.cgresult = 0
     else:
         if pullop.heads is None and list(pullop.common) == [nullid]:
             pullop.repo.ui.status(_("requesting all changes\n"))
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
         if obsolete.commonversion(remoteversions) is not None:
             kwargs['obsmarkers'] = True
             pullop.stepsdone.add('obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args['source'] = 'pull'
         bundle = e.callcommand('getbundle', args).result()
 
         try:
             op = bundle2.bundleoperation(pullop.repo, pullop.gettransaction,
                                          source='pull')
             op.modes['bookmarks'] = 'records'
             bundle2.processbundle(pullop.repo, bundle, op=op)
         except bundle2.AbortFromPart as exc:
             pullop.repo.ui.status(_('remote: abort: %s\n') % exc)
             raise error.Abort(_('pull failed on remote'), hint=exc.hint)
         except error.BundleValueError as exc:
             raise error.Abort(_('missing support for %s') % exc)
 
     if pullop.fetch:
         pullop.cgresult = bundle2.combinechangegroupresults(op)
 
     # processing phases change
     for namespace, value in op.records['listkeys']:
         if namespace == 'phases':
             _pullapplyphases(pullop, value)
 
     # processing bookmark update
     if bookmarksrequested:
         books = {}
         for record in op.records['bookmarks']:
             books[record['bookmark']] = record["node"]
         pullop.remotebookmarks = books
     else:
         for namespace, value in op.records['listkeys']:
             if namespace == 'bookmarks':
                 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
 
     # bookmark data were either already there or pulled in the bundle
     if pullop.remotebookmarks is not None:
         _pullbookmarks(pullop)
 
 def _pullbundle2extraprepare(pullop, kwargs):
     """hook function so that extensions can extend the getbundle call"""
 
 def _pullchangeset(pullop):
     """pull changeset from unbundle into the local repo"""
     # We delay the open of the transaction as late as possible so we
     # don't open transaction for nothing or you break future useful
     # rollback call
     if 'changegroup' in pullop.stepsdone:
         return
     pullop.stepsdone.add('changegroup')
     if not pullop.fetch:
         pullop.repo.ui.status(_("no changes found\n"))
         pullop.cgresult = 0
         return
     tr = pullop.gettransaction()
     if pullop.heads is None and list(pullop.common) == [nullid]:
         pullop.repo.ui.status(_("requesting all changes\n"))
     elif pullop.heads is None and pullop.remote.capable('changegroupsubset'):
         # issue1320, avoid a race if remote changed after discovery
         pullop.heads = pullop.rheads
 
     if pullop.remote.capable('getbundle'):
         # TODO: get bundlecaps from remote
         cg = pullop.remote.getbundle('pull', common=pullop.common,
                                      heads=pullop.heads or pullop.rheads)
     elif pullop.heads is None:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand('changegroup', {
                 'nodes': pullop.fetch,
                 'source': 'pull',
             }).result()
 
     elif not pullop.remote.capable('changegroupsubset'):
         raise error.Abort(_("partial pull cannot be done because "
                            "other repository doesn't support "
                            "changegroupsubset."))
     else:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand('changegroupsubset', {
                 'bases': pullop.fetch,
                 'heads': pullop.heads,
                 'source': 'pull',
             }).result()
 
     bundleop = bundle2.applybundle(pullop.repo, cg, tr, 'pull',
                                    pullop.remote.url())
     pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
 
 def _pullphase(pullop):
     # Get remote phases data from remote
     if 'phases' in pullop.stepsdone:
         return
     remotephases = listkeys(pullop.remote, 'phases')
     _pullapplyphases(pullop, remotephases)
 
 def _pullapplyphases(pullop, remotephases):
     """apply phase movement from observed remote state"""
     if 'phases' in pullop.stepsdone:
         return
     pullop.stepsdone.add('phases')
     publishing = bool(remotephases.get('publishing', False))
     if remotephases and not publishing:
         # remote is new and non-publishing
         pheads, _dr = phases.analyzeremotephases(pullop.repo,
                                                  pullop.pulledsubset,
                                                  remotephases)
         dheads = pullop.pulledsubset
     else:
         # Remote is old or publishing all common changesets
         # should be seen as public
         pheads = pullop.pulledsubset
         dheads = []
     unfi = pullop.repo.unfiltered()
     phase = unfi._phasecache.phase
     rev = unfi.changelog.nodemap.get
     public = phases.public
     draft = phases.draft
 
     # exclude changesets already public locally and update the others
     pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
     if pheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, public, pheads)
 
     # exclude changesets already draft locally and update the others
     dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
     if dheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, draft, dheads)
 
 def _pullbookmarks(pullop):
     """process the remote bookmark information to update the local one"""
     if 'bookmarks' in pullop.stepsdone:
         return
     pullop.stepsdone.add('bookmarks')
     repo = pullop.repo
     remotebookmarks = pullop.remotebookmarks
     bookmod.updatefromremote(repo.ui, repo, remotebookmarks,
                              pullop.remote.url(),
                              pullop.gettransaction,
                              explicit=pullop.explicitbookmarks)
 
 def _pullobsolete(pullop):
     """utility function to pull obsolete markers from a remote
 
     The `gettransaction` is function that return the pull transaction, creating
     one if necessary. We return the transaction to inform the calling code that
     a new transaction have been created (when applicable).
 
     Exists mostly to allow overriding for experimentation purpose"""
     if 'obsmarkers' in pullop.stepsdone:
         return
     pullop.stepsdone.add('obsmarkers')
     tr = None
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         pullop.repo.ui.debug('fetching remote obsolete markers\n')
         remoteobs = listkeys(pullop.remote, 'obsolete')
         if 'dump0' in remoteobs:
             tr = pullop.gettransaction()
             markers = []
             for key in sorted(remoteobs, reverse=True):
                 if key.startswith('dump'):
                     data = util.b85decode(remoteobs[key])
                     version, newmarks = obsolete._readmarkers(data)
                     markers += newmarks
             if markers:
                 pullop.repo.obsstore.add(tr, markers)
             pullop.repo.invalidatevolatilesets()
     return tr
 
 def applynarrowacl(repo, kwargs):
     """Apply narrow fetch access control.
 
     This massages the named arguments for getbundle wire protocol commands
     so requested data is filtered through access control rules.
     """
     ui = repo.ui
     # TODO this assumes existence of HTTP and is a layering violation.
     username = ui.shortuser(ui.environ.get('REMOTE_USER') or ui.username())
     user_includes = ui.configlist(
         _NARROWACL_SECTION, username + '.includes',
         ui.configlist(_NARROWACL_SECTION, 'default.includes'))
     user_excludes = ui.configlist(
         _NARROWACL_SECTION, username + '.excludes',
         ui.configlist(_NARROWACL_SECTION, 'default.excludes'))
     if not user_includes:
         raise error.Abort(_("{} configuration for user {} is empty")
                           .format(_NARROWACL_SECTION, username))
 
     user_includes = [
         'path:.' if p == '*' else 'path:' + p for p in user_includes]
     user_excludes = [
         'path:.' if p == '*' else 'path:' + p for p in user_excludes]
 
     req_includes = set(kwargs.get(r'includepats', []))
     req_excludes = set(kwargs.get(r'excludepats', []))
 
     req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
         req_includes, req_excludes, user_includes, user_excludes)
 
     if invalid_includes:
         raise error.Abort(
             _("The following includes are not accessible for {}: {}")
             .format(username, invalid_includes))
 
     new_args = {}
     new_args.update(kwargs)
     new_args[r'narrow'] = True
     new_args[r'narrow_acl'] = True
     new_args[r'includepats'] = req_includes
     if req_excludes:
         new_args[r'excludepats'] = req_excludes
 
     return new_args
 
 def _computeellipsis(repo, common, heads, known, match, depth=None):
     """Compute the shape of a narrowed DAG.
 
     Args:
       repo: The repository we're transferring.
       common: The roots of the DAG range we're transferring.
               May be just [nullid], which means all ancestors of heads.
       heads: The heads of the DAG range we're transferring.
       match: The narrowmatcher that allows us to identify relevant changes.
       depth: If not None, only consider nodes to be full nodes if they are at
              most depth changesets away from one of heads.
 
     Returns:
       A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
 
         visitnodes: The list of nodes (either full or ellipsis) which
                     need to be sent to the client.
         relevant_nodes: The set of changelog nodes which change a file inside
                  the narrowspec. The client needs these as non-ellipsis nodes.
         ellipsisroots: A dict of {rev: parents} that is used in
                        narrowchangegroup to produce ellipsis nodes with the
                        correct parents.
     """
     cl = repo.changelog
     mfl = repo.manifestlog
 
     clrev = cl.rev
 
     commonrevs = {clrev(n) for n in common} | {nullrev}
     headsrevs = {clrev(n) for n in heads}
 
     if depth:
         revdepth = {h: 0 for h in headsrevs}
 
     ellipsisheads = collections.defaultdict(set)
     ellipsisroots = collections.defaultdict(set)
 
     def addroot(head, curchange):
         """Add a root to an ellipsis head, splitting heads with 3 roots."""
         ellipsisroots[head].add(curchange)
         # Recursively split ellipsis heads with 3 roots by finding the
         # roots' youngest common descendant which is an elided merge commit.
         # That descendant takes 2 of the 3 roots as its own, and becomes a
         # root of the head.
         while len(ellipsisroots[head]) > 2:
             child, roots = splithead(head)
             splitroots(head, child, roots)
             head = child  # Recurse in case we just added a 3rd root
 
     def splitroots(head, child, roots):
         ellipsisroots[head].difference_update(roots)
         ellipsisroots[head].add(child)
         ellipsisroots[child].update(roots)
         ellipsisroots[child].discard(child)
 
     def splithead(head):
         r1, r2, r3 = sorted(ellipsisroots[head])
         for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
             mid = repo.revs('sort(merge() & %d::%d & %d::%d, -rev)',
                             nr1, head, nr2, head)
             for j in mid:
                 if j == nr2:
                     return nr2, (nr1, nr2)
                 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
                     return j, (nr1, nr2)
         raise error.Abort(_('Failed to split up ellipsis node! head: %d, '
                             'roots: %d %d %d') % (head, r1, r2, r3))
 
     missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
     visit = reversed(missing)
     relevant_nodes = set()
     visitnodes = [cl.node(m) for m in missing]
     required = set(headsrevs) | known
     for rev in visit:
         clrev = cl.changelogrevision(rev)
         ps = [prev for prev in cl.parentrevs(rev) if prev != nullrev]
         if depth is not None:
             curdepth = revdepth[rev]
             for p in ps:
                 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
         needed = False
         shallow_enough = depth is None or revdepth[rev] <= depth
         if shallow_enough:
             curmf = mfl[clrev.manifest].read()
             if ps:
                 # We choose to not trust the changed files list in
                 # changesets because it's not always correct. TODO: could
                 # we trust it for the non-merge case?
                 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
                 needed = bool(curmf.diff(p1mf, match))
                 if not needed and len(ps) > 1:
                     # For merge changes, the list of changed files is not
                     # helpful, since we need to emit the merge if a file
                     # in the narrow spec has changed on either side of the
                     # merge. As a result, we do a manifest diff to check.
                     p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
                     needed = bool(curmf.diff(p2mf, match))
             else:
                 # For a root node, we need to include the node if any
                 # files in the node match the narrowspec.
                 needed = any(curmf.walk(match))
 
         if needed:
             for head in ellipsisheads[rev]:
                 addroot(head, rev)
             for p in ps:
                 required.add(p)
             relevant_nodes.add(cl.node(rev))
         else:
             if not ps:
                 ps = [nullrev]
             if rev in required:
                 for head in ellipsisheads[rev]:
                     addroot(head, rev)
                 for p in ps:
                     ellipsisheads[p].add(rev)
             else:
                 for p in ps:
                     ellipsisheads[p] |= ellipsisheads[rev]
 
     # add common changesets as roots of their reachable ellipsis heads
     for c in commonrevs:
         for head in ellipsisheads[c]:
             addroot(head, c)
     return visitnodes, relevant_nodes, ellipsisroots
 
 def caps20to10(repo, role):
     """return a set with appropriate options to use bundle20 during getbundle"""
     caps = {'HG20'}
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
     caps.add('bundle2=' + urlreq.quote(capsblob))
     return caps
 
 # List of names of steps to perform for a bundle2 for getbundle, order matters.
 getbundle2partsorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 getbundle2partsmapping = {}
 
 def getbundle2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part for getbundle
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the getbundle2partsmapping dictionary directly."""
     def dec(func):
         assert stepname not in getbundle2partsmapping
         getbundle2partsmapping[stepname] = func
         if idx is None:
             getbundle2partsorder.append(stepname)
         else:
             getbundle2partsorder.insert(idx, stepname)
         return func
     return dec
 
 def bundle2requested(bundlecaps):
     if bundlecaps is not None:
         return any(cap.startswith('HG2') for cap in bundlecaps)
     return False
 
 def getbundlechunks(repo, source, heads=None, common=None, bundlecaps=None,
                     **kwargs):
     """Return chunks constituting a bundle's raw data.
 
     Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
     passed.
 
     Returns a 2-tuple of a dict with metadata about the generated bundle
     and an iterator over raw chunks (of varying sizes).
     """
     kwargs = pycompat.byteskwargs(kwargs)
     info = {}
     usebundle2 = bundle2requested(bundlecaps)
     # bundle10 case
     if not usebundle2:
         if bundlecaps and not kwargs.get('cg', True):
             raise ValueError(_('request for bundle10 must include changegroup'))
 
         if kwargs:
             raise ValueError(_('unsupported getbundle arguments: %s')
                              % ', '.join(sorted(kwargs.keys())))
         outgoing = _computeoutgoing(repo, heads, common)
         info['bundleversion'] = 1
         return info, changegroup.makestream(repo, outgoing, '01', source,
                                             bundlecaps=bundlecaps)
 
     # bundle20 case
     info['bundleversion'] = 2
     b2caps = {}
     for bcaps in bundlecaps:
         if bcaps.startswith('bundle2='):
             blob = urlreq.unquote(bcaps[len('bundle2='):])
             b2caps.update(bundle2.decodecaps(blob))
     bundler = bundle2.bundle20(repo.ui, b2caps)
 
     kwargs['heads'] = heads
     kwargs['common'] = common
 
     for name in getbundle2partsorder:
         func = getbundle2partsmapping[name]
         func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps,
              **pycompat.strkwargs(kwargs))
 
     info['prefercompressed'] = bundler.prefercompressed
 
     return info, bundler.getchunks()
 
 @getbundle2partsgenerator('stream2')
 def _getbundlestream2(bundler, repo, *args, **kwargs):
     return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
 
 @getbundle2partsgenerator('changegroup')
 def _getbundlechangegrouppart(bundler, repo, source, bundlecaps=None,
                               b2caps=None, heads=None, common=None, **kwargs):
     """add a changegroup part to the requested bundle"""
     if not kwargs.get(r'cg', True):
         return
 
     version = '01'
     cgversions = b2caps.get('changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [v for v in cgversions
                       if v in changegroup.supportedoutgoingversions(repo)]
         if not cgversions:
             raise error.Abort(_('no common changegroup version'))
         version = max(cgversions)
 
     outgoing = _computeoutgoing(repo, heads, common)
     if not outgoing.missing:
         return
 
     if kwargs.get(r'narrow', False):
         include = sorted(filter(bool, kwargs.get(r'includepats', [])))
         exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
         matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
     else:
         matcher = None
 
     cgstream = changegroup.makestream(repo, outgoing, version, source,
                                       bundlecaps=bundlecaps, matcher=matcher)
 
     part = bundler.newpart('changegroup', data=cgstream)
     if cgversions:
         part.addparam('version', version)
 
     part.addparam('nbchanges', '%d' % len(outgoing.missing),
                   mandatory=False)
 
     if 'treemanifest' in repo.requirements:
         part.addparam('treemanifest', '1')
 
     if (kwargs.get(r'narrow', False) and kwargs.get(r'narrow_acl', False)
         and (include or exclude)):
         # this is mandatory because otherwise ACL clients won't work
         narrowspecpart = bundler.newpart('Narrow:responsespec')
         narrowspecpart.data = '%s\0%s' % ('\n'.join(include),
                                            '\n'.join(exclude))
 
 @getbundle2partsgenerator('bookmarks')
 def _getbundlebookmarkpart(bundler, repo, source, bundlecaps=None,
                               b2caps=None, **kwargs):
     """add a bookmark part to the requested bundle"""
     if not kwargs.get(r'bookmarks', False):
         return
     if 'bookmarks' not in b2caps:
         raise error.Abort(_('no common bookmarks exchange method'))
     books  = bookmod.listbinbookmarks(repo)
     data = bookmod.binaryencode(books)
     if data:
         bundler.newpart('bookmarks', data=data)
 
 @getbundle2partsgenerator('listkeys')
 def _getbundlelistkeysparts(bundler, repo, source, bundlecaps=None,
                             b2caps=None, **kwargs):
     """add parts containing listkeys namespaces to the requested bundle"""
     listkeys = kwargs.get(r'listkeys', ())
     for namespace in listkeys:
         part = bundler.newpart('listkeys')
         part.addparam('namespace', namespace)
         keys = repo.listkeys(namespace).items()
         part.data = pushkey.encodekeys(keys)
 
 @getbundle2partsgenerator('obsmarkers')
 def _getbundleobsmarkerpart(bundler, repo, source, bundlecaps=None,
                             b2caps=None, heads=None, **kwargs):
     """add an obsolescence markers part to the requested bundle"""
     if kwargs.get(r'obsmarkers', False):
         if heads is None:
             heads = repo.heads()
         subset = [c.node() for c in repo.set('::%ln', heads)]
         markers = repo.obsstore.relevantmarkers(subset)
         markers = sorted(markers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 @getbundle2partsgenerator('phases')
 def _getbundlephasespart(bundler, repo, source, bundlecaps=None,
                             b2caps=None, heads=None, **kwargs):
     """add phase heads part to the requested bundle"""
     if kwargs.get(r'phases', False):
         if not 'heads' in b2caps.get('phases'):
             raise error.Abort(_('no common phases exchange method'))
         if heads is None:
             heads = repo.heads()
 
         headsbyphase = collections.defaultdict(set)
         if repo.publishing():
             headsbyphase[phases.public] = heads
         else:
             # find the appropriate heads to move
 
             phase = repo._phasecache.phase
             node = repo.changelog.node
             rev = repo.changelog.rev
             for h in heads:
                 headsbyphase[phase(repo, rev(h))].add(h)
             seenphases = list(headsbyphase.keys())
 
             # We do not handle anything but public and draft phase for now)
             if seenphases:
                 assert max(seenphases) <= phases.draft
 
             # if client is pulling non-public changesets, we need to find
             # intermediate public heads.
             draftheads = headsbyphase.get(phases.draft, set())
             if draftheads:
                 publicheads = headsbyphase.get(phases.public, set())
 
                 revset = 'heads(only(%ln, %ln) and public())'
                 extraheads = repo.revs(revset, draftheads, publicheads)
                 for r in extraheads:
                     headsbyphase[phases.public].add(node(r))
 
         # transform data in a format used by the encoding function
         phasemapping = []
         for phase in phases.allphases:
             phasemapping.append(sorted(headsbyphase[phase]))
 
         # generate the actual part
         phasedata = phases.binaryencode(phasemapping)
         bundler.newpart('phase-heads', data=phasedata)
 
 @getbundle2partsgenerator('hgtagsfnodes')
 def _getbundletagsfnodes(bundler, repo, source, bundlecaps=None,
                          b2caps=None, heads=None, common=None,
                          **kwargs):
     """Transfer the .hgtags filenodes mapping.
 
     Only values for heads in this bundle will be transferred.
 
     The part data consists of pairs of 20 byte changeset node and .hgtags
     filenodes raw values.
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     if not (kwargs.get(r'cg', True) and 'hgtagsfnodes' in b2caps):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addparttagsfnodescache(repo, bundler, outgoing)
 
 @getbundle2partsgenerator('cache:rev-branch-cache')
 def _getbundlerevbranchcache(bundler, repo, source, bundlecaps=None,
                              b2caps=None, heads=None, common=None,
                              **kwargs):
     """Transfer the rev-branch-cache mapping
 
     The payload is a series of data related to each branch
 
     1) branch name length
     2) number of open heads
     3) number of closed heads
     4) open heads nodes
     5) closed heads nodes
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     # - narrow bundle isn't in play (not currently compatible).
     if (not kwargs.get(r'cg', True)
         or 'rev-branch-cache' not in b2caps
         or kwargs.get(r'narrow', False)
         or repo.ui.has_section(_NARROWACL_SECTION)):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addpartrevbranchcache(repo, bundler, outgoing)
 
 def check_heads(repo, their_heads, context):
     """check if the heads of a repo have been modified
 
     Used by peer for unbundling.
     """
     heads = repo.heads()
     heads_hash = hashlib.sha1(''.join(sorted(heads))).digest()
     if not (their_heads == ['force'] or their_heads == heads or
             their_heads == ['hashed', heads_hash]):
         # someone else committed/pushed/unbundled while we
         # were transferring data
         raise error.PushRaced('repository changed while %s - '
                               'please try again' % context)
 
 def unbundle(repo, cg, heads, source, url):
     """Apply a bundle to a repo.
 
     this function makes sure the repo is locked during the application and have
     mechanism to check that no push race occurred between the creation of the
     bundle and its application.
 
     If the push was raced as PushRaced exception is raised."""
     r = 0
     # need a transaction when processing a bundle2 stream
     # [wlock, lock, tr] - needs to be an array so nested functions can modify it
     lockandtr = [None, None, None]
     recordout = None
     # quick fix for output mismatch with bundle2 in 3.4
     captureoutput = repo.ui.configbool('experimental', 'bundle2-output-capture')
     if url.startswith('remote:http:') or url.startswith('remote:https:'):
         captureoutput = True
     try:
         # note: outside bundle1, 'heads' is expected to be empty and this
         # 'check_heads' call wil be a no-op
         check_heads(repo, heads, 'uploading changes')
         # push can proceed
         if not isinstance(cg, bundle2.unbundle20):
             # legacy case: bundle1 (changegroup 01)
             txnname = "\n".join([source, util.hidepassword(url)])
             with repo.lock(), repo.transaction(txnname) as tr:
                 op = bundle2.applybundle(repo, cg, tr, source, url)
                 r = bundle2.combinechangegroupresults(op)
         else:
             r = None
             try:
                 def gettransaction():
                     if not lockandtr[2]:
                         if not bookmod.bookmarksinstore(repo):
                             lockandtr[0] = repo.wlock()
                         lockandtr[1] = repo.lock()
                         lockandtr[2] = repo.transaction(source)
                         lockandtr[2].hookargs['source'] = source
                         lockandtr[2].hookargs['url'] = url
                         lockandtr[2].hookargs['bundle2'] = '1'
                     return lockandtr[2]
 
                 # Do greedy locking by default until we're satisfied with lazy
                 # locking.
                 if not repo.ui.configbool('experimental', 'bundle2lazylocking'):
                     gettransaction()
 
                 op = bundle2.bundleoperation(repo, gettransaction,
                                              captureoutput=captureoutput,
                                              source='push')
                 try:
                     op = bundle2.processbundle(repo, cg, op=op)
                 finally:
                     r = op.reply
                     if captureoutput and r is not None:
                         repo.ui.pushbuffer(error=True, subproc=True)
                         def recordout(output):
                             r.newpart('output', data=output, mandatory=False)
                 if lockandtr[2] is not None:
                     lockandtr[2].close()
             except BaseException as exc:
                 exc.duringunbundle2 = True
                 if captureoutput and r is not None:
                     parts = exc._bundle2salvagedoutput = r.salvageoutput()
                     def recordout(output):
                         part = bundle2.bundlepart('output', data=output,
                                                   mandatory=False)
                         parts.append(part)
                 raise
     finally:
         lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
         if recordout is not None:
             recordout(repo.ui.popbuffer())
     return r
 
 def _maybeapplyclonebundle(pullop):
     """Apply a clone bundle from a remote, if possible."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     if not repo.ui.configbool('ui', 'clonebundles'):
         return
 
     # Only run if local repo is empty.
     if len(repo):
         return
 
     if pullop.heads:
         return
 
     if not remote.capable('clonebundles'):
         return
 
     with remote.commandexecutor() as e:
         res = e.callcommand('clonebundles', {}).result()
 
     # If we call the wire protocol command, that's good enough to record the
     # attempt.
     pullop.clonebundleattempted = True
 
     entries = parseclonebundlesmanifest(repo, res)
     if not entries:
         repo.ui.note(_('no clone bundles available on remote; '
                        'falling back to regular clone\n'))
         return
 
     entries = filterclonebundleentries(
         repo, entries, streamclonerequested=pullop.streamclonerequested)
 
     if not entries:
         # There is a thundering herd concern here. However, if a server
         # operator doesn't advertise bundles appropriate for its clients,
         # they deserve what's coming. Furthermore, from a client's
         # perspective, no automatic fallback would mean not being able to
         # clone!
         repo.ui.warn(_('no compatible clone bundles available on server; '
                        'falling back to regular clone\n'))
         repo.ui.warn(_('(you may want to report this to the server '
                        'operator)\n'))
         return
 
     entries = sortclonebundleentries(repo.ui, entries)
 
     url = entries[0]['URL']
     repo.ui.status(_('applying clone bundle from %s\n') % url)
     if trypullbundlefromurl(repo.ui, repo, url):
         repo.ui.status(_('finished applying clone bundle\n'))
     # Bundle failed.
     #
     # We abort by default to avoid the thundering herd of
     # clients flooding a server that was expecting expensive
     # clone load to be offloaded.
     elif repo.ui.configbool('ui', 'clonebundlefallback'):
         repo.ui.warn(_('falling back to normal clone\n'))
     else:
         raise error.Abort(_('error applying bundle'),
                           hint=_('if this error persists, consider contacting '
                                  'the server operator or disable clone '
                                  'bundles via '
                                  '"--config ui.clonebundles=false"'))
 
 def parseclonebundlesmanifest(repo, s):
     """Parses the raw text of a clone bundles manifest.
 
     Returns a list of dicts. The dicts have a ``URL`` key corresponding
     to the URL and other keys are the attributes for the entry.
     """
     m = []
     for line in s.splitlines():
         fields = line.split()
         if not fields:
             continue
         attrs = {'URL': fields[0]}
         for rawattr in fields[1:]:
             key, value = rawattr.split('=', 1)
             key = urlreq.unquote(key)
             value = urlreq.unquote(value)
             attrs[key] = value
 
             # Parse BUNDLESPEC into components. This makes client-side
             # preferences easier to specify since you can prefer a single
             # component of the BUNDLESPEC.
             if key == 'BUNDLESPEC':
                 try:
                     bundlespec = parsebundlespec(repo, value)
                     attrs['COMPRESSION'] = bundlespec.compression
                     attrs['VERSION'] = bundlespec.version
                 except error.InvalidBundleSpecification:
                     pass
                 except error.UnsupportedBundleSpecification:
                     pass
 
         m.append(attrs)
 
     return m
 
 def isstreamclonespec(bundlespec):
     # Stream clone v1
     if (bundlespec.wirecompression == 'UN' and bundlespec.wireversion == 's1'):
         return True
 
     # Stream clone v2
     if (bundlespec.wirecompression == 'UN' and
         bundlespec.wireversion == '02' and
         bundlespec.contentopts.get('streamv2')):
         return True
 
     return False
 
 def filterclonebundleentries(repo, entries, streamclonerequested=False):
     """Remove incompatible clone bundle manifest entries.
 
     Accepts a list of entries parsed with ``parseclonebundlesmanifest``
     and returns a new list consisting of only the entries that this client
     should be able to apply.
 
     There is no guarantee we'll be able to apply all returned entries because
     the metadata we use to filter on may be missing or wrong.
     """
     newentries = []
     for entry in entries:
         spec = entry.get('BUNDLESPEC')
         if spec:
             try:
                 bundlespec = parsebundlespec(repo, spec, strict=True)
 
                 # If a stream clone was requested, filter out non-streamclone
                 # entries.
                 if streamclonerequested and not isstreamclonespec(bundlespec):
                     repo.ui.debug('filtering %s because not a stream clone\n' %
                                   entry['URL'])
                     continue
 
             except error.InvalidBundleSpecification as e:
                 repo.ui.debug(stringutil.forcebytestr(e) + '\n')
                 continue
             except error.UnsupportedBundleSpecification as e:
                 repo.ui.debug('filtering %s because unsupported bundle '
                               'spec: %s\n' % (
                                   entry['URL'], stringutil.forcebytestr(e)))
                 continue
         # If we don't have a spec and requested a stream clone, we don't know
         # what the entry is so don't attempt to apply it.
         elif streamclonerequested:
             repo.ui.debug('filtering %s because cannot determine if a stream '
                           'clone bundle\n' % entry['URL'])
             continue
 
         if 'REQUIRESNI' in entry and not sslutil.hassni:
             repo.ui.debug('filtering %s because SNI not supported\n' %
                           entry['URL'])
             continue
 
         newentries.append(entry)
 
     return newentries
 
 class clonebundleentry(object):
     """Represents an item in a clone bundles manifest.
 
     This rich class is needed to support sorting since sorted() in Python 3
     doesn't support ``cmp`` and our comparison is complex enough that ``key=``
     won't work.
     """
 
     def __init__(self, value, prefers):
         self.value = value
         self.prefers = prefers
 
     def _cmp(self, other):
         for prefkey, prefvalue in self.prefers:
             avalue = self.value.get(prefkey)
             bvalue = other.value.get(prefkey)
 
             # Special case for b missing attribute and a matches exactly.
             if avalue is not None and bvalue is None and avalue == prefvalue:
                 return -1
 
             # Special case for a missing attribute and b matches exactly.
             if bvalue is not None and avalue is None and bvalue == prefvalue:
                 return 1
 
             # We can't compare unless attribute present on both.
             if avalue is None or bvalue is None:
                 continue
 
             # Same values should fall back to next attribute.
             if avalue == bvalue:
                 continue
 
             # Exact matches come first.
             if avalue == prefvalue:
                 return -1
             if bvalue == prefvalue:
                 return 1
 
             # Fall back to next attribute.
             continue
 
         # If we got here we couldn't sort by attributes and prefers. Fall
         # back to index order.
         return 0
 
     def __lt__(self, other):
         return self._cmp(other) < 0
 
     def __gt__(self, other):
         return self._cmp(other) > 0
 
     def __eq__(self, other):
         return self._cmp(other) == 0
 
     def __le__(self, other):
         return self._cmp(other) <= 0
 
     def __ge__(self, other):
         return self._cmp(other) >= 0
 
     def __ne__(self, other):
         return self._cmp(other) != 0
 
 def sortclonebundleentries(ui, entries):
     prefers = ui.configlist('ui', 'clonebundleprefers')
     if not prefers:
         return list(entries)
 
     prefers = [p.split('=', 1) for p in prefers]
 
     items = sorted(clonebundleentry(v, prefers) for v in entries)
     return [i.value for i in items]
 
 def trypullbundlefromurl(ui, repo, url):
     """Attempt to apply a bundle from a URL."""
     with repo.lock(), repo.transaction('bundleurl') as tr:
         try:
             fh = urlmod.open(ui, url)
             cg = readbundle(ui, fh, 'stream')
 
             if isinstance(cg, streamclone.streamcloneapplier):
                 cg.apply(repo)
             else:
                 bundle2.applybundle(repo, cg, tr, 'clonebundles', url)
             return True
         except urlerr.httperror as e:
             ui.warn(_('HTTP error fetching bundle: %s\n') %
                     stringutil.forcebytestr(e))
         except urlerr.urlerror as e:
             ui.warn(_('error fetching bundle: %s\n') %
                     stringutil.forcebytestr(e.reason))
 
         return False

mercurial/exchangev2.py

@@ -1,697 +1,699
 # exchangev2.py - repository exchange for wire protocol version 2
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import weakref
 
 from .i18n import _
 from .node import (
     nullid,
     short,
 )
 from . import (
     bookmarks,
     error,
     mdiff,
     narrowspec,
     phases,
     pycompat,
+    setdiscovery,
+)
+from .interfaces import (
     repository,
-    setdiscovery,
 )
 
 def pull(pullop):
     """Pull using wire protocol version 2."""
     repo = pullop.repo
     remote = pullop.remote
 
     usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
 
     # If this is a clone and it was requested to perform a "stream clone",
     # we obtain the raw files data from the remote then fall back to an
     # incremental pull. This is somewhat hacky and is not nearly robust enough
     # for long-term usage.
     if usingrawchangelogandmanifest:
         with repo.transaction('clone'):
             _fetchrawstorefiles(repo, remote)
             repo.invalidate(clearfilecache=True)
 
     tr = pullop.trmanager.transaction()
 
     # We don't use the repo's narrow matcher here because the patterns passed
     # to exchange.pull() could be different.
     narrowmatcher = narrowspec.match(repo.root,
                                      # Empty maps to nevermatcher. So always
                                      # set includes if missing.
                                      pullop.includepats or {'path:.'},
                                      pullop.excludepats)
 
     if pullop.includepats or pullop.excludepats:
         pathfilter = {}
         if pullop.includepats:
             pathfilter[b'include'] = sorted(pullop.includepats)
         if pullop.excludepats:
             pathfilter[b'exclude'] = sorted(pullop.excludepats)
     else:
         pathfilter = None
 
     # Figure out what needs to be fetched.
     common, fetch, remoteheads = _pullchangesetdiscovery(
         repo, remote, pullop.heads, abortwhenunrelated=pullop.force)
 
     # And fetch the data.
     pullheads = pullop.heads or remoteheads
     csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads)
 
     # New revisions are written to the changelog. But all other updates
     # are deferred. Do those now.
 
     # Ensure all new changesets are draft by default. If the repo is
     # publishing, the phase will be adjusted by the loop below.
     if csetres['added']:
         phases.registernew(repo, tr, phases.draft, csetres['added'])
 
     # And adjust the phase of all changesets accordingly.
     for phase in phases.phasenames:
         if phase == b'secret' or not csetres['nodesbyphase'][phase]:
             continue
 
         phases.advanceboundary(repo, tr, phases.phasenames.index(phase),
                                csetres['nodesbyphase'][phase])
 
     # Write bookmark updates.
     bookmarks.updatefromremote(repo.ui, repo, csetres['bookmarks'],
                                remote.url(), pullop.gettransaction,
                                explicit=pullop.explicitbookmarks)
 
     manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
 
     # We don't properly support shallow changeset and manifest yet. So we apply
     # depth limiting locally.
     if pullop.depth:
         relevantcsetnodes = set()
         clnode = repo.changelog.node
 
         for rev in repo.revs(b'ancestors(%ln, %s)',
                              pullheads, pullop.depth - 1):
             relevantcsetnodes.add(clnode(rev))
 
         csetrelevantfilter = lambda n: n in relevantcsetnodes
 
     else:
         csetrelevantfilter = lambda n: True
 
     # If obtaining the raw store files, we need to scan the full repo to
     # derive all the changesets, manifests, and linkrevs.
     if usingrawchangelogandmanifest:
         csetsforfiles = []
         mnodesforfiles = []
         manifestlinkrevs = {}
 
         for rev in repo:
             ctx = repo[rev]
             node = ctx.node()
 
             if not csetrelevantfilter(node):
                 continue
 
             mnode = ctx.manifestnode()
 
             csetsforfiles.append(node)
             mnodesforfiles.append(mnode)
             manifestlinkrevs[mnode] = rev
 
     else:
         csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)]
         mnodesforfiles = manres['added']
         manifestlinkrevs = manres['linkrevs']
 
     # Find all file nodes referenced by added manifests and fetch those
     # revisions.
     fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
     _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles,
                          manifestlinkrevs, shallow=bool(pullop.depth))
 
 def _checkuserawstorefiledata(pullop):
     """Check whether we should use rawstorefiledata command to retrieve data."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     # Command to obtain raw store data isn't available.
     if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
         return False
 
     # Only honor if user requested stream clone operation.
     if not pullop.streamclonerequested:
         return False
 
     # Only works on empty repos.
     if len(repo):
         return False
 
     # TODO This is super hacky. There needs to be a storage API for this. We
     # also need to check for compatibility with the remote.
     if b'revlogv1' not in repo.requirements:
         return False
 
     return True
 
 def _fetchrawstorefiles(repo, remote):
     with remote.commandexecutor() as e:
         objs = e.callcommand(b'rawstorefiledata', {
             b'files': [b'changelog', b'manifestlog'],
         }).result()
 
         # First object is a summary of files data that follows.
         overall = next(objs)
 
         progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'],
                                         unit=_('bytes'))
         with progress:
             progress.update(0)
 
             # Next are pairs of file metadata, data.
             while True:
                 try:
                     filemeta = next(objs)
                 except StopIteration:
                     break
 
                 for k in (b'location', b'path', b'size'):
                     if k not in filemeta:
                         raise error.Abort(_(b'remote file data missing key: %s')
                                           % k)
 
                 if filemeta[b'location'] == b'store':
                     vfs = repo.svfs
                 else:
                     raise error.Abort(_(b'invalid location for raw file data: '
                                         b'%s') % filemeta[b'location'])
 
                 bytesremaining = filemeta[b'size']
 
                 with vfs.open(filemeta[b'path'], b'wb') as fh:
                     while True:
                         try:
                             chunk = next(objs)
                         except StopIteration:
                             break
 
                         bytesremaining -= len(chunk)
 
                         if bytesremaining < 0:
                             raise error.Abort(_(
                                 b'received invalid number of bytes for file '
                                 b'data; expected %d, got extra') %
                                               filemeta[b'size'])
 
                         progress.increment(step=len(chunk))
                         fh.write(chunk)
 
                         try:
                             if chunk.islast:
                                 break
                         except AttributeError:
                             raise error.Abort(_(
                                 b'did not receive indefinite length bytestring '
                                 b'for file data'))
 
                 if bytesremaining:
                     raise error.Abort(_(b'received invalid number of bytes for'
                                         b'file data; expected %d got %d') %
                                       (filemeta[b'size'],
                                        filemeta[b'size'] - bytesremaining))
 
 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
     """Determine which changesets need to be pulled."""
 
     if heads:
         knownnode = repo.changelog.hasnode
         if all(knownnode(head) for head in heads):
             return heads, False, heads
 
     # TODO wire protocol version 2 is capable of more efficient discovery
     # than setdiscovery. Consider implementing something better.
     common, fetch, remoteheads = setdiscovery.findcommonheads(
         repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated)
 
     common = set(common)
     remoteheads = set(remoteheads)
 
     # If a remote head is filtered locally, put it back in the common set.
     # See the comment in exchange._pulldiscoverychangegroup() for more.
 
     if fetch and remoteheads:
         nodemap = repo.unfiltered().changelog.nodemap
 
         common |= {head for head in remoteheads if head in nodemap}
 
         if set(remoteheads).issubset(common):
             fetch = []
 
     common.discard(nullid)
 
     return common, fetch, remoteheads
 
 def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads):
     # TODO consider adding a step here where we obtain the DAG shape first
     # (or ask the server to slice changesets into chunks for us) so that
     # we can perform multiple fetches in batches. This will facilitate
     # resuming interrupted clones, higher server-side cache hit rates due
     # to smaller segments, etc.
     with remote.commandexecutor() as e:
         objs = e.callcommand(b'changesetdata', {
             b'revisions': [{
                 b'type': b'changesetdagrange',
                 b'roots': sorted(common),
                 b'heads': sorted(remoteheads),
             }],
             b'fields': {b'bookmarks', b'parents', b'phase', b'revision'},
         }).result()
 
         # The context manager waits on all response data when exiting. So
         # we need to remain in the context manager in order to stream data.
         return _processchangesetdata(repo, tr, objs)
 
 def _processchangesetdata(repo, tr, objs):
     repo.hook('prechangegroup', throw=True,
               **pycompat.strkwargs(tr.hookargs))
 
     urepo = repo.unfiltered()
     cl = urepo.changelog
 
     cl.delayupdate(tr)
 
     # The first emitted object is a header describing the data that
     # follows.
     meta = next(objs)
 
     progress = repo.ui.makeprogress(_('changesets'),
                                     unit=_('chunks'),
                                     total=meta.get(b'totalitems'))
 
     manifestnodes = {}
 
     def linkrev(node):
         repo.ui.debug('add changeset %s\n' % short(node))
         # Linkrev for changelog is always self.
         return len(cl)
 
     def onchangeset(cl, node):
         progress.increment()
 
         revision = cl.changelogrevision(node)
 
         # We need to preserve the mapping of changelog revision to node
         # so we can set the linkrev accordingly when manifests are added.
         manifestnodes[cl.rev(node)] = revision.manifest
 
     nodesbyphase = {phase: set() for phase in phases.phasenames}
     remotebookmarks = {}
 
     # addgroup() expects a 7-tuple describing revisions. This normalizes
     # the wire data to that format.
     #
     # This loop also aggregates non-revision metadata, such as phase
     # data.
     def iterrevisions():
         for cset in objs:
             node = cset[b'node']
 
             if b'phase' in cset:
                 nodesbyphase[cset[b'phase']].add(node)
 
             for mark in cset.get(b'bookmarks', []):
                 remotebookmarks[mark] = node
 
             # TODO add mechanism for extensions to examine records so they
             # can siphon off custom data fields.
 
             extrafields = {}
 
             for field, size in cset.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             # Some entries might only be metadata only updates.
             if b'revision' not in extrafields:
                 continue
 
             data = extrafields[b'revision']
 
             yield (
                 node,
                 cset[b'parents'][0],
                 cset[b'parents'][1],
                 # Linknode is always itself for changesets.
                 cset[b'node'],
                 # We always send full revisions. So delta base is not set.
                 nullid,
                 mdiff.trivialdiffheader(len(data)) + data,
                 # Flags not yet supported.
                 0,
             )
 
     added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr),
                         addrevisioncb=onchangeset)
 
     progress.complete()
 
     return {
         'added': added,
         'nodesbyphase': nodesbyphase,
         'bookmarks': remotebookmarks,
         'manifestnodes': manifestnodes,
     }
 
 def _fetchmanifests(repo, tr, remote, manifestnodes):
     rootmanifest = repo.manifestlog.getstorage(b'')
 
     # Some manifests can be shared between changesets. Filter out revisions
     # we already know about.
     fetchnodes = []
     linkrevs = {}
     seen = set()
 
     for clrev, node in sorted(manifestnodes.iteritems()):
         if node in seen:
             continue
 
         try:
             rootmanifest.rev(node)
         except error.LookupError:
             fetchnodes.append(node)
             linkrevs[node] = clrev
 
         seen.add(node)
 
     # TODO handle tree manifests
 
     # addgroup() expects 7-tuple describing revisions. This normalizes
     # the wire data to that format.
     def iterrevisions(objs, progress):
         for manifest in objs:
             node = manifest[b'node']
 
             extrafields = {}
 
             for field, size in manifest.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             if b'delta' in extrafields:
                 basenode = manifest[b'deltabasenode']
                 delta = extrafields[b'delta']
             elif b'revision' in extrafields:
                 basenode = nullid
                 revision = extrafields[b'revision']
                 delta = mdiff.trivialdiffheader(len(revision)) + revision
             else:
                 continue
 
             yield (
                 node,
                 manifest[b'parents'][0],
                 manifest[b'parents'][1],
                 # The value passed in is passed to the lookup function passed
                 # to addgroup(). We already have a map of manifest node to
                 # changelog revision number. So we just pass in the
                 # manifest node here and use linkrevs.__getitem__ as the
                 # resolution function.
                 node,
                 basenode,
                 delta,
                 # Flags not yet supported.
                 0
             )
 
             progress.increment()
 
     progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
                                     total=len(fetchnodes))
 
     commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
     batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
     # TODO make size configurable on client?
 
     # We send commands 1 at a time to the remote. This is not the most
     # efficient because we incur a round trip at the end of each batch.
     # However, the existing frame-based reactor keeps consuming server
     # data in the background. And this results in response data buffering
     # in memory. This can consume gigabytes of memory.
     # TODO send multiple commands in a request once background buffering
     # issues are resolved.
 
     added = []
 
     for i in pycompat.xrange(0, len(fetchnodes), batchsize):
         batch = [node for node in fetchnodes[i:i + batchsize]]
         if not batch:
             continue
 
         with remote.commandexecutor() as e:
             objs = e.callcommand(b'manifestdata', {
                 b'tree': b'',
                 b'nodes': batch,
                 b'fields': {b'parents', b'revision'},
                 b'haveparents': True,
             }).result()
 
             # Chomp off header object.
             next(objs)
 
             added.extend(rootmanifest.addgroup(
                 iterrevisions(objs, progress),
                 linkrevs.__getitem__,
                 weakref.proxy(tr)))
 
     progress.complete()
 
     return {
         'added': added,
         'linkrevs': linkrevs,
     }
 
 def _derivefilesfrommanifests(repo, matcher, manifestnodes):
     """Determine what file nodes are relevant given a set of manifest nodes.
 
     Returns a dict mapping file paths to dicts of file node to first manifest
     node.
     """
     ml = repo.manifestlog
     fnodes = collections.defaultdict(dict)
 
     progress = repo.ui.makeprogress(
         _('scanning manifests'), total=len(manifestnodes))
 
     with progress:
         for manifestnode in manifestnodes:
             m = ml.get(b'', manifestnode)
 
             # TODO this will pull in unwanted nodes because it takes the storage
             # delta into consideration. What we really want is something that
             # takes the delta between the manifest's parents. And ideally we
             # would ignore file nodes that are known locally. For now, ignore
             # both these limitations. This will result in incremental fetches
             # requesting data we already have. So this is far from ideal.
             md = m.readfast()
 
             for path, fnode in md.items():
                 if matcher(path):
                     fnodes[path].setdefault(fnode, manifestnode)
 
             progress.increment()
 
     return fnodes
 
 def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
     """Fetch file data from explicit file revisions."""
     def iterrevisions(objs, progress):
         for filerevision in objs:
             node = filerevision[b'node']
 
             extrafields = {}
 
             for field, size in filerevision.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             if b'delta' in extrafields:
                 basenode = filerevision[b'deltabasenode']
                 delta = extrafields[b'delta']
             elif b'revision' in extrafields:
                 basenode = nullid
                 revision = extrafields[b'revision']
                 delta = mdiff.trivialdiffheader(len(revision)) + revision
             else:
                 continue
 
             yield (
                 node,
                 filerevision[b'parents'][0],
                 filerevision[b'parents'][1],
                 node,
                 basenode,
                 delta,
                 # Flags not yet supported.
                 0,
             )
 
             progress.increment()
 
     progress = repo.ui.makeprogress(
         _('files'), unit=_('chunks'),
          total=sum(len(v) for v in fnodes.itervalues()))
 
     # TODO make batch size configurable
     batchsize = 10000
     fnodeslist = [x for x in sorted(fnodes.items())]
 
     for i in pycompat.xrange(0, len(fnodeslist), batchsize):
         batch = [x for x in fnodeslist[i:i + batchsize]]
         if not batch:
             continue
 
         with remote.commandexecutor() as e:
             fs = []
             locallinkrevs = {}
 
             for path, nodes in batch:
                 fs.append((path, e.callcommand(b'filedata', {
                     b'path': path,
                     b'nodes': sorted(nodes),
                     b'fields': {b'parents', b'revision'},
                     b'haveparents': True,
                 })))
 
                 locallinkrevs[path] = {
                     node: linkrevs[manifestnode]
                     for node, manifestnode in nodes.iteritems()}
 
             for path, f in fs:
                 objs = f.result()
 
                 # Chomp off header objects.
                 next(objs)
 
                 store = repo.file(path)
                 store.addgroup(
                     iterrevisions(objs, progress),
                     locallinkrevs[path].__getitem__,
                     weakref.proxy(tr))
 
 def _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csets,
                          manlinkrevs, shallow=False):
     """Fetch file data from explicit changeset revisions."""
 
     def iterrevisions(objs, remaining, progress):
         while remaining:
             filerevision = next(objs)
 
             node = filerevision[b'node']
 
             extrafields = {}
 
             for field, size in filerevision.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             if b'delta' in extrafields:
                 basenode = filerevision[b'deltabasenode']
                 delta = extrafields[b'delta']
             elif b'revision' in extrafields:
                 basenode = nullid
                 revision = extrafields[b'revision']
                 delta = mdiff.trivialdiffheader(len(revision)) + revision
             else:
                 continue
 
             if b'linknode' in filerevision:
                 linknode = filerevision[b'linknode']
             else:
                 linknode = node
 
             yield (
                 node,
                 filerevision[b'parents'][0],
                 filerevision[b'parents'][1],
                 linknode,
                 basenode,
                 delta,
                 # Flags not yet supported.
                 0,
             )
 
             progress.increment()
             remaining -= 1
 
     progress = repo.ui.makeprogress(
         _('files'), unit=_('chunks'),
         total=sum(len(v) for v in fnodes.itervalues()))
 
     commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
     batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
 
     shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features
     fields = {b'parents', b'revision'}
     clrev = repo.changelog.rev
 
     # There are no guarantees that we'll have ancestor revisions if
     # a) this repo has shallow file storage b) shallow data fetching is enabled.
     # Force remote to not delta against possibly unknown revisions when these
     # conditions hold.
     haveparents = not (shallowfiles or shallow)
 
     # Similarly, we may not have calculated linkrevs for all incoming file
     # revisions. Ask the remote to do work for us in this case.
     if not haveparents:
         fields.add(b'linknode')
 
     for i in pycompat.xrange(0, len(csets), batchsize):
         batch = [x for x in csets[i:i + batchsize]]
         if not batch:
             continue
 
         with remote.commandexecutor() as e:
             args = {
                 b'revisions': [{
                     b'type': b'changesetexplicit',
                     b'nodes': batch,
                 }],
                 b'fields': fields,
                 b'haveparents': haveparents,
             }
 
             if pathfilter:
                 args[b'pathfilter'] = pathfilter
 
             objs = e.callcommand(b'filesdata', args).result()
 
             # First object is an overall header.
             overall = next(objs)
 
             # We have overall['totalpaths'] segments.
             for i in pycompat.xrange(overall[b'totalpaths']):
                 header = next(objs)
 
                 path = header[b'path']
                 store = repo.file(path)
 
                 linkrevs = {
                     fnode: manlinkrevs[mnode]
                     for fnode, mnode in fnodes[path].iteritems()}
 
                 def getlinkrev(node):
                     if node in linkrevs:
                         return linkrevs[node]
                     else:
                         return clrev(node)
 
                 store.addgroup(iterrevisions(objs, header[b'totalitems'],
                                              progress),
                                getlinkrev,
                                weakref.proxy(tr),
                                maybemissingparents=shallow)

mercurial/filelog.py

@@ -1,242 +1,244
 # filelog.py - file history class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from .i18n import _
 from .node import (
     nullid,
     nullrev,
 )
 from . import (
     error,
+    revlog,
+)
+from .interfaces import (
     repository,
-    revlog,
 )
 from .utils import (
     interfaceutil,
     storageutil,
 )
 
 @interfaceutil.implementer(repository.ifilestorage)
 class filelog(object):
     def __init__(self, opener, path):
         self._revlog = revlog.revlog(opener,
                                      '/'.join(('data', path + '.i')),
                                      censorable=True)
         # Full name of the user visible file, relative to the repository root.
         # Used by LFS.
         self._revlog.filename = path
 
     def __len__(self):
         return len(self._revlog)
 
     def __iter__(self):
         return self._revlog.__iter__()
 
     def hasnode(self, node):
         if node in (nullid, nullrev):
             return False
 
         try:
             self._revlog.rev(node)
             return True
         except (TypeError, ValueError, IndexError, error.LookupError):
             return False
 
     def revs(self, start=0, stop=None):
         return self._revlog.revs(start=start, stop=stop)
 
     def parents(self, node):
         return self._revlog.parents(node)
 
     def parentrevs(self, rev):
         return self._revlog.parentrevs(rev)
 
     def rev(self, node):
         return self._revlog.rev(node)
 
     def node(self, rev):
         return self._revlog.node(rev)
 
     def lookup(self, node):
         return storageutil.fileidlookup(self._revlog, node,
                                         self._revlog.indexfile)
 
     def linkrev(self, rev):
         return self._revlog.linkrev(rev)
 
     def commonancestorsheads(self, node1, node2):
         return self._revlog.commonancestorsheads(node1, node2)
 
     # Used by dagop.blockdescendants().
     def descendants(self, revs):
         return self._revlog.descendants(revs)
 
     def heads(self, start=None, stop=None):
         return self._revlog.heads(start, stop)
 
     # Used by hgweb, children extension.
     def children(self, node):
         return self._revlog.children(node)
 
     def iscensored(self, rev):
         return self._revlog.iscensored(rev)
 
     def revision(self, node, _df=None, raw=False):
         return self._revlog.revision(node, _df=_df, raw=raw)
 
     def rawdata(self, node, _df=None):
         return self._revlog.rawdata(node, _df=_df)
 
     def emitrevisions(self, nodes, nodesorder=None,
                       revisiondata=False, assumehaveparentrevisions=False,
                       deltamode=repository.CG_DELTAMODE_STD):
         return self._revlog.emitrevisions(
             nodes, nodesorder=nodesorder, revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode)
 
     def addrevision(self, revisiondata, transaction, linkrev, p1, p2,
                     node=None, flags=revlog.REVIDX_DEFAULT_FLAGS,
                     cachedelta=None):
         return self._revlog.addrevision(revisiondata, transaction, linkrev,
                                     p1, p2, node=node, flags=flags,
                                     cachedelta=cachedelta)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None,
                  maybemissingparents=False):
         if maybemissingparents:
             raise error.Abort(_('revlog storage does not support missing '
                                 'parents write mode'))
 
         return self._revlog.addgroup(deltas, linkmapper, transaction,
                                      addrevisioncb=addrevisioncb)
 
     def getstrippoint(self, minlink):
         return self._revlog.getstrippoint(minlink)
 
     def strip(self, minlink, transaction):
         return self._revlog.strip(minlink, transaction)
 
     def censorrevision(self, tr, node, tombstone=b''):
         return self._revlog.censorrevision(tr, node, tombstone=tombstone)
 
     def files(self):
         return self._revlog.files()
 
     def read(self, node):
         return storageutil.filtermetadata(self.revision(node))
 
     def add(self, text, meta, transaction, link, p1=None, p2=None):
         if meta or text.startswith('\1\n'):
             text = storageutil.packmeta(meta, text)
         return self.addrevision(text, transaction, link, p1, p2)
 
     def renamed(self, node):
         return storageutil.filerevisioncopied(self, node)
 
     def size(self, rev):
         """return the size of a given revision"""
 
         # for revisions with renames, we have to go the slow way
         node = self.node(rev)
         if self.renamed(node):
             return len(self.read(node))
         if self.iscensored(rev):
             return 0
 
         # XXX if self.read(node).startswith("\1\n"), this returns (size+4)
         return self._revlog.size(rev)
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
         return not storageutil.filedataequivalent(self, node, text)
 
     def verifyintegrity(self, state):
         return self._revlog.verifyintegrity(state)
 
     def storageinfo(self, exclusivefiles=False, sharedfiles=False,
                     revisionscount=False, trackedsize=False,
                     storedsize=False):
         return self._revlog.storageinfo(
             exclusivefiles=exclusivefiles, sharedfiles=sharedfiles,
             revisionscount=revisionscount, trackedsize=trackedsize,
             storedsize=storedsize)
 
     # TODO these aren't part of the interface and aren't internal methods.
     # Callers should be fixed to not use them.
 
     # Used by bundlefilelog, unionfilelog.
     @property
     def indexfile(self):
         return self._revlog.indexfile
 
     @indexfile.setter
     def indexfile(self, value):
         self._revlog.indexfile = value
 
     # Used by repo upgrade.
     def clone(self, tr, destrevlog, **kwargs):
         if not isinstance(destrevlog, filelog):
             raise error.ProgrammingError('expected filelog to clone()')
 
         return self._revlog.clone(tr, destrevlog._revlog, **kwargs)
 
 class narrowfilelog(filelog):
     """Filelog variation to be used with narrow stores."""
 
     def __init__(self, opener, path, narrowmatch):
         super(narrowfilelog, self).__init__(opener, path)
         self._narrowmatch = narrowmatch
 
     def renamed(self, node):
         res = super(narrowfilelog, self).renamed(node)
 
         # Renames that come from outside the narrowspec are problematic
         # because we may lack the base text for the rename. This can result
         # in code attempting to walk the ancestry or compute a diff
         # encountering a missing revision. We address this by silently
         # removing rename metadata if the source file is outside the
         # narrow spec.
         #
         # A better solution would be to see if the base revision is available,
         # rather than assuming it isn't.
         #
         # An even better solution would be to teach all consumers of rename
         # metadata that the base revision may not be available.
         #
         # TODO consider better ways of doing this.
         if res and not self._narrowmatch(res[0]):
             return None
 
         return res
 
     def size(self, rev):
         # Because we have a custom renamed() that may lie, we need to call
         # the base renamed() to report accurate results.
         node = self.node(rev)
         if super(narrowfilelog, self).renamed(node):
             return len(self.read(node))
         else:
             return super(narrowfilelog, self).size(rev)
 
     def cmp(self, node, text):
         different = super(narrowfilelog, self).cmp(node, text)
 
         # Because renamed() may lie, we may get false positives for
         # different content. Check for this by comparing against the original
         # renamed() implementation.
         if different:
             if super(narrowfilelog, self).renamed(node):
                 t2 = self.read(node)
                 return t2 != text
 
         return different

mercurial/hg.py

@@ -1,1237 +1,1240
 # hg.py - repository classes for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import hashlib
 import os
 import shutil
 import stat
 
 from .i18n import _
 from .node import (
     nullid,
 )
 
 from . import (
     bookmarks,
     bundlerepo,
     cacheutil,
     cmdutil,
     destutil,
     discovery,
     error,
     exchange,
     extensions,
     httppeer,
     localrepo,
     lock,
     logcmdutil,
     logexchange,
     merge as mergemod,
     narrowspec,
     node,
     phases,
     pycompat,
-    repository as repositorymod,
     scmutil,
     sshpeer,
     statichttprepo,
     ui as uimod,
     unionrepo,
     url,
     util,
     verify as verifymod,
     vfs as vfsmod,
 )
 
+from .interfaces import (
+    repository as repositorymod,
+)
+
 release = lock.release
 
 # shared features
 sharedbookmarks = 'bookmarks'
 
 def _local(path):
     path = util.expandpath(util.urllocalpath(path))
 
     try:
         isfile = os.path.isfile(path)
     # Python 2 raises TypeError, Python 3 ValueError.
     except (TypeError, ValueError) as e:
         raise error.Abort(_('invalid path %s: %s') % (
             path, pycompat.bytestr(e)))
 
     return isfile and bundlerepo or localrepo
 
 def addbranchrevs(lrepo, other, branches, revs):
     peer = other.peer() # a courtesy to callers using a localrepo for other
     hashbranch, branches = branches
     if not hashbranch and not branches:
         x = revs or None
         if revs:
             y = revs[0]
         else:
             y = None
         return x, y
     if revs:
         revs = list(revs)
     else:
         revs = []
 
     if not peer.capable('branchmap'):
         if branches:
             raise error.Abort(_("remote branch lookup not supported"))
         revs.append(hashbranch)
         return revs, revs[0]
 
     with peer.commandexecutor() as e:
         branchmap = e.callcommand('branchmap', {}).result()
 
     def primary(branch):
         if branch == '.':
             if not lrepo:
                 raise error.Abort(_("dirstate branch not accessible"))
             branch = lrepo.dirstate.branch()
         if branch in branchmap:
             revs.extend(node.hex(r) for r in reversed(branchmap[branch]))
             return True
         else:
             return False
 
     for branch in branches:
         if not primary(branch):
             raise error.RepoLookupError(_("unknown branch '%s'") % branch)
     if hashbranch:
         if not primary(hashbranch):
             revs.append(hashbranch)
     return revs, revs[0]
 
 def parseurl(path, branches=None):
     '''parse url#branch, returning (url, (branch, branches))'''
 
     u = util.url(path)
     branch = None
     if u.fragment:
         branch = u.fragment
         u.fragment = None
     return bytes(u), (branch, branches or [])
 
 schemes = {
     'bundle': bundlerepo,
     'union': unionrepo,
     'file': _local,
     'http': httppeer,
     'https': httppeer,
     'ssh': sshpeer,
     'static-http': statichttprepo,
 }
 
 def _peerlookup(path):
     u = util.url(path)
     scheme = u.scheme or 'file'
     thing = schemes.get(scheme) or schemes['file']
     try:
         return thing(path)
     except TypeError:
         # we can't test callable(thing) because 'thing' can be an unloaded
         # module that implements __call__
         if not util.safehasattr(thing, 'instance'):
             raise
         return thing
 
 def islocal(repo):
     '''return true if repo (or path pointing to repo) is local'''
     if isinstance(repo, bytes):
         try:
             return _peerlookup(repo).islocal(repo)
         except AttributeError:
             return False
     return repo.local()
 
 def openpath(ui, path, sendaccept=True):
     '''open path with open if local, url.open if remote'''
     pathurl = util.url(path, parsequery=False, parsefragment=False)
     if pathurl.islocal():
         return util.posixfile(pathurl.localpath(), 'rb')
     else:
         return url.open(ui, path, sendaccept=sendaccept)
 
 # a list of (ui, repo) functions called for wire peer initialization
 wirepeersetupfuncs = []
 
 def _peerorrepo(ui, path, create=False, presetupfuncs=None,
                 intents=None, createopts=None):
     """return a repository object for the specified path"""
     obj = _peerlookup(path).instance(ui, path, create, intents=intents,
                                      createopts=createopts)
     ui = getattr(obj, "ui", ui)
     for f in presetupfuncs or []:
         f(ui, obj)
     ui.log(b'extension', b'- executing reposetup hooks\n')
     with util.timedcm('all reposetup') as allreposetupstats:
         for name, module in extensions.extensions(ui):
             ui.log(b'extension', b'  - running reposetup for %s\n', name)
             hook = getattr(module, 'reposetup', None)
             if hook:
                 with util.timedcm('reposetup %r', name) as stats:
                     hook(ui, obj)
                 ui.log(b'extension', b'  > reposetup for %s took %s\n',
                        name, stats)
     ui.log(b'extension', b'> all reposetup took %s\n', allreposetupstats)
     if not obj.local():
         for f in wirepeersetupfuncs:
             f(ui, obj)
     return obj
 
 def repository(ui, path='', create=False, presetupfuncs=None, intents=None,
                createopts=None):
     """return a repository object for the specified path"""
     peer = _peerorrepo(ui, path, create, presetupfuncs=presetupfuncs,
                        intents=intents, createopts=createopts)
     repo = peer.local()
     if not repo:
         raise error.Abort(_("repository '%s' is not local") %
                          (path or peer.url()))
     return repo.filtered('visible')
 
 def peer(uiorrepo, opts, path, create=False, intents=None, createopts=None):
     '''return a repository peer for the specified path'''
     rui = remoteui(uiorrepo, opts)
     return _peerorrepo(rui, path, create, intents=intents,
                        createopts=createopts).peer()
 
 def defaultdest(source):
     '''return default destination of clone if none is given
 
     >>> defaultdest(b'foo')
     'foo'
     >>> defaultdest(b'/foo/bar')
     'bar'
     >>> defaultdest(b'/')
     ''
     >>> defaultdest(b'')
     ''
     >>> defaultdest(b'http://example.org/')
     ''
     >>> defaultdest(b'http://example.org/foo/')
     'foo'
     '''
     path = util.url(source).path
     if not path:
         return ''
     return os.path.basename(os.path.normpath(path))
 
 def sharedreposource(repo):
     """Returns repository object for source repository of a shared repo.
 
     If repo is not a shared repository, returns None.
     """
     if repo.sharedpath == repo.path:
         return None
 
     if util.safehasattr(repo, 'srcrepo') and repo.srcrepo:
         return repo.srcrepo
 
     # the sharedpath always ends in the .hg; we want the path to the repo
     source = repo.vfs.split(repo.sharedpath)[0]
     srcurl, branches = parseurl(source)
     srcrepo = repository(repo.ui, srcurl)
     repo.srcrepo = srcrepo
     return srcrepo
 
 def share(ui, source, dest=None, update=True, bookmarks=True, defaultpath=None,
           relative=False):
     '''create a shared repository'''
 
     if not islocal(source):
         raise error.Abort(_('can only share local repositories'))
 
     if not dest:
         dest = defaultdest(source)
     else:
         dest = ui.expandpath(dest)
 
     if isinstance(source, bytes):
         origsource = ui.expandpath(source)
         source, branches = parseurl(origsource)
         srcrepo = repository(ui, source)
         rev, checkout = addbranchrevs(srcrepo, srcrepo, branches, None)
     else:
         srcrepo = source.local()
         checkout = None
 
     shareditems = set()
     if bookmarks:
         shareditems.add(sharedbookmarks)
 
     r = repository(ui, dest, create=True, createopts={
         'sharedrepo': srcrepo,
         'sharedrelative': relative,
         'shareditems': shareditems,
     })
 
     postshare(srcrepo, r, defaultpath=defaultpath)
     r = repository(ui, dest)
     _postshareupdate(r, update, checkout=checkout)
     return r
 
 def unshare(ui, repo):
     """convert a shared repository to a normal one
 
     Copy the store data to the repo and remove the sharedpath data.
 
     Returns a new repository object representing the unshared repository.
 
     The passed repository object is not usable after this function is
     called.
     """
 
     with repo.lock():
         # we use locks here because if we race with commit, we
         # can end up with extra data in the cloned revlogs that's
         # not pointed to by changesets, thus causing verify to
         # fail
         destlock = copystore(ui, repo, repo.path)
         with destlock or util.nullcontextmanager():
 
             sharefile = repo.vfs.join('sharedpath')
             util.rename(sharefile, sharefile + '.old')
 
             repo.requirements.discard('shared')
             repo.requirements.discard('relshared')
             repo._writerequirements()
 
     # Removing share changes some fundamental properties of the repo instance.
     # So we instantiate a new repo object and operate on it rather than
     # try to keep the existing repo usable.
     newrepo = repository(repo.baseui, repo.root, create=False)
 
     # TODO: figure out how to access subrepos that exist, but were previously
     #       removed from .hgsub
     c = newrepo['.']
     subs = c.substate
     for s in sorted(subs):
         c.sub(s).unshare()
 
     localrepo.poisonrepository(repo)
 
     return newrepo
 
 def postshare(sourcerepo, destrepo, defaultpath=None):
     """Called after a new shared repo is created.
 
     The new repo only has a requirements file and pointer to the source.
     This function configures additional shared data.
 
     Extensions can wrap this function and write additional entries to
     destrepo/.hg/shared to indicate additional pieces of data to be shared.
     """
     default = defaultpath or sourcerepo.ui.config('paths', 'default')
     if default:
         template = ('[paths]\n'
                     'default = %s\n')
         destrepo.vfs.write('hgrc', util.tonativeeol(template % default))
     if repositorymod.NARROW_REQUIREMENT in sourcerepo.requirements:
         with destrepo.wlock():
             narrowspec.copytoworkingcopy(destrepo)
 
 def _postshareupdate(repo, update, checkout=None):
     """Maybe perform a working directory update after a shared repo is created.
 
     ``update`` can be a boolean or a revision to update to.
     """
     if not update:
         return
 
     repo.ui.status(_("updating working directory\n"))
     if update is not True:
         checkout = update
     for test in (checkout, 'default', 'tip'):
         if test is None:
             continue
         try:
             uprev = repo.lookup(test)
             break
         except error.RepoLookupError:
             continue
     _update(repo, uprev)
 
 def copystore(ui, srcrepo, destpath):
     '''copy files from store of srcrepo in destpath
 
     returns destlock
     '''
     destlock = None
     try:
         hardlink = None
         topic = _('linking') if hardlink else _('copying')
         with ui.makeprogress(topic, unit=_('files')) as progress:
             num = 0
             srcpublishing = srcrepo.publishing()
             srcvfs = vfsmod.vfs(srcrepo.sharedpath)
             dstvfs = vfsmod.vfs(destpath)
             for f in srcrepo.store.copylist():
                 if srcpublishing and f.endswith('phaseroots'):
                     continue
                 dstbase = os.path.dirname(f)
                 if dstbase and not dstvfs.exists(dstbase):
                     dstvfs.mkdir(dstbase)
                 if srcvfs.exists(f):
                     if f.endswith('data'):
                         # 'dstbase' may be empty (e.g. revlog format 0)
                         lockfile = os.path.join(dstbase, "lock")
                         # lock to avoid premature writing to the target
                         destlock = lock.lock(dstvfs, lockfile)
                     hardlink, n = util.copyfiles(srcvfs.join(f), dstvfs.join(f),
                                                  hardlink, progress)
                     num += n
             if hardlink:
                 ui.debug("linked %d files\n" % num)
             else:
                 ui.debug("copied %d files\n" % num)
         return destlock
     except: # re-raises
         release(destlock)
         raise
 
 def clonewithshare(ui, peeropts, sharepath, source, srcpeer, dest, pull=False,
                    rev=None, update=True, stream=False):
     """Perform a clone using a shared repo.
 
     The store for the repository will be located at <sharepath>/.hg. The
     specified revisions will be cloned or pulled from "source". A shared repo
     will be created at "dest" and a working copy will be created if "update" is
     True.
     """
     revs = None
     if rev:
         if not srcpeer.capable('lookup'):
             raise error.Abort(_("src repository does not support "
                                "revision lookup and so doesn't "
                                "support clone by revision"))
 
         # TODO this is batchable.
         remoterevs = []
         for r in rev:
             with srcpeer.commandexecutor() as e:
                 remoterevs.append(e.callcommand('lookup', {
                     'key': r,
                 }).result())
         revs = remoterevs
 
     # Obtain a lock before checking for or cloning the pooled repo otherwise
     # 2 clients may race creating or populating it.
     pooldir = os.path.dirname(sharepath)
     # lock class requires the directory to exist.
     try:
         util.makedir(pooldir, False)
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
 
     poolvfs = vfsmod.vfs(pooldir)
     basename = os.path.basename(sharepath)
 
     with lock.lock(poolvfs, '%s.lock' % basename):
         if os.path.exists(sharepath):
             ui.status(_('(sharing from existing pooled repository %s)\n') %
                       basename)
         else:
             ui.status(_('(sharing from new pooled repository %s)\n') % basename)
             # Always use pull mode because hardlinks in share mode don't work
             # well. Never update because working copies aren't necessary in
             # share mode.
             clone(ui, peeropts, source, dest=sharepath, pull=True,
                   revs=rev, update=False, stream=stream)
 
     # Resolve the value to put in [paths] section for the source.
     if islocal(source):
         defaultpath = os.path.abspath(util.urllocalpath(source))
     else:
         defaultpath = source
 
     sharerepo = repository(ui, path=sharepath)
     destrepo = share(ui, sharerepo, dest=dest, update=False, bookmarks=False,
                      defaultpath=defaultpath)
 
     # We need to perform a pull against the dest repo to fetch bookmarks
     # and other non-store data that isn't shared by default. In the case of
     # non-existing shared repo, this means we pull from the remote twice. This
     # is a bit weird. But at the time it was implemented, there wasn't an easy
     # way to pull just non-changegroup data.
     exchange.pull(destrepo, srcpeer, heads=revs)
 
     _postshareupdate(destrepo, update)
 
     return srcpeer, peer(ui, peeropts, dest)
 
 # Recomputing branch cache might be slow on big repos,
 # so just copy it
 def _copycache(srcrepo, dstcachedir, fname):
     """copy a cache from srcrepo to destcachedir (if it exists)"""
     srcbranchcache = srcrepo.vfs.join('cache/%s' % fname)
     dstbranchcache = os.path.join(dstcachedir, fname)
     if os.path.exists(srcbranchcache):
         if not os.path.exists(dstcachedir):
             os.mkdir(dstcachedir)
         util.copyfile(srcbranchcache, dstbranchcache)
 
 def clone(ui, peeropts, source, dest=None, pull=False, revs=None,
           update=True, stream=False, branch=None, shareopts=None,
           storeincludepats=None, storeexcludepats=None, depth=None):
     """Make a copy of an existing repository.
 
     Create a copy of an existing repository in a new directory.  The
     source and destination are URLs, as passed to the repository
     function.  Returns a pair of repository peers, the source and
     newly created destination.
 
     The location of the source is added to the new repository's
     .hg/hgrc file, as the default to be used for future pulls and
     pushes.
 
     If an exception is raised, the partly cloned/updated destination
     repository will be deleted.
 
     Arguments:
 
     source: repository object or URL
 
     dest: URL of destination repository to create (defaults to base
     name of source repository)
 
     pull: always pull from source repository, even in local case or if the
     server prefers streaming
 
     stream: stream raw data uncompressed from repository (fast over
     LAN, slow over WAN)
 
     revs: revision to clone up to (implies pull=True)
 
     update: update working directory after clone completes, if
     destination is local repository (True means update to default rev,
     anything else is treated as a revision)
 
     branch: branches to clone
 
     shareopts: dict of options to control auto sharing behavior. The "pool" key
     activates auto sharing mode and defines the directory for stores. The
     "mode" key determines how to construct the directory name of the shared
     repository. "identity" means the name is derived from the node of the first
     changeset in the repository. "remote" means the name is derived from the
     remote's path/URL. Defaults to "identity."
 
     storeincludepats and storeexcludepats: sets of file patterns to include and
     exclude in the repository copy, respectively. If not defined, all files
     will be included (a "full" clone). Otherwise a "narrow" clone containing
     only the requested files will be performed. If ``storeincludepats`` is not
     defined but ``storeexcludepats`` is, ``storeincludepats`` is assumed to be
     ``path:.``. If both are empty sets, no files will be cloned.
     """
 
     if isinstance(source, bytes):
         origsource = ui.expandpath(source)
         source, branches = parseurl(origsource, branch)
         srcpeer = peer(ui, peeropts, source)
     else:
         srcpeer = source.peer() # in case we were called with a localrepo
         branches = (None, branch or [])
         origsource = source = srcpeer.url()
     revs, checkout = addbranchrevs(srcpeer, srcpeer, branches, revs)
 
     if dest is None:
         dest = defaultdest(source)
         if dest:
             ui.status(_("destination directory: %s\n") % dest)
     else:
         dest = ui.expandpath(dest)
 
     dest = util.urllocalpath(dest)
     source = util.urllocalpath(source)
 
     if not dest:
         raise error.Abort(_("empty destination path is not valid"))
 
     destvfs = vfsmod.vfs(dest, expandpath=True)
     if destvfs.lexists():
         if not destvfs.isdir():
             raise error.Abort(_("destination '%s' already exists") % dest)
         elif destvfs.listdir():
             raise error.Abort(_("destination '%s' is not empty") % dest)
 
     createopts = {}
     narrow = False
 
     if storeincludepats is not None:
         narrowspec.validatepatterns(storeincludepats)
         narrow = True
 
     if storeexcludepats is not None:
         narrowspec.validatepatterns(storeexcludepats)
         narrow = True
 
     if narrow:
         # Include everything by default if only exclusion patterns defined.
         if storeexcludepats and not storeincludepats:
             storeincludepats = {'path:.'}
 
         createopts['narrowfiles'] = True
 
     if depth:
         createopts['shallowfilestore'] = True
 
     if srcpeer.capable(b'lfs-serve'):
         # Repository creation honors the config if it disabled the extension, so
         # we can't just announce that lfs will be enabled.  This check avoids
         # saying that lfs will be enabled, and then saying it's an unknown
         # feature.  The lfs creation option is set in either case so that a
         # requirement is added.  If the extension is explicitly disabled but the
         # requirement is set, the clone aborts early, before transferring any
         # data.
         createopts['lfs'] = True
 
         if extensions.disabledext('lfs'):
             ui.status(_('(remote is using large file support (lfs), but it is '
                         'explicitly disabled in the local configuration)\n'))
         else:
             ui.status(_('(remote is using large file support (lfs); lfs will '
                         'be enabled for this repository)\n'))
 
     shareopts = shareopts or {}
     sharepool = shareopts.get('pool')
     sharenamemode = shareopts.get('mode')
     if sharepool and islocal(dest):
         sharepath = None
         if sharenamemode == 'identity':
             # Resolve the name from the initial changeset in the remote
             # repository. This returns nullid when the remote is empty. It
             # raises RepoLookupError if revision 0 is filtered or otherwise
             # not available. If we fail to resolve, sharing is not enabled.
             try:
                 with srcpeer.commandexecutor() as e:
                     rootnode = e.callcommand('lookup', {
                         'key': '0',
                     }).result()
 
                 if rootnode != node.nullid:
                     sharepath = os.path.join(sharepool, node.hex(rootnode))
                 else:
                     ui.status(_('(not using pooled storage: '
                                 'remote appears to be empty)\n'))
             except error.RepoLookupError:
                 ui.status(_('(not using pooled storage: '
                             'unable to resolve identity of remote)\n'))
         elif sharenamemode == 'remote':
             sharepath = os.path.join(
                 sharepool, node.hex(hashlib.sha1(source).digest()))
         else:
             raise error.Abort(_('unknown share naming mode: %s') %
                               sharenamemode)
 
         # TODO this is a somewhat arbitrary restriction.
         if narrow:
             ui.status(_('(pooled storage not supported for narrow clones)\n'))
             sharepath = None
 
         if sharepath:
             return clonewithshare(ui, peeropts, sharepath, source, srcpeer,
                                   dest, pull=pull, rev=revs, update=update,
                                   stream=stream)
 
     srclock = destlock = cleandir = None
     srcrepo = srcpeer.local()
     try:
         abspath = origsource
         if islocal(origsource):
             abspath = os.path.abspath(util.urllocalpath(origsource))
 
         if islocal(dest):
             cleandir = dest
 
         copy = False
         if (srcrepo and srcrepo.cancopy() and islocal(dest)
             and not phases.hassecret(srcrepo)):
             copy = not pull and not revs
 
         # TODO this is a somewhat arbitrary restriction.
         if narrow:
             copy = False
 
         if copy:
             try:
                 # we use a lock here because if we race with commit, we
                 # can end up with extra data in the cloned revlogs that's
                 # not pointed to by changesets, thus causing verify to
                 # fail
                 srclock = srcrepo.lock(wait=False)
             except error.LockError:
                 copy = False
 
         if copy:
             srcrepo.hook('preoutgoing', throw=True, source='clone')
             hgdir = os.path.realpath(os.path.join(dest, ".hg"))
             if not os.path.exists(dest):
                 util.makedirs(dest)
             else:
                 # only clean up directories we create ourselves
                 cleandir = hgdir
             try:
                 destpath = hgdir
                 util.makedir(destpath, notindexed=True)
             except OSError as inst:
                 if inst.errno == errno.EEXIST:
                     cleandir = None
                     raise error.Abort(_("destination '%s' already exists")
                                      % dest)
                 raise
 
             destlock = copystore(ui, srcrepo, destpath)
             # copy bookmarks over
             srcbookmarks = srcrepo.vfs.join('bookmarks')
             dstbookmarks = os.path.join(destpath, 'bookmarks')
             if os.path.exists(srcbookmarks):
                 util.copyfile(srcbookmarks, dstbookmarks)
 
             dstcachedir = os.path.join(destpath, 'cache')
             for cache in cacheutil.cachetocopy(srcrepo):
                 _copycache(srcrepo, dstcachedir, cache)
 
             # we need to re-init the repo after manually copying the data
             # into it
             destpeer = peer(srcrepo, peeropts, dest)
             srcrepo.hook('outgoing', source='clone',
                           node=node.hex(node.nullid))
         else:
             try:
                 # only pass ui when no srcrepo
                 destpeer = peer(srcrepo or ui, peeropts, dest, create=True,
                                 createopts=createopts)
             except OSError as inst:
                 if inst.errno == errno.EEXIST:
                     cleandir = None
                     raise error.Abort(_("destination '%s' already exists")
                                      % dest)
                 raise
 
             if revs:
                 if not srcpeer.capable('lookup'):
                     raise error.Abort(_("src repository does not support "
                                        "revision lookup and so doesn't "
                                        "support clone by revision"))
 
                 # TODO this is batchable.
                 remoterevs = []
                 for rev in revs:
                     with srcpeer.commandexecutor() as e:
                         remoterevs.append(e.callcommand('lookup', {
                             'key': rev,
                         }).result())
                 revs = remoterevs
 
                 checkout = revs[0]
             else:
                 revs = None
             local = destpeer.local()
             if local:
                 if narrow:
                     with local.wlock(), local.lock():
                         local.setnarrowpats(storeincludepats, storeexcludepats)
                         narrowspec.copytoworkingcopy(local)
 
                 u = util.url(abspath)
                 defaulturl = bytes(u)
                 local.ui.setconfig('paths', 'default', defaulturl, 'clone')
                 if not stream:
                     if pull:
                         stream = False
                     else:
                         stream = None
                 # internal config: ui.quietbookmarkmove
                 overrides = {('ui', 'quietbookmarkmove'): True}
                 with local.ui.configoverride(overrides, 'clone'):
                     exchange.pull(local, srcpeer, revs,
                                   streamclonerequested=stream,
                                   includepats=storeincludepats,
                                   excludepats=storeexcludepats,
                                   depth=depth)
             elif srcrepo:
                 # TODO lift restriction once exchange.push() accepts narrow
                 # push.
                 if narrow:
                     raise error.Abort(_('narrow clone not available for '
                                         'remote destinations'))
 
                 exchange.push(srcrepo, destpeer, revs=revs,
                               bookmarks=srcrepo._bookmarks.keys())
             else:
                 raise error.Abort(_("clone from remote to remote not supported")
                                  )
 
         cleandir = None
 
         destrepo = destpeer.local()
         if destrepo:
             template = uimod.samplehgrcs['cloned']
             u = util.url(abspath)
             u.passwd = None
             defaulturl = bytes(u)
             destrepo.vfs.write('hgrc', util.tonativeeol(template % defaulturl))
             destrepo.ui.setconfig('paths', 'default', defaulturl, 'clone')
 
             if ui.configbool('experimental', 'remotenames'):
                 logexchange.pullremotenames(destrepo, srcpeer)
 
             if update:
                 if update is not True:
                     with srcpeer.commandexecutor() as e:
                         checkout = e.callcommand('lookup', {
                             'key': update,
                         }).result()
 
                 uprev = None
                 status = None
                 if checkout is not None:
                     # Some extensions (at least hg-git and hg-subversion) have
                     # a peer.lookup() implementation that returns a name instead
                     # of a nodeid. We work around it here until we've figured
                     # out a better solution.
                     if len(checkout) == 20 and checkout in destrepo:
                         uprev = checkout
                     elif scmutil.isrevsymbol(destrepo, checkout):
                         uprev = scmutil.revsymbol(destrepo, checkout).node()
                     else:
                         if update is not True:
                             try:
                                 uprev = destrepo.lookup(update)
                             except error.RepoLookupError:
                                 pass
                 if uprev is None:
                     try:
                         uprev = destrepo._bookmarks['@']
                         update = '@'
                         bn = destrepo[uprev].branch()
                         if bn == 'default':
                             status = _("updating to bookmark @\n")
                         else:
                             status = (_("updating to bookmark @ on branch %s\n")
                                       % bn)
                     except KeyError:
                         try:
                             uprev = destrepo.branchtip('default')
                         except error.RepoLookupError:
                             uprev = destrepo.lookup('tip')
                 if not status:
                     bn = destrepo[uprev].branch()
                     status = _("updating to branch %s\n") % bn
                 destrepo.ui.status(status)
                 _update(destrepo, uprev)
                 if update in destrepo._bookmarks:
                     bookmarks.activate(destrepo, update)
     finally:
         release(srclock, destlock)
         if cleandir is not None:
             shutil.rmtree(cleandir, True)
         if srcpeer is not None:
             srcpeer.close()
     return srcpeer, destpeer
 
 def _showstats(repo, stats, quietempty=False):
     if quietempty and stats.isempty():
         return
     repo.ui.status(_("%d files updated, %d files merged, "
                      "%d files removed, %d files unresolved\n") % (
                    stats.updatedcount, stats.mergedcount,
                    stats.removedcount, stats.unresolvedcount))
 
 def updaterepo(repo, node, overwrite, updatecheck=None):
     """Update the working directory to node.
 
     When overwrite is set, changes are clobbered, merged else
 
     returns stats (see pydoc mercurial.merge.applyupdates)"""
     return mergemod.update(repo, node, branchmerge=False, force=overwrite,
                            labels=['working copy', 'destination'],
                            updatecheck=updatecheck)
 
 def update(repo, node, quietempty=False, updatecheck=None):
     """update the working directory to node"""
     stats = updaterepo(repo, node, False, updatecheck=updatecheck)
     _showstats(repo, stats, quietempty)
     if stats.unresolvedcount:
         repo.ui.status(_("use 'hg resolve' to retry unresolved file merges\n"))
     return stats.unresolvedcount > 0
 
 # naming conflict in clone()
 _update = update
 
 def clean(repo, node, show_stats=True, quietempty=False):
     """forcibly switch the working directory to node, clobbering changes"""
     stats = updaterepo(repo, node, True)
     repo.vfs.unlinkpath('graftstate', ignoremissing=True)
     if show_stats:
         _showstats(repo, stats, quietempty)
     return stats.unresolvedcount > 0
 
 # naming conflict in updatetotally()
 _clean = clean
 
 def updatetotally(ui, repo, checkout, brev, clean=False, updatecheck=None):
     """Update the working directory with extra care for non-file components
 
     This takes care of non-file components below:
 
     :bookmark: might be advanced or (in)activated
 
     This takes arguments below:
 
     :checkout: to which revision the working directory is updated
     :brev: a name, which might be a bookmark to be activated after updating
     :clean: whether changes in the working directory can be discarded
     :updatecheck: how to deal with a dirty working directory
 
     Valid values for updatecheck are (None => linear):
 
      * abort: abort if the working directory is dirty
      * none: don't check (merge working directory changes into destination)
      * linear: check that update is linear before merging working directory
                changes into destination
      * noconflict: check that the update does not result in file merges
 
     This returns whether conflict is detected at updating or not.
     """
     if updatecheck is None:
         updatecheck = ui.config('commands', 'update.check')
         if updatecheck not in ('abort', 'none', 'linear', 'noconflict'):
             # If not configured, or invalid value configured
             updatecheck = 'linear'
     with repo.wlock():
         movemarkfrom = None
         warndest = False
         if checkout is None:
             updata = destutil.destupdate(repo, clean=clean)
             checkout, movemarkfrom, brev = updata
             warndest = True
 
         if clean:
             ret = _clean(repo, checkout)
         else:
             if updatecheck == 'abort':
                 cmdutil.bailifchanged(repo, merge=False)
                 updatecheck = 'none'
             ret = _update(repo, checkout, updatecheck=updatecheck)
 
         if not ret and movemarkfrom:
             if movemarkfrom == repo['.'].node():
                 pass # no-op update
             elif bookmarks.update(repo, [movemarkfrom], repo['.'].node()):
                 b = ui.label(repo._activebookmark, 'bookmarks.active')
                 ui.status(_("updating bookmark %s\n") % b)
             else:
                 # this can happen with a non-linear update
                 b = ui.label(repo._activebookmark, 'bookmarks')
                 ui.status(_("(leaving bookmark %s)\n") % b)
                 bookmarks.deactivate(repo)
         elif brev in repo._bookmarks:
             if brev != repo._activebookmark:
                 b = ui.label(brev, 'bookmarks.active')
                 ui.status(_("(activating bookmark %s)\n") % b)
             bookmarks.activate(repo, brev)
         elif brev:
             if repo._activebookmark:
                 b = ui.label(repo._activebookmark, 'bookmarks')
                 ui.status(_("(leaving bookmark %s)\n") % b)
             bookmarks.deactivate(repo)
 
         if warndest:
             destutil.statusotherdests(ui, repo)
 
     return ret
 
 def merge(repo, node, force=None, remind=True, mergeforce=False, labels=None,
           abort=False):
     """Branch merge with node, resolving changes. Return true if any
     unresolved conflicts."""
     if abort:
         return abortmerge(repo.ui, repo)
 
     stats = mergemod.update(repo, node, branchmerge=True, force=force,
                             mergeforce=mergeforce, labels=labels)
     _showstats(repo, stats)
     if stats.unresolvedcount:
         repo.ui.status(_("use 'hg resolve' to retry unresolved file merges "
                          "or 'hg merge --abort' to abandon\n"))
     elif remind:
         repo.ui.status(_("(branch merge, don't forget to commit)\n"))
     return stats.unresolvedcount > 0
 
 def abortmerge(ui, repo):
     ms = mergemod.mergestate.read(repo)
     if ms.active():
         # there were conflicts
         node = ms.localctx.hex()
     else:
         # there were no conficts, mergestate was not stored
         node = repo['.'].hex()
 
     repo.ui.status(_("aborting the merge, updating back to"
                      " %s\n") % node[:12])
     stats = mergemod.update(repo, node, branchmerge=False, force=True)
     _showstats(repo, stats)
     return stats.unresolvedcount > 0
 
 def _incoming(displaychlist, subreporecurse, ui, repo, source,
         opts, buffered=False):
     """
     Helper for incoming / gincoming.
     displaychlist gets called with
         (remoterepo, incomingchangesetlist, displayer) parameters,
     and is supposed to contain only code that can't be unified.
     """
     source, branches = parseurl(ui.expandpath(source), opts.get('branch'))
     other = peer(repo, opts, source)
     ui.status(_('comparing with %s\n') % util.hidepassword(source))
     revs, checkout = addbranchrevs(repo, other, branches, opts.get('rev'))
 
     if revs:
         revs = [other.lookup(rev) for rev in revs]
     other, chlist, cleanupfn = bundlerepo.getremotechanges(ui, repo, other,
                                 revs, opts["bundle"], opts["force"])
     try:
         if not chlist:
             ui.status(_("no changes found\n"))
             return subreporecurse()
         ui.pager('incoming')
         displayer = logcmdutil.changesetdisplayer(ui, other, opts,
                                                   buffered=buffered)
         displaychlist(other, chlist, displayer)
         displayer.close()
     finally:
         cleanupfn()
     subreporecurse()
     return 0 # exit code is zero since we found incoming changes
 
 def incoming(ui, repo, source, opts):
     def subreporecurse():
         ret = 1
         if opts.get('subrepos'):
             ctx = repo[None]
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 ret = min(ret, sub.incoming(ui, source, opts))
         return ret
 
     def display(other, chlist, displayer):
         limit = logcmdutil.getlimit(opts)
         if opts.get('newest_first'):
             chlist.reverse()
         count = 0
         for n in chlist:
             if limit is not None and count >= limit:
                 break
             parents = [p for p in other.changelog.parents(n) if p != nullid]
             if opts.get('no_merges') and len(parents) == 2:
                 continue
             count += 1
             displayer.show(other[n])
     return _incoming(display, subreporecurse, ui, repo, source, opts)
 
 def _outgoing(ui, repo, dest, opts):
     path = ui.paths.getpath(dest, default=('default-push', 'default'))
     if not path:
         raise error.Abort(_('default repository not configured!'),
                 hint=_("see 'hg help config.paths'"))
     dest = path.pushloc or path.loc
     branches = path.branch, opts.get('branch') or []
 
     ui.status(_('comparing with %s\n') % util.hidepassword(dest))
     revs, checkout = addbranchrevs(repo, repo, branches, opts.get('rev'))
     if revs:
         revs = [repo[rev].node() for rev in scmutil.revrange(repo, revs)]
 
     other = peer(repo, opts, dest)
     outgoing = discovery.findcommonoutgoing(repo, other, revs,
                                             force=opts.get('force'))
     o = outgoing.missing
     if not o:
         scmutil.nochangesfound(repo.ui, repo, outgoing.excluded)
     return o, other
 
 def outgoing(ui, repo, dest, opts):
     def recurse():
         ret = 1
         if opts.get('subrepos'):
             ctx = repo[None]
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 ret = min(ret, sub.outgoing(ui, dest, opts))
         return ret
 
     limit = logcmdutil.getlimit(opts)
     o, other = _outgoing(ui, repo, dest, opts)
     if not o:
         cmdutil.outgoinghooks(ui, repo, other, opts, o)
         return recurse()
 
     if opts.get('newest_first'):
         o.reverse()
     ui.pager('outgoing')
     displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
     count = 0
     for n in o:
         if limit is not None and count >= limit:
             break
         parents = [p for p in repo.changelog.parents(n) if p != nullid]
         if opts.get('no_merges') and len(parents) == 2:
             continue
         count += 1
         displayer.show(repo[n])
     displayer.close()
     cmdutil.outgoinghooks(ui, repo, other, opts, o)
     recurse()
     return 0 # exit code is zero since we found outgoing changes
 
 def verify(repo, level=None):
     """verify the consistency of a repository"""
     ret = verifymod.verify(repo, level=level)
 
     # Broken subrepo references in hidden csets don't seem worth worrying about,
     # since they can't be pushed/pulled, and --hidden can be used if they are a
     # concern.
 
     # pathto() is needed for -R case
     revs = repo.revs("filelog(%s)",
                      util.pathto(repo.root, repo.getcwd(), '.hgsubstate'))
 
     if revs:
         repo.ui.status(_('checking subrepo links\n'))
         for rev in revs:
             ctx = repo[rev]
             try:
                 for subpath in ctx.substate:
                     try:
                         ret = (ctx.sub(subpath, allowcreate=False).verify()
                                or ret)
                     except error.RepoError as e:
                         repo.ui.warn(('%d: %s\n') % (rev, e))
             except Exception:
                 repo.ui.warn(_('.hgsubstate is corrupt in revision %s\n') %
                              node.short(ctx.node()))
 
     return ret
 
 def remoteui(src, opts):
     'build a remote ui from ui or repo and opts'
     if util.safehasattr(src, 'baseui'): # looks like a repository
         dst = src.baseui.copy() # drop repo-specific config
         src = src.ui # copy target options from repo
     else: # assume it's a global ui object
         dst = src.copy() # keep all global options
 
     # copy ssh-specific options
     for o in 'ssh', 'remotecmd':
         v = opts.get(o) or src.config('ui', o)
         if v:
             dst.setconfig("ui", o, v, 'copied')
 
     # copy bundle-specific options
     r = src.config('bundle', 'mainreporoot')
     if r:
         dst.setconfig('bundle', 'mainreporoot', r, 'copied')
 
     # copy selected local settings to the remote ui
     for sect in ('auth', 'hostfingerprints', 'hostsecurity', 'http_proxy'):
         for key, val in src.configitems(sect):
             dst.setconfig(sect, key, val, 'copied')
     v = src.config('web', 'cacerts')
     if v:
         dst.setconfig('web', 'cacerts', util.expandpath(v), 'copied')
 
     return dst
 
 # Files of interest
 # Used to check if the repository has changed looking at mtime and size of
 # these files.
 foi = [('spath', '00changelog.i'),
        ('spath', 'phaseroots'), # ! phase can change content at the same size
        ('spath', 'obsstore'),
        ('path', 'bookmarks'), # ! bookmark can change content at the same size
       ]
 
 class cachedlocalrepo(object):
     """Holds a localrepository that can be cached and reused."""
 
     def __init__(self, repo):
         """Create a new cached repo from an existing repo.
 
         We assume the passed in repo was recently created. If the
         repo has changed between when it was created and when it was
         turned into a cache, it may not refresh properly.
         """
         assert isinstance(repo, localrepo.localrepository)
         self._repo = repo
         self._state, self.mtime = self._repostate()
         self._filtername = repo.filtername
 
     def fetch(self):
         """Refresh (if necessary) and return a repository.
 
         If the cached instance is out of date, it will be recreated
         automatically and returned.
 
         Returns a tuple of the repo and a boolean indicating whether a new
         repo instance was created.
         """
         # We compare the mtimes and sizes of some well-known files to
         # determine if the repo changed. This is not precise, as mtimes
         # are susceptible to clock skew and imprecise filesystems and
         # file content can change while maintaining the same size.
 
         state, mtime = self._repostate()
         if state == self._state:
             return self._repo, False
 
         repo = repository(self._repo.baseui, self._repo.url())
         if self._filtername:
             self._repo = repo.filtered(self._filtername)
         else:
             self._repo = repo.unfiltered()
         self._state = state
         self.mtime = mtime
 
         return self._repo, True
 
     def _repostate(self):
         state = []
         maxmtime = -1
         for attr, fname in foi:
             prefix = getattr(self._repo, attr)
             p = os.path.join(prefix, fname)
             try:
                 st = os.stat(p)
             except OSError:
                 st = os.stat(prefix)
             state.append((st[stat.ST_MTIME], st.st_size))
             maxmtime = max(maxmtime, st[stat.ST_MTIME])
 
         return tuple(state), maxmtime
 
     def copy(self):
         """Obtain a copy of this class instance.
 
         A new localrepository instance is obtained. The new instance should be
         completely independent of the original.
         """
         repo = repository(self._repo.baseui, self._repo.origroot)
         if self._filtername:
             repo = repo.filtered(self._filtername)
         else:
             repo = repo.unfiltered()
         c = cachedlocalrepo(repo)
         c._state = self._state
         c.mtime = self.mtime
         return c

mercurial/httppeer.py

@@ -1,1010 +1,1012
 # httppeer.py - HTTP repository proxy classes for mercurial
 #
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import io
 import os
 import socket
 import struct
 import weakref
 
 from .i18n import _
+from .interfaces import (
+    repository,
+)
 from . import (
     bundle2,
     error,
     httpconnection,
     pycompat,
-    repository,
     statichttprepo,
     url as urlmod,
     util,
     wireprotoframing,
     wireprototypes,
     wireprotov1peer,
     wireprotov2peer,
     wireprotov2server,
 )
 from .utils import (
     cborutil,
     interfaceutil,
     stringutil,
 )
 
 httplib = util.httplib
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 def encodevalueinheaders(value, header, limit):
     """Encode a string value into multiple HTTP headers.
 
     ``value`` will be encoded into 1 or more HTTP headers with the names
     ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
     name + value will be at most ``limit`` bytes long.
 
     Returns an iterable of 2-tuples consisting of header names and
     values as native strings.
     """
     # HTTP Headers are ASCII. Python 3 requires them to be unicodes,
     # not bytes. This function always takes bytes in as arguments.
     fmt = pycompat.strurl(header) + r'-%s'
     # Note: it is *NOT* a bug that the last bit here is a bytestring
     # and not a unicode: we're just getting the encoded length anyway,
     # and using an r-string to make it portable between Python 2 and 3
     # doesn't work because then the \r is a literal backslash-r
     # instead of a carriage return.
     valuelen = limit - len(fmt % r'000') - len(': \r\n')
     result = []
 
     n = 0
     for i in pycompat.xrange(0, len(value), valuelen):
         n += 1
         result.append((fmt % str(n), pycompat.strurl(value[i:i + valuelen])))
 
     return result
 
 class _multifile(object):
     def __init__(self, *fileobjs):
         for f in fileobjs:
             if not util.safehasattr(f, 'length'):
                 raise ValueError(
                     '_multifile only supports file objects that '
                     'have a length but this one does not:', type(f), f)
         self._fileobjs = fileobjs
         self._index = 0
 
     @property
     def length(self):
         return sum(f.length for f in self._fileobjs)
 
     def read(self, amt=None):
         if amt <= 0:
             return ''.join(f.read() for f in self._fileobjs)
         parts = []
         while amt and self._index < len(self._fileobjs):
             parts.append(self._fileobjs[self._index].read(amt))
             got = len(parts[-1])
             if got < amt:
                 self._index += 1
             amt -= got
         return ''.join(parts)
 
     def seek(self, offset, whence=os.SEEK_SET):
         if whence != os.SEEK_SET:
             raise NotImplementedError(
                 '_multifile does not support anything other'
                 ' than os.SEEK_SET for whence on seek()')
         if offset != 0:
             raise NotImplementedError(
                 '_multifile only supports seeking to start, but that '
                 'could be fixed if you need it')
         for f in self._fileobjs:
             f.seek(0)
         self._index = 0
 
 def makev1commandrequest(ui, requestbuilder, caps, capablefn,
                          repobaseurl, cmd, args):
     """Make an HTTP request to run a command for a version 1 client.
 
     ``caps`` is a set of known server capabilities. The value may be
     None if capabilities are not yet known.
 
     ``capablefn`` is a function to evaluate a capability.
 
     ``cmd``, ``args``, and ``data`` define the command, its arguments, and
     raw data to pass to it.
     """
     if cmd == 'pushkey':
         args['data'] = ''
     data = args.pop('data', None)
     headers = args.pop('headers', {})
 
     ui.debug("sending %s command\n" % cmd)
     q = [('cmd', cmd)]
     headersize = 0
     # Important: don't use self.capable() here or else you end up
     # with infinite recursion when trying to look up capabilities
     # for the first time.
     postargsok = caps is not None and 'httppostargs' in caps
 
     # Send arguments via POST.
     if postargsok and args:
         strargs = urlreq.urlencode(sorted(args.items()))
         if not data:
             data = strargs
         else:
             if isinstance(data, bytes):
                 i = io.BytesIO(data)
                 i.length = len(data)
                 data = i
             argsio = io.BytesIO(strargs)
             argsio.length = len(strargs)
             data = _multifile(argsio, data)
         headers[r'X-HgArgs-Post'] = len(strargs)
     elif args:
         # Calling self.capable() can infinite loop if we are calling
         # "capabilities". But that command should never accept wire
         # protocol arguments. So this should never happen.
         assert cmd != 'capabilities'
         httpheader = capablefn('httpheader')
         if httpheader:
             headersize = int(httpheader.split(',', 1)[0])
 
         # Send arguments via HTTP headers.
         if headersize > 0:
             # The headers can typically carry more data than the URL.
             encargs = urlreq.urlencode(sorted(args.items()))
             for header, value in encodevalueinheaders(encargs, 'X-HgArg',
                                                       headersize):
                 headers[header] = value
         # Send arguments via query string (Mercurial <1.9).
         else:
             q += sorted(args.items())
 
     qs = '?%s' % urlreq.urlencode(q)
     cu = "%s%s" % (repobaseurl, qs)
     size = 0
     if util.safehasattr(data, 'length'):
         size = data.length
     elif data is not None:
         size = len(data)
     if data is not None and r'Content-Type' not in headers:
         headers[r'Content-Type'] = r'application/mercurial-0.1'
 
     # Tell the server we accept application/mercurial-0.2 and multiple
     # compression formats if the server is capable of emitting those
     # payloads.
     # Note: Keep this set empty by default, as client advertisement of
     # protocol parameters should only occur after the handshake.
     protoparams = set()
 
     mediatypes = set()
     if caps is not None:
         mt = capablefn('httpmediatype')
         if mt:
             protoparams.add('0.1')
             mediatypes = set(mt.split(','))
 
         protoparams.add('partial-pull')
 
     if '0.2tx' in mediatypes:
         protoparams.add('0.2')
 
     if '0.2tx' in mediatypes and capablefn('compression'):
         # We /could/ compare supported compression formats and prune
         # non-mutually supported or error if nothing is mutually supported.
         # For now, send the full list to the server and have it error.
         comps = [e.wireprotosupport().name for e in
                  util.compengines.supportedwireengines(util.CLIENTROLE)]
         protoparams.add('comp=%s' % ','.join(comps))
 
     if protoparams:
         protoheaders = encodevalueinheaders(' '.join(sorted(protoparams)),
                                             'X-HgProto',
                                             headersize or 1024)
         for header, value in protoheaders:
             headers[header] = value
 
     varyheaders = []
     for header in headers:
         if header.lower().startswith(r'x-hg'):
             varyheaders.append(header)
 
     if varyheaders:
         headers[r'Vary'] = r','.join(sorted(varyheaders))
 
     req = requestbuilder(pycompat.strurl(cu), data, headers)
 
     if data is not None:
         ui.debug("sending %d bytes\n" % size)
         req.add_unredirected_header(r'Content-Length', r'%d' % size)
 
     return req, cu, qs
 
 def _reqdata(req):
     """Get request data, if any. If no data, returns None."""
     if pycompat.ispy3:
         return req.data
     if not req.has_data():
         return None
     return req.get_data()
 
 def sendrequest(ui, opener, req):
     """Send a prepared HTTP request.
 
     Returns the response object.
     """
     dbg = ui.debug
     if (ui.debugflag
         and ui.configbool('devel', 'debug.peer-request')):
         line = 'devel-peer-request: %s\n'
         dbg(line % '%s %s' % (pycompat.bytesurl(req.get_method()),
                               pycompat.bytesurl(req.get_full_url())))
         hgargssize = None
 
         for header, value in sorted(req.header_items()):
             header = pycompat.bytesurl(header)
             value = pycompat.bytesurl(value)
             if header.startswith('X-hgarg-'):
                 if hgargssize is None:
                     hgargssize = 0
                 hgargssize += len(value)
             else:
                 dbg(line % '  %s %s' % (header, value))
 
         if hgargssize is not None:
             dbg(line % '  %d bytes of commands arguments in headers'
                 % hgargssize)
         data = _reqdata(req)
         if data is not None:
             length = getattr(data, 'length', None)
             if length is None:
                 length = len(data)
             dbg(line % '  %d bytes of data' % length)
 
         start = util.timer()
 
     res = None
     try:
         res = opener.open(req)
     except urlerr.httperror as inst:
         if inst.code == 401:
             raise error.Abort(_('authorization failed'))
         raise
     except httplib.HTTPException as inst:
         ui.debug('http error requesting %s\n' %
                  util.hidepassword(req.get_full_url()))
         ui.traceback()
         raise IOError(None, inst)
     finally:
         if ui.debugflag and ui.configbool('devel', 'debug.peer-request'):
             code = res.code if res else -1
             dbg(line % '  finished in %.4f seconds (%d)'
                 % (util.timer() - start, code))
 
     # Insert error handlers for common I/O failures.
     urlmod.wrapresponse(res)
 
     return res
 
 class RedirectedRepoError(error.RepoError):
     def __init__(self, msg, respurl):
         super(RedirectedRepoError, self).__init__(msg)
         self.respurl = respurl
 
 def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible,
                            allowcbor=False):
     # record the url we got redirected to
     redirected = False
     respurl = pycompat.bytesurl(resp.geturl())
     if respurl.endswith(qs):
         respurl = respurl[:-len(qs)]
         qsdropped = False
     else:
         qsdropped = True
 
     if baseurl.rstrip('/') != respurl.rstrip('/'):
         redirected = True
         if not ui.quiet:
             ui.warn(_('real URL is %s\n') % respurl)
 
     try:
         proto = pycompat.bytesurl(resp.getheader(r'content-type', r''))
     except AttributeError:
         proto = pycompat.bytesurl(resp.headers.get(r'content-type', r''))
 
     safeurl = util.hidepassword(baseurl)
     if proto.startswith('application/hg-error'):
         raise error.OutOfBandError(resp.read())
 
     # Pre 1.0 versions of Mercurial used text/plain and
     # application/hg-changegroup. We don't support such old servers.
     if not proto.startswith('application/mercurial-'):
         ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl))
         msg = _("'%s' does not appear to be an hg repository:\n"
                 "---%%<--- (%s)\n%s\n---%%<---\n") % (
             safeurl, proto or 'no content-type', resp.read(1024))
 
         # Some servers may strip the query string from the redirect. We
         # raise a special error type so callers can react to this specially.
         if redirected and qsdropped:
             raise RedirectedRepoError(msg, respurl)
         else:
             raise error.RepoError(msg)
 
     try:
         subtype = proto.split('-', 1)[1]
 
         # Unless we end up supporting CBOR in the legacy wire protocol,
         # this should ONLY be encountered for the initial capabilities
         # request during handshake.
         if subtype == 'cbor':
             if allowcbor:
                 return respurl, proto, resp
             else:
                 raise error.RepoError(_('unexpected CBOR response from '
                                         'server'))
 
         version_info = tuple([int(n) for n in subtype.split('.')])
     except ValueError:
         raise error.RepoError(_("'%s' sent a broken Content-Type "
                                 "header (%s)") % (safeurl, proto))
 
     # TODO consider switching to a decompression reader that uses
     # generators.
     if version_info == (0, 1):
         if compressible:
             resp = util.compengines['zlib'].decompressorreader(resp)
 
     elif version_info == (0, 2):
         # application/mercurial-0.2 always identifies the compression
         # engine in the payload header.
         elen = struct.unpack('B', util.readexactly(resp, 1))[0]
         ename = util.readexactly(resp, elen)
         engine = util.compengines.forwiretype(ename)
 
         resp = engine.decompressorreader(resp)
     else:
         raise error.RepoError(_("'%s' uses newer protocol %s") %
                               (safeurl, subtype))
 
     return respurl, proto, resp
 
 class httppeer(wireprotov1peer.wirepeer):
     def __init__(self, ui, path, url, opener, requestbuilder, caps):
         self.ui = ui
         self._path = path
         self._url = url
         self._caps = caps
         self.limitedarguments = caps is not None and 'httppostargs' not in caps
         self._urlopener = opener
         self._requestbuilder = requestbuilder
 
     def __del__(self):
         for h in self._urlopener.handlers:
             h.close()
             getattr(h, "close_all", lambda: None)()
 
     # Begin of ipeerconnection interface.
 
     def url(self):
         return self._path
 
     def local(self):
         return None
 
     def peer(self):
         return self
 
     def canpush(self):
         return True
 
     def close(self):
         try:
             reqs, sent, recv = (self._urlopener.requestscount,
                                 self._urlopener.sentbytescount,
                                 self._urlopener.receivedbytescount)
         except AttributeError:
             return
         self.ui.note(_('(sent %d HTTP requests and %d bytes; '
                        'received %d bytes in responses)\n') %
                      (reqs, sent, recv))
 
     # End of ipeerconnection interface.
 
     # Begin of ipeercommands interface.
 
     def capabilities(self):
         return self._caps
 
     # End of ipeercommands interface.
 
     def _callstream(self, cmd, _compressible=False, **args):
         args = pycompat.byteskwargs(args)
 
         req, cu, qs = makev1commandrequest(self.ui, self._requestbuilder,
                                            self._caps, self.capable,
                                            self._url, cmd, args)
 
         resp = sendrequest(self.ui, self._urlopener, req)
 
         self._url, ct, resp = parsev1commandresponse(self.ui, self._url, cu, qs,
                                                      resp, _compressible)
 
         return resp
 
     def _call(self, cmd, **args):
         fp = self._callstream(cmd, **args)
         try:
             return fp.read()
         finally:
             # if using keepalive, allow connection to be reused
             fp.close()
 
     def _callpush(self, cmd, cg, **args):
         # have to stream bundle to a temp file because we do not have
         # http 1.1 chunked transfer.
 
         types = self.capable('unbundle')
         try:
             types = types.split(',')
         except AttributeError:
             # servers older than d1b16a746db6 will send 'unbundle' as a
             # boolean capability. They only support headerless/uncompressed
             # bundles.
             types = [""]
         for x in types:
             if x in bundle2.bundletypes:
                 type = x
                 break
 
         tempname = bundle2.writebundle(self.ui, cg, None, type)
         fp = httpconnection.httpsendfile(self.ui, tempname, "rb")
         headers = {r'Content-Type': r'application/mercurial-0.1'}
 
         try:
             r = self._call(cmd, data=fp, headers=headers, **args)
             vals = r.split('\n', 1)
             if len(vals) < 2:
                 raise error.ResponseError(_("unexpected response:"), r)
             return vals
         except urlerr.httperror:
             # Catch and re-raise these so we don't try and treat them
             # like generic socket errors. They lack any values in
             # .args on Python 3 which breaks our socket.error block.
             raise
         except socket.error as err:
             if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
                 raise error.Abort(_('push failed: %s') % err.args[1])
             raise error.Abort(err.args[1])
         finally:
             fp.close()
             os.unlink(tempname)
 
     def _calltwowaystream(self, cmd, fp, **args):
         fh = None
         fp_ = None
         filename = None
         try:
             # dump bundle to disk
             fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
             fh = os.fdopen(fd, r"wb")
             d = fp.read(4096)
             while d:
                 fh.write(d)
                 d = fp.read(4096)
             fh.close()
             # start http push
             fp_ = httpconnection.httpsendfile(self.ui, filename, "rb")
             headers = {r'Content-Type': r'application/mercurial-0.1'}
             return self._callstream(cmd, data=fp_, headers=headers, **args)
         finally:
             if fp_ is not None:
                 fp_.close()
             if fh is not None:
                 fh.close()
                 os.unlink(filename)
 
     def _callcompressable(self, cmd, **args):
         return self._callstream(cmd, _compressible=True, **args)
 
     def _abort(self, exception):
         raise exception
 
 def sendv2request(ui, opener, requestbuilder, apiurl, permission, requests,
                   redirect):
     wireprotoframing.populatestreamencoders()
 
     uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order')
 
     if uiencoders:
         encoders = []
 
         for encoder in uiencoders:
             if encoder not in wireprotoframing.STREAM_ENCODERS:
                 ui.warn(_(b'wire protocol version 2 encoder referenced in '
                           b'config (%s) is not known; ignoring\n') % encoder)
             else:
                 encoders.append(encoder)
 
     else:
         encoders = wireprotoframing.STREAM_ENCODERS_ORDER
 
     reactor = wireprotoframing.clientreactor(ui,
                                              hasmultiplesend=False,
                                              buffersends=True,
                                              clientcontentencoders=encoders)
 
     handler = wireprotov2peer.clienthandler(ui, reactor,
                                             opener=opener,
                                             requestbuilder=requestbuilder)
 
     url = '%s/%s' % (apiurl, permission)
 
     if len(requests) > 1:
         url += '/multirequest'
     else:
         url += '/%s' % requests[0][0]
 
     ui.debug('sending %d commands\n' % len(requests))
     for command, args, f in requests:
         ui.debug('sending command %s: %s\n' % (
             command, stringutil.pprint(args, indent=2)))
         assert not list(handler.callcommand(command, args, f,
                                             redirect=redirect))
 
     # TODO stream this.
     body = b''.join(map(bytes, handler.flushcommands()))
 
     # TODO modify user-agent to reflect v2
     headers = {
         r'Accept': wireprotov2server.FRAMINGTYPE,
         r'Content-Type': wireprotov2server.FRAMINGTYPE,
     }
 
     req = requestbuilder(pycompat.strurl(url), body, headers)
     req.add_unredirected_header(r'Content-Length', r'%d' % len(body))
 
     try:
         res = opener.open(req)
     except urlerr.httperror as e:
         if e.code == 401:
             raise error.Abort(_('authorization failed'))
 
         raise
     except httplib.HTTPException as e:
         ui.traceback()
         raise IOError(None, e)
 
     return handler, res
 
 class queuedcommandfuture(pycompat.futures.Future):
     """Wraps result() on command futures to trigger submission on call."""
 
     def result(self, timeout=None):
         if self.done():
             return pycompat.futures.Future.result(self, timeout)
 
         self._peerexecutor.sendcommands()
 
         # sendcommands() will restore the original __class__ and self.result
         # will resolve to Future.result.
         return self.result(timeout)
 
 @interfaceutil.implementer(repository.ipeercommandexecutor)
 class httpv2executor(object):
     def __init__(self, ui, opener, requestbuilder, apiurl, descriptor,
                  redirect):
         self._ui = ui
         self._opener = opener
         self._requestbuilder = requestbuilder
         self._apiurl = apiurl
         self._descriptor = descriptor
         self._redirect = redirect
         self._sent = False
         self._closed = False
         self._neededpermissions = set()
         self._calls = []
         self._futures = weakref.WeakSet()
         self._responseexecutor = None
         self._responsef = None
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, exctb):
         self.close()
 
     def callcommand(self, command, args):
         if self._sent:
             raise error.ProgrammingError('callcommand() cannot be used after '
                                          'commands are sent')
 
         if self._closed:
             raise error.ProgrammingError('callcommand() cannot be used after '
                                          'close()')
 
         # The service advertises which commands are available. So if we attempt
         # to call an unknown command or pass an unknown argument, we can screen
         # for this.
         if command not in self._descriptor['commands']:
             raise error.ProgrammingError(
                 'wire protocol command %s is not available' % command)
 
         cmdinfo = self._descriptor['commands'][command]
         unknownargs = set(args.keys()) - set(cmdinfo.get('args', {}))
 
         if unknownargs:
             raise error.ProgrammingError(
                 'wire protocol command %s does not accept argument: %s' % (
                     command, ', '.join(sorted(unknownargs))))
 
         self._neededpermissions |= set(cmdinfo['permissions'])
 
         # TODO we /could/ also validate types here, since the API descriptor
         # includes types...
 
         f = pycompat.futures.Future()
 
         # Monkeypatch it so result() triggers sendcommands(), otherwise result()
         # could deadlock.
         f.__class__ = queuedcommandfuture
         f._peerexecutor = self
 
         self._futures.add(f)
         self._calls.append((command, args, f))
 
         return f
 
     def sendcommands(self):
         if self._sent:
             return
 
         if not self._calls:
             return
 
         self._sent = True
 
         # Unhack any future types so caller sees a clean type and so we
         # break reference cycle.
         for f in self._futures:
             if isinstance(f, queuedcommandfuture):
                 f.__class__ = pycompat.futures.Future
                 f._peerexecutor = None
 
         # Mark the future as running and filter out cancelled futures.
         calls = [(command, args, f)
                  for command, args, f in self._calls
                  if f.set_running_or_notify_cancel()]
 
         # Clear out references, prevent improper object usage.
         self._calls = None
 
         if not calls:
             return
 
         permissions = set(self._neededpermissions)
 
         if 'push' in permissions and 'pull' in permissions:
             permissions.remove('pull')
 
         if len(permissions) > 1:
             raise error.RepoError(_('cannot make request requiring multiple '
                                     'permissions: %s') %
                                   _(', ').join(sorted(permissions)))
 
         permission = {
             'push': 'rw',
             'pull': 'ro',
         }[permissions.pop()]
 
         handler, resp = sendv2request(
             self._ui, self._opener, self._requestbuilder, self._apiurl,
             permission, calls, self._redirect)
 
         # TODO we probably want to validate the HTTP code, media type, etc.
 
         self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
         self._responsef = self._responseexecutor.submit(self._handleresponse,
                                                         handler, resp)
 
     def close(self):
         if self._closed:
             return
 
         self.sendcommands()
 
         self._closed = True
 
         if not self._responsef:
             return
 
         # TODO ^C here may not result in immediate program termination.
 
         try:
             self._responsef.result()
         finally:
             self._responseexecutor.shutdown(wait=True)
             self._responsef = None
             self._responseexecutor = None
 
             # If any of our futures are still in progress, mark them as
             # errored, otherwise a result() could wait indefinitely.
             for f in self._futures:
                 if not f.done():
                     f.set_exception(error.ResponseError(
                         _('unfulfilled command response')))
 
             self._futures = None
 
     def _handleresponse(self, handler, resp):
         # Called in a thread to read the response.
 
         while handler.readdata(resp):
             pass
 
 @interfaceutil.implementer(repository.ipeerv2)
 class httpv2peer(object):
 
     limitedarguments = False
 
     def __init__(self, ui, repourl, apipath, opener, requestbuilder,
                  apidescriptor):
         self.ui = ui
         self.apidescriptor = apidescriptor
 
         if repourl.endswith('/'):
             repourl = repourl[:-1]
 
         self._url = repourl
         self._apipath = apipath
         self._apiurl = '%s/%s' % (repourl, apipath)
         self._opener = opener
         self._requestbuilder = requestbuilder
 
         self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
 
     # Start of ipeerconnection.
 
     def url(self):
         return self._url
 
     def local(self):
         return None
 
     def peer(self):
         return self
 
     def canpush(self):
         # TODO change once implemented.
         return False
 
     def close(self):
         self.ui.note(_('(sent %d HTTP requests and %d bytes; '
                        'received %d bytes in responses)\n') %
                      (self._opener.requestscount,
                       self._opener.sentbytescount,
                       self._opener.receivedbytescount))
 
     # End of ipeerconnection.
 
     # Start of ipeercapabilities.
 
     def capable(self, name):
         # The capabilities used internally historically map to capabilities
         # advertised from the "capabilities" wire protocol command. However,
         # version 2 of that command works differently.
 
         # Maps to commands that are available.
         if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'):
             return True
 
         # Other concepts.
         if name in ('bundle2'):
             return True
 
         # Alias command-* to presence of command of that name.
         if name.startswith('command-'):
             return name[len('command-'):] in self.apidescriptor['commands']
 
         return False
 
     def requirecap(self, name, purpose):
         if self.capable(name):
             return
 
         raise error.CapabilityError(
             _('cannot %s; client or remote repository does not support the '
               '\'%s\' capability') % (purpose, name))
 
     # End of ipeercapabilities.
 
     def _call(self, name, **args):
         with self.commandexecutor() as e:
             return e.callcommand(name, args).result()
 
     def commandexecutor(self):
         return httpv2executor(self.ui, self._opener, self._requestbuilder,
                               self._apiurl, self.apidescriptor, self._redirect)
 
 # Registry of API service names to metadata about peers that handle it.
 #
 # The following keys are meaningful:
 #
 # init
 #    Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
 #                        apidescriptor) to create a peer.
 #
 # priority
 #    Integer priority for the service. If we could choose from multiple
 #    services, we choose the one with the highest priority.
 API_PEERS = {
     wireprototypes.HTTP_WIREPROTO_V2: {
         'init': httpv2peer,
         'priority': 50,
     },
 }
 
 def performhandshake(ui, url, opener, requestbuilder):
     # The handshake is a request to the capabilities command.
 
     caps = None
     def capable(x):
         raise error.ProgrammingError('should not be called')
 
     args = {}
 
     # The client advertises support for newer protocols by adding an
     # X-HgUpgrade-* header with a list of supported APIs and an
     # X-HgProto-* header advertising which serializing formats it supports.
     # We only support the HTTP version 2 transport and CBOR responses for
     # now.
     advertisev2 = ui.configbool('experimental', 'httppeer.advertise-v2')
 
     if advertisev2:
         args['headers'] = {
             r'X-HgProto-1': r'cbor',
         }
 
         args['headers'].update(
             encodevalueinheaders(' '.join(sorted(API_PEERS)),
                                  'X-HgUpgrade',
                                  # We don't know the header limit this early.
                                  # So make it small.
                                  1024))
 
     req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
                                            capable, url, 'capabilities',
                                            args)
     resp = sendrequest(ui, opener, req)
 
     # The server may redirect us to the repo root, stripping the
     # ?cmd=capabilities query string from the URL. The server would likely
     # return HTML in this case and ``parsev1commandresponse()`` would raise.
     # We catch this special case and re-issue the capabilities request against
     # the new URL.
     #
     # We should ideally not do this, as a redirect that drops the query
     # string from the URL is arguably a server bug. (Garbage in, garbage out).
     # However,  Mercurial clients for several years appeared to handle this
     # issue without behavior degradation. And according to issue 5860, it may
     # be a longstanding bug in some server implementations. So we allow a
     # redirect that drops the query string to "just work."
     try:
         respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
                                                    compressible=False,
                                                    allowcbor=advertisev2)
     except RedirectedRepoError as e:
         req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
                                                capable, e.respurl,
                                                'capabilities', args)
         resp = sendrequest(ui, opener, req)
         respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
                                                    compressible=False,
                                                    allowcbor=advertisev2)
 
     try:
         rawdata = resp.read()
     finally:
         resp.close()
 
     if not ct.startswith('application/mercurial-'):
         raise error.ProgrammingError('unexpected content-type: %s' % ct)
 
     if advertisev2:
         if ct == 'application/mercurial-cbor':
             try:
                 info = cborutil.decodeall(rawdata)[0]
             except cborutil.CBORDecodeError:
                 raise error.Abort(_('error decoding CBOR from remote server'),
                                   hint=_('try again and consider contacting '
                                          'the server operator'))
 
         # We got a legacy response. That's fine.
         elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'):
             info = {
                 'v1capabilities': set(rawdata.split())
             }
 
         else:
             raise error.RepoError(
                 _('unexpected response type from server: %s') % ct)
     else:
         info = {
             'v1capabilities': set(rawdata.split())
         }
 
     return respurl, info
 
 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
     """Construct an appropriate HTTP peer instance.
 
     ``opener`` is an ``url.opener`` that should be used to establish
     connections, perform HTTP requests.
 
     ``requestbuilder`` is the type used for constructing HTTP requests.
     It exists as an argument so extensions can override the default.
     """
     u = util.url(path)
     if u.query or u.fragment:
         raise error.Abort(_('unsupported URL component: "%s"') %
                           (u.query or u.fragment))
 
     # urllib cannot handle URLs with embedded user or passwd.
     url, authinfo = u.authinfo()
     ui.debug('using %s\n' % url)
 
     opener = opener or urlmod.opener(ui, authinfo)
 
     respurl, info = performhandshake(ui, url, opener, requestbuilder)
 
     # Given the intersection of APIs that both we and the server support,
     # sort by their advertised priority and pick the first one.
     #
     # TODO consider making this request-based and interface driven. For
     # example, the caller could say "I want a peer that does X." It's quite
     # possible that not all peers would do that. Since we know the service
     # capabilities, we could filter out services not meeting the
     # requirements. Possibly by consulting the interfaces defined by the
     # peer type.
     apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys())
 
     preferredchoices = sorted(apipeerchoices,
                               key=lambda x: API_PEERS[x]['priority'],
                               reverse=True)
 
     for service in preferredchoices:
         apipath = '%s/%s' % (info['apibase'].rstrip('/'), service)
 
         return API_PEERS[service]['init'](ui, respurl, apipath, opener,
                                           requestbuilder,
                                           info['apis'][service])
 
     # Failed to construct an API peer. Fall back to legacy.
     return httppeer(ui, path, respurl, opener, requestbuilder,
                     info['v1capabilities'])
 
 def instance(ui, path, create, intents=None, createopts=None):
     if create:
         raise error.Abort(_('cannot create new http repository'))
     try:
         if path.startswith('https:') and not urlmod.has_https:
             raise error.Abort(_('Python support for SSL and HTTPS '
                                 'is not installed'))
 
         inst = makepeer(ui, path)
 
         return inst
     except error.RepoError as httpexception:
         try:
             r = statichttprepo.instance(ui, "static-" + path, create)
             ui.note(_('(falling back to static-http)\n'))
             return r
         except error.RepoError:
             raise httpexception # use the original http RepoError instead

mercurial/interfaces/repository.py

@@ -1,1877 +1,1877
 # repository.py - Interfaces and base classes for repositories and peers.
 #
 # Copyright 2017 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
-from .i18n import _
-from . import (
+from ..i18n import _
+from .. import (
     error,
 )
-from .utils import (
+from ..utils import (
     interfaceutil,
 )
 
 # When narrowing is finalized and no longer subject to format changes,
 # we should move this to just "narrow" or similar.
 NARROW_REQUIREMENT = 'narrowhg-experimental'
 
 # Local repository feature string.
 
 # Revlogs are being used for file storage.
 REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage'
 # The storage part of the repository is shared from an external source.
 REPO_FEATURE_SHARED_STORAGE = b'sharedstore'
 # LFS supported for backing file storage.
 REPO_FEATURE_LFS = b'lfs'
 # Repository supports being stream cloned.
 REPO_FEATURE_STREAM_CLONE = b'streamclone'
 # Files storage may lack data for all ancestors.
 REPO_FEATURE_SHALLOW_FILE_STORAGE = b'shallowfilestorage'
 
 REVISION_FLAG_CENSORED = 1 << 15
 REVISION_FLAG_ELLIPSIS = 1 << 14
 REVISION_FLAG_EXTSTORED = 1 << 13
 
 REVISION_FLAGS_KNOWN = (
     REVISION_FLAG_CENSORED | REVISION_FLAG_ELLIPSIS | REVISION_FLAG_EXTSTORED)
 
 CG_DELTAMODE_STD = b'default'
 CG_DELTAMODE_PREV = b'previous'
 CG_DELTAMODE_FULL = b'fulltext'
 CG_DELTAMODE_P1 = b'p1'
 
 class ipeerconnection(interfaceutil.Interface):
     """Represents a "connection" to a repository.
 
     This is the base interface for representing a connection to a repository.
     It holds basic properties and methods applicable to all peer types.
 
     This is not a complete interface definition and should not be used
     outside of this module.
     """
     ui = interfaceutil.Attribute("""ui.ui instance""")
 
     def url():
         """Returns a URL string representing this peer.
 
         Currently, implementations expose the raw URL used to construct the
         instance. It may contain credentials as part of the URL. The
         expectations of the value aren't well-defined and this could lead to
         data leakage.
 
         TODO audit/clean consumers and more clearly define the contents of this
         value.
         """
 
     def local():
         """Returns a local repository instance.
 
         If the peer represents a local repository, returns an object that
         can be used to interface with it. Otherwise returns ``None``.
         """
 
     def peer():
         """Returns an object conforming to this interface.
 
         Most implementations will ``return self``.
         """
 
     def canpush():
         """Returns a boolean indicating if this peer can be pushed to."""
 
     def close():
         """Close the connection to this peer.
 
         This is called when the peer will no longer be used. Resources
         associated with the peer should be cleaned up.
         """
 
 class ipeercapabilities(interfaceutil.Interface):
     """Peer sub-interface related to capabilities."""
 
     def capable(name):
         """Determine support for a named capability.
 
         Returns ``False`` if capability not supported.
 
         Returns ``True`` if boolean capability is supported. Returns a string
         if capability support is non-boolean.
 
         Capability strings may or may not map to wire protocol capabilities.
         """
 
     def requirecap(name, purpose):
         """Require a capability to be present.
 
         Raises a ``CapabilityError`` if the capability isn't present.
         """
 
 class ipeercommands(interfaceutil.Interface):
     """Client-side interface for communicating over the wire protocol.
 
     This interface is used as a gateway to the Mercurial wire protocol.
     methods commonly call wire protocol commands of the same name.
     """
 
     def branchmap():
         """Obtain heads in named branches.
 
         Returns a dict mapping branch name to an iterable of nodes that are
         heads on that branch.
         """
 
     def capabilities():
         """Obtain capabilities of the peer.
 
         Returns a set of string capabilities.
         """
 
     def clonebundles():
         """Obtains the clone bundles manifest for the repo.
 
         Returns the manifest as unparsed bytes.
         """
 
     def debugwireargs(one, two, three=None, four=None, five=None):
         """Used to facilitate debugging of arguments passed over the wire."""
 
     def getbundle(source, **kwargs):
         """Obtain remote repository data as a bundle.
 
         This command is how the bulk of repository data is transferred from
         the peer to the local repository
 
         Returns a generator of bundle data.
         """
 
     def heads():
         """Determine all known head revisions in the peer.
 
         Returns an iterable of binary nodes.
         """
 
     def known(nodes):
         """Determine whether multiple nodes are known.
 
         Accepts an iterable of nodes whose presence to check for.
 
         Returns an iterable of booleans indicating of the corresponding node
         at that index is known to the peer.
         """
 
     def listkeys(namespace):
         """Obtain all keys in a pushkey namespace.
 
         Returns an iterable of key names.
         """
 
     def lookup(key):
         """Resolve a value to a known revision.
 
         Returns a binary node of the resolved revision on success.
         """
 
     def pushkey(namespace, key, old, new):
         """Set a value using the ``pushkey`` protocol.
 
         Arguments correspond to the pushkey namespace and key to operate on and
         the old and new values for that key.
 
         Returns a string with the peer result. The value inside varies by the
         namespace.
         """
 
     def stream_out():
         """Obtain streaming clone data.
 
         Successful result should be a generator of data chunks.
         """
 
     def unbundle(bundle, heads, url):
         """Transfer repository data to the peer.
 
         This is how the bulk of data during a push is transferred.
 
         Returns the integer number of heads added to the peer.
         """
 
 class ipeerlegacycommands(interfaceutil.Interface):
     """Interface for implementing support for legacy wire protocol commands.
 
     Wire protocol commands transition to legacy status when they are no longer
     used by modern clients. To facilitate identifying which commands are
     legacy, the interfaces are split.
     """
 
     def between(pairs):
         """Obtain nodes between pairs of nodes.
 
         ``pairs`` is an iterable of node pairs.
 
         Returns an iterable of iterables of nodes corresponding to each
         requested pair.
         """
 
     def branches(nodes):
         """Obtain ancestor changesets of specific nodes back to a branch point.
 
         For each requested node, the peer finds the first ancestor node that is
         a DAG root or is a merge.
 
         Returns an iterable of iterables with the resolved values for each node.
         """
 
     def changegroup(nodes, source):
         """Obtain a changegroup with data for descendants of specified nodes."""
 
     def changegroupsubset(bases, heads, source):
         pass
 
 class ipeercommandexecutor(interfaceutil.Interface):
     """Represents a mechanism to execute remote commands.
 
     This is the primary interface for requesting that wire protocol commands
     be executed. Instances of this interface are active in a context manager
     and have a well-defined lifetime. When the context manager exits, all
     outstanding requests are waited on.
     """
 
     def callcommand(name, args):
         """Request that a named command be executed.
 
         Receives the command name and a dictionary of command arguments.
 
         Returns a ``concurrent.futures.Future`` that will resolve to the
         result of that command request. That exact value is left up to
         the implementation and possibly varies by command.
 
         Not all commands can coexist with other commands in an executor
         instance: it depends on the underlying wire protocol transport being
         used and the command itself.
 
         Implementations MAY call ``sendcommands()`` automatically if the
         requested command can not coexist with other commands in this executor.
 
         Implementations MAY call ``sendcommands()`` automatically when the
         future's ``result()`` is called. So, consumers using multiple
         commands with an executor MUST ensure that ``result()`` is not called
         until all command requests have been issued.
         """
 
     def sendcommands():
         """Trigger submission of queued command requests.
 
         Not all transports submit commands as soon as they are requested to
         run. When called, this method forces queued command requests to be
         issued. It will no-op if all commands have already been sent.
 
         When called, no more new commands may be issued with this executor.
         """
 
     def close():
         """Signal that this command request is finished.
 
         When called, no more new commands may be issued. All outstanding
         commands that have previously been issued are waited on before
         returning. This not only includes waiting for the futures to resolve,
         but also waiting for all response data to arrive. In other words,
         calling this waits for all on-wire state for issued command requests
         to finish.
 
         When used as a context manager, this method is called when exiting the
         context manager.
 
         This method may call ``sendcommands()`` if there are buffered commands.
         """
 
 class ipeerrequests(interfaceutil.Interface):
     """Interface for executing commands on a peer."""
 
     limitedarguments = interfaceutil.Attribute(
         """True if the peer cannot receive large argument value for commands."""
     )
 
     def commandexecutor():
         """A context manager that resolves to an ipeercommandexecutor.
 
         The object this resolves to can be used to issue command requests
         to the peer.
 
         Callers should call its ``callcommand`` method to issue command
         requests.
 
         A new executor should be obtained for each distinct set of commands
         (possibly just a single command) that the consumer wants to execute
         as part of a single operation or round trip. This is because some
         peers are half-duplex and/or don't support persistent connections.
         e.g. in the case of HTTP peers, commands sent to an executor represent
         a single HTTP request. While some peers may support multiple command
         sends over the wire per executor, consumers need to code to the least
         capable peer. So it should be assumed that command executors buffer
         called commands until they are told to send them and that each
         command executor could result in a new connection or wire-level request
         being issued.
         """
 
 class ipeerbase(ipeerconnection, ipeercapabilities, ipeerrequests):
     """Unified interface for peer repositories.
 
     All peer instances must conform to this interface.
     """
 
 class ipeerv2(ipeerconnection, ipeercapabilities, ipeerrequests):
     """Unified peer interface for wire protocol version 2 peers."""
 
     apidescriptor = interfaceutil.Attribute(
         """Data structure holding description of server API.""")
 
 @interfaceutil.implementer(ipeerbase)
 class peer(object):
     """Base class for peer repositories."""
 
     limitedarguments = False
 
     def capable(self, name):
         caps = self.capabilities()
         if name in caps:
             return True
 
         name = '%s=' % name
         for cap in caps:
             if cap.startswith(name):
                 return cap[len(name):]
 
         return False
 
     def requirecap(self, name, purpose):
         if self.capable(name):
             return
 
         raise error.CapabilityError(
             _('cannot %s; remote repository does not support the '
               '\'%s\' capability') % (purpose, name))
 
 class iverifyproblem(interfaceutil.Interface):
     """Represents a problem with the integrity of the repository.
 
     Instances of this interface are emitted to describe an integrity issue
     with a repository (e.g. corrupt storage, missing data, etc).
 
     Instances are essentially messages associated with severity.
     """
     warning = interfaceutil.Attribute(
         """Message indicating a non-fatal problem.""")
 
     error = interfaceutil.Attribute(
         """Message indicating a fatal problem.""")
 
     node = interfaceutil.Attribute(
         """Revision encountering the problem.
 
         ``None`` means the problem doesn't apply to a single revision.
         """)
 
 class irevisiondelta(interfaceutil.Interface):
     """Represents a delta between one revision and another.
 
     Instances convey enough information to allow a revision to be exchanged
     with another repository.
 
     Instances represent the fulltext revision data or a delta against
     another revision. Therefore the ``revision`` and ``delta`` attributes
     are mutually exclusive.
 
     Typically used for changegroup generation.
     """
 
     node = interfaceutil.Attribute(
         """20 byte node of this revision.""")
 
     p1node = interfaceutil.Attribute(
         """20 byte node of 1st parent of this revision.""")
 
     p2node = interfaceutil.Attribute(
         """20 byte node of 2nd parent of this revision.""")
 
     linknode = interfaceutil.Attribute(
         """20 byte node of the changelog revision this node is linked to.""")
 
     flags = interfaceutil.Attribute(
         """2 bytes of integer flags that apply to this revision.
 
         This is a bitwise composition of the ``REVISION_FLAG_*`` constants.
         """)
 
     basenode = interfaceutil.Attribute(
         """20 byte node of the revision this data is a delta against.
 
         ``nullid`` indicates that the revision is a full revision and not
         a delta.
         """)
 
     baserevisionsize = interfaceutil.Attribute(
         """Size of base revision this delta is against.
 
         May be ``None`` if ``basenode`` is ``nullid``.
         """)
 
     revision = interfaceutil.Attribute(
         """Raw fulltext of revision data for this node.""")
 
     delta = interfaceutil.Attribute(
         """Delta between ``basenode`` and ``node``.
 
         Stored in the bdiff delta format.
         """)
 
 class ifilerevisionssequence(interfaceutil.Interface):
     """Contains index data for all revisions of a file.
 
     Types implementing this behave like lists of tuples. The index
     in the list corresponds to the revision number. The values contain
     index metadata.
 
     The *null* revision (revision number -1) is always the last item
     in the index.
     """
 
     def __len__():
         """The total number of revisions."""
 
     def __getitem__(rev):
         """Returns the object having a specific revision number.
 
         Returns an 8-tuple with the following fields:
 
         offset+flags
            Contains the offset and flags for the revision. 64-bit unsigned
            integer where first 6 bytes are the offset and the next 2 bytes
            are flags. The offset can be 0 if it is not used by the store.
         compressed size
             Size of the revision data in the store. It can be 0 if it isn't
             needed by the store.
         uncompressed size
             Fulltext size. It can be 0 if it isn't needed by the store.
         base revision
             Revision number of revision the delta for storage is encoded
             against. -1 indicates not encoded against a base revision.
         link revision
             Revision number of changelog revision this entry is related to.
         p1 revision
             Revision number of 1st parent. -1 if no 1st parent.
         p2 revision
             Revision number of 2nd parent. -1 if no 1st parent.
         node
             Binary node value for this revision number.
 
         Negative values should index off the end of the sequence. ``-1``
         should return the null revision. ``-2`` should return the most
         recent revision.
         """
 
     def __contains__(rev):
         """Whether a revision number exists."""
 
     def insert(self, i, entry):
         """Add an item to the index at specific revision."""
 
 class ifileindex(interfaceutil.Interface):
     """Storage interface for index data of a single file.
 
     File storage data is divided into index metadata and data storage.
     This interface defines the index portion of the interface.
 
     The index logically consists of:
 
     * A mapping between revision numbers and nodes.
     * DAG data (storing and querying the relationship between nodes).
     * Metadata to facilitate storage.
     """
     def __len__():
         """Obtain the number of revisions stored for this file."""
 
     def __iter__():
         """Iterate over revision numbers for this file."""
 
     def hasnode(node):
         """Returns a bool indicating if a node is known to this store.
 
         Implementations must only return True for full, binary node values:
         hex nodes, revision numbers, and partial node matches must be
         rejected.
 
         The null node is never present.
         """
 
     def revs(start=0, stop=None):
         """Iterate over revision numbers for this file, with control."""
 
     def parents(node):
         """Returns a 2-tuple of parent nodes for a revision.
 
         Values will be ``nullid`` if the parent is empty.
         """
 
     def parentrevs(rev):
         """Like parents() but operates on revision numbers."""
 
     def rev(node):
         """Obtain the revision number given a node.
 
         Raises ``error.LookupError`` if the node is not known.
         """
 
     def node(rev):
         """Obtain the node value given a revision number.
 
         Raises ``IndexError`` if the node is not known.
         """
 
     def lookup(node):
         """Attempt to resolve a value to a node.
 
         Value can be a binary node, hex node, revision number, or a string
         that can be converted to an integer.
 
         Raises ``error.LookupError`` if a node could not be resolved.
         """
 
     def linkrev(rev):
         """Obtain the changeset revision number a revision is linked to."""
 
     def iscensored(rev):
         """Return whether a revision's content has been censored."""
 
     def commonancestorsheads(node1, node2):
         """Obtain an iterable of nodes containing heads of common ancestors.
 
         See ``ancestor.commonancestorsheads()``.
         """
 
     def descendants(revs):
         """Obtain descendant revision numbers for a set of revision numbers.
 
         If ``nullrev`` is in the set, this is equivalent to ``revs()``.
         """
 
     def heads(start=None, stop=None):
         """Obtain a list of nodes that are DAG heads, with control.
 
         The set of revisions examined can be limited by specifying
         ``start`` and ``stop``. ``start`` is a node. ``stop`` is an
         iterable of nodes. DAG traversal starts at earlier revision
         ``start`` and iterates forward until any node in ``stop`` is
         encountered.
         """
 
     def children(node):
         """Obtain nodes that are children of a node.
 
         Returns a list of nodes.
         """
 
 class ifiledata(interfaceutil.Interface):
     """Storage interface for data storage of a specific file.
 
     This complements ``ifileindex`` and provides an interface for accessing
     data for a tracked file.
     """
     def size(rev):
         """Obtain the fulltext size of file data.
 
         Any metadata is excluded from size measurements.
         """
 
     def revision(node, raw=False):
         """"Obtain fulltext data for a node.
 
         By default, any storage transformations are applied before the data
         is returned. If ``raw`` is True, non-raw storage transformations
         are not applied.
 
         The fulltext data may contain a header containing metadata. Most
         consumers should use ``read()`` to obtain the actual file data.
         """
 
     def rawdata(node):
         """Obtain raw data for a node.
         """
 
     def read(node):
         """Resolve file fulltext data.
 
         This is similar to ``revision()`` except any metadata in the data
         headers is stripped.
         """
 
     def renamed(node):
         """Obtain copy metadata for a node.
 
         Returns ``False`` if no copy metadata is stored or a 2-tuple of
         (path, node) from which this revision was copied.
         """
 
     def cmp(node, fulltext):
         """Compare fulltext to another revision.
 
         Returns True if the fulltext is different from what is stored.
 
         This takes copy metadata into account.
 
         TODO better document the copy metadata and censoring logic.
         """
 
     def emitrevisions(nodes,
                       nodesorder=None,
                       revisiondata=False,
                       assumehaveparentrevisions=False,
                       deltamode=CG_DELTAMODE_STD):
         """Produce ``irevisiondelta`` for revisions.
 
         Given an iterable of nodes, emits objects conforming to the
         ``irevisiondelta`` interface that describe revisions in storage.
 
         This method is a generator.
 
         The input nodes may be unordered. Implementations must ensure that a
         node's parents are emitted before the node itself. Transitively, this
         means that a node may only be emitted once all its ancestors in
         ``nodes`` have also been emitted.
 
         By default, emits "index" data (the ``node``, ``p1node``, and
         ``p2node`` attributes). If ``revisiondata`` is set, revision data
         will also be present on the emitted objects.
 
         With default argument values, implementations can choose to emit
         either fulltext revision data or a delta. When emitting deltas,
         implementations must consider whether the delta's base revision
         fulltext is available to the receiver.
 
         The base revision fulltext is guaranteed to be available if any of
         the following are met:
 
         * Its fulltext revision was emitted by this method call.
         * A delta for that revision was emitted by this method call.
         * ``assumehaveparentrevisions`` is True and the base revision is a
           parent of the node.
 
         ``nodesorder`` can be used to control the order that revisions are
         emitted. By default, revisions can be reordered as long as they are
         in DAG topological order (see above). If the value is ``nodes``,
         the iteration order from ``nodes`` should be used. If the value is
         ``storage``, then the native order from the backing storage layer
         is used. (Not all storage layers will have strong ordering and behavior
         of this mode is storage-dependent.) ``nodes`` ordering can force
         revisions to be emitted before their ancestors, so consumers should
         use it with care.
 
         The ``linknode`` attribute on the returned ``irevisiondelta`` may not
         be set and it is the caller's responsibility to resolve it, if needed.
 
         If ``deltamode`` is CG_DELTAMODE_PREV and revision data is requested,
         all revision data should be emitted as deltas against the revision
         emitted just prior. The initial revision should be a delta against its
         1st parent.
         """
 
 class ifilemutation(interfaceutil.Interface):
     """Storage interface for mutation events of a tracked file."""
 
     def add(filedata, meta, transaction, linkrev, p1, p2):
         """Add a new revision to the store.
 
         Takes file data, dictionary of metadata, a transaction, linkrev,
         and parent nodes.
 
         Returns the node that was added.
 
         May no-op if a revision matching the supplied data is already stored.
         """
 
     def addrevision(revisiondata, transaction, linkrev, p1, p2, node=None,
                     flags=0, cachedelta=None):
         """Add a new revision to the store.
 
         This is similar to ``add()`` except it operates at a lower level.
 
         The data passed in already contains a metadata header, if any.
 
         ``node`` and ``flags`` can be used to define the expected node and
         the flags to use with storage. ``flags`` is a bitwise value composed
         of the various ``REVISION_FLAG_*`` constants.
 
         ``add()`` is usually called when adding files from e.g. the working
         directory. ``addrevision()`` is often called by ``add()`` and for
         scenarios where revision data has already been computed, such as when
         applying raw data from a peer repo.
         """
 
     def addgroup(deltas, linkmapper, transaction, addrevisioncb=None,
                  maybemissingparents=False):
         """Process a series of deltas for storage.
 
         ``deltas`` is an iterable of 7-tuples of
         (node, p1, p2, linknode, deltabase, delta, flags) defining revisions
         to add.
 
         The ``delta`` field contains ``mpatch`` data to apply to a base
         revision, identified by ``deltabase``. The base node can be
         ``nullid``, in which case the header from the delta can be ignored
         and the delta used as the fulltext.
 
         ``addrevisioncb`` should be called for each node as it is committed.
 
         ``maybemissingparents`` is a bool indicating whether the incoming
         data may reference parents/ancestor revisions that aren't present.
         This flag is set when receiving data into a "shallow" store that
         doesn't hold all history.
 
         Returns a list of nodes that were processed. A node will be in the list
         even if it existed in the store previously.
         """
 
     def censorrevision(tr, node, tombstone=b''):
         """Remove the content of a single revision.
 
         The specified ``node`` will have its content purged from storage.
         Future attempts to access the revision data for this node will
         result in failure.
 
         A ``tombstone`` message can optionally be stored. This message may be
         displayed to users when they attempt to access the missing revision
         data.
 
         Storage backends may have stored deltas against the previous content
         in this revision. As part of censoring a revision, these storage
         backends are expected to rewrite any internally stored deltas such
         that they no longer reference the deleted content.
         """
 
     def getstrippoint(minlink):
         """Find the minimum revision that must be stripped to strip a linkrev.
 
         Returns a 2-tuple containing the minimum revision number and a set
         of all revisions numbers that would be broken by this strip.
 
         TODO this is highly revlog centric and should be abstracted into
         a higher-level deletion API. ``repair.strip()`` relies on this.
         """
 
     def strip(minlink, transaction):
         """Remove storage of items starting at a linkrev.
 
         This uses ``getstrippoint()`` to determine the first node to remove.
         Then it effectively truncates storage for all revisions after that.
 
         TODO this is highly revlog centric and should be abstracted into a
         higher-level deletion API.
         """
 
 class ifilestorage(ifileindex, ifiledata, ifilemutation):
     """Complete storage interface for a single tracked file."""
 
     def files():
         """Obtain paths that are backing storage for this file.
 
         TODO this is used heavily by verify code and there should probably
         be a better API for that.
         """
 
     def storageinfo(exclusivefiles=False, sharedfiles=False,
                     revisionscount=False, trackedsize=False,
                     storedsize=False):
         """Obtain information about storage for this file's data.
 
         Returns a dict describing storage for this tracked path. The keys
         in the dict map to arguments of the same. The arguments are bools
         indicating whether to calculate and obtain that data.
 
         exclusivefiles
            Iterable of (vfs, path) describing files that are exclusively
            used to back storage for this tracked path.
 
         sharedfiles
            Iterable of (vfs, path) describing files that are used to back
            storage for this tracked path. Those files may also provide storage
            for other stored entities.
 
         revisionscount
            Number of revisions available for retrieval.
 
         trackedsize
            Total size in bytes of all tracked revisions. This is a sum of the
            length of the fulltext of all revisions.
 
         storedsize
            Total size in bytes used to store data for all tracked revisions.
            This is commonly less than ``trackedsize`` due to internal usage
            of deltas rather than fulltext revisions.
 
         Not all storage backends may support all queries are have a reasonable
         value to use. In that case, the value should be set to ``None`` and
         callers are expected to handle this special value.
         """
 
     def verifyintegrity(state):
         """Verifies the integrity of file storage.
 
         ``state`` is a dict holding state of the verifier process. It can be
         used to communicate data between invocations of multiple storage
         primitives.
 
         If individual revisions cannot have their revision content resolved,
         the method is expected to set the ``skipread`` key to a set of nodes
         that encountered problems.
 
         The method yields objects conforming to the ``iverifyproblem``
         interface.
         """
 
 class idirs(interfaceutil.Interface):
     """Interface representing a collection of directories from paths.
 
     This interface is essentially a derived data structure representing
     directories from a collection of paths.
     """
 
     def addpath(path):
         """Add a path to the collection.
 
         All directories in the path will be added to the collection.
         """
 
     def delpath(path):
         """Remove a path from the collection.
 
         If the removal was the last path in a particular directory, the
         directory is removed from the collection.
         """
 
     def __iter__():
         """Iterate over the directories in this collection of paths."""
 
     def __contains__(path):
         """Whether a specific directory is in this collection."""
 
 class imanifestdict(interfaceutil.Interface):
     """Interface representing a manifest data structure.
 
     A manifest is effectively a dict mapping paths to entries. Each entry
     consists of a binary node and extra flags affecting that entry.
     """
 
     def __getitem__(path):
         """Returns the binary node value for a path in the manifest.
 
         Raises ``KeyError`` if the path does not exist in the manifest.
 
         Equivalent to ``self.find(path)[0]``.
         """
 
     def find(path):
         """Returns the entry for a path in the manifest.
 
         Returns a 2-tuple of (node, flags).
 
         Raises ``KeyError`` if the path does not exist in the manifest.
         """
 
     def __len__():
         """Return the number of entries in the manifest."""
 
     def __nonzero__():
         """Returns True if the manifest has entries, False otherwise."""
 
     __bool__ = __nonzero__
 
     def __setitem__(path, node):
         """Define the node value for a path in the manifest.
 
         If the path is already in the manifest, its flags will be copied to
         the new entry.
         """
 
     def __contains__(path):
         """Whether a path exists in the manifest."""
 
     def __delitem__(path):
         """Remove a path from the manifest.
 
         Raises ``KeyError`` if the path is not in the manifest.
         """
 
     def __iter__():
         """Iterate over paths in the manifest."""
 
     def iterkeys():
         """Iterate over paths in the manifest."""
 
     def keys():
         """Obtain a list of paths in the manifest."""
 
     def filesnotin(other, match=None):
         """Obtain the set of paths in this manifest but not in another.
 
         ``match`` is an optional matcher function to be applied to both
         manifests.
 
         Returns a set of paths.
         """
 
     def dirs():
         """Returns an object implementing the ``idirs`` interface."""
 
     def hasdir(dir):
         """Returns a bool indicating if a directory is in this manifest."""
 
     def matches(match):
         """Generate a new manifest filtered through a matcher.
 
         Returns an object conforming to the ``imanifestdict`` interface.
         """
 
     def walk(match):
         """Generator of paths in manifest satisfying a matcher.
 
         This is equivalent to ``self.matches(match).iterkeys()`` except a new
         manifest object is not created.
 
         If the matcher has explicit files listed and they don't exist in
         the manifest, ``match.bad()`` is called for each missing file.
         """
 
     def diff(other, match=None, clean=False):
         """Find differences between this manifest and another.
 
         This manifest is compared to ``other``.
 
         If ``match`` is provided, the two manifests are filtered against this
         matcher and only entries satisfying the matcher are compared.
 
         If ``clean`` is True, unchanged files are included in the returned
         object.
 
         Returns a dict with paths as keys and values of 2-tuples of 2-tuples of
         the form ``((node1, flag1), (node2, flag2))`` where ``(node1, flag1)``
         represents the node and flags for this manifest and ``(node2, flag2)``
         are the same for the other manifest.
         """
 
     def setflag(path, flag):
         """Set the flag value for a given path.
 
         Raises ``KeyError`` if the path is not already in the manifest.
         """
 
     def get(path, default=None):
         """Obtain the node value for a path or a default value if missing."""
 
     def flags(path, default=''):
         """Return the flags value for a path or a default value if missing."""
 
     def copy():
         """Return a copy of this manifest."""
 
     def items():
         """Returns an iterable of (path, node) for items in this manifest."""
 
     def iteritems():
         """Identical to items()."""
 
     def iterentries():
         """Returns an iterable of (path, node, flags) for this manifest.
 
         Similar to ``iteritems()`` except items are a 3-tuple and include
         flags.
         """
 
     def text():
         """Obtain the raw data representation for this manifest.
 
         Result is used to create a manifest revision.
         """
 
     def fastdelta(base, changes):
         """Obtain a delta between this manifest and another given changes.
 
         ``base`` in the raw data representation for another manifest.
 
         ``changes`` is an iterable of ``(path, to_delete)``.
 
         Returns a 2-tuple containing ``bytearray(self.text())`` and the
         delta between ``base`` and this manifest.
         """
 
 class imanifestrevisionbase(interfaceutil.Interface):
     """Base interface representing a single revision of a manifest.
 
     Should not be used as a primary interface: should always be inherited
     as part of a larger interface.
     """
 
     def new():
         """Obtain a new manifest instance.
 
         Returns an object conforming to the ``imanifestrevisionwritable``
         interface. The instance will be associated with the same
         ``imanifestlog`` collection as this instance.
         """
 
     def copy():
         """Obtain a copy of this manifest instance.
 
         Returns an object conforming to the ``imanifestrevisionwritable``
         interface. The instance will be associated with the same
         ``imanifestlog`` collection as this instance.
         """
 
     def read():
         """Obtain the parsed manifest data structure.
 
         The returned object conforms to the ``imanifestdict`` interface.
         """
 
 class imanifestrevisionstored(imanifestrevisionbase):
     """Interface representing a manifest revision committed to storage."""
 
     def node():
         """The binary node for this manifest."""
 
     parents = interfaceutil.Attribute(
         """List of binary nodes that are parents for this manifest revision."""
     )
 
     def readdelta(shallow=False):
         """Obtain the manifest data structure representing changes from parent.
 
         This manifest is compared to its 1st parent. A new manifest representing
         those differences is constructed.
 
         The returned object conforms to the ``imanifestdict`` interface.
         """
 
     def readfast(shallow=False):
         """Calls either ``read()`` or ``readdelta()``.
 
         The faster of the two options is called.
         """
 
     def find(key):
         """Calls self.read().find(key)``.
 
         Returns a 2-tuple of ``(node, flags)`` or raises ``KeyError``.
         """
 
 class imanifestrevisionwritable(imanifestrevisionbase):
     """Interface representing a manifest revision that can be committed."""
 
     def write(transaction, linkrev, p1node, p2node, added, removed, match=None):
         """Add this revision to storage.
 
         Takes a transaction object, the changeset revision number it will
         be associated with, its parent nodes, and lists of added and
         removed paths.
 
         If match is provided, storage can choose not to inspect or write out
         items that do not match. Storage is still required to be able to provide
         the full manifest in the future for any directories written (these
         manifests should not be "narrowed on disk").
 
         Returns the binary node of the created revision.
         """
 
 class imanifeststorage(interfaceutil.Interface):
     """Storage interface for manifest data."""
 
     tree = interfaceutil.Attribute(
         """The path to the directory this manifest tracks.
 
         The empty bytestring represents the root manifest.
         """)
 
     index = interfaceutil.Attribute(
         """An ``ifilerevisionssequence`` instance.""")
 
     indexfile = interfaceutil.Attribute(
         """Path of revlog index file.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     opener = interfaceutil.Attribute(
         """VFS opener to use to access underlying files used for storage.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     version = interfaceutil.Attribute(
         """Revlog version number.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     _generaldelta = interfaceutil.Attribute(
         """Whether generaldelta storage is being used.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     fulltextcache = interfaceutil.Attribute(
         """Dict with cache of fulltexts.
 
         TODO this doesn't feel appropriate for the storage interface.
         """)
 
     def __len__():
         """Obtain the number of revisions stored for this manifest."""
 
     def __iter__():
         """Iterate over revision numbers for this manifest."""
 
     def rev(node):
         """Obtain the revision number given a binary node.
 
         Raises ``error.LookupError`` if the node is not known.
         """
 
     def node(rev):
         """Obtain the node value given a revision number.
 
         Raises ``error.LookupError`` if the revision is not known.
         """
 
     def lookup(value):
         """Attempt to resolve a value to a node.
 
         Value can be a binary node, hex node, revision number, or a bytes
         that can be converted to an integer.
 
         Raises ``error.LookupError`` if a ndoe could not be resolved.
         """
 
     def parents(node):
         """Returns a 2-tuple of parent nodes for a node.
 
         Values will be ``nullid`` if the parent is empty.
         """
 
     def parentrevs(rev):
         """Like parents() but operates on revision numbers."""
 
     def linkrev(rev):
         """Obtain the changeset revision number a revision is linked to."""
 
     def revision(node, _df=None, raw=False):
         """Obtain fulltext data for a node."""
 
     def rawdata(node, _df=None):
         """Obtain raw data for a node."""
 
     def revdiff(rev1, rev2):
         """Obtain a delta between two revision numbers.
 
         The returned data is the result of ``bdiff.bdiff()`` on the raw
         revision data.
         """
 
     def cmp(node, fulltext):
         """Compare fulltext to another revision.
 
         Returns True if the fulltext is different from what is stored.
         """
 
     def emitrevisions(nodes,
                       nodesorder=None,
                       revisiondata=False,
                       assumehaveparentrevisions=False):
         """Produce ``irevisiondelta`` describing revisions.
 
         See the documentation for ``ifiledata`` for more.
         """
 
     def addgroup(deltas, linkmapper, transaction, addrevisioncb=None):
         """Process a series of deltas for storage.
 
         See the documentation in ``ifilemutation`` for more.
         """
 
     def rawsize(rev):
         """Obtain the size of tracked data.
 
         Is equivalent to ``len(m.rawdata(node))``.
 
         TODO this method is only used by upgrade code and may be removed.
         """
 
     def getstrippoint(minlink):
         """Find minimum revision that must be stripped to strip a linkrev.
 
         See the documentation in ``ifilemutation`` for more.
         """
 
     def strip(minlink, transaction):
         """Remove storage of items starting at a linkrev.
 
         See the documentation in ``ifilemutation`` for more.
         """
 
     def checksize():
         """Obtain the expected sizes of backing files.
 
         TODO this is used by verify and it should not be part of the interface.
         """
 
     def files():
         """Obtain paths that are backing storage for this manifest.
 
         TODO this is used by verify and there should probably be a better API
         for this functionality.
         """
 
     def deltaparent(rev):
         """Obtain the revision that a revision is delta'd against.
 
         TODO delta encoding is an implementation detail of storage and should
         not be exposed to the storage interface.
         """
 
     def clone(tr, dest, **kwargs):
         """Clone this instance to another."""
 
     def clearcaches(clear_persisted_data=False):
         """Clear any caches associated with this instance."""
 
     def dirlog(d):
         """Obtain a manifest storage instance for a tree."""
 
     def add(m, transaction, link, p1, p2, added, removed, readtree=None,
             match=None):
         """Add a revision to storage.
 
         ``m`` is an object conforming to ``imanifestdict``.
 
         ``link`` is the linkrev revision number.
 
         ``p1`` and ``p2`` are the parent revision numbers.
 
         ``added`` and ``removed`` are iterables of added and removed paths,
         respectively.
 
         ``readtree`` is a function that can be used to read the child tree(s)
         when recursively writing the full tree structure when using
         treemanifets.
 
         ``match`` is a matcher that can be used to hint to storage that not all
         paths must be inspected; this is an optimization and can be safely
         ignored. Note that the storage must still be able to reproduce a full
         manifest including files that did not match.
         """
 
     def storageinfo(exclusivefiles=False, sharedfiles=False,
                     revisionscount=False, trackedsize=False,
                     storedsize=False):
         """Obtain information about storage for this manifest's data.
 
         See ``ifilestorage.storageinfo()`` for a description of this method.
         This one behaves the same way, except for manifest data.
         """
 
 class imanifestlog(interfaceutil.Interface):
     """Interface representing a collection of manifest snapshots.
 
     Represents the root manifest in a repository.
 
     Also serves as a means to access nested tree manifests and to cache
     tree manifests.
     """
 
     def __getitem__(node):
         """Obtain a manifest instance for a given binary node.
 
         Equivalent to calling ``self.get('', node)``.
 
         The returned object conforms to the ``imanifestrevisionstored``
         interface.
         """
 
     def get(tree, node, verify=True):
         """Retrieve the manifest instance for a given directory and binary node.
 
         ``node`` always refers to the node of the root manifest (which will be
         the only manifest if flat manifests are being used).
 
         If ``tree`` is the empty string, the root manifest is returned.
         Otherwise the manifest for the specified directory will be returned
         (requires tree manifests).
 
         If ``verify`` is True, ``LookupError`` is raised if the node is not
         known.
 
         The returned object conforms to the ``imanifestrevisionstored``
         interface.
         """
 
     def getstorage(tree):
         """Retrieve an interface to storage for a particular tree.
 
         If ``tree`` is the empty bytestring, storage for the root manifest will
         be returned. Otherwise storage for a tree manifest is returned.
 
         TODO formalize interface for returned object.
         """
 
     def clearcaches():
         """Clear caches associated with this collection."""
 
     def rev(node):
         """Obtain the revision number for a binary node.
 
         Raises ``error.LookupError`` if the node is not known.
         """
 
 class ilocalrepositoryfilestorage(interfaceutil.Interface):
     """Local repository sub-interface providing access to tracked file storage.
 
     This interface defines how a repository accesses storage for a single
     tracked file path.
     """
 
     def file(f):
         """Obtain a filelog for a tracked path.
 
         The returned type conforms to the ``ifilestorage`` interface.
         """
 
 class ilocalrepositorymain(interfaceutil.Interface):
     """Main interface for local repositories.
 
     This currently captures the reality of things - not how things should be.
     """
 
     supportedformats = interfaceutil.Attribute(
         """Set of requirements that apply to stream clone.
 
         This is actually a class attribute and is shared among all instances.
         """)
 
     supported = interfaceutil.Attribute(
         """Set of requirements that this repo is capable of opening.""")
 
     requirements = interfaceutil.Attribute(
         """Set of requirements this repo uses.""")
 
     features = interfaceutil.Attribute(
         """Set of "features" this repository supports.
 
         A "feature" is a loosely-defined term. It can refer to a feature
         in the classical sense or can describe an implementation detail
         of the repository. For example, a ``readonly`` feature may denote
         the repository as read-only. Or a ``revlogfilestore`` feature may
         denote that the repository is using revlogs for file storage.
 
         The intent of features is to provide a machine-queryable mechanism
         for repo consumers to test for various repository characteristics.
 
         Features are similar to ``requirements``. The main difference is that
         requirements are stored on-disk and represent requirements to open the
         repository. Features are more run-time capabilities of the repository
         and more granular capabilities (which may be derived from requirements).
         """)
 
     filtername = interfaceutil.Attribute(
         """Name of the repoview that is active on this repo.""")
 
     wvfs = interfaceutil.Attribute(
         """VFS used to access the working directory.""")
 
     vfs = interfaceutil.Attribute(
         """VFS rooted at the .hg directory.
 
         Used to access repository data not in the store.
         """)
 
     svfs = interfaceutil.Attribute(
         """VFS rooted at the store.
 
         Used to access repository data in the store. Typically .hg/store.
         But can point elsewhere if the store is shared.
         """)
 
     root = interfaceutil.Attribute(
         """Path to the root of the working directory.""")
 
     path = interfaceutil.Attribute(
         """Path to the .hg directory.""")
 
     origroot = interfaceutil.Attribute(
         """The filesystem path that was used to construct the repo.""")
 
     auditor = interfaceutil.Attribute(
         """A pathauditor for the working directory.
 
         This checks if a path refers to a nested repository.
 
         Operates on the filesystem.
         """)
 
     nofsauditor = interfaceutil.Attribute(
         """A pathauditor for the working directory.
 
         This is like ``auditor`` except it doesn't do filesystem checks.
         """)
 
     baseui = interfaceutil.Attribute(
         """Original ui instance passed into constructor.""")
 
     ui = interfaceutil.Attribute(
         """Main ui instance for this instance.""")
 
     sharedpath = interfaceutil.Attribute(
         """Path to the .hg directory of the repo this repo was shared from.""")
 
     store = interfaceutil.Attribute(
         """A store instance.""")
 
     spath = interfaceutil.Attribute(
         """Path to the store.""")
 
     sjoin = interfaceutil.Attribute(
         """Alias to self.store.join.""")
 
     cachevfs = interfaceutil.Attribute(
         """A VFS used to access the cache directory.
 
         Typically .hg/cache.
         """)
 
     wcachevfs = interfaceutil.Attribute(
         """A VFS used to access the cache directory dedicated to working copy
 
         Typically .hg/wcache.
         """)
 
     filteredrevcache = interfaceutil.Attribute(
         """Holds sets of revisions to be filtered.""")
 
     names = interfaceutil.Attribute(
         """A ``namespaces`` instance.""")
 
     def close():
         """Close the handle on this repository."""
 
     def peer():
         """Obtain an object conforming to the ``peer`` interface."""
 
     def unfiltered():
         """Obtain an unfiltered/raw view of this repo."""
 
     def filtered(name, visibilityexceptions=None):
         """Obtain a named view of this repository."""
 
     obsstore = interfaceutil.Attribute(
         """A store of obsolescence data.""")
 
     changelog = interfaceutil.Attribute(
         """A handle on the changelog revlog.""")
 
     manifestlog = interfaceutil.Attribute(
         """An instance conforming to the ``imanifestlog`` interface.
 
         Provides access to manifests for the repository.
         """)
 
     dirstate = interfaceutil.Attribute(
         """Working directory state.""")
 
     narrowpats = interfaceutil.Attribute(
         """Matcher patterns for this repository's narrowspec.""")
 
     def narrowmatch(match=None, includeexact=False):
         """Obtain a matcher for the narrowspec."""
 
     def setnarrowpats(newincludes, newexcludes):
         """Define the narrowspec for this repository."""
 
     def __getitem__(changeid):
         """Try to resolve a changectx."""
 
     def __contains__(changeid):
         """Whether a changeset exists."""
 
     def __nonzero__():
         """Always returns True."""
         return True
 
     __bool__ = __nonzero__
 
     def __len__():
         """Returns the number of changesets in the repo."""
 
     def __iter__():
         """Iterate over revisions in the changelog."""
 
     def revs(expr, *args):
         """Evaluate a revset.
 
         Emits revisions.
         """
 
     def set(expr, *args):
         """Evaluate a revset.
 
         Emits changectx instances.
         """
 
     def anyrevs(specs, user=False, localalias=None):
         """Find revisions matching one of the given revsets."""
 
     def url():
         """Returns a string representing the location of this repo."""
 
     def hook(name, throw=False, **args):
         """Call a hook."""
 
     def tags():
         """Return a mapping of tag to node."""
 
     def tagtype(tagname):
         """Return the type of a given tag."""
 
     def tagslist():
         """Return a list of tags ordered by revision."""
 
     def nodetags(node):
         """Return the tags associated with a node."""
 
     def nodebookmarks(node):
         """Return the list of bookmarks pointing to the specified node."""
 
     def branchmap():
         """Return a mapping of branch to heads in that branch."""
 
     def revbranchcache():
         pass
 
     def branchtip(branchtip, ignoremissing=False):
         """Return the tip node for a given branch."""
 
     def lookup(key):
         """Resolve the node for a revision."""
 
     def lookupbranch(key):
         """Look up the branch name of the given revision or branch name."""
 
     def known(nodes):
         """Determine whether a series of nodes is known.
 
         Returns a list of bools.
         """
 
     def local():
         """Whether the repository is local."""
         return True
 
     def publishing():
         """Whether the repository is a publishing repository."""
 
     def cancopy():
         pass
 
     def shared():
         """The type of shared repository or None."""
 
     def wjoin(f, *insidef):
         """Calls self.vfs.reljoin(self.root, f, *insidef)"""
 
     def setparents(p1, p2):
         """Set the parent nodes of the working directory."""
 
     def filectx(path, changeid=None, fileid=None):
         """Obtain a filectx for the given file revision."""
 
     def getcwd():
         """Obtain the current working directory from the dirstate."""
 
     def pathto(f, cwd=None):
         """Obtain the relative path to a file."""
 
     def adddatafilter(name, fltr):
         pass
 
     def wread(filename):
         """Read a file from wvfs, using data filters."""
 
     def wwrite(filename, data, flags, backgroundclose=False, **kwargs):
         """Write data to a file in the wvfs, using data filters."""
 
     def wwritedata(filename, data):
         """Resolve data for writing to the wvfs, using data filters."""
 
     def currenttransaction():
         """Obtain the current transaction instance or None."""
 
     def transaction(desc, report=None):
         """Open a new transaction to write to the repository."""
 
     def undofiles():
         """Returns a list of (vfs, path) for files to undo transactions."""
 
     def recover():
         """Roll back an interrupted transaction."""
 
     def rollback(dryrun=False, force=False):
         """Undo the last transaction.
 
         DANGEROUS.
         """
 
     def updatecaches(tr=None, full=False):
         """Warm repo caches."""
 
     def invalidatecaches():
         """Invalidate cached data due to the repository mutating."""
 
     def invalidatevolatilesets():
         pass
 
     def invalidatedirstate():
         """Invalidate the dirstate."""
 
     def invalidate(clearfilecache=False):
         pass
 
     def invalidateall():
         pass
 
     def lock(wait=True):
         """Lock the repository store and return a lock instance."""
 
     def wlock(wait=True):
         """Lock the non-store parts of the repository."""
 
     def currentwlock():
         """Return the wlock if it's held or None."""
 
     def checkcommitpatterns(wctx, vdirs, match, status, fail):
         pass
 
     def commit(text='', user=None, date=None, match=None, force=False,
                editor=False, extra=None):
         """Add a new revision to the repository."""
 
     def commitctx(ctx, error=False, origctx=None):
         """Commit a commitctx instance to the repository."""
 
     def destroying():
         """Inform the repository that nodes are about to be destroyed."""
 
     def destroyed():
         """Inform the repository that nodes have been destroyed."""
 
     def status(node1='.', node2=None, match=None, ignored=False,
                clean=False, unknown=False, listsubrepos=False):
         """Convenience method to call repo[x].status()."""
 
     def addpostdsstatus(ps):
         pass
 
     def postdsstatus():
         pass
 
     def clearpostdsstatus():
         pass
 
     def heads(start=None):
         """Obtain list of nodes that are DAG heads."""
 
     def branchheads(branch=None, start=None, closed=False):
         pass
 
     def branches(nodes):
         pass
 
     def between(pairs):
         pass
 
     def checkpush(pushop):
         pass
 
     prepushoutgoinghooks = interfaceutil.Attribute(
         """util.hooks instance.""")
 
     def pushkey(namespace, key, old, new):
         pass
 
     def listkeys(namespace):
         pass
 
     def debugwireargs(one, two, three=None, four=None, five=None):
         pass
 
     def savecommitmessage(text):
         pass
 
 class completelocalrepository(ilocalrepositorymain,
                               ilocalrepositoryfilestorage):
     """Complete interface for a local repository."""
 
 class iwireprotocolcommandcacher(interfaceutil.Interface):
     """Represents a caching backend for wire protocol commands.
 
     Wire protocol version 2 supports transparent caching of many commands.
     To leverage this caching, servers can activate objects that cache
     command responses. Objects handle both cache writing and reading.
     This interface defines how that response caching mechanism works.
 
     Wire protocol version 2 commands emit a series of objects that are
     serialized and sent to the client. The caching layer exists between
     the invocation of the command function and the sending of its output
     objects to an output layer.
 
     Instances of this interface represent a binding to a cache that
     can serve a response (in place of calling a command function) and/or
     write responses to a cache for subsequent use.
 
     When a command request arrives, the following happens with regards
     to this interface:
 
     1. The server determines whether the command request is cacheable.
     2. If it is, an instance of this interface is spawned.
     3. The cacher is activated in a context manager (``__enter__`` is called).
     4. A cache *key* for that request is derived. This will call the
        instance's ``adjustcachekeystate()`` method so the derivation
        can be influenced.
     5. The cacher is informed of the derived cache key via a call to
        ``setcachekey()``.
     6. The cacher's ``lookup()`` method is called to test for presence of
        the derived key in the cache.
     7. If ``lookup()`` returns a hit, that cached result is used in place
        of invoking the command function. ``__exit__`` is called and the instance
        is discarded.
     8. The command function is invoked.
     9. ``onobject()`` is called for each object emitted by the command
        function.
     10. After the final object is seen, ``onfinished()`` is called.
     11. ``__exit__`` is called to signal the end of use of the instance.
 
     Cache *key* derivation can be influenced by the instance.
 
     Cache keys are initially derived by a deterministic representation of
     the command request. This includes the command name, arguments, protocol
     version, etc. This initial key derivation is performed by CBOR-encoding a
     data structure and feeding that output into a hasher.
 
     Instances of this interface can influence this initial key derivation
     via ``adjustcachekeystate()``.
 
     The instance is informed of the derived cache key via a call to
     ``setcachekey()``. The instance must store the key locally so it can
     be consulted on subsequent operations that may require it.
 
     When constructed, the instance has access to a callable that can be used
     for encoding response objects. This callable receives as its single
     argument an object emitted by a command function. It returns an iterable
     of bytes chunks representing the encoded object. Unless the cacher is
     caching native Python objects in memory or has a way of reconstructing
     the original Python objects, implementations typically call this function
     to produce bytes from the output objects and then store those bytes in
     the cache. When it comes time to re-emit those bytes, they are wrapped
     in a ``wireprototypes.encodedresponse`` instance to tell the output
     layer that they are pre-encoded.
 
     When receiving the objects emitted by the command function, instances
     can choose what to do with those objects. The simplest thing to do is
     re-emit the original objects. They will be forwarded to the output
     layer and will be processed as if the cacher did not exist.
 
     Implementations could also choose to not emit objects - instead locally
     buffering objects or their encoded representation. They could then emit
     a single "coalesced" object when ``onfinished()`` is called. In
     this way, the implementation would function as a filtering layer of
     sorts.
 
     When caching objects, typically the encoded form of the object will
     be stored. Keep in mind that if the original object is forwarded to
     the output layer, it will need to be encoded there as well. For large
     output, this redundant encoding could add overhead. Implementations
     could wrap the encoded object data in ``wireprototypes.encodedresponse``
     instances to avoid this overhead.
     """
     def __enter__():
         """Marks the instance as active.
 
         Should return self.
         """
 
     def __exit__(exctype, excvalue, exctb):
         """Called when cacher is no longer used.
 
         This can be used by implementations to perform cleanup actions (e.g.
         disconnecting network sockets, aborting a partially cached response.
         """
 
     def adjustcachekeystate(state):
         """Influences cache key derivation by adjusting state to derive key.
 
         A dict defining the state used to derive the cache key is passed.
 
         Implementations can modify this dict to record additional state that
         is wanted to influence key derivation.
 
         Implementations are *highly* encouraged to not modify or delete
         existing keys.
         """
 
     def setcachekey(key):
         """Record the derived cache key for this request.
 
         Instances may mutate the key for internal usage, as desired. e.g.
         instances may wish to prepend the repo name, introduce path
         components for filesystem or URL addressing, etc. Behavior is up to
         the cache.
 
         Returns a bool indicating if the request is cacheable by this
         instance.
         """
 
     def lookup():
         """Attempt to resolve an entry in the cache.
 
         The instance is instructed to look for the cache key that it was
         informed about via the call to ``setcachekey()``.
 
         If there's no cache hit or the cacher doesn't wish to use the cached
         entry, ``None`` should be returned.
 
         Else, a dict defining the cached result should be returned. The
         dict may have the following keys:
 
         objs
            An iterable of objects that should be sent to the client. That
            iterable of objects is expected to be what the command function
            would return if invoked or an equivalent representation thereof.
         """
 
     def onobject(obj):
         """Called when a new object is emitted from the command function.
 
         Receives as its argument the object that was emitted from the
         command function.
 
         This method returns an iterator of objects to forward to the output
         layer. The easiest implementation is a generator that just
         ``yield obj``.
         """
 
     def onfinished():
         """Called after all objects have been emitted from the command function.
 
         Implementations should return an iterator of objects to forward to
         the output layer.
 
         This method can be a generator.
         """