formatting: run black version 19.10b0 on the codebase

The latest version of black is out and contains the change we needed. So we can
start using it now.

note: `test-check-format.t` will complains about this changes because it still
use `grey` and need to be migrated to `black`. See next changesets for this.

hgext/gpg.py

@@ -1,382 +1,381 b''
 # Copyright 2005, 2006 Benoit Boissinot <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''commands to sign and verify changesets'''
 
 from __future__ import absolute_import
 
 import binascii
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     error,
     help,
     match,
     node as hgnode,
     pycompat,
     registrar,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'gpg', b'cmd', default=b'gpg',
 )
 configitem(
     b'gpg', b'key', default=None,
 )
 configitem(
     b'gpg', b'.*', default=None, generic=True,
 )
 
 # Custom help category
 _HELP_CATEGORY = b'gpg'
 help.CATEGORY_ORDER.insert(
     help.CATEGORY_ORDER.index(registrar.command.CATEGORY_HELP), _HELP_CATEGORY
 )
 help.CATEGORY_NAMES[_HELP_CATEGORY] = b'Signing changes (GPG)'
 
 
 class gpg(object):
     def __init__(self, path, key=None):
         self.path = path
         self.key = (key and b" --local-user \"%s\"" % key) or b""
 
     def sign(self, data):
         gpgcmd = b"%s --sign --detach-sign%s" % (self.path, self.key)
         return procutil.filter(data, gpgcmd)
 
     def verify(self, data, sig):
         """ returns of the good and bad signatures"""
         sigfile = datafile = None
         try:
             # create temporary files
             fd, sigfile = pycompat.mkstemp(prefix=b"hg-gpg-", suffix=b".sig")
             fp = os.fdopen(fd, r'wb')
             fp.write(sig)
             fp.close()
             fd, datafile = pycompat.mkstemp(prefix=b"hg-gpg-", suffix=b".txt")
             fp = os.fdopen(fd, r'wb')
             fp.write(data)
             fp.close()
-            gpgcmd = b"%s --logger-fd 1 --status-fd 1 --verify \"%s\" \"%s\"" % (
-                self.path,
-                sigfile,
-                datafile,
+            gpgcmd = (
+                b"%s --logger-fd 1 --status-fd 1 --verify \"%s\" \"%s\""
+                % (self.path, sigfile, datafile,)
             )
             ret = procutil.filter(b"", gpgcmd)
         finally:
             for f in (sigfile, datafile):
                 try:
                     if f:
                         os.unlink(f)
                 except OSError:
                     pass
         keys = []
         key, fingerprint = None, None
         for l in ret.splitlines():
             # see DETAILS in the gnupg documentation
             # filter the logger output
             if not l.startswith(b"[GNUPG:]"):
                 continue
             l = l[9:]
             if l.startswith(b"VALIDSIG"):
                 # fingerprint of the primary key
                 fingerprint = l.split()[10]
             elif l.startswith(b"ERRSIG"):
                 key = l.split(b" ", 3)[:2]
                 key.append(b"")
                 fingerprint = None
             elif (
                 l.startswith(b"GOODSIG")
                 or l.startswith(b"EXPSIG")
                 or l.startswith(b"EXPKEYSIG")
                 or l.startswith(b"BADSIG")
             ):
                 if key is not None:
                     keys.append(key + [fingerprint])
                 key = l.split(b" ", 2)
                 fingerprint = None
         if key is not None:
             keys.append(key + [fingerprint])
         return keys
 
 
 def newgpg(ui, **opts):
     """create a new gpg instance"""
     gpgpath = ui.config(b"gpg", b"cmd")
     gpgkey = opts.get(r'key')
     if not gpgkey:
         gpgkey = ui.config(b"gpg", b"key")
     return gpg(gpgpath, gpgkey)
 
 
 def sigwalk(repo):
     """
     walk over every sigs, yields a couple
     ((node, version, sig), (filename, linenumber))
     """
 
     def parsefile(fileiter, context):
         ln = 1
         for l in fileiter:
             if not l:
                 continue
             yield (l.split(b" ", 2), (context, ln))
             ln += 1
 
     # read the heads
     fl = repo.file(b".hgsigs")
     for r in reversed(fl.heads()):
         fn = b".hgsigs|%s" % hgnode.short(r)
         for item in parsefile(fl.read(r).splitlines(), fn):
             yield item
     try:
         # read local signatures
         fn = b"localsigs"
         for item in parsefile(repo.vfs(fn), fn):
             yield item
     except IOError:
         pass
 
 
 def getkeys(ui, repo, mygpg, sigdata, context):
     """get the keys who signed a data"""
     fn, ln = context
     node, version, sig = sigdata
     prefix = b"%s:%d" % (fn, ln)
     node = hgnode.bin(node)
 
     data = node2txt(repo, node, version)
     sig = binascii.a2b_base64(sig)
     keys = mygpg.verify(data, sig)
 
     validkeys = []
     # warn for expired key and/or sigs
     for key in keys:
         if key[0] == b"ERRSIG":
             ui.write(_(b"%s Unknown key ID \"%s\"\n") % (prefix, key[1]))
             continue
         if key[0] == b"BADSIG":
             ui.write(_(b"%s Bad signature from \"%s\"\n") % (prefix, key[2]))
             continue
         if key[0] == b"EXPSIG":
             ui.write(
                 _(b"%s Note: Signature has expired (signed by: \"%s\")\n")
                 % (prefix, key[2])
             )
         elif key[0] == b"EXPKEYSIG":
             ui.write(
                 _(b"%s Note: This key has expired (signed by: \"%s\")\n")
                 % (prefix, key[2])
             )
         validkeys.append((key[1], key[2], key[3]))
     return validkeys
 
 
 @command(b"sigs", [], _(b'hg sigs'), helpcategory=_HELP_CATEGORY)
 def sigs(ui, repo):
     """list signed changesets"""
     mygpg = newgpg(ui)
     revs = {}
 
     for data, context in sigwalk(repo):
         node, version, sig = data
         fn, ln = context
         try:
             n = repo.lookup(node)
         except KeyError:
             ui.warn(_(b"%s:%d node does not exist\n") % (fn, ln))
             continue
         r = repo.changelog.rev(n)
         keys = getkeys(ui, repo, mygpg, data, context)
         if not keys:
             continue
         revs.setdefault(r, [])
         revs[r].extend(keys)
     for rev in sorted(revs, reverse=True):
         for k in revs[rev]:
             r = b"%5d:%s" % (rev, hgnode.hex(repo.changelog.node(rev)))
             ui.write(b"%-30s %s\n" % (keystr(ui, k), r))
 
 
 @command(b"sigcheck", [], _(b'hg sigcheck REV'), helpcategory=_HELP_CATEGORY)
 def sigcheck(ui, repo, rev):
     """verify all the signatures there may be for a particular revision"""
     mygpg = newgpg(ui)
     rev = repo.lookup(rev)
     hexrev = hgnode.hex(rev)
     keys = []
 
     for data, context in sigwalk(repo):
         node, version, sig = data
         if node == hexrev:
             k = getkeys(ui, repo, mygpg, data, context)
             if k:
                 keys.extend(k)
 
     if not keys:
         ui.write(_(b"no valid signature for %s\n") % hgnode.short(rev))
         return
 
     # print summary
     ui.write(_(b"%s is signed by:\n") % hgnode.short(rev))
     for key in keys:
         ui.write(b" %s\n" % keystr(ui, key))
 
 
 def keystr(ui, key):
     """associate a string to a key (username, comment)"""
     keyid, user, fingerprint = key
     comment = ui.config(b"gpg", fingerprint)
     if comment:
         return b"%s (%s)" % (user, comment)
     else:
         return user
 
 
 @command(
     b"sign",
     [
         (b'l', b'local', None, _(b'make the signature local')),
         (b'f', b'force', None, _(b'sign even if the sigfile is modified')),
         (
             b'',
             b'no-commit',
             None,
             _(b'do not commit the sigfile after signing'),
         ),
         (b'k', b'key', b'', _(b'the key id to sign with'), _(b'ID')),
         (b'm', b'message', b'', _(b'use text as commit message'), _(b'TEXT')),
         (b'e', b'edit', False, _(b'invoke editor on commit messages')),
     ]
     + cmdutil.commitopts2,
     _(b'hg sign [OPTION]... [REV]...'),
     helpcategory=_HELP_CATEGORY,
 )
 def sign(ui, repo, *revs, **opts):
     """add a signature for the current or given revision
 
     If no revision is given, the parent of the working directory is used,
     or tip if no revision is checked out.
 
     The ``gpg.cmd`` config setting can be used to specify the command
     to run. A default key can be specified with ``gpg.key``.
 
     See :hg:`help dates` for a list of formats valid for -d/--date.
     """
     with repo.wlock():
         return _dosign(ui, repo, *revs, **opts)
 
 
 def _dosign(ui, repo, *revs, **opts):
     mygpg = newgpg(ui, **opts)
     opts = pycompat.byteskwargs(opts)
     sigver = b"0"
     sigmessage = b""
 
     date = opts.get(b'date')
     if date:
         opts[b'date'] = dateutil.parsedate(date)
 
     if revs:
         nodes = [repo.lookup(n) for n in revs]
     else:
         nodes = [
             node for node in repo.dirstate.parents() if node != hgnode.nullid
         ]
         if len(nodes) > 1:
             raise error.Abort(
                 _(b'uncommitted merge - please provide a specific revision')
             )
         if not nodes:
             nodes = [repo.changelog.tip()]
 
     for n in nodes:
         hexnode = hgnode.hex(n)
         ui.write(
             _(b"signing %d:%s\n") % (repo.changelog.rev(n), hgnode.short(n))
         )
         # build data
         data = node2txt(repo, n, sigver)
         sig = mygpg.sign(data)
         if not sig:
             raise error.Abort(_(b"error while signing"))
         sig = binascii.b2a_base64(sig)
         sig = sig.replace(b"\n", b"")
         sigmessage += b"%s %s %s\n" % (hexnode, sigver, sig)
 
     # write it
     if opts[b'local']:
         repo.vfs.append(b"localsigs", sigmessage)
         return
 
     if not opts[b"force"]:
         msigs = match.exact([b'.hgsigs'])
         if any(repo.status(match=msigs, unknown=True, ignored=True)):
             raise error.Abort(
                 _(b"working copy of .hgsigs is changed "),
                 hint=_(b"please commit .hgsigs manually"),
             )
 
     sigsfile = repo.wvfs(b".hgsigs", b"ab")
     sigsfile.write(sigmessage)
     sigsfile.close()
 
     if b'.hgsigs' not in repo.dirstate:
         repo[None].add([b".hgsigs"])
 
     if opts[b"no_commit"]:
         return
 
     message = opts[b'message']
     if not message:
         # we don't translate commit messages
         message = b"\n".join(
             [
                 b"Added signature for changeset %s" % hgnode.short(n)
                 for n in nodes
             ]
         )
     try:
         editor = cmdutil.getcommiteditor(
             editform=b'gpg.sign', **pycompat.strkwargs(opts)
         )
         repo.commit(
             message, opts[b'user'], opts[b'date'], match=msigs, editor=editor
         )
     except ValueError as inst:
         raise error.Abort(pycompat.bytestr(inst))
 
 
 def node2txt(repo, node, ver):
     """map a manifest into some text"""
     if ver == b"0":
         return b"%s\n" % hgnode.hex(node)
     else:
         raise error.Abort(_(b"unknown signature version"))
 
 
 def extsetup(ui):
     # Add our category before "Repository maintenance".
     help.CATEGORY_ORDER.insert(
         help.CATEGORY_ORDER.index(command.CATEGORY_MAINTENANCE), _HELP_CATEGORY
     )
     help.CATEGORY_NAMES[_HELP_CATEGORY] = b'GPG signing'

mercurial/config.py

@@ -1,313 +1,311 b''
 # config.py - configuration parsing for Mercurial
 #
 #  Copyright 2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import os
 
 from .i18n import _
 from .pycompat import getattr
 from . import (
     encoding,
     error,
     pycompat,
     util,
 )
 
 
 class config(object):
     def __init__(self, data=None, includepaths=None):
         self._data = {}
         self._unset = []
         self._includepaths = includepaths or []
         if data:
             for k in data._data:
                 self._data[k] = data[k].copy()
             self._source = data._source.copy()
         else:
             self._source = util.cowdict()
 
     def copy(self):
         return config(self)
 
     def __contains__(self, section):
         return section in self._data
 
     def hasitem(self, section, item):
         return item in self._data.get(section, {})
 
     def __getitem__(self, section):
         return self._data.get(section, {})
 
     def __iter__(self):
         for d in self.sections():
             yield d
 
     def update(self, src):
         self._source = self._source.preparewrite()
         for s, n in src._unset:
             ds = self._data.get(s, None)
             if ds is not None and n in ds:
                 self._data[s] = ds.preparewrite()
                 del self._data[s][n]
                 del self._source[(s, n)]
         for s in src:
             ds = self._data.get(s, None)
             if ds:
                 self._data[s] = ds.preparewrite()
             else:
                 self._data[s] = util.cowsortdict()
             self._data[s].update(src._data[s])
         self._source.update(src._source)
 
     def get(self, section, item, default=None):
         return self._data.get(section, {}).get(item, default)
 
     def backup(self, section, item):
         """return a tuple allowing restore to reinstall a previous value
 
         The main reason we need it is because it handles the "no data" case.
         """
         try:
             value = self._data[section][item]
             source = self.source(section, item)
             return (section, item, value, source)
         except KeyError:
             return (section, item)
 
     def source(self, section, item):
         return self._source.get((section, item), b"")
 
     def sections(self):
         return sorted(self._data.keys())
 
     def items(self, section):
         return list(pycompat.iteritems(self._data.get(section, {})))
 
     def set(self, section, item, value, source=b""):
         if pycompat.ispy3:
             assert not isinstance(
                 section, str
             ), b'config section may not be unicode strings on Python 3'
             assert not isinstance(
                 item, str
             ), b'config item may not be unicode strings on Python 3'
             assert not isinstance(
                 value, str
             ), b'config values may not be unicode strings on Python 3'
         if section not in self:
             self._data[section] = util.cowsortdict()
         else:
             self._data[section] = self._data[section].preparewrite()
         self._data[section][item] = value
         if source:
             self._source = self._source.preparewrite()
             self._source[(section, item)] = source
 
     def restore(self, data):
         """restore data returned by self.backup"""
         self._source = self._source.preparewrite()
         if len(data) == 4:
             # restore old data
             section, item, value, source = data
             self._data[section] = self._data[section].preparewrite()
             self._data[section][item] = value
             self._source[(section, item)] = source
         else:
             # no data before, remove everything
             section, item = data
             if section in self._data:
                 self._data[section].pop(item, None)
             self._source.pop((section, item), None)
 
     def parse(self, src, data, sections=None, remap=None, include=None):
         sectionre = util.re.compile(br'\[([^\[]+)\]')
         itemre = util.re.compile(br'([^=\s][^=]*?)\s*=\s*(.*\S|)')
         contre = util.re.compile(br'\s+(\S|\S.*\S)\s*$')
         emptyre = util.re.compile(br'(;|#|\s*$)')
         commentre = util.re.compile(br'(;|#)')
         unsetre = util.re.compile(br'%unset\s+(\S+)')
         includere = util.re.compile(br'%include\s+(\S|\S.*\S)\s*$')
         section = b""
         item = None
         line = 0
         cont = False
 
         if remap:
             section = remap.get(section, section)
 
         for l in data.splitlines(True):
             line += 1
             if line == 1 and l.startswith(b'\xef\xbb\xbf'):
                 # Someone set us up the BOM
                 l = l[3:]
             if cont:
                 if commentre.match(l):
                     continue
                 m = contre.match(l)
                 if m:
                     if sections and section not in sections:
                         continue
                     v = self.get(section, item) + b"\n" + m.group(1)
                     self.set(section, item, v, b"%s:%d" % (src, line))
                     continue
                 item = None
                 cont = False
             m = includere.match(l)
 
             if m and include:
                 expanded = util.expandpath(m.group(1))
                 includepaths = [os.path.dirname(src)] + self._includepaths
 
                 for base in includepaths:
                     inc = os.path.normpath(os.path.join(base, expanded))
 
                     try:
                         include(inc, remap=remap, sections=sections)
                         break
                     except IOError as inst:
                         if inst.errno != errno.ENOENT:
                             raise error.ParseError(
                                 _(b"cannot include %s (%s)")
                                 % (inc, encoding.strtolocal(inst.strerror)),
                                 b"%s:%d" % (src, line),
                             )
                 continue
             if emptyre.match(l):
                 continue
             m = sectionre.match(l)
             if m:
                 section = m.group(1)
                 if remap:
                     section = remap.get(section, section)
                 if section not in self:
                     self._data[section] = util.cowsortdict()
                 continue
             m = itemre.match(l)
             if m:
                 item = m.group(1)
                 cont = True
                 if sections and section not in sections:
                     continue
                 self.set(section, item, m.group(2), b"%s:%d" % (src, line))
                 continue
             m = unsetre.match(l)
             if m:
                 name = m.group(1)
                 if sections and section not in sections:
                     continue
                 if self.get(section, name) is not None:
                     self._data[section] = self._data[section].preparewrite()
                     del self._data[section][name]
                 self._unset.append((section, name))
                 continue
 
             raise error.ParseError(l.rstrip(), (b"%s:%d" % (src, line)))
 
     def read(self, path, fp=None, sections=None, remap=None):
         if not fp:
             fp = util.posixfile(path, b'rb')
-        assert (
-            getattr(fp, 'mode', r'rb') == r'rb'
-        ), b'config files must be opened in binary mode, got fp=%r mode=%r' % (
-            fp,
-            fp.mode,
+        assert getattr(fp, 'mode', r'rb') == r'rb', (
+            b'config files must be opened in binary mode, got fp=%r mode=%r'
+            % (fp, fp.mode,)
         )
         self.parse(
             path, fp.read(), sections=sections, remap=remap, include=self.read
         )
 
 
 def parselist(value):
     """parse a configuration value as a list of comma/space separated strings
 
     >>> parselist(b'this,is "a small" ,test')
     ['this', 'is', 'a small', 'test']
     """
 
     def _parse_plain(parts, s, offset):
         whitespace = False
         while offset < len(s) and (
             s[offset : offset + 1].isspace() or s[offset : offset + 1] == b','
         ):
             whitespace = True
             offset += 1
         if offset >= len(s):
             return None, parts, offset
         if whitespace:
             parts.append(b'')
         if s[offset : offset + 1] == b'"' and not parts[-1]:
             return _parse_quote, parts, offset + 1
         elif s[offset : offset + 1] == b'"' and parts[-1][-1:] == b'\\':
             parts[-1] = parts[-1][:-1] + s[offset : offset + 1]
             return _parse_plain, parts, offset + 1
         parts[-1] += s[offset : offset + 1]
         return _parse_plain, parts, offset + 1
 
     def _parse_quote(parts, s, offset):
         if offset < len(s) and s[offset : offset + 1] == b'"':  # ""
             parts.append(b'')
             offset += 1
             while offset < len(s) and (
                 s[offset : offset + 1].isspace()
                 or s[offset : offset + 1] == b','
             ):
                 offset += 1
             return _parse_plain, parts, offset
 
         while offset < len(s) and s[offset : offset + 1] != b'"':
             if (
                 s[offset : offset + 1] == b'\\'
                 and offset + 1 < len(s)
                 and s[offset + 1 : offset + 2] == b'"'
             ):
                 offset += 1
                 parts[-1] += b'"'
             else:
                 parts[-1] += s[offset : offset + 1]
             offset += 1
 
         if offset >= len(s):
             real_parts = _configlist(parts[-1])
             if not real_parts:
                 parts[-1] = b'"'
             else:
                 real_parts[0] = b'"' + real_parts[0]
                 parts = parts[:-1]
                 parts.extend(real_parts)
             return None, parts, offset
 
         offset += 1
         while offset < len(s) and s[offset : offset + 1] in [b' ', b',']:
             offset += 1
 
         if offset < len(s):
             if offset + 1 == len(s) and s[offset : offset + 1] == b'"':
                 parts[-1] += b'"'
                 offset += 1
             else:
                 parts.append(b'')
         else:
             return None, parts, offset
 
         return _parse_plain, parts, offset
 
     def _configlist(s):
         s = s.rstrip(b' ,')
         if not s:
             return []
         parser, parts, offset = _parse_plain, [b''], 0
         while parser:
             parser, parts, offset = parser(parts, s, offset)
         return parts
 
     if value is not None and isinstance(value, bytes):
         result = _configlist(value.lstrip(b' ,\n'))
     else:
         result = value
     return result or []

mercurial/context.py

@@ -1,2987 +1,2986 b''
 # context.py - changeset and file context objects for mercurial
 #
 # Copyright 2006, 2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import filecmp
 import os
 import stat
 
 from .i18n import _
 from .node import (
     addednodeid,
     hex,
     modifiednodeid,
     nullid,
     nullrev,
     short,
     wdirfilenodeids,
     wdirhex,
 )
 from .pycompat import (
     getattr,
     open,
 )
 from . import (
     copies,
     dagop,
     encoding,
     error,
     fileset,
     match as matchmod,
     obsolete as obsmod,
     patch,
     pathutil,
     phases,
     pycompat,
     repoview,
     scmutil,
     sparse,
     subrepo,
     subrepoutil,
     util,
 )
 from .utils import (
     dateutil,
     stringutil,
 )
 
 propertycache = util.propertycache
 
 
 class basectx(object):
     """A basectx object represents the common logic for its children:
     changectx: read-only context that is already present in the repo,
     workingctx: a context that represents the working directory and can
                 be committed,
     memctx: a context that represents changes in-memory and can also
             be committed."""
 
     def __init__(self, repo):
         self._repo = repo
 
     def __bytes__(self):
         return short(self.node())
 
     __str__ = encoding.strmethod(__bytes__)
 
     def __repr__(self):
         return r"<%s %s>" % (type(self).__name__, str(self))
 
     def __eq__(self, other):
         try:
             return type(self) == type(other) and self._rev == other._rev
         except AttributeError:
             return False
 
     def __ne__(self, other):
         return not (self == other)
 
     def __contains__(self, key):
         return key in self._manifest
 
     def __getitem__(self, key):
         return self.filectx(key)
 
     def __iter__(self):
         return iter(self._manifest)
 
     def _buildstatusmanifest(self, status):
         """Builds a manifest that includes the given status results, if this is
         a working copy context. For non-working copy contexts, it just returns
         the normal manifest."""
         return self.manifest()
 
     def _matchstatus(self, other, match):
         """This internal method provides a way for child objects to override the
         match operator.
         """
         return match
 
     def _buildstatus(
         self, other, s, match, listignored, listclean, listunknown
     ):
         """build a status with respect to another context"""
         # Load earliest manifest first for caching reasons. More specifically,
         # if you have revisions 1000 and 1001, 1001 is probably stored as a
         # delta against 1000. Thus, if you read 1000 first, we'll reconstruct
         # 1000 and cache it so that when you read 1001, we just need to apply a
         # delta to what's in the cache. So that's one full reconstruction + one
         # delta application.
         mf2 = None
         if self.rev() is not None and self.rev() < other.rev():
             mf2 = self._buildstatusmanifest(s)
         mf1 = other._buildstatusmanifest(s)
         if mf2 is None:
             mf2 = self._buildstatusmanifest(s)
 
         modified, added = [], []
         removed = []
         clean = []
         deleted, unknown, ignored = s.deleted, s.unknown, s.ignored
         deletedset = set(deleted)
         d = mf1.diff(mf2, match=match, clean=listclean)
         for fn, value in pycompat.iteritems(d):
             if fn in deletedset:
                 continue
             if value is None:
                 clean.append(fn)
                 continue
             (node1, flag1), (node2, flag2) = value
             if node1 is None:
                 added.append(fn)
             elif node2 is None:
                 removed.append(fn)
             elif flag1 != flag2:
                 modified.append(fn)
             elif node2 not in wdirfilenodeids:
                 # When comparing files between two commits, we save time by
                 # not comparing the file contents when the nodeids differ.
                 # Note that this means we incorrectly report a reverted change
                 # to a file as a modification.
                 modified.append(fn)
             elif self[fn].cmp(other[fn]):
                 modified.append(fn)
             else:
                 clean.append(fn)
 
         if removed:
             # need to filter files if they are already reported as removed
             unknown = [
                 fn
                 for fn in unknown
                 if fn not in mf1 and (not match or match(fn))
             ]
             ignored = [
                 fn
                 for fn in ignored
                 if fn not in mf1 and (not match or match(fn))
             ]
             # if they're deleted, don't report them as removed
             removed = [fn for fn in removed if fn not in deletedset]
 
         return scmutil.status(
             modified, added, removed, deleted, unknown, ignored, clean
         )
 
     @propertycache
     def substate(self):
         return subrepoutil.state(self, self._repo.ui)
 
     def subrev(self, subpath):
         return self.substate[subpath][1]
 
     def rev(self):
         return self._rev
 
     def node(self):
         return self._node
 
     def hex(self):
         return hex(self.node())
 
     def manifest(self):
         return self._manifest
 
     def manifestctx(self):
         return self._manifestctx
 
     def repo(self):
         return self._repo
 
     def phasestr(self):
         return phases.phasenames[self.phase()]
 
     def mutable(self):
         return self.phase() > phases.public
 
     def matchfileset(self, expr, badfn=None):
         return fileset.match(self, expr, badfn=badfn)
 
     def obsolete(self):
         """True if the changeset is obsolete"""
         return self.rev() in obsmod.getrevs(self._repo, b'obsolete')
 
     def extinct(self):
         """True if the changeset is extinct"""
         return self.rev() in obsmod.getrevs(self._repo, b'extinct')
 
     def orphan(self):
         """True if the changeset is not obsolete, but its ancestor is"""
         return self.rev() in obsmod.getrevs(self._repo, b'orphan')
 
     def phasedivergent(self):
         """True if the changeset tries to be a successor of a public changeset
 
         Only non-public and non-obsolete changesets may be phase-divergent.
         """
         return self.rev() in obsmod.getrevs(self._repo, b'phasedivergent')
 
     def contentdivergent(self):
         """Is a successor of a changeset with multiple possible successor sets
 
         Only non-public and non-obsolete changesets may be content-divergent.
         """
         return self.rev() in obsmod.getrevs(self._repo, b'contentdivergent')
 
     def isunstable(self):
         """True if the changeset is either orphan, phase-divergent or
         content-divergent"""
         return self.orphan() or self.phasedivergent() or self.contentdivergent()
 
     def instabilities(self):
         """return the list of instabilities affecting this changeset.
 
         Instabilities are returned as strings. possible values are:
         - orphan,
         - phase-divergent,
         - content-divergent.
         """
         instabilities = []
         if self.orphan():
             instabilities.append(b'orphan')
         if self.phasedivergent():
             instabilities.append(b'phase-divergent')
         if self.contentdivergent():
             instabilities.append(b'content-divergent')
         return instabilities
 
     def parents(self):
         """return contexts for each parent changeset"""
         return self._parents
 
     def p1(self):
         return self._parents[0]
 
     def p2(self):
         parents = self._parents
         if len(parents) == 2:
             return parents[1]
         return self._repo[nullrev]
 
     def _fileinfo(self, path):
         if r'_manifest' in self.__dict__:
             try:
                 return self._manifest[path], self._manifest.flags(path)
             except KeyError:
                 raise error.ManifestLookupError(
                     self._node, path, _(b'not found in manifest')
                 )
         if r'_manifestdelta' in self.__dict__ or path in self.files():
             if path in self._manifestdelta:
                 return (
                     self._manifestdelta[path],
                     self._manifestdelta.flags(path),
                 )
         mfl = self._repo.manifestlog
         try:
             node, flag = mfl[self._changeset.manifest].find(path)
         except KeyError:
             raise error.ManifestLookupError(
                 self._node, path, _(b'not found in manifest')
             )
 
         return node, flag
 
     def filenode(self, path):
         return self._fileinfo(path)[0]
 
     def flags(self, path):
         try:
             return self._fileinfo(path)[1]
         except error.LookupError:
             return b''
 
     @propertycache
     def _copies(self):
         return copies.computechangesetcopies(self)
 
     def p1copies(self):
         return self._copies[0]
 
     def p2copies(self):
         return self._copies[1]
 
     def sub(self, path, allowcreate=True):
         '''return a subrepo for the stored revision of path, never wdir()'''
         return subrepo.subrepo(self, path, allowcreate=allowcreate)
 
     def nullsub(self, path, pctx):
         return subrepo.nullsubrepo(self, path, pctx)
 
     def workingsub(self, path):
         '''return a subrepo for the stored revision, or wdir if this is a wdir
         context.
         '''
         return subrepo.subrepo(self, path, allowwdir=True)
 
     def match(
         self,
         pats=None,
         include=None,
         exclude=None,
         default=b'glob',
         listsubrepos=False,
         badfn=None,
     ):
         r = self._repo
         return matchmod.match(
             r.root,
             r.getcwd(),
             pats,
             include,
             exclude,
             default,
             auditor=r.nofsauditor,
             ctx=self,
             listsubrepos=listsubrepos,
             badfn=badfn,
         )
 
     def diff(
         self,
         ctx2=None,
         match=None,
         changes=None,
         opts=None,
         losedatafn=None,
         pathfn=None,
         copy=None,
         copysourcematch=None,
         hunksfilterfn=None,
     ):
         """Returns a diff generator for the given contexts and matcher"""
         if ctx2 is None:
             ctx2 = self.p1()
         if ctx2 is not None:
             ctx2 = self._repo[ctx2]
         return patch.diff(
             self._repo,
             ctx2,
             self,
             match=match,
             changes=changes,
             opts=opts,
             losedatafn=losedatafn,
             pathfn=pathfn,
             copy=copy,
             copysourcematch=copysourcematch,
             hunksfilterfn=hunksfilterfn,
         )
 
     def dirs(self):
         return self._manifest.dirs()
 
     def hasdir(self, dir):
         return self._manifest.hasdir(dir)
 
     def status(
         self,
         other=None,
         match=None,
         listignored=False,
         listclean=False,
         listunknown=False,
         listsubrepos=False,
     ):
         """return status of files between two nodes or node and working
         directory.
 
         If other is None, compare this node with working directory.
 
         returns (modified, added, removed, deleted, unknown, ignored, clean)
         """
 
         ctx1 = self
         ctx2 = self._repo[other]
 
         # This next code block is, admittedly, fragile logic that tests for
         # reversing the contexts and wouldn't need to exist if it weren't for
         # the fast (and common) code path of comparing the working directory
         # with its first parent.
         #
         # What we're aiming for here is the ability to call:
         #
         # workingctx.status(parentctx)
         #
         # If we always built the manifest for each context and compared those,
         # then we'd be done. But the special case of the above call means we
         # just copy the manifest of the parent.
         reversed = False
         if not isinstance(ctx1, changectx) and isinstance(ctx2, changectx):
             reversed = True
             ctx1, ctx2 = ctx2, ctx1
 
         match = self._repo.narrowmatch(match)
         match = ctx2._matchstatus(ctx1, match)
         r = scmutil.status([], [], [], [], [], [], [])
         r = ctx2._buildstatus(
             ctx1, r, match, listignored, listclean, listunknown
         )
 
         if reversed:
             # Reverse added and removed. Clear deleted, unknown and ignored as
             # these make no sense to reverse.
             r = scmutil.status(
                 r.modified, r.removed, r.added, [], [], [], r.clean
             )
 
         if listsubrepos:
             for subpath, sub in scmutil.itersubrepos(ctx1, ctx2):
                 try:
                     rev2 = ctx2.subrev(subpath)
                 except KeyError:
                     # A subrepo that existed in node1 was deleted between
                     # node1 and node2 (inclusive). Thus, ctx2's substate
                     # won't contain that subpath. The best we can do ignore it.
                     rev2 = None
                 submatch = matchmod.subdirmatcher(subpath, match)
                 s = sub.status(
                     rev2,
                     match=submatch,
                     ignored=listignored,
                     clean=listclean,
                     unknown=listunknown,
                     listsubrepos=True,
                 )
                 for rfiles, sfiles in zip(r, s):
                     rfiles.extend(b"%s/%s" % (subpath, f) for f in sfiles)
 
         for l in r:
             l.sort()
 
         return r
 
 
 class changectx(basectx):
     """A changecontext object makes access to data related to a particular
     changeset convenient. It represents a read-only context already present in
     the repo."""
 
     def __init__(self, repo, rev, node):
         super(changectx, self).__init__(repo)
         self._rev = rev
         self._node = node
 
     def __hash__(self):
         try:
             return hash(self._rev)
         except AttributeError:
             return id(self)
 
     def __nonzero__(self):
         return self._rev != nullrev
 
     __bool__ = __nonzero__
 
     @propertycache
     def _changeset(self):
         return self._repo.changelog.changelogrevision(self.rev())
 
     @propertycache
     def _manifest(self):
         return self._manifestctx.read()
 
     @property
     def _manifestctx(self):
         return self._repo.manifestlog[self._changeset.manifest]
 
     @propertycache
     def _manifestdelta(self):
         return self._manifestctx.readdelta()
 
     @propertycache
     def _parents(self):
         repo = self._repo
         p1, p2 = repo.changelog.parentrevs(self._rev)
         if p2 == nullrev:
             return [repo[p1]]
         return [repo[p1], repo[p2]]
 
     def changeset(self):
         c = self._changeset
         return (
             c.manifest,
             c.user,
             c.date,
             c.files,
             c.description,
             c.extra,
         )
 
     def manifestnode(self):
         return self._changeset.manifest
 
     def user(self):
         return self._changeset.user
 
     def date(self):
         return self._changeset.date
 
     def files(self):
         return self._changeset.files
 
     def filesmodified(self):
         modified = set(self.files())
         modified.difference_update(self.filesadded())
         modified.difference_update(self.filesremoved())
         return sorted(modified)
 
     def filesadded(self):
         filesadded = self._changeset.filesadded
         compute_on_none = True
         if self._repo.filecopiesmode == b'changeset-sidedata':
             compute_on_none = False
         else:
             source = self._repo.ui.config(b'experimental', b'copies.read-from')
             if source == b'changeset-only':
                 compute_on_none = False
             elif source != b'compatibility':
                 # filelog mode, ignore any changelog content
                 filesadded = None
         if filesadded is None:
             if compute_on_none:
                 filesadded = copies.computechangesetfilesadded(self)
             else:
                 filesadded = []
         return filesadded
 
     def filesremoved(self):
         filesremoved = self._changeset.filesremoved
         compute_on_none = True
         if self._repo.filecopiesmode == b'changeset-sidedata':
             compute_on_none = False
         else:
             source = self._repo.ui.config(b'experimental', b'copies.read-from')
             if source == b'changeset-only':
                 compute_on_none = False
             elif source != b'compatibility':
                 # filelog mode, ignore any changelog content
                 filesremoved = None
         if filesremoved is None:
             if compute_on_none:
                 filesremoved = copies.computechangesetfilesremoved(self)
             else:
                 filesremoved = []
         return filesremoved
 
     @propertycache
     def _copies(self):
         p1copies = self._changeset.p1copies
         p2copies = self._changeset.p2copies
         compute_on_none = True
         if self._repo.filecopiesmode == b'changeset-sidedata':
             compute_on_none = False
         else:
             source = self._repo.ui.config(b'experimental', b'copies.read-from')
             # If config says to get copy metadata only from changeset, then
             # return that, defaulting to {} if there was no copy metadata.  In
             # compatibility mode, we return copy data from the changeset if it
             # was recorded there, and otherwise we fall back to getting it from
             # the filelogs (below).
             #
             # If we are in compatiblity mode and there is not data in the
             # changeset), we get the copy metadata from the filelogs.
             #
             # otherwise, when config said to read only from filelog, we get the
             # copy metadata from the filelogs.
             if source == b'changeset-only':
                 compute_on_none = False
             elif source != b'compatibility':
                 # filelog mode, ignore any changelog content
                 p1copies = p2copies = None
         if p1copies is None:
             if compute_on_none:
                 p1copies, p2copies = super(changectx, self)._copies
             else:
                 if p1copies is None:
                     p1copies = {}
         if p2copies is None:
             p2copies = {}
         return p1copies, p2copies
 
     def description(self):
         return self._changeset.description
 
     def branch(self):
         return encoding.tolocal(self._changeset.extra.get(b"branch"))
 
     def closesbranch(self):
         return b'close' in self._changeset.extra
 
     def extra(self):
         """Return a dict of extra information."""
         return self._changeset.extra
 
     def tags(self):
         """Return a list of byte tag names"""
         return self._repo.nodetags(self._node)
 
     def bookmarks(self):
         """Return a list of byte bookmark names."""
         return self._repo.nodebookmarks(self._node)
 
     def phase(self):
         return self._repo._phasecache.phase(self._repo, self._rev)
 
     def hidden(self):
         return self._rev in repoview.filterrevs(self._repo, b'visible')
 
     def isinmemory(self):
         return False
 
     def children(self):
         """return list of changectx contexts for each child changeset.
 
         This returns only the immediate child changesets. Use descendants() to
         recursively walk children.
         """
         c = self._repo.changelog.children(self._node)
         return [self._repo[x] for x in c]
 
     def ancestors(self):
         for a in self._repo.changelog.ancestors([self._rev]):
             yield self._repo[a]
 
     def descendants(self):
         """Recursively yield all children of the changeset.
 
         For just the immediate children, use children()
         """
         for d in self._repo.changelog.descendants([self._rev]):
             yield self._repo[d]
 
     def filectx(self, path, fileid=None, filelog=None):
         """get a file context from this changeset"""
         if fileid is None:
             fileid = self.filenode(path)
         return filectx(
             self._repo, path, fileid=fileid, changectx=self, filelog=filelog
         )
 
     def ancestor(self, c2, warn=False):
         """return the "best" ancestor context of self and c2
 
         If there are multiple candidates, it will show a message and check
         merge.preferancestor configuration before falling back to the
         revlog ancestor."""
         # deal with workingctxs
         n2 = c2._node
         if n2 is None:
             n2 = c2._parents[0]._node
         cahs = self._repo.changelog.commonancestorsheads(self._node, n2)
         if not cahs:
             anc = nullid
         elif len(cahs) == 1:
             anc = cahs[0]
         else:
             # experimental config: merge.preferancestor
             for r in self._repo.ui.configlist(b'merge', b'preferancestor'):
                 try:
                     ctx = scmutil.revsymbol(self._repo, r)
                 except error.RepoLookupError:
                     continue
                 anc = ctx.node()
                 if anc in cahs:
                     break
             else:
                 anc = self._repo.changelog.ancestor(self._node, n2)
             if warn:
                 self._repo.ui.status(
                     (
                         _(b"note: using %s as ancestor of %s and %s\n")
                         % (short(anc), short(self._node), short(n2))
                     )
                     + b''.join(
                         _(
                             b"      alternatively, use --config "
                             b"merge.preferancestor=%s\n"
                         )
                         % short(n)
                         for n in sorted(cahs)
                         if n != anc
                     )
                 )
         return self._repo[anc]
 
     def isancestorof(self, other):
         """True if this changeset is an ancestor of other"""
         return self._repo.changelog.isancestorrev(self._rev, other._rev)
 
     def walk(self, match):
         '''Generates matching file names.'''
 
         # Wrap match.bad method to have message with nodeid
         def bad(fn, msg):
             # The manifest doesn't know about subrepos, so don't complain about
             # paths into valid subrepos.
             if any(fn == s or fn.startswith(s + b'/') for s in self.substate):
                 return
             match.bad(fn, _(b'no such file in rev %s') % self)
 
         m = matchmod.badmatch(self._repo.narrowmatch(match), bad)
         return self._manifest.walk(m)
 
     def matches(self, match):
         return self.walk(match)
 
 
 class basefilectx(object):
     """A filecontext object represents the common logic for its children:
     filectx: read-only access to a filerevision that is already present
              in the repo,
     workingfilectx: a filecontext that represents files from the working
                     directory,
     memfilectx: a filecontext that represents files in-memory,
     """
 
     @propertycache
     def _filelog(self):
         return self._repo.file(self._path)
 
     @propertycache
     def _changeid(self):
         if r'_changectx' in self.__dict__:
             return self._changectx.rev()
         elif r'_descendantrev' in self.__dict__:
             # this file context was created from a revision with a known
             # descendant, we can (lazily) correct for linkrev aliases
             return self._adjustlinkrev(self._descendantrev)
         else:
             return self._filelog.linkrev(self._filerev)
 
     @propertycache
     def _filenode(self):
         if r'_fileid' in self.__dict__:
             return self._filelog.lookup(self._fileid)
         else:
             return self._changectx.filenode(self._path)
 
     @propertycache
     def _filerev(self):
         return self._filelog.rev(self._filenode)
 
     @propertycache
     def _repopath(self):
         return self._path
 
     def __nonzero__(self):
         try:
             self._filenode
             return True
         except error.LookupError:
             # file is missing
             return False
 
     __bool__ = __nonzero__
 
     def __bytes__(self):
         try:
             return b"%s@%s" % (self.path(), self._changectx)
         except error.LookupError:
             return b"%s@???" % self.path()
 
     __str__ = encoding.strmethod(__bytes__)
 
     def __repr__(self):
         return r"<%s %s>" % (type(self).__name__, str(self))
 
     def __hash__(self):
         try:
             return hash((self._path, self._filenode))
         except AttributeError:
             return id(self)
 
     def __eq__(self, other):
         try:
             return (
                 type(self) == type(other)
                 and self._path == other._path
                 and self._filenode == other._filenode
             )
         except AttributeError:
             return False
 
     def __ne__(self, other):
         return not (self == other)
 
     def filerev(self):
         return self._filerev
 
     def filenode(self):
         return self._filenode
 
     @propertycache
     def _flags(self):
         return self._changectx.flags(self._path)
 
     def flags(self):
         return self._flags
 
     def filelog(self):
         return self._filelog
 
     def rev(self):
         return self._changeid
 
     def linkrev(self):
         return self._filelog.linkrev(self._filerev)
 
     def node(self):
         return self._changectx.node()
 
     def hex(self):
         return self._changectx.hex()
 
     def user(self):
         return self._changectx.user()
 
     def date(self):
         return self._changectx.date()
 
     def files(self):
         return self._changectx.files()
 
     def description(self):
         return self._changectx.description()
 
     def branch(self):
         return self._changectx.branch()
 
     def extra(self):
         return self._changectx.extra()
 
     def phase(self):
         return self._changectx.phase()
 
     def phasestr(self):
         return self._changectx.phasestr()
 
     def obsolete(self):
         return self._changectx.obsolete()
 
     def instabilities(self):
         return self._changectx.instabilities()
 
     def manifest(self):
         return self._changectx.manifest()
 
     def changectx(self):
         return self._changectx
 
     def renamed(self):
         return self._copied
 
     def copysource(self):
         return self._copied and self._copied[0]
 
     def repo(self):
         return self._repo
 
     def size(self):
         return len(self.data())
 
     def path(self):
         return self._path
 
     def isbinary(self):
         try:
             return stringutil.binary(self.data())
         except IOError:
             return False
 
     def isexec(self):
         return b'x' in self.flags()
 
     def islink(self):
         return b'l' in self.flags()
 
     def isabsent(self):
         """whether this filectx represents a file not in self._changectx
 
         This is mainly for merge code to detect change/delete conflicts. This is
         expected to be True for all subclasses of basectx."""
         return False
 
     _customcmp = False
 
     def cmp(self, fctx):
         """compare with other file context
 
         returns True if different than fctx.
         """
         if fctx._customcmp:
             return fctx.cmp(self)
 
         if self._filenode is None:
             raise error.ProgrammingError(
                 b'filectx.cmp() must be reimplemented if not backed by revlog'
             )
 
         if fctx._filenode is None:
             if self._repo._encodefilterpats:
                 # can't rely on size() because wdir content may be decoded
                 return self._filelog.cmp(self._filenode, fctx.data())
             if self.size() - 4 == fctx.size():
                 # size() can match:
                 # if file data starts with '\1\n', empty metadata block is
                 # prepended, which adds 4 bytes to filelog.size().
                 return self._filelog.cmp(self._filenode, fctx.data())
         if self.size() == fctx.size():
             # size() matches: need to compare content
             return self._filelog.cmp(self._filenode, fctx.data())
 
         # size() differs
         return True
 
     def _adjustlinkrev(self, srcrev, inclusive=False, stoprev=None):
         """return the first ancestor of <srcrev> introducing <fnode>
 
         If the linkrev of the file revision does not point to an ancestor of
         srcrev, we'll walk down the ancestors until we find one introducing
         this file revision.
 
         :srcrev: the changeset revision we search ancestors from
         :inclusive: if true, the src revision will also be checked
         :stoprev: an optional revision to stop the walk at. If no introduction
                   of this file content could be found before this floor
                   revision, the function will returns "None" and stops its
                   iteration.
         """
         repo = self._repo
         cl = repo.unfiltered().changelog
         mfl = repo.manifestlog
         # fetch the linkrev
         lkr = self.linkrev()
         if srcrev == lkr:
             return lkr
         # hack to reuse ancestor computation when searching for renames
         memberanc = getattr(self, '_ancestrycontext', None)
         iteranc = None
         if srcrev is None:
             # wctx case, used by workingfilectx during mergecopy
             revs = [p.rev() for p in self._repo[None].parents()]
             inclusive = True  # we skipped the real (revless) source
         else:
             revs = [srcrev]
         if memberanc is None:
             memberanc = iteranc = cl.ancestors(revs, lkr, inclusive=inclusive)
         # check if this linkrev is an ancestor of srcrev
         if lkr not in memberanc:
             if iteranc is None:
                 iteranc = cl.ancestors(revs, lkr, inclusive=inclusive)
             fnode = self._filenode
             path = self._path
             for a in iteranc:
                 if stoprev is not None and a < stoprev:
                     return None
                 ac = cl.read(a)  # get changeset data (we avoid object creation)
                 if path in ac[3]:  # checking the 'files' field.
                     # The file has been touched, check if the content is
                     # similar to the one we search for.
                     if fnode == mfl[ac[0]].readfast().get(path):
                         return a
             # In theory, we should never get out of that loop without a result.
             # But if manifest uses a buggy file revision (not children of the
             # one it replaces) we could. Such a buggy situation will likely
             # result is crash somewhere else at to some point.
         return lkr
 
     def isintroducedafter(self, changelogrev):
         """True if a filectx has been introduced after a given floor revision
         """
         if self.linkrev() >= changelogrev:
             return True
         introrev = self._introrev(stoprev=changelogrev)
         if introrev is None:
             return False
         return introrev >= changelogrev
 
     def introrev(self):
         """return the rev of the changeset which introduced this file revision
 
         This method is different from linkrev because it take into account the
         changeset the filectx was created from. It ensures the returned
         revision is one of its ancestors. This prevents bugs from
         'linkrev-shadowing' when a file revision is used by multiple
         changesets.
         """
         return self._introrev()
 
     def _introrev(self, stoprev=None):
         """
         Same as `introrev` but, with an extra argument to limit changelog
         iteration range in some internal usecase.
 
         If `stoprev` is set, the `introrev` will not be searched past that
         `stoprev` revision and "None" might be returned. This is useful to
         limit the iteration range.
         """
         toprev = None
         attrs = vars(self)
         if r'_changeid' in attrs:
             # We have a cached value already
             toprev = self._changeid
         elif r'_changectx' in attrs:
             # We know which changelog entry we are coming from
             toprev = self._changectx.rev()
 
         if toprev is not None:
             return self._adjustlinkrev(toprev, inclusive=True, stoprev=stoprev)
         elif r'_descendantrev' in attrs:
             introrev = self._adjustlinkrev(self._descendantrev, stoprev=stoprev)
             # be nice and cache the result of the computation
             if introrev is not None:
                 self._changeid = introrev
             return introrev
         else:
             return self.linkrev()
 
     def introfilectx(self):
         """Return filectx having identical contents, but pointing to the
         changeset revision where this filectx was introduced"""
         introrev = self.introrev()
         if self.rev() == introrev:
             return self
         return self.filectx(self.filenode(), changeid=introrev)
 
     def _parentfilectx(self, path, fileid, filelog):
         """create parent filectx keeping ancestry info for _adjustlinkrev()"""
         fctx = filectx(self._repo, path, fileid=fileid, filelog=filelog)
         if r'_changeid' in vars(self) or r'_changectx' in vars(self):
             # If self is associated with a changeset (probably explicitly
             # fed), ensure the created filectx is associated with a
             # changeset that is an ancestor of self.changectx.
             # This lets us later use _adjustlinkrev to get a correct link.
             fctx._descendantrev = self.rev()
             fctx._ancestrycontext = getattr(self, '_ancestrycontext', None)
         elif r'_descendantrev' in vars(self):
             # Otherwise propagate _descendantrev if we have one associated.
             fctx._descendantrev = self._descendantrev
             fctx._ancestrycontext = getattr(self, '_ancestrycontext', None)
         return fctx
 
     def parents(self):
         _path = self._path
         fl = self._filelog
         parents = self._filelog.parents(self._filenode)
         pl = [(_path, node, fl) for node in parents if node != nullid]
 
         r = fl.renamed(self._filenode)
         if r:
             # - In the simple rename case, both parent are nullid, pl is empty.
             # - In case of merge, only one of the parent is null id and should
             # be replaced with the rename information. This parent is -always-
             # the first one.
             #
             # As null id have always been filtered out in the previous list
             # comprehension, inserting to 0 will always result in "replacing
             # first nullid parent with rename information.
             pl.insert(0, (r[0], r[1], self._repo.file(r[0])))
 
         return [self._parentfilectx(path, fnode, l) for path, fnode, l in pl]
 
     def p1(self):
         return self.parents()[0]
 
     def p2(self):
         p = self.parents()
         if len(p) == 2:
             return p[1]
         return filectx(self._repo, self._path, fileid=-1, filelog=self._filelog)
 
     def annotate(self, follow=False, skiprevs=None, diffopts=None):
         """Returns a list of annotateline objects for each line in the file
 
         - line.fctx is the filectx of the node where that line was last changed
         - line.lineno is the line number at the first appearance in the managed
           file
         - line.text is the data on that line (including newline character)
         """
         getlog = util.lrucachefunc(lambda x: self._repo.file(x))
 
         def parents(f):
             # Cut _descendantrev here to mitigate the penalty of lazy linkrev
             # adjustment. Otherwise, p._adjustlinkrev() would walk changelog
             # from the topmost introrev (= srcrev) down to p.linkrev() if it
             # isn't an ancestor of the srcrev.
             f._changeid
             pl = f.parents()
 
             # Don't return renamed parents if we aren't following.
             if not follow:
                 pl = [p for p in pl if p.path() == f.path()]
 
             # renamed filectx won't have a filelog yet, so set it
             # from the cache to save time
             for p in pl:
                 if not r'_filelog' in p.__dict__:
                     p._filelog = getlog(p.path())
 
             return pl
 
         # use linkrev to find the first changeset where self appeared
         base = self.introfilectx()
         if getattr(base, '_ancestrycontext', None) is None:
             cl = self._repo.changelog
             if base.rev() is None:
                 # wctx is not inclusive, but works because _ancestrycontext
                 # is used to test filelog revisions
                 ac = cl.ancestors(
                     [p.rev() for p in base.parents()], inclusive=True
                 )
             else:
                 ac = cl.ancestors([base.rev()], inclusive=True)
             base._ancestrycontext = ac
 
         return dagop.annotate(
             base, parents, skiprevs=skiprevs, diffopts=diffopts
         )
 
     def ancestors(self, followfirst=False):
         visit = {}
         c = self
         if followfirst:
             cut = 1
         else:
             cut = None
 
         while True:
             for parent in c.parents()[:cut]:
                 visit[(parent.linkrev(), parent.filenode())] = parent
             if not visit:
                 break
             c = visit.pop(max(visit))
             yield c
 
     def decodeddata(self):
         """Returns `data()` after running repository decoding filters.
 
         This is often equivalent to how the data would be expressed on disk.
         """
         return self._repo.wwritedata(self.path(), self.data())
 
 
 class filectx(basefilectx):
     """A filecontext object makes access to data related to a particular
        filerevision convenient."""
 
     def __init__(
         self,
         repo,
         path,
         changeid=None,
         fileid=None,
         filelog=None,
         changectx=None,
     ):
         """changeid must be a revision number, if specified.
            fileid can be a file revision or node."""
         self._repo = repo
         self._path = path
 
         assert (
             changeid is not None or fileid is not None or changectx is not None
-        ), b"bad args: changeid=%r, fileid=%r, changectx=%r" % (
-            changeid,
-            fileid,
-            changectx,
+        ), (
+            b"bad args: changeid=%r, fileid=%r, changectx=%r"
+            % (changeid, fileid, changectx,)
         )
 
         if filelog is not None:
             self._filelog = filelog
 
         if changeid is not None:
             self._changeid = changeid
         if changectx is not None:
             self._changectx = changectx
         if fileid is not None:
             self._fileid = fileid
 
     @propertycache
     def _changectx(self):
         try:
             return self._repo[self._changeid]
         except error.FilteredRepoLookupError:
             # Linkrev may point to any revision in the repository.  When the
             # repository is filtered this may lead to `filectx` trying to build
             # `changectx` for filtered revision. In such case we fallback to
             # creating `changectx` on the unfiltered version of the reposition.
             # This fallback should not be an issue because `changectx` from
             # `filectx` are not used in complex operations that care about
             # filtering.
             #
             # This fallback is a cheap and dirty fix that prevent several
             # crashes. It does not ensure the behavior is correct. However the
             # behavior was not correct before filtering either and "incorrect
             # behavior" is seen as better as "crash"
             #
             # Linkrevs have several serious troubles with filtering that are
             # complicated to solve. Proper handling of the issue here should be
             # considered when solving linkrev issue are on the table.
             return self._repo.unfiltered()[self._changeid]
 
     def filectx(self, fileid, changeid=None):
         '''opens an arbitrary revision of the file without
         opening a new filelog'''
         return filectx(
             self._repo,
             self._path,
             fileid=fileid,
             filelog=self._filelog,
             changeid=changeid,
         )
 
     def rawdata(self):
         return self._filelog.rawdata(self._filenode)
 
     def rawflags(self):
         """low-level revlog flags"""
         return self._filelog.flags(self._filerev)
 
     def data(self):
         try:
             return self._filelog.read(self._filenode)
         except error.CensoredNodeError:
             if self._repo.ui.config(b"censor", b"policy") == b"ignore":
                 return b""
             raise error.Abort(
                 _(b"censored node: %s") % short(self._filenode),
                 hint=_(b"set censor.policy to ignore errors"),
             )
 
     def size(self):
         return self._filelog.size(self._filerev)
 
     @propertycache
     def _copied(self):
         """check if file was actually renamed in this changeset revision
 
         If rename logged in file revision, we report copy for changeset only
         if file revisions linkrev points back to the changeset in question
         or both changeset parents contain different file revisions.
         """
 
         renamed = self._filelog.renamed(self._filenode)
         if not renamed:
             return None
 
         if self.rev() == self.linkrev():
             return renamed
 
         name = self.path()
         fnode = self._filenode
         for p in self._changectx.parents():
             try:
                 if fnode == p.filenode(name):
                     return None
             except error.LookupError:
                 pass
         return renamed
 
     def children(self):
         # hard for renames
         c = self._filelog.children(self._filenode)
         return [
             filectx(self._repo, self._path, fileid=x, filelog=self._filelog)
             for x in c
         ]
 
 
 class committablectx(basectx):
     """A committablectx object provides common functionality for a context that
     wants the ability to commit, e.g. workingctx or memctx."""
 
     def __init__(
         self,
         repo,
         text=b"",
         user=None,
         date=None,
         extra=None,
         changes=None,
         branch=None,
     ):
         super(committablectx, self).__init__(repo)
         self._rev = None
         self._node = None
         self._text = text
         if date:
             self._date = dateutil.parsedate(date)
         if user:
             self._user = user
         if changes:
             self._status = changes
 
         self._extra = {}
         if extra:
             self._extra = extra.copy()
         if branch is not None:
             self._extra[b'branch'] = encoding.fromlocal(branch)
         if not self._extra.get(b'branch'):
             self._extra[b'branch'] = b'default'
 
     def __bytes__(self):
         return bytes(self._parents[0]) + b"+"
 
     __str__ = encoding.strmethod(__bytes__)
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     @propertycache
     def _status(self):
         return self._repo.status()
 
     @propertycache
     def _user(self):
         return self._repo.ui.username()
 
     @propertycache
     def _date(self):
         ui = self._repo.ui
         date = ui.configdate(b'devel', b'default-date')
         if date is None:
             date = dateutil.makedate()
         return date
 
     def subrev(self, subpath):
         return None
 
     def manifestnode(self):
         return None
 
     def user(self):
         return self._user or self._repo.ui.username()
 
     def date(self):
         return self._date
 
     def description(self):
         return self._text
 
     def files(self):
         return sorted(
             self._status.modified + self._status.added + self._status.removed
         )
 
     def modified(self):
         return self._status.modified
 
     def added(self):
         return self._status.added
 
     def removed(self):
         return self._status.removed
 
     def deleted(self):
         return self._status.deleted
 
     filesmodified = modified
     filesadded = added
     filesremoved = removed
 
     def branch(self):
         return encoding.tolocal(self._extra[b'branch'])
 
     def closesbranch(self):
         return b'close' in self._extra
 
     def extra(self):
         return self._extra
 
     def isinmemory(self):
         return False
 
     def tags(self):
         return []
 
     def bookmarks(self):
         b = []
         for p in self.parents():
             b.extend(p.bookmarks())
         return b
 
     def phase(self):
         phase = phases.draft  # default phase to draft
         for p in self.parents():
             phase = max(phase, p.phase())
         return phase
 
     def hidden(self):
         return False
 
     def children(self):
         return []
 
     def ancestor(self, c2):
         """return the "best" ancestor context of self and c2"""
         return self._parents[0].ancestor(c2)  # punt on two parents for now
 
     def ancestors(self):
         for p in self._parents:
             yield p
         for a in self._repo.changelog.ancestors(
             [p.rev() for p in self._parents]
         ):
             yield self._repo[a]
 
     def markcommitted(self, node):
         """Perform post-commit cleanup necessary after committing this ctx
 
         Specifically, this updates backing stores this working context
         wraps to reflect the fact that the changes reflected by this
         workingctx have been committed.  For example, it marks
         modified and added files as normal in the dirstate.
 
         """
 
     def dirty(self, missing=False, merge=True, branch=True):
         return False
 
 
 class workingctx(committablectx):
     """A workingctx object makes access to data related to
     the current working directory convenient.
     date - any valid date string or (unixtime, offset), or None.
     user - username string, or None.
     extra - a dictionary of extra values, or None.
     changes - a list of file lists as returned by localrepo.status()
                or None to use the repository status.
     """
 
     def __init__(
         self, repo, text=b"", user=None, date=None, extra=None, changes=None
     ):
         branch = None
         if not extra or b'branch' not in extra:
             try:
                 branch = repo.dirstate.branch()
             except UnicodeDecodeError:
                 raise error.Abort(_(b'branch name not in UTF-8!'))
         super(workingctx, self).__init__(
             repo, text, user, date, extra, changes, branch=branch
         )
 
     def __iter__(self):
         d = self._repo.dirstate
         for f in d:
             if d[f] != b'r':
                 yield f
 
     def __contains__(self, key):
         return self._repo.dirstate[key] not in b"?r"
 
     def hex(self):
         return wdirhex
 
     @propertycache
     def _parents(self):
         p = self._repo.dirstate.parents()
         if p[1] == nullid:
             p = p[:-1]
         # use unfiltered repo to delay/avoid loading obsmarkers
         unfi = self._repo.unfiltered()
         return [changectx(self._repo, unfi.changelog.rev(n), n) for n in p]
 
     def _fileinfo(self, path):
         # populate __dict__['_manifest'] as workingctx has no _manifestdelta
         self._manifest
         return super(workingctx, self)._fileinfo(path)
 
     def _buildflagfunc(self):
         # Create a fallback function for getting file flags when the
         # filesystem doesn't support them
 
         copiesget = self._repo.dirstate.copies().get
         parents = self.parents()
         if len(parents) < 2:
             # when we have one parent, it's easy: copy from parent
             man = parents[0].manifest()
 
             def func(f):
                 f = copiesget(f, f)
                 return man.flags(f)
 
         else:
             # merges are tricky: we try to reconstruct the unstored
             # result from the merge (issue1802)
             p1, p2 = parents
             pa = p1.ancestor(p2)
             m1, m2, ma = p1.manifest(), p2.manifest(), pa.manifest()
 
             def func(f):
                 f = copiesget(f, f)  # may be wrong for merges with copies
                 fl1, fl2, fla = m1.flags(f), m2.flags(f), ma.flags(f)
                 if fl1 == fl2:
                     return fl1
                 if fl1 == fla:
                     return fl2
                 if fl2 == fla:
                     return fl1
                 return b''  # punt for conflicts
 
         return func
 
     @propertycache
     def _flagfunc(self):
         return self._repo.dirstate.flagfunc(self._buildflagfunc)
 
     def flags(self, path):
         if r'_manifest' in self.__dict__:
             try:
                 return self._manifest.flags(path)
             except KeyError:
                 return b''
 
         try:
             return self._flagfunc(path)
         except OSError:
             return b''
 
     def filectx(self, path, filelog=None):
         """get a file context from the working directory"""
         return workingfilectx(
             self._repo, path, workingctx=self, filelog=filelog
         )
 
     def dirty(self, missing=False, merge=True, branch=True):
         b"check whether a working directory is modified"
         # check subrepos first
         for s in sorted(self.substate):
             if self.sub(s).dirty(missing=missing):
                 return True
         # check current working dir
         return (
             (merge and self.p2())
             or (branch and self.branch() != self.p1().branch())
             or self.modified()
             or self.added()
             or self.removed()
             or (missing and self.deleted())
         )
 
     def add(self, list, prefix=b""):
         with self._repo.wlock():
             ui, ds = self._repo.ui, self._repo.dirstate
             uipath = lambda f: ds.pathto(pathutil.join(prefix, f))
             rejected = []
             lstat = self._repo.wvfs.lstat
             for f in list:
                 # ds.pathto() returns an absolute file when this is invoked from
                 # the keyword extension.  That gets flagged as non-portable on
                 # Windows, since it contains the drive letter and colon.
                 scmutil.checkportable(ui, os.path.join(prefix, f))
                 try:
                     st = lstat(f)
                 except OSError:
                     ui.warn(_(b"%s does not exist!\n") % uipath(f))
                     rejected.append(f)
                     continue
                 limit = ui.configbytes(b'ui', b'large-file-limit')
                 if limit != 0 and st.st_size > limit:
                     ui.warn(
                         _(
                             b"%s: up to %d MB of RAM may be required "
                             b"to manage this file\n"
                             b"(use 'hg revert %s' to cancel the "
                             b"pending addition)\n"
                         )
                         % (f, 3 * st.st_size // 1000000, uipath(f))
                     )
                 if not (stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode)):
                     ui.warn(
                         _(
                             b"%s not added: only files and symlinks "
                             b"supported currently\n"
                         )
                         % uipath(f)
                     )
                     rejected.append(f)
                 elif ds[f] in b'amn':
                     ui.warn(_(b"%s already tracked!\n") % uipath(f))
                 elif ds[f] == b'r':
                     ds.normallookup(f)
                 else:
                     ds.add(f)
             return rejected
 
     def forget(self, files, prefix=b""):
         with self._repo.wlock():
             ds = self._repo.dirstate
             uipath = lambda f: ds.pathto(pathutil.join(prefix, f))
             rejected = []
             for f in files:
                 if f not in ds:
                     self._repo.ui.warn(_(b"%s not tracked!\n") % uipath(f))
                     rejected.append(f)
                 elif ds[f] != b'a':
                     ds.remove(f)
                 else:
                     ds.drop(f)
             return rejected
 
     def copy(self, source, dest):
         try:
             st = self._repo.wvfs.lstat(dest)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             self._repo.ui.warn(
                 _(b"%s does not exist!\n") % self._repo.dirstate.pathto(dest)
             )
             return
         if not (stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode)):
             self._repo.ui.warn(
                 _(b"copy failed: %s is not a file or a symbolic link\n")
                 % self._repo.dirstate.pathto(dest)
             )
         else:
             with self._repo.wlock():
                 ds = self._repo.dirstate
                 if ds[dest] in b'?':
                     ds.add(dest)
                 elif ds[dest] in b'r':
                     ds.normallookup(dest)
                 ds.copy(source, dest)
 
     def match(
         self,
         pats=None,
         include=None,
         exclude=None,
         default=b'glob',
         listsubrepos=False,
         badfn=None,
     ):
         r = self._repo
 
         # Only a case insensitive filesystem needs magic to translate user input
         # to actual case in the filesystem.
         icasefs = not util.fscasesensitive(r.root)
         return matchmod.match(
             r.root,
             r.getcwd(),
             pats,
             include,
             exclude,
             default,
             auditor=r.auditor,
             ctx=self,
             listsubrepos=listsubrepos,
             badfn=badfn,
             icasefs=icasefs,
         )
 
     def _filtersuspectsymlink(self, files):
         if not files or self._repo.dirstate._checklink:
             return files
 
         # Symlink placeholders may get non-symlink-like contents
         # via user error or dereferencing by NFS or Samba servers,
         # so we filter out any placeholders that don't look like a
         # symlink
         sane = []
         for f in files:
             if self.flags(f) == b'l':
                 d = self[f].data()
                 if (
                     d == b''
                     or len(d) >= 1024
                     or b'\n' in d
                     or stringutil.binary(d)
                 ):
                     self._repo.ui.debug(
                         b'ignoring suspect symlink placeholder "%s"\n' % f
                     )
                     continue
             sane.append(f)
         return sane
 
     def _checklookup(self, files):
         # check for any possibly clean files
         if not files:
             return [], [], []
 
         modified = []
         deleted = []
         fixup = []
         pctx = self._parents[0]
         # do a full compare of any files that might have changed
         for f in sorted(files):
             try:
                 # This will return True for a file that got replaced by a
                 # directory in the interim, but fixing that is pretty hard.
                 if (
                     f not in pctx
                     or self.flags(f) != pctx.flags(f)
                     or pctx[f].cmp(self[f])
                 ):
                     modified.append(f)
                 else:
                     fixup.append(f)
             except (IOError, OSError):
                 # A file become inaccessible in between? Mark it as deleted,
                 # matching dirstate behavior (issue5584).
                 # The dirstate has more complex behavior around whether a
                 # missing file matches a directory, etc, but we don't need to
                 # bother with that: if f has made it to this point, we're sure
                 # it's in the dirstate.
                 deleted.append(f)
 
         return modified, deleted, fixup
 
     def _poststatusfixup(self, status, fixup):
         """update dirstate for files that are actually clean"""
         poststatus = self._repo.postdsstatus()
         if fixup or poststatus:
             try:
                 oldid = self._repo.dirstate.identity()
 
                 # updating the dirstate is optional
                 # so we don't wait on the lock
                 # wlock can invalidate the dirstate, so cache normal _after_
                 # taking the lock
                 with self._repo.wlock(False):
                     if self._repo.dirstate.identity() == oldid:
                         if fixup:
                             normal = self._repo.dirstate.normal
                             for f in fixup:
                                 normal(f)
                             # write changes out explicitly, because nesting
                             # wlock at runtime may prevent 'wlock.release()'
                             # after this block from doing so for subsequent
                             # changing files
                             tr = self._repo.currenttransaction()
                             self._repo.dirstate.write(tr)
 
                         if poststatus:
                             for ps in poststatus:
                                 ps(self, status)
                     else:
                         # in this case, writing changes out breaks
                         # consistency, because .hg/dirstate was
                         # already changed simultaneously after last
                         # caching (see also issue5584 for detail)
                         self._repo.ui.debug(
                             b'skip updating dirstate: identity mismatch\n'
                         )
             except error.LockError:
                 pass
             finally:
                 # Even if the wlock couldn't be grabbed, clear out the list.
                 self._repo.clearpostdsstatus()
 
     def _dirstatestatus(self, match, ignored=False, clean=False, unknown=False):
         '''Gets the status from the dirstate -- internal use only.'''
         subrepos = []
         if b'.hgsub' in self:
             subrepos = sorted(self.substate)
         cmp, s = self._repo.dirstate.status(
             match, subrepos, ignored=ignored, clean=clean, unknown=unknown
         )
 
         # check for any possibly clean files
         fixup = []
         if cmp:
             modified2, deleted2, fixup = self._checklookup(cmp)
             s.modified.extend(modified2)
             s.deleted.extend(deleted2)
 
             if fixup and clean:
                 s.clean.extend(fixup)
 
         self._poststatusfixup(s, fixup)
 
         if match.always():
             # cache for performance
             if s.unknown or s.ignored or s.clean:
                 # "_status" is cached with list*=False in the normal route
                 self._status = scmutil.status(
                     s.modified, s.added, s.removed, s.deleted, [], [], []
                 )
             else:
                 self._status = s
 
         return s
 
     @propertycache
     def _copies(self):
         p1copies = {}
         p2copies = {}
         parents = self._repo.dirstate.parents()
         p1manifest = self._repo[parents[0]].manifest()
         p2manifest = self._repo[parents[1]].manifest()
         changedset = set(self.added()) | set(self.modified())
         narrowmatch = self._repo.narrowmatch()
         for dst, src in self._repo.dirstate.copies().items():
             if dst not in changedset or not narrowmatch(dst):
                 continue
             if src in p1manifest:
                 p1copies[dst] = src
             elif src in p2manifest:
                 p2copies[dst] = src
         return p1copies, p2copies
 
     @propertycache
     def _manifest(self):
         """generate a manifest corresponding to the values in self._status
 
         This reuse the file nodeid from parent, but we use special node
         identifiers for added and modified files. This is used by manifests
         merge to see that files are different and by update logic to avoid
         deleting newly added files.
         """
         return self._buildstatusmanifest(self._status)
 
     def _buildstatusmanifest(self, status):
         """Builds a manifest that includes the given status results."""
         parents = self.parents()
 
         man = parents[0].manifest().copy()
 
         ff = self._flagfunc
         for i, l in (
             (addednodeid, status.added),
             (modifiednodeid, status.modified),
         ):
             for f in l:
                 man[f] = i
                 try:
                     man.setflag(f, ff(f))
                 except OSError:
                     pass
 
         for f in status.deleted + status.removed:
             if f in man:
                 del man[f]
 
         return man
 
     def _buildstatus(
         self, other, s, match, listignored, listclean, listunknown
     ):
         """build a status with respect to another context
 
         This includes logic for maintaining the fast path of status when
         comparing the working directory against its parent, which is to skip
         building a new manifest if self (working directory) is not comparing
         against its parent (repo['.']).
         """
         s = self._dirstatestatus(match, listignored, listclean, listunknown)
         # Filter out symlinks that, in the case of FAT32 and NTFS filesystems,
         # might have accidentally ended up with the entire contents of the file
         # they are supposed to be linking to.
         s.modified[:] = self._filtersuspectsymlink(s.modified)
         if other != self._repo[b'.']:
             s = super(workingctx, self)._buildstatus(
                 other, s, match, listignored, listclean, listunknown
             )
         return s
 
     def _matchstatus(self, other, match):
         """override the match method with a filter for directory patterns
 
         We use inheritance to customize the match.bad method only in cases of
         workingctx since it belongs only to the working directory when
         comparing against the parent changeset.
 
         If we aren't comparing against the working directory's parent, then we
         just use the default match object sent to us.
         """
         if other != self._repo[b'.']:
 
             def bad(f, msg):
                 # 'f' may be a directory pattern from 'match.files()',
                 # so 'f not in ctx1' is not enough
                 if f not in other and not other.hasdir(f):
                     self._repo.ui.warn(
                         b'%s: %s\n' % (self._repo.dirstate.pathto(f), msg)
                     )
 
             match.bad = bad
         return match
 
     def walk(self, match):
         '''Generates matching file names.'''
         return sorted(
             self._repo.dirstate.walk(
                 self._repo.narrowmatch(match),
                 subrepos=sorted(self.substate),
                 unknown=True,
                 ignored=False,
             )
         )
 
     def matches(self, match):
         match = self._repo.narrowmatch(match)
         ds = self._repo.dirstate
         return sorted(f for f in ds.matches(match) if ds[f] != b'r')
 
     def markcommitted(self, node):
         with self._repo.dirstate.parentchange():
             for f in self.modified() + self.added():
                 self._repo.dirstate.normal(f)
             for f in self.removed():
                 self._repo.dirstate.drop(f)
             self._repo.dirstate.setparents(node)
 
         # write changes out explicitly, because nesting wlock at
         # runtime may prevent 'wlock.release()' in 'repo.commit()'
         # from immediately doing so for subsequent changing files
         self._repo.dirstate.write(self._repo.currenttransaction())
 
         sparse.aftercommit(self._repo, node)
 
 
 class committablefilectx(basefilectx):
     """A committablefilectx provides common functionality for a file context
     that wants the ability to commit, e.g. workingfilectx or memfilectx."""
 
     def __init__(self, repo, path, filelog=None, ctx=None):
         self._repo = repo
         self._path = path
         self._changeid = None
         self._filerev = self._filenode = None
 
         if filelog is not None:
             self._filelog = filelog
         if ctx:
             self._changectx = ctx
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     def linkrev(self):
         # linked to self._changectx no matter if file is modified or not
         return self.rev()
 
     def renamed(self):
         path = self.copysource()
         if not path:
             return None
         return path, self._changectx._parents[0]._manifest.get(path, nullid)
 
     def parents(self):
         '''return parent filectxs, following copies if necessary'''
 
         def filenode(ctx, path):
             return ctx._manifest.get(path, nullid)
 
         path = self._path
         fl = self._filelog
         pcl = self._changectx._parents
         renamed = self.renamed()
 
         if renamed:
             pl = [renamed + (None,)]
         else:
             pl = [(path, filenode(pcl[0], path), fl)]
 
         for pc in pcl[1:]:
             pl.append((path, filenode(pc, path), fl))
 
         return [
             self._parentfilectx(p, fileid=n, filelog=l)
             for p, n, l in pl
             if n != nullid
         ]
 
     def children(self):
         return []
 
 
 class workingfilectx(committablefilectx):
     """A workingfilectx object makes access to data related to a particular
        file in the working directory convenient."""
 
     def __init__(self, repo, path, filelog=None, workingctx=None):
         super(workingfilectx, self).__init__(repo, path, filelog, workingctx)
 
     @propertycache
     def _changectx(self):
         return workingctx(self._repo)
 
     def data(self):
         return self._repo.wread(self._path)
 
     def copysource(self):
         return self._repo.dirstate.copied(self._path)
 
     def size(self):
         return self._repo.wvfs.lstat(self._path).st_size
 
     def lstat(self):
         return self._repo.wvfs.lstat(self._path)
 
     def date(self):
         t, tz = self._changectx.date()
         try:
             return (self._repo.wvfs.lstat(self._path)[stat.ST_MTIME], tz)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             return (t, tz)
 
     def exists(self):
         return self._repo.wvfs.exists(self._path)
 
     def lexists(self):
         return self._repo.wvfs.lexists(self._path)
 
     def audit(self):
         return self._repo.wvfs.audit(self._path)
 
     def cmp(self, fctx):
         """compare with other file context
 
         returns True if different than fctx.
         """
         # fctx should be a filectx (not a workingfilectx)
         # invert comparison to reuse the same code path
         return fctx.cmp(self)
 
     def remove(self, ignoremissing=False):
         """wraps unlink for a repo's working directory"""
         rmdir = self._repo.ui.configbool(b'experimental', b'removeemptydirs')
         self._repo.wvfs.unlinkpath(
             self._path, ignoremissing=ignoremissing, rmdir=rmdir
         )
 
     def write(self, data, flags, backgroundclose=False, **kwargs):
         """wraps repo.wwrite"""
         return self._repo.wwrite(
             self._path, data, flags, backgroundclose=backgroundclose, **kwargs
         )
 
     def markcopied(self, src):
         """marks this file a copy of `src`"""
         self._repo.dirstate.copy(src, self._path)
 
     def clearunknown(self):
         """Removes conflicting items in the working directory so that
         ``write()`` can be called successfully.
         """
         wvfs = self._repo.wvfs
         f = self._path
         wvfs.audit(f)
         if self._repo.ui.configbool(
             b'experimental', b'merge.checkpathconflicts'
         ):
             # remove files under the directory as they should already be
             # warned and backed up
             if wvfs.isdir(f) and not wvfs.islink(f):
                 wvfs.rmtree(f, forcibly=True)
             for p in reversed(list(util.finddirs(f))):
                 if wvfs.isfileorlink(p):
                     wvfs.unlink(p)
                     break
         else:
             # don't remove files if path conflicts are not processed
             if wvfs.isdir(f) and not wvfs.islink(f):
                 wvfs.removedirs(f)
 
     def setflags(self, l, x):
         self._repo.wvfs.setflags(self._path, l, x)
 
 
 class overlayworkingctx(committablectx):
     """Wraps another mutable context with a write-back cache that can be
     converted into a commit context.
 
     self._cache[path] maps to a dict with keys: {
         'exists': bool?
         'date': date?
         'data': str?
         'flags': str?
         'copied': str? (path or None)
     }
     If `exists` is True, `flags` must be non-None and 'date' is non-None. If it
     is `False`, the file was deleted.
     """
 
     def __init__(self, repo):
         super(overlayworkingctx, self).__init__(repo)
         self.clean()
 
     def setbase(self, wrappedctx):
         self._wrappedctx = wrappedctx
         self._parents = [wrappedctx]
         # Drop old manifest cache as it is now out of date.
         # This is necessary when, e.g., rebasing several nodes with one
         # ``overlayworkingctx`` (e.g. with --collapse).
         util.clearcachedproperty(self, b'_manifest')
 
     def data(self, path):
         if self.isdirty(path):
             if self._cache[path][b'exists']:
                 if self._cache[path][b'data'] is not None:
                     return self._cache[path][b'data']
                 else:
                     # Must fallback here, too, because we only set flags.
                     return self._wrappedctx[path].data()
             else:
                 raise error.ProgrammingError(
                     b"No such file or directory: %s" % path
                 )
         else:
             return self._wrappedctx[path].data()
 
     @propertycache
     def _manifest(self):
         parents = self.parents()
         man = parents[0].manifest().copy()
 
         flag = self._flagfunc
         for path in self.added():
             man[path] = addednodeid
             man.setflag(path, flag(path))
         for path in self.modified():
             man[path] = modifiednodeid
             man.setflag(path, flag(path))
         for path in self.removed():
             del man[path]
         return man
 
     @propertycache
     def _flagfunc(self):
         def f(path):
             return self._cache[path][b'flags']
 
         return f
 
     def files(self):
         return sorted(self.added() + self.modified() + self.removed())
 
     def modified(self):
         return [
             f
             for f in self._cache.keys()
             if self._cache[f][b'exists'] and self._existsinparent(f)
         ]
 
     def added(self):
         return [
             f
             for f in self._cache.keys()
             if self._cache[f][b'exists'] and not self._existsinparent(f)
         ]
 
     def removed(self):
         return [
             f
             for f in self._cache.keys()
             if not self._cache[f][b'exists'] and self._existsinparent(f)
         ]
 
     def p1copies(self):
         copies = self._repo._wrappedctx.p1copies().copy()
         narrowmatch = self._repo.narrowmatch()
         for f in self._cache.keys():
             if not narrowmatch(f):
                 continue
             copies.pop(f, None)  # delete if it exists
             source = self._cache[f][b'copied']
             if source:
                 copies[f] = source
         return copies
 
     def p2copies(self):
         copies = self._repo._wrappedctx.p2copies().copy()
         narrowmatch = self._repo.narrowmatch()
         for f in self._cache.keys():
             if not narrowmatch(f):
                 continue
             copies.pop(f, None)  # delete if it exists
             source = self._cache[f][b'copied']
             if source:
                 copies[f] = source
         return copies
 
     def isinmemory(self):
         return True
 
     def filedate(self, path):
         if self.isdirty(path):
             return self._cache[path][b'date']
         else:
             return self._wrappedctx[path].date()
 
     def markcopied(self, path, origin):
         self._markdirty(
             path,
             exists=True,
             date=self.filedate(path),
             flags=self.flags(path),
             copied=origin,
         )
 
     def copydata(self, path):
         if self.isdirty(path):
             return self._cache[path][b'copied']
         else:
             return None
 
     def flags(self, path):
         if self.isdirty(path):
             if self._cache[path][b'exists']:
                 return self._cache[path][b'flags']
             else:
                 raise error.ProgrammingError(
                     b"No such file or directory: %s" % self._path
                 )
         else:
             return self._wrappedctx[path].flags()
 
     def __contains__(self, key):
         if key in self._cache:
             return self._cache[key][b'exists']
         return key in self.p1()
 
     def _existsinparent(self, path):
         try:
             # ``commitctx` raises a ``ManifestLookupError`` if a path does not
             # exist, unlike ``workingctx``, which returns a ``workingfilectx``
             # with an ``exists()`` function.
             self._wrappedctx[path]
             return True
         except error.ManifestLookupError:
             return False
 
     def _auditconflicts(self, path):
         """Replicates conflict checks done by wvfs.write().
 
         Since we never write to the filesystem and never call `applyupdates` in
         IMM, we'll never check that a path is actually writable -- e.g., because
         it adds `a/foo`, but `a` is actually a file in the other commit.
         """
 
         def fail(path, component):
             # p1() is the base and we're receiving "writes" for p2()'s
             # files.
             if b'l' in self.p1()[component].flags():
                 raise error.Abort(
                     b"error: %s conflicts with symlink %s "
                     b"in %d." % (path, component, self.p1().rev())
                 )
             else:
                 raise error.Abort(
                     b"error: '%s' conflicts with file '%s' in "
                     b"%d." % (path, component, self.p1().rev())
                 )
 
         # Test that each new directory to be created to write this path from p2
         # is not a file in p1.
         components = path.split(b'/')
         for i in pycompat.xrange(len(components)):
             component = b"/".join(components[0:i])
             if component in self:
                 fail(path, component)
 
         # Test the other direction -- that this path from p2 isn't a directory
         # in p1 (test that p1 doesn't have any paths matching `path/*`).
         match = self.match([path], default=b'path')
         matches = self.p1().manifest().matches(match)
         mfiles = matches.keys()
         if len(mfiles) > 0:
             if len(mfiles) == 1 and mfiles[0] == path:
                 return
             # omit the files which are deleted in current IMM wctx
             mfiles = [m for m in mfiles if m in self]
             if not mfiles:
                 return
             raise error.Abort(
                 b"error: file '%s' cannot be written because "
                 b" '%s/' is a directory in %s (containing %d "
                 b"entries: %s)"
                 % (path, path, self.p1(), len(mfiles), b', '.join(mfiles))
             )
 
     def write(self, path, data, flags=b'', **kwargs):
         if data is None:
             raise error.ProgrammingError(b"data must be non-None")
         self._auditconflicts(path)
         self._markdirty(
             path, exists=True, data=data, date=dateutil.makedate(), flags=flags
         )
 
     def setflags(self, path, l, x):
         flag = b''
         if l:
             flag = b'l'
         elif x:
             flag = b'x'
         self._markdirty(path, exists=True, date=dateutil.makedate(), flags=flag)
 
     def remove(self, path):
         self._markdirty(path, exists=False)
 
     def exists(self, path):
         """exists behaves like `lexists`, but needs to follow symlinks and
         return False if they are broken.
         """
         if self.isdirty(path):
             # If this path exists and is a symlink, "follow" it by calling
             # exists on the destination path.
             if (
                 self._cache[path][b'exists']
                 and b'l' in self._cache[path][b'flags']
             ):
                 return self.exists(self._cache[path][b'data'].strip())
             else:
                 return self._cache[path][b'exists']
 
         return self._existsinparent(path)
 
     def lexists(self, path):
         """lexists returns True if the path exists"""
         if self.isdirty(path):
             return self._cache[path][b'exists']
 
         return self._existsinparent(path)
 
     def size(self, path):
         if self.isdirty(path):
             if self._cache[path][b'exists']:
                 return len(self._cache[path][b'data'])
             else:
                 raise error.ProgrammingError(
                     b"No such file or directory: %s" % self._path
                 )
         return self._wrappedctx[path].size()
 
     def tomemctx(
         self,
         text,
         branch=None,
         extra=None,
         date=None,
         parents=None,
         user=None,
         editor=None,
     ):
         """Converts this ``overlayworkingctx`` into a ``memctx`` ready to be
         committed.
 
         ``text`` is the commit message.
         ``parents`` (optional) are rev numbers.
         """
         # Default parents to the wrapped contexts' if not passed.
         if parents is None:
             parents = self._wrappedctx.parents()
             if len(parents) == 1:
                 parents = (parents[0], None)
 
         # ``parents`` is passed as rev numbers; convert to ``commitctxs``.
         if parents[1] is None:
             parents = (self._repo[parents[0]], None)
         else:
             parents = (self._repo[parents[0]], self._repo[parents[1]])
 
         files = self.files()
 
         def getfile(repo, memctx, path):
             if self._cache[path][b'exists']:
                 return memfilectx(
                     repo,
                     memctx,
                     path,
                     self._cache[path][b'data'],
                     b'l' in self._cache[path][b'flags'],
                     b'x' in self._cache[path][b'flags'],
                     self._cache[path][b'copied'],
                 )
             else:
                 # Returning None, but including the path in `files`, is
                 # necessary for memctx to register a deletion.
                 return None
 
         return memctx(
             self._repo,
             parents,
             text,
             files,
             getfile,
             date=date,
             extra=extra,
             user=user,
             branch=branch,
             editor=editor,
         )
 
     def isdirty(self, path):
         return path in self._cache
 
     def isempty(self):
         # We need to discard any keys that are actually clean before the empty
         # commit check.
         self._compact()
         return len(self._cache) == 0
 
     def clean(self):
         self._cache = {}
 
     def _compact(self):
         """Removes keys from the cache that are actually clean, by comparing
         them with the underlying context.
 
         This can occur during the merge process, e.g. by passing --tool :local
         to resolve a conflict.
         """
         keys = []
         # This won't be perfect, but can help performance significantly when
         # using things like remotefilelog.
         scmutil.prefetchfiles(
             self.repo(),
             [self.p1().rev()],
             scmutil.matchfiles(self.repo(), self._cache.keys()),
         )
 
         for path in self._cache.keys():
             cache = self._cache[path]
             try:
                 underlying = self._wrappedctx[path]
                 if (
                     underlying.data() == cache[b'data']
                     and underlying.flags() == cache[b'flags']
                 ):
                     keys.append(path)
             except error.ManifestLookupError:
                 # Path not in the underlying manifest (created).
                 continue
 
         for path in keys:
             del self._cache[path]
         return keys
 
     def _markdirty(
         self, path, exists, data=None, date=None, flags=b'', copied=None
     ):
         # data not provided, let's see if we already have some; if not, let's
         # grab it from our underlying context, so that we always have data if
         # the file is marked as existing.
         if exists and data is None:
             oldentry = self._cache.get(path) or {}
             data = oldentry.get(b'data')
             if data is None:
                 data = self._wrappedctx[path].data()
 
         self._cache[path] = {
             b'exists': exists,
             b'data': data,
             b'date': date,
             b'flags': flags,
             b'copied': copied,
         }
 
     def filectx(self, path, filelog=None):
         return overlayworkingfilectx(
             self._repo, path, parent=self, filelog=filelog
         )
 
 
 class overlayworkingfilectx(committablefilectx):
     """Wrap a ``workingfilectx`` but intercepts all writes into an in-memory
     cache, which can be flushed through later by calling ``flush()``."""
 
     def __init__(self, repo, path, filelog=None, parent=None):
         super(overlayworkingfilectx, self).__init__(repo, path, filelog, parent)
         self._repo = repo
         self._parent = parent
         self._path = path
 
     def cmp(self, fctx):
         return self.data() != fctx.data()
 
     def changectx(self):
         return self._parent
 
     def data(self):
         return self._parent.data(self._path)
 
     def date(self):
         return self._parent.filedate(self._path)
 
     def exists(self):
         return self.lexists()
 
     def lexists(self):
         return self._parent.exists(self._path)
 
     def copysource(self):
         return self._parent.copydata(self._path)
 
     def size(self):
         return self._parent.size(self._path)
 
     def markcopied(self, origin):
         self._parent.markcopied(self._path, origin)
 
     def audit(self):
         pass
 
     def flags(self):
         return self._parent.flags(self._path)
 
     def setflags(self, islink, isexec):
         return self._parent.setflags(self._path, islink, isexec)
 
     def write(self, data, flags, backgroundclose=False, **kwargs):
         return self._parent.write(self._path, data, flags, **kwargs)
 
     def remove(self, ignoremissing=False):
         return self._parent.remove(self._path)
 
     def clearunknown(self):
         pass
 
 
 class workingcommitctx(workingctx):
     """A workingcommitctx object makes access to data related to
     the revision being committed convenient.
 
     This hides changes in the working directory, if they aren't
     committed in this context.
     """
 
     def __init__(
         self, repo, changes, text=b"", user=None, date=None, extra=None
     ):
         super(workingcommitctx, self).__init__(
             repo, text, user, date, extra, changes
         )
 
     def _dirstatestatus(self, match, ignored=False, clean=False, unknown=False):
         """Return matched files only in ``self._status``
 
         Uncommitted files appear "clean" via this context, even if
         they aren't actually so in the working directory.
         """
         if clean:
             clean = [f for f in self._manifest if f not in self._changedset]
         else:
             clean = []
         return scmutil.status(
             [f for f in self._status.modified if match(f)],
             [f for f in self._status.added if match(f)],
             [f for f in self._status.removed if match(f)],
             [],
             [],
             [],
             clean,
         )
 
     @propertycache
     def _changedset(self):
         """Return the set of files changed in this context
         """
         changed = set(self._status.modified)
         changed.update(self._status.added)
         changed.update(self._status.removed)
         return changed
 
 
 def makecachingfilectxfn(func):
     """Create a filectxfn that caches based on the path.
 
     We can't use util.cachefunc because it uses all arguments as the cache
     key and this creates a cycle since the arguments include the repo and
     memctx.
     """
     cache = {}
 
     def getfilectx(repo, memctx, path):
         if path not in cache:
             cache[path] = func(repo, memctx, path)
         return cache[path]
 
     return getfilectx
 
 
 def memfilefromctx(ctx):
     """Given a context return a memfilectx for ctx[path]
 
     This is a convenience method for building a memctx based on another
     context.
     """
 
     def getfilectx(repo, memctx, path):
         fctx = ctx[path]
         copysource = fctx.copysource()
         return memfilectx(
             repo,
             memctx,
             path,
             fctx.data(),
             islink=fctx.islink(),
             isexec=fctx.isexec(),
             copysource=copysource,
         )
 
     return getfilectx
 
 
 def memfilefrompatch(patchstore):
     """Given a patch (e.g. patchstore object) return a memfilectx
 
     This is a convenience method for building a memctx based on a patchstore.
     """
 
     def getfilectx(repo, memctx, path):
         data, mode, copysource = patchstore.getfile(path)
         if data is None:
             return None
         islink, isexec = mode
         return memfilectx(
             repo,
             memctx,
             path,
             data,
             islink=islink,
             isexec=isexec,
             copysource=copysource,
         )
 
     return getfilectx
 
 
 class memctx(committablectx):
     """Use memctx to perform in-memory commits via localrepo.commitctx().
 
     Revision information is supplied at initialization time while
     related files data and is made available through a callback
     mechanism.  'repo' is the current localrepo, 'parents' is a
     sequence of two parent revisions identifiers (pass None for every
     missing parent), 'text' is the commit message and 'files' lists
     names of files touched by the revision (normalized and relative to
     repository root).
 
     filectxfn(repo, memctx, path) is a callable receiving the
     repository, the current memctx object and the normalized path of
     requested file, relative to repository root. It is fired by the
     commit function for every file in 'files', but calls order is
     undefined. If the file is available in the revision being
     committed (updated or added), filectxfn returns a memfilectx
     object. If the file was removed, filectxfn return None for recent
     Mercurial. Moved files are represented by marking the source file
     removed and the new file added with copy information (see
     memfilectx).
 
     user receives the committer name and defaults to current
     repository username, date is the commit date in any format
     supported by dateutil.parsedate() and defaults to current date, extra
     is a dictionary of metadata or is left empty.
     """
 
     # Mercurial <= 3.1 expects the filectxfn to raise IOError for missing files.
     # Extensions that need to retain compatibility across Mercurial 3.1 can use
     # this field to determine what to do in filectxfn.
     _returnnoneformissingfiles = True
 
     def __init__(
         self,
         repo,
         parents,
         text,
         files,
         filectxfn,
         user=None,
         date=None,
         extra=None,
         branch=None,
         editor=False,
     ):
         super(memctx, self).__init__(
             repo, text, user, date, extra, branch=branch
         )
         self._rev = None
         self._node = None
         parents = [(p or nullid) for p in parents]
         p1, p2 = parents
         self._parents = [self._repo[p] for p in (p1, p2)]
         files = sorted(set(files))
         self._files = files
         self.substate = {}
 
         if isinstance(filectxfn, patch.filestore):
             filectxfn = memfilefrompatch(filectxfn)
         elif not callable(filectxfn):
             # if store is not callable, wrap it in a function
             filectxfn = memfilefromctx(filectxfn)
 
         # memoizing increases performance for e.g. vcs convert scenarios.
         self._filectxfn = makecachingfilectxfn(filectxfn)
 
         if editor:
             self._text = editor(self._repo, self, [])
             self._repo.savecommitmessage(self._text)
 
     def filectx(self, path, filelog=None):
         """get a file context from the working directory
 
         Returns None if file doesn't exist and should be removed."""
         return self._filectxfn(self._repo, self, path)
 
     def commit(self):
         """commit context to the repo"""
         return self._repo.commitctx(self)
 
     @propertycache
     def _manifest(self):
         """generate a manifest based on the return values of filectxfn"""
 
         # keep this simple for now; just worry about p1
         pctx = self._parents[0]
         man = pctx.manifest().copy()
 
         for f in self._status.modified:
             man[f] = modifiednodeid
 
         for f in self._status.added:
             man[f] = addednodeid
 
         for f in self._status.removed:
             if f in man:
                 del man[f]
 
         return man
 
     @propertycache
     def _status(self):
         """Calculate exact status from ``files`` specified at construction
         """
         man1 = self.p1().manifest()
         p2 = self._parents[1]
         # "1 < len(self._parents)" can't be used for checking
         # existence of the 2nd parent, because "memctx._parents" is
         # explicitly initialized by the list, of which length is 2.
         if p2.node() != nullid:
             man2 = p2.manifest()
             managing = lambda f: f in man1 or f in man2
         else:
             managing = lambda f: f in man1
 
         modified, added, removed = [], [], []
         for f in self._files:
             if not managing(f):
                 added.append(f)
             elif self[f]:
                 modified.append(f)
             else:
                 removed.append(f)
 
         return scmutil.status(modified, added, removed, [], [], [], [])
 
 
 class memfilectx(committablefilectx):
     """memfilectx represents an in-memory file to commit.
 
     See memctx and committablefilectx for more details.
     """
 
     def __init__(
         self,
         repo,
         changectx,
         path,
         data,
         islink=False,
         isexec=False,
         copysource=None,
     ):
         """
         path is the normalized file path relative to repository root.
         data is the file content as a string.
         islink is True if the file is a symbolic link.
         isexec is True if the file is executable.
         copied is the source file path if current file was copied in the
         revision being committed, or None."""
         super(memfilectx, self).__init__(repo, path, None, changectx)
         self._data = data
         if islink:
             self._flags = b'l'
         elif isexec:
             self._flags = b'x'
         else:
             self._flags = b''
         self._copysource = copysource
 
     def copysource(self):
         return self._copysource
 
     def cmp(self, fctx):
         return self.data() != fctx.data()
 
     def data(self):
         return self._data
 
     def remove(self, ignoremissing=False):
         """wraps unlink for a repo's working directory"""
         # need to figure out what to do here
         del self._changectx[self._path]
 
     def write(self, data, flags, **kwargs):
         """wraps repo.wwrite"""
         self._data = data
 
 
 class metadataonlyctx(committablectx):
     """Like memctx but it's reusing the manifest of different commit.
     Intended to be used by lightweight operations that are creating
     metadata-only changes.
 
     Revision information is supplied at initialization time.  'repo' is the
     current localrepo, 'ctx' is original revision which manifest we're reuisng
     'parents' is a sequence of two parent revisions identifiers (pass None for
     every missing parent), 'text' is the commit.
 
     user receives the committer name and defaults to current repository
     username, date is the commit date in any format supported by
     dateutil.parsedate() and defaults to current date, extra is a dictionary of
     metadata or is left empty.
     """
 
     def __init__(
         self,
         repo,
         originalctx,
         parents=None,
         text=None,
         user=None,
         date=None,
         extra=None,
         editor=False,
     ):
         if text is None:
             text = originalctx.description()
         super(metadataonlyctx, self).__init__(repo, text, user, date, extra)
         self._rev = None
         self._node = None
         self._originalctx = originalctx
         self._manifestnode = originalctx.manifestnode()
         if parents is None:
             parents = originalctx.parents()
         else:
             parents = [repo[p] for p in parents if p is not None]
         parents = parents[:]
         while len(parents) < 2:
             parents.append(repo[nullid])
         p1, p2 = self._parents = parents
 
         # sanity check to ensure that the reused manifest parents are
         # manifests of our commit parents
         mp1, mp2 = self.manifestctx().parents
         if p1 != nullid and p1.manifestnode() != mp1:
             raise RuntimeError(
                 r"can't reuse the manifest: its p1 "
                 r"doesn't match the new ctx p1"
             )
         if p2 != nullid and p2.manifestnode() != mp2:
             raise RuntimeError(
                 r"can't reuse the manifest: "
                 r"its p2 doesn't match the new ctx p2"
             )
 
         self._files = originalctx.files()
         self.substate = {}
 
         if editor:
             self._text = editor(self._repo, self, [])
             self._repo.savecommitmessage(self._text)
 
     def manifestnode(self):
         return self._manifestnode
 
     @property
     def _manifestctx(self):
         return self._repo.manifestlog[self._manifestnode]
 
     def filectx(self, path, filelog=None):
         return self._originalctx.filectx(path, filelog=filelog)
 
     def commit(self):
         """commit context to the repo"""
         return self._repo.commitctx(self)
 
     @property
     def _manifest(self):
         return self._originalctx.manifest()
 
     @propertycache
     def _status(self):
         """Calculate exact status from ``files`` specified in the ``origctx``
         and parents manifests.
         """
         man1 = self.p1().manifest()
         p2 = self._parents[1]
         # "1 < len(self._parents)" can't be used for checking
         # existence of the 2nd parent, because "metadataonlyctx._parents" is
         # explicitly initialized by the list, of which length is 2.
         if p2.node() != nullid:
             man2 = p2.manifest()
             managing = lambda f: f in man1 or f in man2
         else:
             managing = lambda f: f in man1
 
         modified, added, removed = [], [], []
         for f in self._files:
             if not managing(f):
                 added.append(f)
             elif f in self:
                 modified.append(f)
             else:
                 removed.append(f)
 
         return scmutil.status(modified, added, removed, [], [], [], [])
 
 
 class arbitraryfilectx(object):
     """Allows you to use filectx-like functions on a file in an arbitrary
     location on disk, possibly not in the working directory.
     """
 
     def __init__(self, path, repo=None):
         # Repo is optional because contrib/simplemerge uses this class.
         self._repo = repo
         self._path = path
 
     def cmp(self, fctx):
         # filecmp follows symlinks whereas `cmp` should not, so skip the fast
         # path if either side is a symlink.
         symlinks = b'l' in self.flags() or b'l' in fctx.flags()
         if not symlinks and isinstance(fctx, workingfilectx) and self._repo:
             # Add a fast-path for merge if both sides are disk-backed.
             # Note that filecmp uses the opposite return values (True if same)
             # from our cmp functions (True if different).
             return not filecmp.cmp(self.path(), self._repo.wjoin(fctx.path()))
         return self.data() != fctx.data()
 
     def path(self):
         return self._path
 
     def flags(self):
         return b''
 
     def data(self):
         return util.readfile(self._path)
 
     def decodeddata(self):
         with open(self._path, b"rb") as f:
             return f.read()
 
     def remove(self):
         util.unlink(self._path)
 
     def write(self, data, flags, **kwargs):
         assert not flags
         with open(self._path, b"wb") as f:
             f.write(data)

mercurial/manifest.py

@@ -1,2265 +1,2268 b''
 # manifest.py - manifest revision class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import heapq
 import itertools
 import struct
 import weakref
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
     nullrev,
 )
 from .pycompat import getattr
 from . import (
     error,
     mdiff,
     policy,
     pycompat,
     revlog,
     util,
 )
 from .interfaces import (
     repository,
     util as interfaceutil,
 )
 
 parsers = policy.importmod(r'parsers')
 propertycache = util.propertycache
 
 # Allow tests to more easily test the alternate path in manifestdict.fastdelta()
 FASTDELTA_TEXTDIFF_THRESHOLD = 1000
 
 
 def _parse(data):
     # This method does a little bit of excessive-looking
     # precondition checking. This is so that the behavior of this
     # class exactly matches its C counterpart to try and help
     # prevent surprise breakage for anyone that develops against
     # the pure version.
     if data and data[-1:] != b'\n':
         raise ValueError(b'Manifest did not end in a newline.')
     prev = None
     for l in data.splitlines():
         if prev is not None and prev > l:
             raise ValueError(b'Manifest lines not in sorted order.')
         prev = l
         f, n = l.split(b'\0')
         if len(n) > 40:
             yield f, bin(n[:40]), n[40:]
         else:
             yield f, bin(n), b''
 
 
 def _text(it):
     files = []
     lines = []
     for f, n, fl in it:
         files.append(f)
         # if this is changed to support newlines in filenames,
         # be sure to check the templates/ dir again (especially *-raw.tmpl)
         lines.append(b"%s\0%s%s\n" % (f, hex(n), fl))
 
     _checkforbidden(files)
     return b''.join(lines)
 
 
 class lazymanifestiter(object):
     def __init__(self, lm):
         self.pos = 0
         self.lm = lm
 
     def __iter__(self):
         return self
 
     def next(self):
         try:
             data, pos = self.lm._get(self.pos)
         except IndexError:
             raise StopIteration
         if pos == -1:
             self.pos += 1
             return data[0]
         self.pos += 1
         zeropos = data.find(b'\x00', pos)
         return data[pos:zeropos]
 
     __next__ = next
 
 
 class lazymanifestiterentries(object):
     def __init__(self, lm):
         self.lm = lm
         self.pos = 0
 
     def __iter__(self):
         return self
 
     def next(self):
         try:
             data, pos = self.lm._get(self.pos)
         except IndexError:
             raise StopIteration
         if pos == -1:
             self.pos += 1
             return data
         zeropos = data.find(b'\x00', pos)
         hashval = unhexlify(data, self.lm.extrainfo[self.pos], zeropos + 1, 40)
         flags = self.lm._getflags(data, self.pos, zeropos)
         self.pos += 1
         return (data[pos:zeropos], hashval, flags)
 
     __next__ = next
 
 
 def unhexlify(data, extra, pos, length):
     s = bin(data[pos : pos + length])
     if extra:
         s += chr(extra & 0xFF)
     return s
 
 
 def _cmp(a, b):
     return (a > b) - (a < b)
 
 
 class _lazymanifest(object):
     """A pure python manifest backed by a byte string.  It is supplimented with
     internal lists as it is modified, until it is compacted back to a pure byte
     string.
 
     ``data`` is the initial manifest data.
 
     ``positions`` is a list of offsets, one per manifest entry.  Positive
     values are offsets into ``data``, negative values are offsets into the
     ``extradata`` list.  When an entry is removed, its entry is dropped from
     ``positions``.  The values are encoded such that when walking the list and
     indexing into ``data`` or ``extradata`` as appropriate, the entries are
     sorted by filename.
 
     ``extradata`` is a list of (key, hash, flags) for entries that were added or
     modified since the manifest was created or compacted.
     """
 
     def __init__(
         self,
         data,
         positions=None,
         extrainfo=None,
         extradata=None,
         hasremovals=False,
     ):
         if positions is None:
             self.positions = self.findlines(data)
             self.extrainfo = [0] * len(self.positions)
             self.data = data
             self.extradata = []
             self.hasremovals = False
         else:
             self.positions = positions[:]
             self.extrainfo = extrainfo[:]
             self.extradata = extradata[:]
             self.data = data
             self.hasremovals = hasremovals
 
     def findlines(self, data):
         if not data:
             return []
         pos = data.find(b"\n")
         if pos == -1 or data[-1:] != b'\n':
             raise ValueError(b"Manifest did not end in a newline.")
         positions = [0]
         prev = data[: data.find(b'\x00')]
         while pos < len(data) - 1 and pos != -1:
             positions.append(pos + 1)
             nexts = data[pos + 1 : data.find(b'\x00', pos + 1)]
             if nexts < prev:
                 raise ValueError(b"Manifest lines not in sorted order.")
             prev = nexts
             pos = data.find(b"\n", pos + 1)
         return positions
 
     def _get(self, index):
         # get the position encoded in pos:
         #   positive number is an index in 'data'
         #   negative number is in extrapieces
         pos = self.positions[index]
         if pos >= 0:
             return self.data, pos
         return self.extradata[-pos - 1], -1
 
     def _getkey(self, pos):
         if pos >= 0:
             return self.data[pos : self.data.find(b'\x00', pos + 1)]
         return self.extradata[-pos - 1][0]
 
     def bsearch(self, key):
         first = 0
         last = len(self.positions) - 1
 
         while first <= last:
             midpoint = (first + last) // 2
             nextpos = self.positions[midpoint]
             candidate = self._getkey(nextpos)
             r = _cmp(key, candidate)
             if r == 0:
                 return midpoint
             else:
                 if r < 0:
                     last = midpoint - 1
                 else:
                     first = midpoint + 1
         return -1
 
     def bsearch2(self, key):
         # same as the above, but will always return the position
         # done for performance reasons
         first = 0
         last = len(self.positions) - 1
 
         while first <= last:
             midpoint = (first + last) // 2
             nextpos = self.positions[midpoint]
             candidate = self._getkey(nextpos)
             r = _cmp(key, candidate)
             if r == 0:
                 return (midpoint, True)
             else:
                 if r < 0:
                     last = midpoint - 1
                 else:
                     first = midpoint + 1
         return (first, False)
 
     def __contains__(self, key):
         return self.bsearch(key) != -1
 
     def _getflags(self, data, needle, pos):
         start = pos + 41
         end = data.find(b"\n", start)
         if end == -1:
             end = len(data) - 1
         if start == end:
             return b''
         return self.data[start:end]
 
     def __getitem__(self, key):
         if not isinstance(key, bytes):
             raise TypeError(b"getitem: manifest keys must be a bytes.")
         needle = self.bsearch(key)
         if needle == -1:
             raise KeyError
         data, pos = self._get(needle)
         if pos == -1:
             return (data[1], data[2])
         zeropos = data.find(b'\x00', pos)
         assert 0 <= needle <= len(self.positions)
         assert len(self.extrainfo) == len(self.positions)
         hashval = unhexlify(data, self.extrainfo[needle], zeropos + 1, 40)
         flags = self._getflags(data, needle, zeropos)
         return (hashval, flags)
 
     def __delitem__(self, key):
         needle, found = self.bsearch2(key)
         if not found:
             raise KeyError
         cur = self.positions[needle]
         self.positions = self.positions[:needle] + self.positions[needle + 1 :]
         self.extrainfo = self.extrainfo[:needle] + self.extrainfo[needle + 1 :]
         if cur >= 0:
             # This does NOT unsort the list as far as the search functions are
             # concerned, as they only examine lines mapped by self.positions.
             self.data = self.data[:cur] + b'\x00' + self.data[cur + 1 :]
             self.hasremovals = True
 
     def __setitem__(self, key, value):
         if not isinstance(key, bytes):
             raise TypeError(b"setitem: manifest keys must be a byte string.")
         if not isinstance(value, tuple) or len(value) != 2:
             raise TypeError(
                 b"Manifest values must be a tuple of (node, flags)."
             )
         hashval = value[0]
         if not isinstance(hashval, bytes) or not 20 <= len(hashval) <= 22:
             raise TypeError(b"node must be a 20-byte byte string")
         flags = value[1]
         if len(hashval) == 22:
             hashval = hashval[:-1]
         if not isinstance(flags, bytes) or len(flags) > 1:
             raise TypeError(b"flags must a 0 or 1 byte string, got %r", flags)
         needle, found = self.bsearch2(key)
         if found:
             # put the item
             pos = self.positions[needle]
             if pos < 0:
                 self.extradata[-pos - 1] = (key, hashval, value[1])
             else:
                 # just don't bother
                 self.extradata.append((key, hashval, value[1]))
                 self.positions[needle] = -len(self.extradata)
         else:
             # not found, put it in with extra positions
             self.extradata.append((key, hashval, value[1]))
             self.positions = (
                 self.positions[:needle]
                 + [-len(self.extradata)]
                 + self.positions[needle:]
             )
             self.extrainfo = (
                 self.extrainfo[:needle] + [0] + self.extrainfo[needle:]
             )
 
     def copy(self):
         # XXX call _compact like in C?
         return _lazymanifest(
             self.data,
             self.positions,
             self.extrainfo,
             self.extradata,
             self.hasremovals,
         )
 
     def _compact(self):
         # hopefully not called TOO often
         if len(self.extradata) == 0 and not self.hasremovals:
             return
         l = []
         i = 0
         offset = 0
         self.extrainfo = [0] * len(self.positions)
         while i < len(self.positions):
             if self.positions[i] >= 0:
                 cur = self.positions[i]
                 last_cut = cur
 
                 # Collect all contiguous entries in the buffer at the current
                 # offset, breaking out only for added/modified items held in
                 # extradata, or a deleted line prior to the next position.
                 while True:
                     self.positions[i] = offset
                     i += 1
                     if i == len(self.positions) or self.positions[i] < 0:
                         break
 
                     # A removed file has no positions[] entry, but does have an
                     # overwritten first byte.  Break out and find the end of the
                     # current good entry/entries if there is a removed file
                     # before the next position.
                     if (
                         self.hasremovals
                         and self.data.find(b'\n\x00', cur, self.positions[i])
                         != -1
                     ):
                         break
 
                     offset += self.positions[i] - cur
                     cur = self.positions[i]
                 end_cut = self.data.find(b'\n', cur)
                 if end_cut != -1:
                     end_cut += 1
                 offset += end_cut - cur
                 l.append(self.data[last_cut:end_cut])
             else:
                 while i < len(self.positions) and self.positions[i] < 0:
                     cur = self.positions[i]
                     t = self.extradata[-cur - 1]
                     l.append(self._pack(t))
                     self.positions[i] = offset
                     if len(t[1]) > 20:
                         self.extrainfo[i] = ord(t[1][21])
                     offset += len(l[-1])
                     i += 1
         self.data = b''.join(l)
         self.hasremovals = False
         self.extradata = []
 
     def _pack(self, d):
         return d[0] + b'\x00' + hex(d[1][:20]) + d[2] + b'\n'
 
     def text(self):
         self._compact()
         return self.data
 
     def diff(self, m2, clean=False):
         '''Finds changes between the current manifest and m2.'''
         # XXX think whether efficiency matters here
         diff = {}
 
         for fn, e1, flags in self.iterentries():
             if fn not in m2:
                 diff[fn] = (e1, flags), (None, b'')
             else:
                 e2 = m2[fn]
                 if (e1, flags) != e2:
                     diff[fn] = (e1, flags), e2
                 elif clean:
                     diff[fn] = None
 
         for fn, e2, flags in m2.iterentries():
             if fn not in self:
                 diff[fn] = (None, b''), (e2, flags)
 
         return diff
 
     def iterentries(self):
         return lazymanifestiterentries(self)
 
     def iterkeys(self):
         return lazymanifestiter(self)
 
     def __iter__(self):
         return lazymanifestiter(self)
 
     def __len__(self):
         return len(self.positions)
 
     def filtercopy(self, filterfn):
         # XXX should be optimized
         c = _lazymanifest(b'')
         for f, n, fl in self.iterentries():
             if filterfn(f):
                 c[f] = n, fl
         return c
 
 
 try:
     _lazymanifest = parsers.lazymanifest
 except AttributeError:
     pass
 
 
 @interfaceutil.implementer(repository.imanifestdict)
 class manifestdict(object):
     def __init__(self, data=b''):
         self._lm = _lazymanifest(data)
 
     def __getitem__(self, key):
         return self._lm[key][0]
 
     def find(self, key):
         return self._lm[key]
 
     def __len__(self):
         return len(self._lm)
 
     def __nonzero__(self):
         # nonzero is covered by the __len__ function, but implementing it here
         # makes it easier for extensions to override.
         return len(self._lm) != 0
 
     __bool__ = __nonzero__
 
     def __setitem__(self, key, node):
         self._lm[key] = node, self.flags(key, b'')
 
     def __contains__(self, key):
         if key is None:
             return False
         return key in self._lm
 
     def __delitem__(self, key):
         del self._lm[key]
 
     def __iter__(self):
         return self._lm.__iter__()
 
     def iterkeys(self):
         return self._lm.iterkeys()
 
     def keys(self):
         return list(self.iterkeys())
 
     def filesnotin(self, m2, match=None):
         '''Set of files in this manifest that are not in the other'''
         if match:
             m1 = self.matches(match)
             m2 = m2.matches(match)
             return m1.filesnotin(m2)
         diff = self.diff(m2)
         files = set(
             filepath
             for filepath, hashflags in pycompat.iteritems(diff)
             if hashflags[1][0] is None
         )
         return files
 
     @propertycache
     def _dirs(self):
         return util.dirs(self)
 
     def dirs(self):
         return self._dirs
 
     def hasdir(self, dir):
         return dir in self._dirs
 
     def _filesfastpath(self, match):
         '''Checks whether we can correctly and quickly iterate over matcher
         files instead of over manifest files.'''
         files = match.files()
         return len(files) < 100 and (
             match.isexact()
             or (match.prefix() and all(fn in self for fn in files))
         )
 
     def walk(self, match):
         '''Generates matching file names.
 
         Equivalent to manifest.matches(match).iterkeys(), but without creating
         an entirely new manifest.
 
         It also reports nonexistent files by marking them bad with match.bad().
         '''
         if match.always():
             for f in iter(self):
                 yield f
             return
 
         fset = set(match.files())
 
         # avoid the entire walk if we're only looking for specific files
         if self._filesfastpath(match):
             for fn in sorted(fset):
                 yield fn
             return
 
         for fn in self:
             if fn in fset:
                 # specified pattern is the exact name
                 fset.remove(fn)
             if match(fn):
                 yield fn
 
         # for dirstate.walk, files=[''] means "walk the whole tree".
         # follow that here, too
         fset.discard(b'')
 
         for fn in sorted(fset):
             if not self.hasdir(fn):
                 match.bad(fn, None)
 
     def matches(self, match):
         '''generate a new manifest filtered by the match argument'''
         if match.always():
             return self.copy()
 
         if self._filesfastpath(match):
             m = manifestdict()
             lm = self._lm
             for fn in match.files():
                 if fn in lm:
                     m._lm[fn] = lm[fn]
             return m
 
         m = manifestdict()
         m._lm = self._lm.filtercopy(match)
         return m
 
     def diff(self, m2, match=None, clean=False):
         '''Finds changes between the current manifest and m2.
 
         Args:
           m2: the manifest to which this manifest should be compared.
           clean: if true, include files unchanged between these manifests
                  with a None value in the returned dictionary.
 
         The result is returned as a dict with filename as key and
         values of the form ((n1,fl1),(n2,fl2)), where n1/n2 is the
         nodeid in the current/other manifest and fl1/fl2 is the flag
         in the current/other manifest. Where the file does not exist,
         the nodeid will be None and the flags will be the empty
         string.
         '''
         if match:
             m1 = self.matches(match)
             m2 = m2.matches(match)
             return m1.diff(m2, clean=clean)
         return self._lm.diff(m2._lm, clean)
 
     def setflag(self, key, flag):
         self._lm[key] = self[key], flag
 
     def get(self, key, default=None):
         try:
             return self._lm[key][0]
         except KeyError:
             return default
 
     def flags(self, key, default=b''):
         try:
             return self._lm[key][1]
         except KeyError:
             return default
 
     def copy(self):
         c = manifestdict()
         c._lm = self._lm.copy()
         return c
 
     def items(self):
         return (x[:2] for x in self._lm.iterentries())
 
     def iteritems(self):
         return (x[:2] for x in self._lm.iterentries())
 
     def iterentries(self):
         return self._lm.iterentries()
 
     def text(self):
         # most likely uses native version
         return self._lm.text()
 
     def fastdelta(self, base, changes):
         """Given a base manifest text as a bytearray and a list of changes
         relative to that text, compute a delta that can be used by revlog.
         """
         delta = []
         dstart = None
         dend = None
         dline = [b""]
         start = 0
         # zero copy representation of base as a buffer
         addbuf = util.buffer(base)
 
         changes = list(changes)
         if len(changes) < FASTDELTA_TEXTDIFF_THRESHOLD:
             # start with a readonly loop that finds the offset of
             # each line and creates the deltas
             for f, todelete in changes:
                 # bs will either be the index of the item or the insert point
                 start, end = _msearch(addbuf, f, start)
                 if not todelete:
                     h, fl = self._lm[f]
                     l = b"%s\0%s%s\n" % (f, hex(h), fl)
                 else:
                     if start == end:
                         # item we want to delete was not found, error out
                         raise AssertionError(
                             _(b"failed to remove %s from manifest") % f
                         )
                     l = b""
                 if dstart is not None and dstart <= start and dend >= start:
                     if dend < end:
                         dend = end
                     if l:
                         dline.append(l)
                 else:
                     if dstart is not None:
                         delta.append([dstart, dend, b"".join(dline)])
                     dstart = start
                     dend = end
                     dline = [l]
 
             if dstart is not None:
                 delta.append([dstart, dend, b"".join(dline)])
             # apply the delta to the base, and get a delta for addrevision
             deltatext, arraytext = _addlistdelta(base, delta)
         else:
             # For large changes, it's much cheaper to just build the text and
             # diff it.
             arraytext = bytearray(self.text())
             deltatext = mdiff.textdiff(
                 util.buffer(base), util.buffer(arraytext)
             )
 
         return arraytext, deltatext
 
 
 def _msearch(m, s, lo=0, hi=None):
     '''return a tuple (start, end) that says where to find s within m.
 
     If the string is found m[start:end] are the line containing
     that string.  If start == end the string was not found and
     they indicate the proper sorted insertion point.
 
     m should be a buffer, a memoryview or a byte string.
     s is a byte string'''
 
     def advance(i, c):
         while i < lenm and m[i : i + 1] != c:
             i += 1
         return i
 
     if not s:
         return (lo, lo)
     lenm = len(m)
     if not hi:
         hi = lenm
     while lo < hi:
         mid = (lo + hi) // 2
         start = mid
         while start > 0 and m[start - 1 : start] != b'\n':
             start -= 1
         end = advance(start, b'\0')
         if bytes(m[start:end]) < s:
             # we know that after the null there are 40 bytes of sha1
             # this translates to the bisect lo = mid + 1
             lo = advance(end + 40, b'\n') + 1
         else:
             # this translates to the bisect hi = mid
             hi = start
     end = advance(lo, b'\0')
     found = m[lo:end]
     if s == found:
         # we know that after the null there are 40 bytes of sha1
         end = advance(end + 40, b'\n')
         return (lo, end + 1)
     else:
         return (lo, lo)
 
 
 def _checkforbidden(l):
     """Check filenames for illegal characters."""
     for f in l:
         if b'\n' in f or b'\r' in f:
             raise error.StorageError(
                 _(b"'\\n' and '\\r' disallowed in filenames: %r")
                 % pycompat.bytestr(f)
             )
 
 
 # apply the changes collected during the bisect loop to our addlist
 # return a delta suitable for addrevision
 def _addlistdelta(addlist, x):
     # for large addlist arrays, building a new array is cheaper
     # than repeatedly modifying the existing one
     currentposition = 0
     newaddlist = bytearray()
 
     for start, end, content in x:
         newaddlist += addlist[currentposition:start]
         if content:
             newaddlist += bytearray(content)
 
         currentposition = end
 
     newaddlist += addlist[currentposition:]
 
     deltatext = b"".join(
         struct.pack(b">lll", start, end, len(content)) + content
         for start, end, content in x
     )
     return deltatext, newaddlist
 
 
 def _splittopdir(f):
     if b'/' in f:
         dir, subpath = f.split(b'/', 1)
         return dir + b'/', subpath
     else:
         return b'', f
 
 
 _noop = lambda s: None
 
 
 class treemanifest(object):
     def __init__(self, dir=b'', text=b''):
         self._dir = dir
         self._node = nullid
         self._loadfunc = _noop
         self._copyfunc = _noop
         self._dirty = False
         self._dirs = {}
         self._lazydirs = {}
         # Using _lazymanifest here is a little slower than plain old dicts
         self._files = {}
         self._flags = {}
         if text:
 
             def readsubtree(subdir, subm):
                 raise AssertionError(
                     b'treemanifest constructor only accepts flat manifests'
                 )
 
             self.parse(text, readsubtree)
             self._dirty = True  # Mark flat manifest dirty after parsing
 
     def _subpath(self, path):
         return self._dir + path
 
     def _loadalllazy(self):
         selfdirs = self._dirs
         for d, (path, node, readsubtree, docopy) in pycompat.iteritems(
             self._lazydirs
         ):
             if docopy:
                 selfdirs[d] = readsubtree(path, node).copy()
             else:
                 selfdirs[d] = readsubtree(path, node)
         self._lazydirs = {}
 
     def _loadlazy(self, d):
         v = self._lazydirs.get(d)
         if v:
             path, node, readsubtree, docopy = v
             if docopy:
                 self._dirs[d] = readsubtree(path, node).copy()
             else:
                 self._dirs[d] = readsubtree(path, node)
             del self._lazydirs[d]
 
     def _loadchildrensetlazy(self, visit):
         if not visit:
             return None
         if visit == b'all' or visit == b'this':
             self._loadalllazy()
             return None
 
         loadlazy = self._loadlazy
         for k in visit:
             loadlazy(k + b'/')
         return visit
 
     def _loaddifflazy(self, t1, t2):
         """load items in t1 and t2 if they're needed for diffing.
 
         The criteria currently is:
         - if it's not present in _lazydirs in either t1 or t2, load it in the
           other (it may already be loaded or it may not exist, doesn't matter)
         - if it's present in _lazydirs in both, compare the nodeid; if it
           differs, load it in both
         """
         toloadlazy = []
         for d, v1 in pycompat.iteritems(t1._lazydirs):
             v2 = t2._lazydirs.get(d)
             if not v2 or v2[1] != v1[1]:
                 toloadlazy.append(d)
         for d, v1 in pycompat.iteritems(t2._lazydirs):
             if d not in t1._lazydirs:
                 toloadlazy.append(d)
 
         for d in toloadlazy:
             t1._loadlazy(d)
             t2._loadlazy(d)
 
     def __len__(self):
         self._load()
         size = len(self._files)
         self._loadalllazy()
         for m in self._dirs.values():
             size += m.__len__()
         return size
 
     def __nonzero__(self):
         # Faster than "__len() != 0" since it avoids loading sub-manifests
         return not self._isempty()
 
     __bool__ = __nonzero__
 
     def _isempty(self):
         self._load()  # for consistency; already loaded by all callers
         # See if we can skip loading everything.
         if self._files or (
             self._dirs and any(not m._isempty() for m in self._dirs.values())
         ):
             return False
         self._loadalllazy()
         return not self._dirs or all(m._isempty() for m in self._dirs.values())
 
     def __repr__(self):
-        return b'<treemanifest dir=%s, node=%s, loaded=%s, dirty=%s at 0x%x>' % (
-            self._dir,
-            hex(self._node),
-            bool(self._loadfunc is _noop),
-            self._dirty,
-            id(self),
+        return (
+            b'<treemanifest dir=%s, node=%s, loaded=%s, dirty=%s at 0x%x>'
+            % (
+                self._dir,
+                hex(self._node),
+                bool(self._loadfunc is _noop),
+                self._dirty,
+                id(self),
+            )
         )
 
     def dir(self):
         '''The directory that this tree manifest represents, including a
         trailing '/'. Empty string for the repo root directory.'''
         return self._dir
 
     def node(self):
         '''This node of this instance. nullid for unsaved instances. Should
         be updated when the instance is read or written from a revlog.
         '''
         assert not self._dirty
         return self._node
 
     def setnode(self, node):
         self._node = node
         self._dirty = False
 
     def iterentries(self):
         self._load()
         self._loadalllazy()
         for p, n in sorted(
             itertools.chain(self._dirs.items(), self._files.items())
         ):
             if p in self._files:
                 yield self._subpath(p), n, self._flags.get(p, b'')
             else:
                 for x in n.iterentries():
                     yield x
 
     def items(self):
         self._load()
         self._loadalllazy()
         for p, n in sorted(
             itertools.chain(self._dirs.items(), self._files.items())
         ):
             if p in self._files:
                 yield self._subpath(p), n
             else:
                 for f, sn in pycompat.iteritems(n):
                     yield f, sn
 
     iteritems = items
 
     def iterkeys(self):
         self._load()
         self._loadalllazy()
         for p in sorted(itertools.chain(self._dirs, self._files)):
             if p in self._files:
                 yield self._subpath(p)
             else:
                 for f in self._dirs[p]:
                     yield f
 
     def keys(self):
         return list(self.iterkeys())
 
     def __iter__(self):
         return self.iterkeys()
 
     def __contains__(self, f):
         if f is None:
             return False
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
 
             if dir not in self._dirs:
                 return False
 
             return self._dirs[dir].__contains__(subpath)
         else:
             return f in self._files
 
     def get(self, f, default=None):
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
 
             if dir not in self._dirs:
                 return default
             return self._dirs[dir].get(subpath, default)
         else:
             return self._files.get(f, default)
 
     def __getitem__(self, f):
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
 
             return self._dirs[dir].__getitem__(subpath)
         else:
             return self._files[f]
 
     def flags(self, f):
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
 
             if dir not in self._dirs:
                 return b''
             return self._dirs[dir].flags(subpath)
         else:
             if f in self._lazydirs or f in self._dirs:
                 return b''
             return self._flags.get(f, b'')
 
     def find(self, f):
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
 
             return self._dirs[dir].find(subpath)
         else:
             return self._files[f], self._flags.get(f, b'')
 
     def __delitem__(self, f):
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
 
             self._dirs[dir].__delitem__(subpath)
             # If the directory is now empty, remove it
             if self._dirs[dir]._isempty():
                 del self._dirs[dir]
         else:
             del self._files[f]
             if f in self._flags:
                 del self._flags[f]
         self._dirty = True
 
     def __setitem__(self, f, n):
         assert n is not None
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
             if dir not in self._dirs:
                 self._dirs[dir] = treemanifest(self._subpath(dir))
             self._dirs[dir].__setitem__(subpath, n)
         else:
             self._files[f] = n[:21]  # to match manifestdict's behavior
         self._dirty = True
 
     def _load(self):
         if self._loadfunc is not _noop:
             lf, self._loadfunc = self._loadfunc, _noop
             lf(self)
         elif self._copyfunc is not _noop:
             cf, self._copyfunc = self._copyfunc, _noop
             cf(self)
 
     def setflag(self, f, flags):
         """Set the flags (symlink, executable) for path f."""
         self._load()
         dir, subpath = _splittopdir(f)
         if dir:
             self._loadlazy(dir)
             if dir not in self._dirs:
                 self._dirs[dir] = treemanifest(self._subpath(dir))
             self._dirs[dir].setflag(subpath, flags)
         else:
             self._flags[f] = flags
         self._dirty = True
 
     def copy(self):
         copy = treemanifest(self._dir)
         copy._node = self._node
         copy._dirty = self._dirty
         if self._copyfunc is _noop:
 
             def _copyfunc(s):
                 self._load()
                 s._lazydirs = {
                     d: (p, n, r, True)
                     for d, (p, n, r, c) in pycompat.iteritems(self._lazydirs)
                 }
                 sdirs = s._dirs
                 for d, v in pycompat.iteritems(self._dirs):
                     sdirs[d] = v.copy()
                 s._files = dict.copy(self._files)
                 s._flags = dict.copy(self._flags)
 
             if self._loadfunc is _noop:
                 _copyfunc(copy)
             else:
                 copy._copyfunc = _copyfunc
         else:
             copy._copyfunc = self._copyfunc
         return copy
 
     def filesnotin(self, m2, match=None):
         '''Set of files in this manifest that are not in the other'''
         if match and not match.always():
             m1 = self.matches(match)
             m2 = m2.matches(match)
             return m1.filesnotin(m2)
 
         files = set()
 
         def _filesnotin(t1, t2):
             if t1._node == t2._node and not t1._dirty and not t2._dirty:
                 return
             t1._load()
             t2._load()
             self._loaddifflazy(t1, t2)
             for d, m1 in pycompat.iteritems(t1._dirs):
                 if d in t2._dirs:
                     m2 = t2._dirs[d]
                     _filesnotin(m1, m2)
                 else:
                     files.update(m1.iterkeys())
 
             for fn in t1._files:
                 if fn not in t2._files:
                     files.add(t1._subpath(fn))
 
         _filesnotin(self, m2)
         return files
 
     @propertycache
     def _alldirs(self):
         return util.dirs(self)
 
     def dirs(self):
         return self._alldirs
 
     def hasdir(self, dir):
         self._load()
         topdir, subdir = _splittopdir(dir)
         if topdir:
             self._loadlazy(topdir)
             if topdir in self._dirs:
                 return self._dirs[topdir].hasdir(subdir)
             return False
         dirslash = dir + b'/'
         return dirslash in self._dirs or dirslash in self._lazydirs
 
     def walk(self, match):
         '''Generates matching file names.
 
         Equivalent to manifest.matches(match).iterkeys(), but without creating
         an entirely new manifest.
 
         It also reports nonexistent files by marking them bad with match.bad().
         '''
         if match.always():
             for f in iter(self):
                 yield f
             return
 
         fset = set(match.files())
 
         for fn in self._walk(match):
             if fn in fset:
                 # specified pattern is the exact name
                 fset.remove(fn)
             yield fn
 
         # for dirstate.walk, files=[''] means "walk the whole tree".
         # follow that here, too
         fset.discard(b'')
 
         for fn in sorted(fset):
             if not self.hasdir(fn):
                 match.bad(fn, None)
 
     def _walk(self, match):
         '''Recursively generates matching file names for walk().'''
         visit = match.visitchildrenset(self._dir[:-1])
         if not visit:
             return
 
         # yield this dir's files and walk its submanifests
         self._load()
         visit = self._loadchildrensetlazy(visit)
         for p in sorted(list(self._dirs) + list(self._files)):
             if p in self._files:
                 fullp = self._subpath(p)
                 if match(fullp):
                     yield fullp
             else:
                 if not visit or p[:-1] in visit:
                     for f in self._dirs[p]._walk(match):
                         yield f
 
     def matches(self, match):
         '''generate a new manifest filtered by the match argument'''
         if match.always():
             return self.copy()
 
         return self._matches(match)
 
     def _matches(self, match):
         '''recursively generate a new manifest filtered by the match argument.
         '''
 
         visit = match.visitchildrenset(self._dir[:-1])
         if visit == b'all':
             return self.copy()
         ret = treemanifest(self._dir)
         if not visit:
             return ret
 
         self._load()
         for fn in self._files:
             # While visitchildrenset *usually* lists only subdirs, this is
             # actually up to the matcher and may have some files in the set().
             # If visit == 'this', we should obviously look at the files in this
             # directory; if visit is a set, and fn is in it, we should inspect
             # fn (but no need to inspect things not in the set).
             if visit != b'this' and fn not in visit:
                 continue
             fullp = self._subpath(fn)
             # visitchildrenset isn't perfect, we still need to call the regular
             # matcher code to further filter results.
             if not match(fullp):
                 continue
             ret._files[fn] = self._files[fn]
             if fn in self._flags:
                 ret._flags[fn] = self._flags[fn]
 
         visit = self._loadchildrensetlazy(visit)
         for dir, subm in pycompat.iteritems(self._dirs):
             if visit and dir[:-1] not in visit:
                 continue
             m = subm._matches(match)
             if not m._isempty():
                 ret._dirs[dir] = m
 
         if not ret._isempty():
             ret._dirty = True
         return ret
 
     def diff(self, m2, match=None, clean=False):
         '''Finds changes between the current manifest and m2.
 
         Args:
           m2: the manifest to which this manifest should be compared.
           clean: if true, include files unchanged between these manifests
                  with a None value in the returned dictionary.
 
         The result is returned as a dict with filename as key and
         values of the form ((n1,fl1),(n2,fl2)), where n1/n2 is the
         nodeid in the current/other manifest and fl1/fl2 is the flag
         in the current/other manifest. Where the file does not exist,
         the nodeid will be None and the flags will be the empty
         string.
         '''
         if match and not match.always():
             m1 = self.matches(match)
             m2 = m2.matches(match)
             return m1.diff(m2, clean=clean)
         result = {}
         emptytree = treemanifest()
 
         def _iterativediff(t1, t2, stack):
             """compares two tree manifests and append new tree-manifests which
             needs to be compared to stack"""
             if t1._node == t2._node and not t1._dirty and not t2._dirty:
                 return
             t1._load()
             t2._load()
             self._loaddifflazy(t1, t2)
 
             for d, m1 in pycompat.iteritems(t1._dirs):
                 m2 = t2._dirs.get(d, emptytree)
                 stack.append((m1, m2))
 
             for d, m2 in pycompat.iteritems(t2._dirs):
                 if d not in t1._dirs:
                     stack.append((emptytree, m2))
 
             for fn, n1 in pycompat.iteritems(t1._files):
                 fl1 = t1._flags.get(fn, b'')
                 n2 = t2._files.get(fn, None)
                 fl2 = t2._flags.get(fn, b'')
                 if n1 != n2 or fl1 != fl2:
                     result[t1._subpath(fn)] = ((n1, fl1), (n2, fl2))
                 elif clean:
                     result[t1._subpath(fn)] = None
 
             for fn, n2 in pycompat.iteritems(t2._files):
                 if fn not in t1._files:
                     fl2 = t2._flags.get(fn, b'')
                     result[t2._subpath(fn)] = ((None, b''), (n2, fl2))
 
         stackls = []
         _iterativediff(self, m2, stackls)
         while stackls:
             t1, t2 = stackls.pop()
             # stackls is populated in the function call
             _iterativediff(t1, t2, stackls)
         return result
 
     def unmodifiedsince(self, m2):
         return not self._dirty and not m2._dirty and self._node == m2._node
 
     def parse(self, text, readsubtree):
         selflazy = self._lazydirs
         subpath = self._subpath
         for f, n, fl in _parse(text):
             if fl == b't':
                 f = f + b'/'
                 # False below means "doesn't need to be copied" and can use the
                 # cached value from readsubtree directly.
                 selflazy[f] = (subpath(f), n, readsubtree, False)
             elif b'/' in f:
                 # This is a flat manifest, so use __setitem__ and setflag rather
                 # than assigning directly to _files and _flags, so we can
                 # assign a path in a subdirectory, and to mark dirty (compared
                 # to nullid).
                 self[f] = n
                 if fl:
                     self.setflag(f, fl)
             else:
                 # Assigning to _files and _flags avoids marking as dirty,
                 # and should be a little faster.
                 self._files[f] = n
                 if fl:
                     self._flags[f] = fl
 
     def text(self):
         """Get the full data of this manifest as a bytestring."""
         self._load()
         return _text(self.iterentries())
 
     def dirtext(self):
         """Get the full data of this directory as a bytestring. Make sure that
         any submanifests have been written first, so their nodeids are correct.
         """
         self._load()
         flags = self.flags
         lazydirs = [
             (d[:-1], v[1], b't') for d, v in pycompat.iteritems(self._lazydirs)
         ]
         dirs = [(d[:-1], self._dirs[d]._node, b't') for d in self._dirs]
         files = [(f, self._files[f], flags(f)) for f in self._files]
         return _text(sorted(dirs + files + lazydirs))
 
     def read(self, gettext, readsubtree):
         def _load_for_read(s):
             s.parse(gettext(), readsubtree)
             s._dirty = False
 
         self._loadfunc = _load_for_read
 
     def writesubtrees(self, m1, m2, writesubtree, match):
         self._load()  # for consistency; should never have any effect here
         m1._load()
         m2._load()
         emptytree = treemanifest()
 
         def getnode(m, d):
             ld = m._lazydirs.get(d)
             if ld:
                 return ld[1]
             return m._dirs.get(d, emptytree)._node
 
         # let's skip investigating things that `match` says we do not need.
         visit = match.visitchildrenset(self._dir[:-1])
         visit = self._loadchildrensetlazy(visit)
         if visit == b'this' or visit == b'all':
             visit = None
         for d, subm in pycompat.iteritems(self._dirs):
             if visit and d[:-1] not in visit:
                 continue
             subp1 = getnode(m1, d)
             subp2 = getnode(m2, d)
             if subp1 == nullid:
                 subp1, subp2 = subp2, subp1
             writesubtree(subm, subp1, subp2, match)
 
     def walksubtrees(self, matcher=None):
         """Returns an iterator of the subtrees of this manifest, including this
         manifest itself.
 
         If `matcher` is provided, it only returns subtrees that match.
         """
         if matcher and not matcher.visitdir(self._dir[:-1]):
             return
         if not matcher or matcher(self._dir[:-1]):
             yield self
 
         self._load()
         # OPT: use visitchildrenset to avoid loading everything.
         self._loadalllazy()
         for d, subm in pycompat.iteritems(self._dirs):
             for subtree in subm.walksubtrees(matcher=matcher):
                 yield subtree
 
 
 class manifestfulltextcache(util.lrucachedict):
     """File-backed LRU cache for the manifest cache
 
     File consists of entries, up to EOF:
 
     - 20 bytes node, 4 bytes length, <length> manifest data
 
     These are written in reverse cache order (oldest to newest).
 
     """
 
     _file = b'manifestfulltextcache'
 
     def __init__(self, max):
         super(manifestfulltextcache, self).__init__(max)
         self._dirty = False
         self._read = False
         self._opener = None
 
     def read(self):
         if self._read or self._opener is None:
             return
 
         try:
             with self._opener(self._file) as fp:
                 set = super(manifestfulltextcache, self).__setitem__
                 # ignore trailing data, this is a cache, corruption is skipped
                 while True:
                     node = fp.read(20)
                     if len(node) < 20:
                         break
                     try:
                         size = struct.unpack(b'>L', fp.read(4))[0]
                     except struct.error:
                         break
                     value = bytearray(fp.read(size))
                     if len(value) != size:
                         break
                     set(node, value)
         except IOError:
             # the file is allowed to be missing
             pass
 
         self._read = True
         self._dirty = False
 
     def write(self):
         if not self._dirty or self._opener is None:
             return
         # rotate backwards to the first used node
         with self._opener(
             self._file, b'w', atomictemp=True, checkambig=True
         ) as fp:
             node = self._head.prev
             while True:
                 if node.key in self._cache:
                     fp.write(node.key)
                     fp.write(struct.pack(b'>L', len(node.value)))
                     fp.write(node.value)
                 if node is self._head:
                     break
                 node = node.prev
 
     def __len__(self):
         if not self._read:
             self.read()
         return super(manifestfulltextcache, self).__len__()
 
     def __contains__(self, k):
         if not self._read:
             self.read()
         return super(manifestfulltextcache, self).__contains__(k)
 
     def __iter__(self):
         if not self._read:
             self.read()
         return super(manifestfulltextcache, self).__iter__()
 
     def __getitem__(self, k):
         if not self._read:
             self.read()
         # the cache lru order can change on read
         setdirty = self._cache.get(k) is not self._head
         value = super(manifestfulltextcache, self).__getitem__(k)
         if setdirty:
             self._dirty = True
         return value
 
     def __setitem__(self, k, v):
         if not self._read:
             self.read()
         super(manifestfulltextcache, self).__setitem__(k, v)
         self._dirty = True
 
     def __delitem__(self, k):
         if not self._read:
             self.read()
         super(manifestfulltextcache, self).__delitem__(k)
         self._dirty = True
 
     def get(self, k, default=None):
         if not self._read:
             self.read()
         return super(manifestfulltextcache, self).get(k, default=default)
 
     def clear(self, clear_persisted_data=False):
         super(manifestfulltextcache, self).clear()
         if clear_persisted_data:
             self._dirty = True
             self.write()
         self._read = False
 
 
 # and upper bound of what we expect from compression
 # (real live value seems to be "3")
 MAXCOMPRESSION = 3
 
 
 @interfaceutil.implementer(repository.imanifeststorage)
 class manifestrevlog(object):
     '''A revlog that stores manifest texts. This is responsible for caching the
     full-text manifest contents.
     '''
 
     def __init__(
         self,
         opener,
         tree=b'',
         dirlogcache=None,
         indexfile=None,
         treemanifest=False,
     ):
         """Constructs a new manifest revlog
 
         `indexfile` - used by extensions to have two manifests at once, like
         when transitioning between flatmanifeset and treemanifests.
 
         `treemanifest` - used to indicate this is a tree manifest revlog. Opener
         options can also be used to make this a tree manifest revlog. The opener
         option takes precedence, so if it is set to True, we ignore whatever
         value is passed in to the constructor.
         """
         # During normal operations, we expect to deal with not more than four
         # revs at a time (such as during commit --amend). When rebasing large
         # stacks of commits, the number can go up, hence the config knob below.
         cachesize = 4
         optiontreemanifest = False
         opts = getattr(opener, 'options', None)
         if opts is not None:
             cachesize = opts.get(b'manifestcachesize', cachesize)
             optiontreemanifest = opts.get(b'treemanifest', False)
 
         self._treeondisk = optiontreemanifest or treemanifest
 
         self._fulltextcache = manifestfulltextcache(cachesize)
 
         if tree:
             assert self._treeondisk, b'opts is %r' % opts
 
         if indexfile is None:
             indexfile = b'00manifest.i'
             if tree:
                 indexfile = b"meta/" + tree + indexfile
 
         self.tree = tree
 
         # The dirlogcache is kept on the root manifest log
         if tree:
             self._dirlogcache = dirlogcache
         else:
             self._dirlogcache = {b'': self}
 
         self._revlog = revlog.revlog(
             opener,
             indexfile,
             # only root indexfile is cached
             checkambig=not bool(tree),
             mmaplargeindex=True,
             upperboundcomp=MAXCOMPRESSION,
         )
 
         self.index = self._revlog.index
         self.version = self._revlog.version
         self._generaldelta = self._revlog._generaldelta
 
     def _setupmanifestcachehooks(self, repo):
         """Persist the manifestfulltextcache on lock release"""
         if not util.safehasattr(repo, b'_wlockref'):
             return
 
         self._fulltextcache._opener = repo.wcachevfs
         if repo._currentlock(repo._wlockref) is None:
             return
 
         reporef = weakref.ref(repo)
         manifestrevlogref = weakref.ref(self)
 
         def persistmanifestcache():
             repo = reporef()
             self = manifestrevlogref()
             if repo is None or self is None:
                 return
             if repo.manifestlog.getstorage(b'') is not self:
                 # there's a different manifest in play now, abort
                 return
             self._fulltextcache.write()
 
         repo._afterlock(persistmanifestcache)
 
     @property
     def fulltextcache(self):
         return self._fulltextcache
 
     def clearcaches(self, clear_persisted_data=False):
         self._revlog.clearcaches()
         self._fulltextcache.clear(clear_persisted_data=clear_persisted_data)
         self._dirlogcache = {self.tree: self}
 
     def dirlog(self, d):
         if d:
             assert self._treeondisk
         if d not in self._dirlogcache:
             mfrevlog = manifestrevlog(
                 self.opener, d, self._dirlogcache, treemanifest=self._treeondisk
             )
             self._dirlogcache[d] = mfrevlog
         return self._dirlogcache[d]
 
     def add(
         self,
         m,
         transaction,
         link,
         p1,
         p2,
         added,
         removed,
         readtree=None,
         match=None,
     ):
         if p1 in self.fulltextcache and util.safehasattr(m, b'fastdelta'):
             # If our first parent is in the manifest cache, we can
             # compute a delta here using properties we know about the
             # manifest up-front, which may save time later for the
             # revlog layer.
 
             _checkforbidden(added)
             # combine the changed lists into one sorted iterator
             work = heapq.merge(
                 [(x, False) for x in sorted(added)],
                 [(x, True) for x in sorted(removed)],
             )
 
             arraytext, deltatext = m.fastdelta(self.fulltextcache[p1], work)
             cachedelta = self._revlog.rev(p1), deltatext
             text = util.buffer(arraytext)
             n = self._revlog.addrevision(
                 text, transaction, link, p1, p2, cachedelta
             )
         else:
             # The first parent manifest isn't already loaded, so we'll
             # just encode a fulltext of the manifest and pass that
             # through to the revlog layer, and let it handle the delta
             # process.
             if self._treeondisk:
                 assert readtree, b"readtree must be set for treemanifest writes"
                 assert match, b"match must be specified for treemanifest writes"
                 m1 = readtree(self.tree, p1)
                 m2 = readtree(self.tree, p2)
                 n = self._addtree(
                     m, transaction, link, m1, m2, readtree, match=match
                 )
                 arraytext = None
             else:
                 text = m.text()
                 n = self._revlog.addrevision(text, transaction, link, p1, p2)
                 arraytext = bytearray(text)
 
         if arraytext is not None:
             self.fulltextcache[n] = arraytext
 
         return n
 
     def _addtree(self, m, transaction, link, m1, m2, readtree, match):
         # If the manifest is unchanged compared to one parent,
         # don't write a new revision
         if self.tree != b'' and (
             m.unmodifiedsince(m1) or m.unmodifiedsince(m2)
         ):
             return m.node()
 
         def writesubtree(subm, subp1, subp2, match):
             sublog = self.dirlog(subm.dir())
             sublog.add(
                 subm,
                 transaction,
                 link,
                 subp1,
                 subp2,
                 None,
                 None,
                 readtree=readtree,
                 match=match,
             )
 
         m.writesubtrees(m1, m2, writesubtree, match)
         text = m.dirtext()
         n = None
         if self.tree != b'':
             # Double-check whether contents are unchanged to one parent
             if text == m1.dirtext():
                 n = m1.node()
             elif text == m2.dirtext():
                 n = m2.node()
 
         if not n:
             n = self._revlog.addrevision(
                 text, transaction, link, m1.node(), m2.node()
             )
 
         # Save nodeid so parent manifest can calculate its nodeid
         m.setnode(n)
         return n
 
     def __len__(self):
         return len(self._revlog)
 
     def __iter__(self):
         return self._revlog.__iter__()
 
     def rev(self, node):
         return self._revlog.rev(node)
 
     def node(self, rev):
         return self._revlog.node(rev)
 
     def lookup(self, value):
         return self._revlog.lookup(value)
 
     def parentrevs(self, rev):
         return self._revlog.parentrevs(rev)
 
     def parents(self, node):
         return self._revlog.parents(node)
 
     def linkrev(self, rev):
         return self._revlog.linkrev(rev)
 
     def checksize(self):
         return self._revlog.checksize()
 
     def revision(self, node, _df=None, raw=False):
         return self._revlog.revision(node, _df=_df, raw=raw)
 
     def rawdata(self, node, _df=None):
         return self._revlog.rawdata(node, _df=_df)
 
     def revdiff(self, rev1, rev2):
         return self._revlog.revdiff(rev1, rev2)
 
     def cmp(self, node, text):
         return self._revlog.cmp(node, text)
 
     def deltaparent(self, rev):
         return self._revlog.deltaparent(rev)
 
     def emitrevisions(
         self,
         nodes,
         nodesorder=None,
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
     ):
         return self._revlog.emitrevisions(
             nodes,
             nodesorder=nodesorder,
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode,
         )
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         return self._revlog.addgroup(
             deltas, linkmapper, transaction, addrevisioncb=addrevisioncb
         )
 
     def rawsize(self, rev):
         return self._revlog.rawsize(rev)
 
     def getstrippoint(self, minlink):
         return self._revlog.getstrippoint(minlink)
 
     def strip(self, minlink, transaction):
         return self._revlog.strip(minlink, transaction)
 
     def files(self):
         return self._revlog.files()
 
     def clone(self, tr, destrevlog, **kwargs):
         if not isinstance(destrevlog, manifestrevlog):
             raise error.ProgrammingError(b'expected manifestrevlog to clone()')
 
         return self._revlog.clone(tr, destrevlog._revlog, **kwargs)
 
     def storageinfo(
         self,
         exclusivefiles=False,
         sharedfiles=False,
         revisionscount=False,
         trackedsize=False,
         storedsize=False,
     ):
         return self._revlog.storageinfo(
             exclusivefiles=exclusivefiles,
             sharedfiles=sharedfiles,
             revisionscount=revisionscount,
             trackedsize=trackedsize,
             storedsize=storedsize,
         )
 
     @property
     def indexfile(self):
         return self._revlog.indexfile
 
     @indexfile.setter
     def indexfile(self, value):
         self._revlog.indexfile = value
 
     @property
     def opener(self):
         return self._revlog.opener
 
     @opener.setter
     def opener(self, value):
         self._revlog.opener = value
 
 
 @interfaceutil.implementer(repository.imanifestlog)
 class manifestlog(object):
     """A collection class representing the collection of manifest snapshots
     referenced by commits in the repository.
 
     In this situation, 'manifest' refers to the abstract concept of a snapshot
     of the list of files in the given commit. Consumers of the output of this
     class do not care about the implementation details of the actual manifests
     they receive (i.e. tree or flat or lazily loaded, etc)."""
 
     def __init__(self, opener, repo, rootstore, narrowmatch):
         usetreemanifest = False
         cachesize = 4
 
         opts = getattr(opener, 'options', None)
         if opts is not None:
             usetreemanifest = opts.get(b'treemanifest', usetreemanifest)
             cachesize = opts.get(b'manifestcachesize', cachesize)
 
         self._treemanifests = usetreemanifest
 
         self._rootstore = rootstore
         self._rootstore._setupmanifestcachehooks(repo)
         self._narrowmatch = narrowmatch
 
         # A cache of the manifestctx or treemanifestctx for each directory
         self._dirmancache = {}
         self._dirmancache[b''] = util.lrucachedict(cachesize)
 
         self._cachesize = cachesize
 
     def __getitem__(self, node):
         """Retrieves the manifest instance for the given node. Throws a
         LookupError if not found.
         """
         return self.get(b'', node)
 
     def get(self, tree, node, verify=True):
         """Retrieves the manifest instance for the given node. Throws a
         LookupError if not found.
 
         `verify` - if True an exception will be thrown if the node is not in
                    the revlog
         """
         if node in self._dirmancache.get(tree, ()):
             return self._dirmancache[tree][node]
 
         if not self._narrowmatch.always():
             if not self._narrowmatch.visitdir(tree[:-1]):
                 return excludeddirmanifestctx(tree, node)
         if tree:
             if self._rootstore._treeondisk:
                 if verify:
                     # Side-effect is LookupError is raised if node doesn't
                     # exist.
                     self.getstorage(tree).rev(node)
 
                 m = treemanifestctx(self, tree, node)
             else:
                 raise error.Abort(
                     _(
                         b"cannot ask for manifest directory '%s' in a flat "
                         b"manifest"
                     )
                     % tree
                 )
         else:
             if verify:
                 # Side-effect is LookupError is raised if node doesn't exist.
                 self._rootstore.rev(node)
 
             if self._treemanifests:
                 m = treemanifestctx(self, b'', node)
             else:
                 m = manifestctx(self, node)
 
         if node != nullid:
             mancache = self._dirmancache.get(tree)
             if not mancache:
                 mancache = util.lrucachedict(self._cachesize)
                 self._dirmancache[tree] = mancache
             mancache[node] = m
         return m
 
     def getstorage(self, tree):
         return self._rootstore.dirlog(tree)
 
     def clearcaches(self, clear_persisted_data=False):
         self._dirmancache.clear()
         self._rootstore.clearcaches(clear_persisted_data=clear_persisted_data)
 
     def rev(self, node):
         return self._rootstore.rev(node)
 
 
 @interfaceutil.implementer(repository.imanifestrevisionwritable)
 class memmanifestctx(object):
     def __init__(self, manifestlog):
         self._manifestlog = manifestlog
         self._manifestdict = manifestdict()
 
     def _storage(self):
         return self._manifestlog.getstorage(b'')
 
     def new(self):
         return memmanifestctx(self._manifestlog)
 
     def copy(self):
         memmf = memmanifestctx(self._manifestlog)
         memmf._manifestdict = self.read().copy()
         return memmf
 
     def read(self):
         return self._manifestdict
 
     def write(self, transaction, link, p1, p2, added, removed, match=None):
         return self._storage().add(
             self._manifestdict,
             transaction,
             link,
             p1,
             p2,
             added,
             removed,
             match=match,
         )
 
 
 @interfaceutil.implementer(repository.imanifestrevisionstored)
 class manifestctx(object):
     """A class representing a single revision of a manifest, including its
     contents, its parent revs, and its linkrev.
     """
 
     def __init__(self, manifestlog, node):
         self._manifestlog = manifestlog
         self._data = None
 
         self._node = node
 
         # TODO: We eventually want p1, p2, and linkrev exposed on this class,
         # but let's add it later when something needs it and we can load it
         # lazily.
         # self.p1, self.p2 = store.parents(node)
         # rev = store.rev(node)
         # self.linkrev = store.linkrev(rev)
 
     def _storage(self):
         return self._manifestlog.getstorage(b'')
 
     def node(self):
         return self._node
 
     def new(self):
         return memmanifestctx(self._manifestlog)
 
     def copy(self):
         memmf = memmanifestctx(self._manifestlog)
         memmf._manifestdict = self.read().copy()
         return memmf
 
     @propertycache
     def parents(self):
         return self._storage().parents(self._node)
 
     def read(self):
         if self._data is None:
             if self._node == nullid:
                 self._data = manifestdict()
             else:
                 store = self._storage()
                 if self._node in store.fulltextcache:
                     text = pycompat.bytestr(store.fulltextcache[self._node])
                 else:
                     text = store.revision(self._node)
                     arraytext = bytearray(text)
                     store.fulltextcache[self._node] = arraytext
                 self._data = manifestdict(text)
         return self._data
 
     def readfast(self, shallow=False):
         '''Calls either readdelta or read, based on which would be less work.
         readdelta is called if the delta is against the p1, and therefore can be
         read quickly.
 
         If `shallow` is True, nothing changes since this is a flat manifest.
         '''
         store = self._storage()
         r = store.rev(self._node)
         deltaparent = store.deltaparent(r)
         if deltaparent != nullrev and deltaparent in store.parentrevs(r):
             return self.readdelta()
         return self.read()
 
     def readdelta(self, shallow=False):
         '''Returns a manifest containing just the entries that are present
         in this manifest, but not in its p1 manifest. This is efficient to read
         if the revlog delta is already p1.
 
         Changing the value of `shallow` has no effect on flat manifests.
         '''
         store = self._storage()
         r = store.rev(self._node)
         d = mdiff.patchtext(store.revdiff(store.deltaparent(r), r))
         return manifestdict(d)
 
     def find(self, key):
         return self.read().find(key)
 
 
 @interfaceutil.implementer(repository.imanifestrevisionwritable)
 class memtreemanifestctx(object):
     def __init__(self, manifestlog, dir=b''):
         self._manifestlog = manifestlog
         self._dir = dir
         self._treemanifest = treemanifest()
 
     def _storage(self):
         return self._manifestlog.getstorage(b'')
 
     def new(self, dir=b''):
         return memtreemanifestctx(self._manifestlog, dir=dir)
 
     def copy(self):
         memmf = memtreemanifestctx(self._manifestlog, dir=self._dir)
         memmf._treemanifest = self._treemanifest.copy()
         return memmf
 
     def read(self):
         return self._treemanifest
 
     def write(self, transaction, link, p1, p2, added, removed, match=None):
         def readtree(dir, node):
             return self._manifestlog.get(dir, node).read()
 
         return self._storage().add(
             self._treemanifest,
             transaction,
             link,
             p1,
             p2,
             added,
             removed,
             readtree=readtree,
             match=match,
         )
 
 
 @interfaceutil.implementer(repository.imanifestrevisionstored)
 class treemanifestctx(object):
     def __init__(self, manifestlog, dir, node):
         self._manifestlog = manifestlog
         self._dir = dir
         self._data = None
 
         self._node = node
 
         # TODO: Load p1/p2/linkrev lazily. They need to be lazily loaded so that
         # we can instantiate treemanifestctx objects for directories we don't
         # have on disk.
         # self.p1, self.p2 = store.parents(node)
         # rev = store.rev(node)
         # self.linkrev = store.linkrev(rev)
 
     def _storage(self):
         narrowmatch = self._manifestlog._narrowmatch
         if not narrowmatch.always():
             if not narrowmatch.visitdir(self._dir[:-1]):
                 return excludedmanifestrevlog(self._dir)
         return self._manifestlog.getstorage(self._dir)
 
     def read(self):
         if self._data is None:
             store = self._storage()
             if self._node == nullid:
                 self._data = treemanifest()
             # TODO accessing non-public API
             elif store._treeondisk:
                 m = treemanifest(dir=self._dir)
 
                 def gettext():
                     return store.revision(self._node)
 
                 def readsubtree(dir, subm):
                     # Set verify to False since we need to be able to create
                     # subtrees for trees that don't exist on disk.
                     return self._manifestlog.get(dir, subm, verify=False).read()
 
                 m.read(gettext, readsubtree)
                 m.setnode(self._node)
                 self._data = m
             else:
                 if self._node in store.fulltextcache:
                     text = pycompat.bytestr(store.fulltextcache[self._node])
                 else:
                     text = store.revision(self._node)
                     arraytext = bytearray(text)
                     store.fulltextcache[self._node] = arraytext
                 self._data = treemanifest(dir=self._dir, text=text)
 
         return self._data
 
     def node(self):
         return self._node
 
     def new(self, dir=b''):
         return memtreemanifestctx(self._manifestlog, dir=dir)
 
     def copy(self):
         memmf = memtreemanifestctx(self._manifestlog, dir=self._dir)
         memmf._treemanifest = self.read().copy()
         return memmf
 
     @propertycache
     def parents(self):
         return self._storage().parents(self._node)
 
     def readdelta(self, shallow=False):
         '''Returns a manifest containing just the entries that are present
         in this manifest, but not in its p1 manifest. This is efficient to read
         if the revlog delta is already p1.
 
         If `shallow` is True, this will read the delta for this directory,
         without recursively reading subdirectory manifests. Instead, any
         subdirectory entry will be reported as it appears in the manifest, i.e.
         the subdirectory will be reported among files and distinguished only by
         its 't' flag.
         '''
         store = self._storage()
         if shallow:
             r = store.rev(self._node)
             d = mdiff.patchtext(store.revdiff(store.deltaparent(r), r))
             return manifestdict(d)
         else:
             # Need to perform a slow delta
             r0 = store.deltaparent(store.rev(self._node))
             m0 = self._manifestlog.get(self._dir, store.node(r0)).read()
             m1 = self.read()
             md = treemanifest(dir=self._dir)
             for f, ((n0, fl0), (n1, fl1)) in pycompat.iteritems(m0.diff(m1)):
                 if n1:
                     md[f] = n1
                     if fl1:
                         md.setflag(f, fl1)
             return md
 
     def readfast(self, shallow=False):
         '''Calls either readdelta or read, based on which would be less work.
         readdelta is called if the delta is against the p1, and therefore can be
         read quickly.
 
         If `shallow` is True, it only returns the entries from this manifest,
         and not any submanifests.
         '''
         store = self._storage()
         r = store.rev(self._node)
         deltaparent = store.deltaparent(r)
         if deltaparent != nullrev and deltaparent in store.parentrevs(r):
             return self.readdelta(shallow=shallow)
 
         if shallow:
             return manifestdict(store.revision(self._node))
         else:
             return self.read()
 
     def find(self, key):
         return self.read().find(key)
 
 
 class excludeddir(treemanifest):
     """Stand-in for a directory that is excluded from the repository.
 
     With narrowing active on a repository that uses treemanifests,
     some of the directory revlogs will be excluded from the resulting
     clone. This is a huge storage win for clients, but means we need
     some sort of pseudo-manifest to surface to internals so we can
     detect a merge conflict outside the narrowspec. That's what this
     class is: it stands in for a directory whose node is known, but
     whose contents are unknown.
     """
 
     def __init__(self, dir, node):
         super(excludeddir, self).__init__(dir)
         self._node = node
         # Add an empty file, which will be included by iterators and such,
         # appearing as the directory itself (i.e. something like "dir/")
         self._files[b''] = node
         self._flags[b''] = b't'
 
     # Manifests outside the narrowspec should never be modified, so avoid
     # copying. This makes a noticeable difference when there are very many
     # directories outside the narrowspec. Also, it makes sense for the copy to
     # be of the same type as the original, which would not happen with the
     # super type's copy().
     def copy(self):
         return self
 
 
 class excludeddirmanifestctx(treemanifestctx):
     """context wrapper for excludeddir - see that docstring for rationale"""
 
     def __init__(self, dir, node):
         self._dir = dir
         self._node = node
 
     def read(self):
         return excludeddir(self._dir, self._node)
 
     def write(self, *args):
         raise error.ProgrammingError(
             b'attempt to write manifest from excluded dir %s' % self._dir
         )
 
 
 class excludedmanifestrevlog(manifestrevlog):
     """Stand-in for excluded treemanifest revlogs.
 
     When narrowing is active on a treemanifest repository, we'll have
     references to directories we can't see due to the revlog being
     skipped. This class exists to conform to the manifestrevlog
     interface for those directories and proactively prevent writes to
     outside the narrowspec.
     """
 
     def __init__(self, dir):
         self._dir = dir
 
     def __len__(self):
         raise error.ProgrammingError(
             b'attempt to get length of excluded dir %s' % self._dir
         )
 
     def rev(self, node):
         raise error.ProgrammingError(
             b'attempt to get rev from excluded dir %s' % self._dir
         )
 
     def linkrev(self, node):
         raise error.ProgrammingError(
             b'attempt to get linkrev from excluded dir %s' % self._dir
         )
 
     def node(self, rev):
         raise error.ProgrammingError(
             b'attempt to get node from excluded dir %s' % self._dir
         )
 
     def add(self, *args, **kwargs):
         # We should never write entries in dirlogs outside the narrow clone.
         # However, the method still gets called from writesubtree() in
         # _addtree(), so we need to handle it. We should possibly make that
         # avoid calling add() with a clean manifest (_dirty is always False
         # in excludeddir instances).
         pass

tests/run-tests.py

@@ -1,3725 +1,3723 b''
 #!/usr/bin/env python
 #
 # run-tests.py - Run a set of tests on Mercurial
 #
 # Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Modifying this script is tricky because it has many modes:
 #   - serial (default) vs parallel (-jN, N > 1)
 #   - no coverage (default) vs coverage (-c, -C, -s)
 #   - temp install (default) vs specific hg script (--with-hg, --local)
 #   - tests are a mix of shell scripts and Python scripts
 #
 # If you change this script, it is recommended that you ensure you
 # haven't broken it by running it in various modes with a representative
 # sample of test scripts.  For example:
 #
 #  1) serial, no coverage, temp install:
 #      ./run-tests.py test-s*
 #  2) serial, no coverage, local hg:
 #      ./run-tests.py --local test-s*
 #  3) serial, coverage, temp install:
 #      ./run-tests.py -c test-s*
 #  4) serial, coverage, local hg:
 #      ./run-tests.py -c --local test-s*      # unsupported
 #  5) parallel, no coverage, temp install:
 #      ./run-tests.py -j2 test-s*
 #  6) parallel, no coverage, local hg:
 #      ./run-tests.py -j2 --local test-s*
 #  7) parallel, coverage, temp install:
 #      ./run-tests.py -j2 -c test-s*          # currently broken
 #  8) parallel, coverage, local install:
 #      ./run-tests.py -j2 -c --local test-s*  # unsupported (and broken)
 #  9) parallel, custom tmp dir:
 #      ./run-tests.py -j2 --tmpdir /tmp/myhgtests
 #  10) parallel, pure, tests that call run-tests:
 #      ./run-tests.py --pure `grep -l run-tests.py *.t`
 #
 # (You could use any subset of the tests: test-s* happens to match
 # enough that it's worth doing parallel runs, few enough that it
 # completes fairly quickly, includes both shell and Python scripts, and
 # includes some scripts that run daemon processes.)
 
 from __future__ import absolute_import, print_function
 
 import argparse
 import collections
 import difflib
 import distutils.version as version
 import errno
 import json
 import multiprocessing
 import os
 import random
 import re
 import shutil
 import signal
 import socket
 import subprocess
 import sys
 import sysconfig
 import tempfile
 import threading
 import time
 import unittest
 import uuid
 import xml.dom.minidom as minidom
 
 try:
     import Queue as queue
 except ImportError:
     import queue
 
 try:
     import shlex
 
     shellquote = shlex.quote
 except (ImportError, AttributeError):
     import pipes
 
     shellquote = pipes.quote
 
 processlock = threading.Lock()
 
 pygmentspresent = False
 # ANSI color is unsupported prior to Windows 10
 if os.name != 'nt':
     try:  # is pygments installed
         import pygments
         import pygments.lexers as lexers
         import pygments.lexer as lexer
         import pygments.formatters as formatters
         import pygments.token as token
         import pygments.style as style
 
         pygmentspresent = True
         difflexer = lexers.DiffLexer()
         terminal256formatter = formatters.Terminal256Formatter()
     except ImportError:
         pass
 
 if pygmentspresent:
 
     class TestRunnerStyle(style.Style):
         default_style = ""
         skipped = token.string_to_tokentype("Token.Generic.Skipped")
         failed = token.string_to_tokentype("Token.Generic.Failed")
         skippedname = token.string_to_tokentype("Token.Generic.SName")
         failedname = token.string_to_tokentype("Token.Generic.FName")
         styles = {
             skipped: '#e5e5e5',
             skippedname: '#00ffff',
             failed: '#7f0000',
             failedname: '#ff0000',
         }
 
     class TestRunnerLexer(lexer.RegexLexer):
         testpattern = r'[\w-]+\.(t|py)(#[a-zA-Z0-9_\-\.]+)?'
         tokens = {
             'root': [
                 (r'^Skipped', token.Generic.Skipped, 'skipped'),
                 (r'^Failed ', token.Generic.Failed, 'failed'),
                 (r'^ERROR: ', token.Generic.Failed, 'failed'),
             ],
             'skipped': [
                 (testpattern, token.Generic.SName),
                 (r':.*', token.Generic.Skipped),
             ],
             'failed': [
                 (testpattern, token.Generic.FName),
                 (r'(:| ).*', token.Generic.Failed),
             ],
         }
 
     runnerformatter = formatters.Terminal256Formatter(style=TestRunnerStyle)
     runnerlexer = TestRunnerLexer()
 
 origenviron = os.environ.copy()
 
 if sys.version_info > (3, 5, 0):
     PYTHON3 = True
     xrange = range  # we use xrange in one place, and we'd rather not use range
 
     def _bytespath(p):
         if p is None:
             return p
         return p.encode('utf-8')
 
     def _strpath(p):
         if p is None:
             return p
         return p.decode('utf-8')
 
     osenvironb = getattr(os, 'environb', None)
     if osenvironb is None:
         # Windows lacks os.environb, for instance.  A proxy over the real thing
         # instead of a copy allows the environment to be updated via bytes on
         # all platforms.
         class environbytes(object):
             def __init__(self, strenv):
                 self.__len__ = strenv.__len__
                 self.clear = strenv.clear
                 self._strenv = strenv
 
             def __getitem__(self, k):
                 v = self._strenv.__getitem__(_strpath(k))
                 return _bytespath(v)
 
             def __setitem__(self, k, v):
                 self._strenv.__setitem__(_strpath(k), _strpath(v))
 
             def __delitem__(self, k):
                 self._strenv.__delitem__(_strpath(k))
 
             def __contains__(self, k):
                 return self._strenv.__contains__(_strpath(k))
 
             def __iter__(self):
                 return iter([_bytespath(k) for k in iter(self._strenv)])
 
             def get(self, k, default=None):
                 v = self._strenv.get(_strpath(k), _strpath(default))
                 return _bytespath(v)
 
             def pop(self, k, default=None):
                 v = self._strenv.pop(_strpath(k), _strpath(default))
                 return _bytespath(v)
 
         osenvironb = environbytes(os.environ)
 
     getcwdb = getattr(os, 'getcwdb')
     if not getcwdb or os.name == 'nt':
         getcwdb = lambda: _bytespath(os.getcwd())
 
 elif sys.version_info >= (3, 0, 0):
     print(
         '%s is only supported on Python 3.5+ and 2.7, not %s'
         % (sys.argv[0], '.'.join(str(v) for v in sys.version_info[:3]))
     )
     sys.exit(70)  # EX_SOFTWARE from `man 3 sysexit`
 else:
     PYTHON3 = False
 
     # In python 2.x, path operations are generally done using
     # bytestrings by default, so we don't have to do any extra
     # fiddling there. We define the wrapper functions anyway just to
     # help keep code consistent between platforms.
     def _bytespath(p):
         return p
 
     _strpath = _bytespath
     osenvironb = os.environ
     getcwdb = os.getcwd
 
 # For Windows support
 wifexited = getattr(os, "WIFEXITED", lambda x: False)
 
 # Whether to use IPv6
 def checksocketfamily(name, port=20058):
     """return true if we can listen on localhost using family=name
 
     name should be either 'AF_INET', or 'AF_INET6'.
     port being used is okay - EADDRINUSE is considered as successful.
     """
     family = getattr(socket, name, None)
     if family is None:
         return False
     try:
         s = socket.socket(family, socket.SOCK_STREAM)
         s.bind(('localhost', port))
         s.close()
         return True
     except socket.error as exc:
         if exc.errno == errno.EADDRINUSE:
             return True
         elif exc.errno in (errno.EADDRNOTAVAIL, errno.EPROTONOSUPPORT):
             return False
         else:
             raise
     else:
         return False
 
 
 # useipv6 will be set by parseargs
 useipv6 = None
 
 
 def checkportisavailable(port):
     """return true if a port seems free to bind on localhost"""
     if useipv6:
         family = socket.AF_INET6
     else:
         family = socket.AF_INET
     try:
         s = socket.socket(family, socket.SOCK_STREAM)
         s.bind(('localhost', port))
         s.close()
         return True
     except socket.error as exc:
         if exc.errno not in (
             errno.EADDRINUSE,
             errno.EADDRNOTAVAIL,
             errno.EPROTONOSUPPORT,
         ):
             raise
     return False
 
 
 closefds = os.name == 'posix'
 
 
 def Popen4(cmd, wd, timeout, env=None):
     processlock.acquire()
     p = subprocess.Popen(
         _strpath(cmd),
         shell=True,
         bufsize=-1,
         cwd=_strpath(wd),
         env=env,
         close_fds=closefds,
         stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
     )
     processlock.release()
 
     p.fromchild = p.stdout
     p.tochild = p.stdin
     p.childerr = p.stderr
 
     p.timeout = False
     if timeout:
 
         def t():
             start = time.time()
             while time.time() - start < timeout and p.returncode is None:
                 time.sleep(0.1)
             p.timeout = True
             if p.returncode is None:
                 terminate(p)
 
         threading.Thread(target=t).start()
 
     return p
 
 
 if sys.executable:
     sysexecutable = sys.executable
 elif os.environ.get('PYTHONEXECUTABLE'):
     sysexecutable = os.environ['PYTHONEXECUTABLE']
 elif os.environ.get('PYTHON'):
     sysexecutable = os.environ['PYTHON']
 else:
     raise AssertionError('Could not find Python interpreter')
 
 PYTHON = _bytespath(sysexecutable.replace('\\', '/'))
 IMPL_PATH = b'PYTHONPATH'
 if 'java' in sys.platform:
     IMPL_PATH = b'JYTHONPATH'
 
 defaults = {
     'jobs': ('HGTEST_JOBS', multiprocessing.cpu_count()),
     'timeout': ('HGTEST_TIMEOUT', 180),
     'slowtimeout': ('HGTEST_SLOWTIMEOUT', 1500),
     'port': ('HGTEST_PORT', 20059),
     'shell': ('HGTEST_SHELL', 'sh'),
 }
 
 
 def canonpath(path):
     return os.path.realpath(os.path.expanduser(path))
 
 
 def parselistfiles(files, listtype, warn=True):
     entries = dict()
     for filename in files:
         try:
             path = os.path.expanduser(os.path.expandvars(filename))
             f = open(path, "rb")
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             if warn:
                 print("warning: no such %s file: %s" % (listtype, filename))
             continue
 
         for line in f.readlines():
             line = line.split(b'#', 1)[0].strip()
             if line:
                 entries[line] = filename
 
         f.close()
     return entries
 
 
 def parsettestcases(path):
     """read a .t test file, return a set of test case names
 
     If path does not exist, return an empty set.
     """
     cases = []
     try:
         with open(path, 'rb') as f:
             for l in f:
                 if l.startswith(b'#testcases '):
                     cases.append(sorted(l[11:].split()))
     except IOError as ex:
         if ex.errno != errno.ENOENT:
             raise
     return cases
 
 
 def getparser():
     """Obtain the OptionParser used by the CLI."""
     parser = argparse.ArgumentParser(usage='%(prog)s [options] [tests]')
 
     selection = parser.add_argument_group('Test Selection')
     selection.add_argument(
         '--allow-slow-tests',
         action='store_true',
         help='allow extremely slow tests',
     )
     selection.add_argument(
         "--blacklist",
         action="append",
         help="skip tests listed in the specified blacklist file",
     )
     selection.add_argument(
         "--changed",
         help="run tests that are changed in parent rev or working directory",
     )
     selection.add_argument(
         "-k", "--keywords", help="run tests matching keywords"
     )
     selection.add_argument(
         "-r", "--retest", action="store_true", help="retest failed tests"
     )
     selection.add_argument(
         "--test-list",
         action="append",
         help="read tests to run from the specified file",
     )
     selection.add_argument(
         "--whitelist",
         action="append",
         help="always run tests listed in the specified whitelist file",
     )
     selection.add_argument(
         'tests', metavar='TESTS', nargs='*', help='Tests to run'
     )
 
     harness = parser.add_argument_group('Test Harness Behavior')
     harness.add_argument(
         '--bisect-repo',
         metavar='bisect_repo',
         help=(
             "Path of a repo to bisect. Use together with " "--known-good-rev"
         ),
     )
     harness.add_argument(
         "-d",
         "--debug",
         action="store_true",
         help="debug mode: write output of test scripts to console"
         " rather than capturing and diffing it (disables timeout)",
     )
     harness.add_argument(
         "-f",
         "--first",
         action="store_true",
         help="exit on the first test failure",
     )
     harness.add_argument(
         "-i",
         "--interactive",
         action="store_true",
         help="prompt to accept changed output",
     )
     harness.add_argument(
         "-j",
         "--jobs",
         type=int,
         help="number of jobs to run in parallel"
         " (default: $%s or %d)" % defaults['jobs'],
     )
     harness.add_argument(
         "--keep-tmpdir",
         action="store_true",
         help="keep temporary directory after running tests",
     )
     harness.add_argument(
         '--known-good-rev',
         metavar="known_good_rev",
         help=(
             "Automatically bisect any failures using this "
             "revision as a known-good revision."
         ),
     )
     harness.add_argument(
         "--list-tests",
         action="store_true",
         help="list tests instead of running them",
     )
     harness.add_argument(
         "--loop", action="store_true", help="loop tests repeatedly"
     )
     harness.add_argument(
         '--random', action="store_true", help='run tests in random order'
     )
     harness.add_argument(
         '--order-by-runtime',
         action="store_true",
         help='run slowest tests first, according to .testtimes',
     )
     harness.add_argument(
         "-p",
         "--port",
         type=int,
         help="port on which servers should listen"
         " (default: $%s or %d)" % defaults['port'],
     )
     harness.add_argument(
         '--profile-runner',
         action='store_true',
         help='run statprof on run-tests',
     )
     harness.add_argument(
         "-R", "--restart", action="store_true", help="restart at last error"
     )
     harness.add_argument(
         "--runs-per-test",
         type=int,
         dest="runs_per_test",
         help="run each test N times (default=1)",
         default=1,
     )
     harness.add_argument(
         "--shell", help="shell to use (default: $%s or %s)" % defaults['shell']
     )
     harness.add_argument(
         '--showchannels', action='store_true', help='show scheduling channels'
     )
     harness.add_argument(
         "--slowtimeout",
         type=int,
         help="kill errant slow tests after SLOWTIMEOUT seconds"
         " (default: $%s or %d)" % defaults['slowtimeout'],
     )
     harness.add_argument(
         "-t",
         "--timeout",
         type=int,
         help="kill errant tests after TIMEOUT seconds"
         " (default: $%s or %d)" % defaults['timeout'],
     )
     harness.add_argument(
         "--tmpdir",
         help="run tests in the given temporary directory"
         " (implies --keep-tmpdir)",
     )
     harness.add_argument(
         "-v", "--verbose", action="store_true", help="output verbose messages"
     )
 
     hgconf = parser.add_argument_group('Mercurial Configuration')
     hgconf.add_argument(
         "--chg",
         action="store_true",
         help="install and use chg wrapper in place of hg",
     )
     hgconf.add_argument("--compiler", help="compiler to build with")
     hgconf.add_argument(
         '--extra-config-opt',
         action="append",
         default=[],
         help='set the given config opt in the test hgrc',
     )
     hgconf.add_argument(
         "-l",
         "--local",
         action="store_true",
         help="shortcut for --with-hg=<testdir>/../hg, "
         "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set",
     )
     hgconf.add_argument(
         "--ipv6",
         action="store_true",
         help="prefer IPv6 to IPv4 for network related tests",
     )
     hgconf.add_argument(
         "--pure",
         action="store_true",
         help="use pure Python code instead of C extensions",
     )
     hgconf.add_argument(
         "-3",
         "--py3-warnings",
         action="store_true",
         help="enable Py3k warnings on Python 2.7+",
     )
     hgconf.add_argument(
         "--with-chg",
         metavar="CHG",
         help="use specified chg wrapper in place of hg",
     )
     hgconf.add_argument(
         "--with-hg",
         metavar="HG",
         help="test using specified hg script rather than a "
         "temporary installation",
     )
 
     reporting = parser.add_argument_group('Results Reporting')
     reporting.add_argument(
         "-C",
         "--annotate",
         action="store_true",
         help="output files annotated with coverage",
     )
     reporting.add_argument(
         "--color",
         choices=["always", "auto", "never"],
         default=os.environ.get('HGRUNTESTSCOLOR', 'auto'),
         help="colorisation: always|auto|never (default: auto)",
     )
     reporting.add_argument(
         "-c",
         "--cover",
         action="store_true",
         help="print a test coverage report",
     )
     reporting.add_argument(
         '--exceptions',
         action='store_true',
         help='log all exceptions and generate an exception report',
     )
     reporting.add_argument(
         "-H",
         "--htmlcov",
         action="store_true",
         help="create an HTML report of the coverage of the files",
     )
     reporting.add_argument(
         "--json",
         action="store_true",
         help="store test result data in 'report.json' file",
     )
     reporting.add_argument(
         "--outputdir",
         help="directory to write error logs to (default=test directory)",
     )
     reporting.add_argument(
         "-n", "--nodiff", action="store_true", help="skip showing test changes"
     )
     reporting.add_argument(
         "-S",
         "--noskips",
         action="store_true",
         help="don't report skip tests verbosely",
     )
     reporting.add_argument(
         "--time", action="store_true", help="time how long each test takes"
     )
     reporting.add_argument("--view", help="external diff viewer")
     reporting.add_argument(
         "--xunit", help="record xunit results at specified path"
     )
 
     for option, (envvar, default) in defaults.items():
         defaults[option] = type(default)(os.environ.get(envvar, default))
     parser.set_defaults(**defaults)
 
     return parser
 
 
 def parseargs(args, parser):
     """Parse arguments with our OptionParser and validate results."""
     options = parser.parse_args(args)
 
     # jython is always pure
     if 'java' in sys.platform or '__pypy__' in sys.modules:
         options.pure = True
 
     if options.local:
         if options.with_hg or options.with_chg:
             parser.error('--local cannot be used with --with-hg or --with-chg')
         testdir = os.path.dirname(_bytespath(canonpath(sys.argv[0])))
         reporootdir = os.path.dirname(testdir)
         pathandattrs = [(b'hg', 'with_hg')]
         if options.chg:
             pathandattrs.append((b'contrib/chg/chg', 'with_chg'))
         for relpath, attr in pathandattrs:
             binpath = os.path.join(reporootdir, relpath)
             if os.name != 'nt' and not os.access(binpath, os.X_OK):
                 parser.error(
                     '--local specified, but %r not found or '
                     'not executable' % binpath
                 )
             setattr(options, attr, _strpath(binpath))
 
     if options.with_hg:
         options.with_hg = canonpath(_bytespath(options.with_hg))
         if not (
             os.path.isfile(options.with_hg)
             and os.access(options.with_hg, os.X_OK)
         ):
             parser.error('--with-hg must specify an executable hg script')
         if os.path.basename(options.with_hg) not in [b'hg', b'hg.exe']:
             sys.stderr.write('warning: --with-hg should specify an hg script\n')
             sys.stderr.flush()
 
     if (options.chg or options.with_chg) and os.name == 'nt':
         parser.error('chg does not work on %s' % os.name)
     if options.with_chg:
         options.chg = False  # no installation to temporary location
         options.with_chg = canonpath(_bytespath(options.with_chg))
         if not (
             os.path.isfile(options.with_chg)
             and os.access(options.with_chg, os.X_OK)
         ):
             parser.error('--with-chg must specify a chg executable')
     if options.chg and options.with_hg:
         # chg shares installation location with hg
         parser.error(
             '--chg does not work when --with-hg is specified '
             '(use --with-chg instead)'
         )
 
     if options.color == 'always' and not pygmentspresent:
         sys.stderr.write(
             'warning: --color=always ignored because '
             'pygments is not installed\n'
         )
 
     if options.bisect_repo and not options.known_good_rev:
         parser.error("--bisect-repo cannot be used without --known-good-rev")
 
     global useipv6
     if options.ipv6:
         useipv6 = checksocketfamily('AF_INET6')
     else:
         # only use IPv6 if IPv4 is unavailable and IPv6 is available
         useipv6 = (not checksocketfamily('AF_INET')) and checksocketfamily(
             'AF_INET6'
         )
 
     options.anycoverage = options.cover or options.annotate or options.htmlcov
     if options.anycoverage:
         try:
             import coverage
 
             covver = version.StrictVersion(coverage.__version__).version
             if covver < (3, 3):
                 parser.error('coverage options require coverage 3.3 or later')
         except ImportError:
             parser.error('coverage options now require the coverage package')
 
     if options.anycoverage and options.local:
         # this needs some path mangling somewhere, I guess
         parser.error(
             "sorry, coverage options do not work when --local " "is specified"
         )
 
     if options.anycoverage and options.with_hg:
         parser.error(
             "sorry, coverage options do not work when --with-hg " "is specified"
         )
 
     global verbose
     if options.verbose:
         verbose = ''
 
     if options.tmpdir:
         options.tmpdir = canonpath(options.tmpdir)
 
     if options.jobs < 1:
         parser.error('--jobs must be positive')
     if options.interactive and options.debug:
         parser.error("-i/--interactive and -d/--debug are incompatible")
     if options.debug:
         if options.timeout != defaults['timeout']:
             sys.stderr.write('warning: --timeout option ignored with --debug\n')
         if options.slowtimeout != defaults['slowtimeout']:
             sys.stderr.write(
                 'warning: --slowtimeout option ignored with --debug\n'
             )
         options.timeout = 0
         options.slowtimeout = 0
     if options.py3_warnings:
         if PYTHON3:
             parser.error('--py3-warnings can only be used on Python 2.7')
 
     if options.blacklist:
         options.blacklist = parselistfiles(options.blacklist, 'blacklist')
     if options.whitelist:
         options.whitelisted = parselistfiles(options.whitelist, 'whitelist')
     else:
         options.whitelisted = {}
 
     if options.showchannels:
         options.nodiff = True
 
     return options
 
 
 def rename(src, dst):
     """Like os.rename(), trade atomicity and opened files friendliness
     for existing destination support.
     """
     shutil.copy(src, dst)
     os.remove(src)
 
 
 def makecleanable(path):
     """Try to fix directory permission recursively so that the entire tree
     can be deleted"""
     for dirpath, dirnames, _filenames in os.walk(path, topdown=True):
         for d in dirnames:
             p = os.path.join(dirpath, d)
             try:
                 os.chmod(p, os.stat(p).st_mode & 0o777 | 0o700)  # chmod u+rwx
             except OSError:
                 pass
 
 
 _unified_diff = difflib.unified_diff
 if PYTHON3:
     import functools
 
     _unified_diff = functools.partial(difflib.diff_bytes, difflib.unified_diff)
 
 
 def getdiff(expected, output, ref, err):
     servefail = False
     lines = []
     for line in _unified_diff(expected, output, ref, err):
         if line.startswith(b'+++') or line.startswith(b'---'):
             line = line.replace(b'\\', b'/')
             if line.endswith(b' \n'):
                 line = line[:-2] + b'\n'
         lines.append(line)
         if not servefail and line.startswith(
             b'+  abort: child process failed to start'
         ):
             servefail = True
 
     return servefail, lines
 
 
 verbose = False
 
 
 def vlog(*msg):
     """Log only when in verbose mode."""
     if verbose is False:
         return
 
     return log(*msg)
 
 
 # Bytes that break XML even in a CDATA block: control characters 0-31
 # sans \t, \n and \r
 CDATA_EVIL = re.compile(br"[\000-\010\013\014\016-\037]")
 
 # Match feature conditionalized output lines in the form, capturing the feature
 # list in group 2, and the preceeding line output in group 1:
 #
 #   output..output (feature !)\n
 optline = re.compile(br'(.*) \((.+?) !\)\n$')
 
 
 def cdatasafe(data):
     """Make a string safe to include in a CDATA block.
 
     Certain control characters are illegal in a CDATA block, and
     there's no way to include a ]]> in a CDATA either. This function
     replaces illegal bytes with ? and adds a space between the ]] so
     that it won't break the CDATA block.
     """
     return CDATA_EVIL.sub(b'?', data).replace(b']]>', b'] ]>')
 
 
 def log(*msg):
     """Log something to stdout.
 
     Arguments are strings to print.
     """
     with iolock:
         if verbose:
             print(verbose, end=' ')
         for m in msg:
             print(m, end=' ')
         print()
         sys.stdout.flush()
 
 
 def highlightdiff(line, color):
     if not color:
         return line
     assert pygmentspresent
     return pygments.highlight(
         line.decode('latin1'), difflexer, terminal256formatter
     ).encode('latin1')
 
 
 def highlightmsg(msg, color):
     if not color:
         return msg
     assert pygmentspresent
     return pygments.highlight(msg, runnerlexer, runnerformatter)
 
 
 def terminate(proc):
     """Terminate subprocess"""
     vlog('# Terminating process %d' % proc.pid)
     try:
         proc.terminate()
     except OSError:
         pass
 
 
 def killdaemons(pidfile):
     import killdaemons as killmod
 
     return killmod.killdaemons(pidfile, tryhard=False, remove=True, logfn=vlog)
 
 
 class Test(unittest.TestCase):
     """Encapsulates a single, runnable test.
 
     While this class conforms to the unittest.TestCase API, it differs in that
     instances need to be instantiated manually. (Typically, unittest.TestCase
     classes are instantiated automatically by scanning modules.)
     """
 
     # Status code reserved for skipped tests (used by hghave).
     SKIPPED_STATUS = 80
 
     def __init__(
         self,
         path,
         outputdir,
         tmpdir,
         keeptmpdir=False,
         debug=False,
         first=False,
         timeout=None,
         startport=None,
         extraconfigopts=None,
         py3warnings=False,
         shell=None,
         hgcommand=None,
         slowtimeout=None,
         usechg=False,
         useipv6=False,
     ):
         """Create a test from parameters.
 
         path is the full path to the file defining the test.
 
         tmpdir is the main temporary directory to use for this test.
 
         keeptmpdir determines whether to keep the test's temporary directory
         after execution. It defaults to removal (False).
 
         debug mode will make the test execute verbosely, with unfiltered
         output.
 
         timeout controls the maximum run time of the test. It is ignored when
         debug is True. See slowtimeout for tests with #require slow.
 
         slowtimeout overrides timeout if the test has #require slow.
 
         startport controls the starting port number to use for this test. Each
         test will reserve 3 port numbers for execution. It is the caller's
         responsibility to allocate a non-overlapping port range to Test
         instances.
 
         extraconfigopts is an iterable of extra hgrc config options. Values
         must have the form "key=value" (something understood by hgrc). Values
         of the form "foo.key=value" will result in "[foo] key=value".
 
         py3warnings enables Py3k warnings.
 
         shell is the shell to execute tests in.
         """
         if timeout is None:
             timeout = defaults['timeout']
         if startport is None:
             startport = defaults['port']
         if slowtimeout is None:
             slowtimeout = defaults['slowtimeout']
         self.path = path
         self.bname = os.path.basename(path)
         self.name = _strpath(self.bname)
         self._testdir = os.path.dirname(path)
         self._outputdir = outputdir
         self._tmpname = os.path.basename(path)
         self.errpath = os.path.join(self._outputdir, b'%s.err' % self.bname)
 
         self._threadtmp = tmpdir
         self._keeptmpdir = keeptmpdir
         self._debug = debug
         self._first = first
         self._timeout = timeout
         self._slowtimeout = slowtimeout
         self._startport = startport
         self._extraconfigopts = extraconfigopts or []
         self._py3warnings = py3warnings
         self._shell = _bytespath(shell)
         self._hgcommand = hgcommand or b'hg'
         self._usechg = usechg
         self._useipv6 = useipv6
 
         self._aborted = False
         self._daemonpids = []
         self._finished = None
         self._ret = None
         self._out = None
         self._skipped = None
         self._testtmp = None
         self._chgsockdir = None
 
         self._refout = self.readrefout()
 
     def readrefout(self):
         """read reference output"""
         # If we're not in --debug mode and reference output file exists,
         # check test output against it.
         if self._debug:
             return None  # to match "out is None"
         elif os.path.exists(self.refpath):
             with open(self.refpath, 'rb') as f:
                 return f.read().splitlines(True)
         else:
             return []
 
     # needed to get base class __repr__ running
     @property
     def _testMethodName(self):
         return self.name
 
     def __str__(self):
         return self.name
 
     def shortDescription(self):
         return self.name
 
     def setUp(self):
         """Tasks to perform before run()."""
         self._finished = False
         self._ret = None
         self._out = None
         self._skipped = None
 
         try:
             os.mkdir(self._threadtmp)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         name = self._tmpname
         self._testtmp = os.path.join(self._threadtmp, name)
         os.mkdir(self._testtmp)
 
         # Remove any previous output files.
         if os.path.exists(self.errpath):
             try:
                 os.remove(self.errpath)
             except OSError as e:
                 # We might have raced another test to clean up a .err
                 # file, so ignore ENOENT when removing a previous .err
                 # file.
                 if e.errno != errno.ENOENT:
                     raise
 
         if self._usechg:
             self._chgsockdir = os.path.join(
                 self._threadtmp, b'%s.chgsock' % name
             )
             os.mkdir(self._chgsockdir)
 
     def run(self, result):
         """Run this test and report results against a TestResult instance."""
         # This function is extremely similar to unittest.TestCase.run(). Once
         # we require Python 2.7 (or at least its version of unittest), this
         # function can largely go away.
         self._result = result
         result.startTest(self)
         try:
             try:
                 self.setUp()
             except (KeyboardInterrupt, SystemExit):
                 self._aborted = True
                 raise
             except Exception:
                 result.addError(self, sys.exc_info())
                 return
 
             success = False
             try:
                 self.runTest()
             except KeyboardInterrupt:
                 self._aborted = True
                 raise
             except unittest.SkipTest as e:
                 result.addSkip(self, str(e))
                 # The base class will have already counted this as a
                 # test we "ran", but we want to exclude skipped tests
                 # from those we count towards those run.
                 result.testsRun -= 1
             except self.failureException as e:
                 # This differs from unittest in that we don't capture
                 # the stack trace. This is for historical reasons and
                 # this decision could be revisited in the future,
                 # especially for PythonTest instances.
                 if result.addFailure(self, str(e)):
                     success = True
             except Exception:
                 result.addError(self, sys.exc_info())
             else:
                 success = True
 
             try:
                 self.tearDown()
             except (KeyboardInterrupt, SystemExit):
                 self._aborted = True
                 raise
             except Exception:
                 result.addError(self, sys.exc_info())
                 success = False
 
             if success:
                 result.addSuccess(self)
         finally:
             result.stopTest(self, interrupted=self._aborted)
 
     def runTest(self):
         """Run this test instance.
 
         This will return a tuple describing the result of the test.
         """
         env = self._getenv()
         self._genrestoreenv(env)
         self._daemonpids.append(env['DAEMON_PIDS'])
         self._createhgrc(env['HGRCPATH'])
 
         vlog('# Test', self.name)
 
         ret, out = self._run(env)
         self._finished = True
         self._ret = ret
         self._out = out
 
         def describe(ret):
             if ret < 0:
                 return 'killed by signal: %d' % -ret
             return 'returned error code %d' % ret
 
         self._skipped = False
 
         if ret == self.SKIPPED_STATUS:
             if out is None:  # Debug mode, nothing to parse.
                 missing = ['unknown']
                 failed = None
             else:
                 missing, failed = TTest.parsehghaveoutput(out)
 
             if not missing:
                 missing = ['skipped']
 
             if failed:
                 self.fail('hg have failed checking for %s' % failed[-1])
             else:
                 self._skipped = True
                 raise unittest.SkipTest(missing[-1])
         elif ret == 'timeout':
             self.fail('timed out')
         elif ret is False:
             self.fail('no result code from test')
         elif out != self._refout:
             # Diff generation may rely on written .err file.
             if (
                 (ret != 0 or out != self._refout)
                 and not self._skipped
                 and not self._debug
             ):
                 with open(self.errpath, 'wb') as f:
                     for line in out:
                         f.write(line)
 
             # The result object handles diff calculation for us.
             with firstlock:
                 if self._result.addOutputMismatch(self, ret, out, self._refout):
                     # change was accepted, skip failing
                     return
                 if self._first:
                     global firsterror
                     firsterror = True
 
             if ret:
                 msg = 'output changed and ' + describe(ret)
             else:
                 msg = 'output changed'
 
             self.fail(msg)
         elif ret:
             self.fail(describe(ret))
 
     def tearDown(self):
         """Tasks to perform after run()."""
         for entry in self._daemonpids:
             killdaemons(entry)
         self._daemonpids = []
 
         if self._keeptmpdir:
             log(
                 '\nKeeping testtmp dir: %s\nKeeping threadtmp dir: %s'
                 % (
                     self._testtmp.decode('utf-8'),
                     self._threadtmp.decode('utf-8'),
                 )
             )
         else:
             try:
                 shutil.rmtree(self._testtmp)
             except OSError:
                 # unreadable directory may be left in $TESTTMP; fix permission
                 # and try again
                 makecleanable(self._testtmp)
                 shutil.rmtree(self._testtmp, True)
             shutil.rmtree(self._threadtmp, True)
 
         if self._usechg:
             # chgservers will stop automatically after they find the socket
             # files are deleted
             shutil.rmtree(self._chgsockdir, True)
 
         if (
             (self._ret != 0 or self._out != self._refout)
             and not self._skipped
             and not self._debug
             and self._out
         ):
             with open(self.errpath, 'wb') as f:
                 for line in self._out:
                     f.write(line)
 
         vlog("# Ret was:", self._ret, '(%s)' % self.name)
 
     def _run(self, env):
         # This should be implemented in child classes to run tests.
         raise unittest.SkipTest('unknown test type')
 
     def abort(self):
         """Terminate execution of this test."""
         self._aborted = True
 
     def _portmap(self, i):
         offset = b'' if i == 0 else b'%d' % i
         return (br':%d\b' % (self._startport + i), b':$HGPORT%s' % offset)
 
     def _getreplacements(self):
         """Obtain a mapping of text replacements to apply to test output.
 
         Test output needs to be normalized so it can be compared to expected
         output. This function defines how some of that normalization will
         occur.
         """
         r = [
             # This list should be parallel to defineport in _getenv
             self._portmap(0),
             self._portmap(1),
             self._portmap(2),
             (br'([^0-9])%s' % re.escape(self._localip()), br'\1$LOCALIP'),
             (br'\bHG_TXNID=TXN:[a-f0-9]{40}\b', br'HG_TXNID=TXN:$ID$'),
         ]
         r.append((self._escapepath(self._testtmp), b'$TESTTMP'))
 
         replacementfile = os.path.join(self._testdir, b'common-pattern.py')
 
         if os.path.exists(replacementfile):
             data = {}
             with open(replacementfile, mode='rb') as source:
                 # the intermediate 'compile' step help with debugging
                 code = compile(source.read(), replacementfile, 'exec')
                 exec(code, data)
                 for value in data.get('substitutions', ()):
                     if len(value) != 2:
                         msg = 'malformatted substitution in %s: %r'
                         msg %= (replacementfile, value)
                         raise ValueError(msg)
                     r.append(value)
         return r
 
     def _escapepath(self, p):
         if os.name == 'nt':
             return b''.join(
                 c.isalpha()
                 and b'[%s%s]' % (c.lower(), c.upper())
                 or c in b'/\\'
                 and br'[/\\]'
                 or c.isdigit()
                 and c
                 or b'\\' + c
                 for c in [p[i : i + 1] for i in range(len(p))]
             )
         else:
             return re.escape(p)
 
     def _localip(self):
         if self._useipv6:
             return b'::1'
         else:
             return b'127.0.0.1'
 
     def _genrestoreenv(self, testenv):
         """Generate a script that can be used by tests to restore the original
         environment."""
         # Put the restoreenv script inside self._threadtmp
         scriptpath = os.path.join(self._threadtmp, b'restoreenv.sh')
         testenv['HGTEST_RESTOREENV'] = _strpath(scriptpath)
 
         # Only restore environment variable names that the shell allows
         # us to export.
         name_regex = re.compile('^[a-zA-Z][a-zA-Z0-9_]*$')
 
         # Do not restore these variables; otherwise tests would fail.
         reqnames = {'PYTHON', 'TESTDIR', 'TESTTMP'}
 
         with open(scriptpath, 'w') as envf:
             for name, value in origenviron.items():
                 if not name_regex.match(name):
                     # Skip environment variables with unusual names not
                     # allowed by most shells.
                     continue
                 if name in reqnames:
                     continue
                 envf.write('%s=%s\n' % (name, shellquote(value)))
 
             for name in testenv:
                 if name in origenviron or name in reqnames:
                     continue
                 envf.write('unset %s\n' % (name,))
 
     def _getenv(self):
         """Obtain environment variables to use during test execution."""
 
         def defineport(i):
             offset = '' if i == 0 else '%s' % i
             env["HGPORT%s" % offset] = '%s' % (self._startport + i)
 
         env = os.environ.copy()
         env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') or ''
         env['HGEMITWARNINGS'] = '1'
         env['TESTTMP'] = _strpath(self._testtmp)
         env['TESTNAME'] = self.name
         env['HOME'] = _strpath(self._testtmp)
         # This number should match portneeded in _getport
         for port in xrange(3):
             # This list should be parallel to _portmap in _getreplacements
             defineport(port)
         env["HGRCPATH"] = _strpath(os.path.join(self._threadtmp, b'.hgrc'))
         env["DAEMON_PIDS"] = _strpath(
             os.path.join(self._threadtmp, b'daemon.pids')
         )
         env["HGEDITOR"] = (
             '"' + sysexecutable + '"' + ' -c "import sys; sys.exit(0)"'
         )
         env["HGUSER"] = "test"
         env["HGENCODING"] = "ascii"
         env["HGENCODINGMODE"] = "strict"
         env["HGHOSTNAME"] = "test-hostname"
         env['HGIPV6'] = str(int(self._useipv6))
         # See contrib/catapipe.py for how to use this functionality.
         if 'HGTESTCATAPULTSERVERPIPE' not in env:
             # If we don't have HGTESTCATAPULTSERVERPIPE explicitly set, pull the
             # non-test one in as a default, otherwise set to devnull
             env['HGTESTCATAPULTSERVERPIPE'] = env.get(
                 'HGCATAPULTSERVERPIPE', os.devnull
             )
 
         extraextensions = []
         for opt in self._extraconfigopts:
             section, key = opt.encode('utf-8').split(b'.', 1)
             if section != 'extensions':
                 continue
             name = key.split(b'=', 1)[0]
             extraextensions.append(name)
 
         if extraextensions:
             env['HGTESTEXTRAEXTENSIONS'] = b' '.join(extraextensions)
 
         # LOCALIP could be ::1 or 127.0.0.1. Useful for tests that require raw
         # IP addresses.
         env['LOCALIP'] = _strpath(self._localip())
 
         # This has the same effect as Py_LegacyWindowsStdioFlag in exewrapper.c,
         # but this is needed for testing python instances like dummyssh,
         # dummysmtpd.py, and dumbhttp.py.
         if PYTHON3 and os.name == 'nt':
             env['PYTHONLEGACYWINDOWSSTDIO'] = '1'
 
         # Reset some environment variables to well-known values so that
         # the tests produce repeatable output.
         env['LANG'] = env['LC_ALL'] = env['LANGUAGE'] = 'C'
         env['TZ'] = 'GMT'
         env["EMAIL"] = "Foo Bar <[e-mail]>"
         env['COLUMNS'] = '80'
         env['TERM'] = 'xterm'
 
         dropped = [
             'CDPATH',
             'CHGDEBUG',
             'EDITOR',
             'GREP_OPTIONS',
             'HG',
             'HGMERGE',
             'HGPLAIN',
             'HGPLAINEXCEPT',
             'HGPROF',
             'http_proxy',
             'no_proxy',
             'NO_PROXY',
             'PAGER',
             'VISUAL',
         ]
 
         for k in dropped:
             if k in env:
                 del env[k]
 
         # unset env related to hooks
         for k in list(env):
             if k.startswith('HG_'):
                 del env[k]
 
         if self._usechg:
             env['CHGSOCKNAME'] = os.path.join(self._chgsockdir, b'server')
 
         return env
 
     def _createhgrc(self, path):
         """Create an hgrc file for this test."""
         with open(path, 'wb') as hgrc:
             hgrc.write(b'[ui]\n')
             hgrc.write(b'slash = True\n')
             hgrc.write(b'interactive = False\n')
             hgrc.write(b'merge = internal:merge\n')
             hgrc.write(b'mergemarkers = detailed\n')
             hgrc.write(b'promptecho = True\n')
             hgrc.write(b'[defaults]\n')
             hgrc.write(b'[devel]\n')
             hgrc.write(b'all-warnings = true\n')
             hgrc.write(b'default-date = 0 0\n')
             hgrc.write(b'[largefiles]\n')
             hgrc.write(
                 b'usercache = %s\n'
                 % (os.path.join(self._testtmp, b'.cache/largefiles'))
             )
             hgrc.write(b'[lfs]\n')
             hgrc.write(
                 b'usercache = %s\n'
                 % (os.path.join(self._testtmp, b'.cache/lfs'))
             )
             hgrc.write(b'[web]\n')
             hgrc.write(b'address = localhost\n')
             hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii'))
             hgrc.write(b'server-header = testing stub value\n')
 
             for opt in self._extraconfigopts:
                 section, key = opt.encode('utf-8').split(b'.', 1)
                 assert b'=' in key, (
                     'extra config opt %s must ' 'have an = for assignment' % opt
                 )
                 hgrc.write(b'[%s]\n%s\n' % (section, key))
 
     def fail(self, msg):
         # unittest differentiates between errored and failed.
         # Failed is denoted by AssertionError (by default at least).
         raise AssertionError(msg)
 
     def _runcommand(self, cmd, env, normalizenewlines=False):
         """Run command in a sub-process, capturing the output (stdout and
         stderr).
 
         Return a tuple (exitcode, output). output is None in debug mode.
         """
         if self._debug:
             proc = subprocess.Popen(
                 _strpath(cmd), shell=True, cwd=_strpath(self._testtmp), env=env
             )
             ret = proc.wait()
             return (ret, None)
 
         proc = Popen4(cmd, self._testtmp, self._timeout, env)
 
         def cleanup():
             terminate(proc)
             ret = proc.wait()
             if ret == 0:
                 ret = signal.SIGTERM << 8
             killdaemons(env['DAEMON_PIDS'])
             return ret
 
         proc.tochild.close()
 
         try:
             output = proc.fromchild.read()
         except KeyboardInterrupt:
             vlog('# Handling keyboard interrupt')
             cleanup()
             raise
 
         ret = proc.wait()
         if wifexited(ret):
             ret = os.WEXITSTATUS(ret)
 
         if proc.timeout:
             ret = 'timeout'
 
         if ret:
             killdaemons(env['DAEMON_PIDS'])
 
         for s, r in self._getreplacements():
             output = re.sub(s, r, output)
 
         if normalizenewlines:
             output = output.replace(b'\r\n', b'\n')
 
         return ret, output.splitlines(True)
 
 
 class PythonTest(Test):
     """A Python-based test."""
 
     @property
     def refpath(self):
         return os.path.join(self._testdir, b'%s.out' % self.bname)
 
     def _run(self, env):
         py3switch = self._py3warnings and b' -3' or b''
         # Quote the python(3) executable for Windows
         cmd = b'"%s"%s "%s"' % (PYTHON, py3switch, self.path)
         vlog("# Running", cmd.decode("utf-8"))
         normalizenewlines = os.name == 'nt'
         result = self._runcommand(cmd, env, normalizenewlines=normalizenewlines)
         if self._aborted:
             raise KeyboardInterrupt()
 
         return result
 
 
 # Some glob patterns apply only in some circumstances, so the script
 # might want to remove (glob) annotations that otherwise should be
 # retained.
 checkcodeglobpats = [
     # On Windows it looks like \ doesn't require a (glob), but we know
     # better.
     re.compile(br'^pushing to \$TESTTMP/.*[^)]$'),
     re.compile(br'^moving \S+/.*[^)]$'),
     re.compile(br'^pulling from \$TESTTMP/.*[^)]$'),
     # Not all platforms have 127.0.0.1 as loopback (though most do),
     # so we always glob that too.
     re.compile(br'.*\$LOCALIP.*$'),
 ]
 
 bchr = chr
 if PYTHON3:
     bchr = lambda x: bytes([x])
 
 WARN_UNDEFINED = 1
 WARN_YES = 2
 WARN_NO = 3
 
 MARK_OPTIONAL = b" (?)\n"
 
 
 def isoptional(line):
     return line.endswith(MARK_OPTIONAL)
 
 
 class TTest(Test):
     """A "t test" is a test backed by a .t file."""
 
     SKIPPED_PREFIX = b'skipped: '
     FAILED_PREFIX = b'hghave check failed: '
     NEEDESCAPE = re.compile(br'[\x00-\x08\x0b-\x1f\x7f-\xff]').search
 
     ESCAPESUB = re.compile(br'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub
     ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256))
     ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
 
     def __init__(self, path, *args, **kwds):
         # accept an extra "case" parameter
         case = kwds.pop('case', [])
         self._case = case
         self._allcases = {x for y in parsettestcases(path) for x in y}
         super(TTest, self).__init__(path, *args, **kwds)
         if case:
             casepath = b'#'.join(case)
             self.name = '%s#%s' % (self.name, _strpath(casepath))
             self.errpath = b'%s#%s.err' % (self.errpath[:-4], casepath)
             self._tmpname += b'-%s' % casepath
         self._have = {}
 
     @property
     def refpath(self):
         return os.path.join(self._testdir, self.bname)
 
     def _run(self, env):
         with open(self.path, 'rb') as f:
             lines = f.readlines()
 
         # .t file is both reference output and the test input, keep reference
         # output updated with the the test input. This avoids some race
         # conditions where the reference output does not match the actual test.
         if self._refout is not None:
             self._refout = lines
 
         salt, script, after, expected = self._parsetest(lines)
 
         # Write out the generated script.
         fname = b'%s.sh' % self._testtmp
         with open(fname, 'wb') as f:
             for l in script:
                 f.write(l)
 
         cmd = b'%s "%s"' % (self._shell, fname)
         vlog("# Running", cmd.decode("utf-8"))
 
         exitcode, output = self._runcommand(cmd, env)
 
         if self._aborted:
             raise KeyboardInterrupt()
 
         # Do not merge output if skipped. Return hghave message instead.
         # Similarly, with --debug, output is None.
         if exitcode == self.SKIPPED_STATUS or output is None:
             return exitcode, output
 
         return self._processoutput(exitcode, output, salt, after, expected)
 
     def _hghave(self, reqs):
         allreqs = b' '.join(reqs)
 
         self._detectslow(reqs)
 
         if allreqs in self._have:
             return self._have.get(allreqs)
 
         # TODO do something smarter when all other uses of hghave are gone.
         runtestdir = os.path.abspath(os.path.dirname(_bytespath(__file__)))
         tdir = runtestdir.replace(b'\\', b'/')
         proc = Popen4(
             b'%s -c "%s/hghave %s"' % (self._shell, tdir, allreqs),
             self._testtmp,
             0,
             self._getenv(),
         )
         stdout, stderr = proc.communicate()
         ret = proc.wait()
         if wifexited(ret):
             ret = os.WEXITSTATUS(ret)
         if ret == 2:
             print(stdout.decode('utf-8'))
             sys.exit(1)
 
         if ret != 0:
             self._have[allreqs] = (False, stdout)
             return False, stdout
 
         self._have[allreqs] = (True, None)
         return True, None
 
     def _detectslow(self, reqs):
         """update the timeout of slow test when appropriate"""
         if b'slow' in reqs:
             self._timeout = self._slowtimeout
 
     def _iftest(self, args):
         # implements "#if"
         reqs = []
         for arg in args:
             if arg.startswith(b'no-') and arg[3:] in self._allcases:
                 if arg[3:] in self._case:
                     return False
             elif arg in self._allcases:
                 if arg not in self._case:
                     return False
             else:
                 reqs.append(arg)
         self._detectslow(reqs)
         return self._hghave(reqs)[0]
 
     def _parsetest(self, lines):
         # We generate a shell script which outputs unique markers to line
         # up script results with our source. These markers include input
         # line number and the last return code.
         salt = b"SALT%d" % time.time()
 
         def addsalt(line, inpython):
             if inpython:
                 script.append(b'%s %d 0\n' % (salt, line))
             else:
                 script.append(b'echo %s %d $?\n' % (salt, line))
 
         activetrace = []
         session = str(uuid.uuid4())
         if PYTHON3:
             session = session.encode('ascii')
         hgcatapult = os.getenv('HGTESTCATAPULTSERVERPIPE') or os.getenv(
             'HGCATAPULTSERVERPIPE'
         )
 
         def toggletrace(cmd=None):
             if not hgcatapult or hgcatapult == os.devnull:
                 return
 
             if activetrace:
                 script.append(
                     b'echo END %s %s >> "$HGTESTCATAPULTSERVERPIPE"\n'
                     % (session, activetrace[0])
                 )
             if cmd is None:
                 return
 
             if isinstance(cmd, str):
                 quoted = shellquote(cmd.strip())
             else:
                 quoted = shellquote(cmd.strip().decode('utf8')).encode('utf8')
             quoted = quoted.replace(b'\\', b'\\\\')
             script.append(
                 b'echo START %s %s >> "$HGTESTCATAPULTSERVERPIPE"\n'
                 % (session, quoted)
             )
             activetrace[0:] = [quoted]
 
         script = []
 
         # After we run the shell script, we re-unify the script output
         # with non-active parts of the source, with synchronization by our
         # SALT line number markers. The after table contains the non-active
         # components, ordered by line number.
         after = {}
 
         # Expected shell script output.
         expected = {}
 
         pos = prepos = -1
 
         # True or False when in a true or false conditional section
         skipping = None
 
         # We keep track of whether or not we're in a Python block so we
         # can generate the surrounding doctest magic.
         inpython = False
 
         if self._debug:
             script.append(b'set -x\n')
         if self._hgcommand != b'hg':
             script.append(b'alias hg="%s"\n' % self._hgcommand)
         if os.getenv('MSYSTEM'):
             script.append(b'alias pwd="pwd -W"\n')
 
         if hgcatapult and hgcatapult != os.devnull:
             if PYTHON3:
                 hgcatapult = hgcatapult.encode('utf8')
                 cataname = self.name.encode('utf8')
             else:
                 cataname = self.name
 
             # Kludge: use a while loop to keep the pipe from getting
             # closed by our echo commands. The still-running file gets
             # reaped at the end of the script, which causes the while
             # loop to exit and closes the pipe. Sigh.
             script.append(
                 b'rtendtracing() {\n'
                 b'  echo END %(session)s %(name)s >> %(catapult)s\n'
                 b'  rm -f "$TESTTMP/.still-running"\n'
                 b'}\n'
                 b'trap "rtendtracing" 0\n'
                 b'touch "$TESTTMP/.still-running"\n'
                 b'while [ -f "$TESTTMP/.still-running" ]; do sleep 1; done '
                 b'> %(catapult)s &\n'
                 b'HGCATAPULTSESSION=%(session)s ; export HGCATAPULTSESSION\n'
                 b'echo START %(session)s %(name)s >> %(catapult)s\n'
                 % {
                     b'name': cataname,
                     b'session': session,
                     b'catapult': hgcatapult,
                 }
             )
 
         if self._case:
             casestr = b'#'.join(self._case)
             if isinstance(self._case, str):
                 quoted = shellquote(casestr)
             else:
                 quoted = shellquote(casestr.decode('utf8')).encode('utf8')
             script.append(b'TESTCASE=%s\n' % quoted)
             script.append(b'export TESTCASE\n')
 
         n = 0
         for n, l in enumerate(lines):
             if not l.endswith(b'\n'):
                 l += b'\n'
             if l.startswith(b'#require'):
                 lsplit = l.split()
                 if len(lsplit) < 2 or lsplit[0] != b'#require':
                     after.setdefault(pos, []).append('  !!! invalid #require\n')
                 if not skipping:
                     haveresult, message = self._hghave(lsplit[1:])
                     if not haveresult:
                         script = [b'echo "%s"\nexit 80\n' % message]
                         break
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'#if'):
                 lsplit = l.split()
                 if len(lsplit) < 2 or lsplit[0] != b'#if':
                     after.setdefault(pos, []).append('  !!! invalid #if\n')
                 if skipping is not None:
                     after.setdefault(pos, []).append('  !!! nested #if\n')
                 skipping = not self._iftest(lsplit[1:])
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'#else'):
                 if skipping is None:
                     after.setdefault(pos, []).append('  !!! missing #if\n')
                 skipping = not skipping
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'#endif'):
                 if skipping is None:
                     after.setdefault(pos, []).append('  !!! missing #if\n')
                 skipping = None
                 after.setdefault(pos, []).append(l)
             elif skipping:
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'  >>> '):  # python inlines
                 after.setdefault(pos, []).append(l)
                 prepos = pos
                 pos = n
                 if not inpython:
                     # We've just entered a Python block. Add the header.
                     inpython = True
                     addsalt(prepos, False)  # Make sure we report the exit code.
                     script.append(b'"%s" -m heredoctest <<EOF\n' % PYTHON)
                 addsalt(n, True)
                 script.append(l[2:])
             elif l.startswith(b'  ... '):  # python inlines
                 after.setdefault(prepos, []).append(l)
                 script.append(l[2:])
             elif l.startswith(b'  $ '):  # commands
                 if inpython:
                     script.append(b'EOF\n')
                     inpython = False
                 after.setdefault(pos, []).append(l)
                 prepos = pos
                 pos = n
                 addsalt(n, False)
                 rawcmd = l[4:]
                 cmd = rawcmd.split()
                 toggletrace(rawcmd)
                 if len(cmd) == 2 and cmd[0] == b'cd':
                     l = b'  $ cd %s || exit 1\n' % cmd[1]
                 script.append(rawcmd)
             elif l.startswith(b'  > '):  # continuations
                 after.setdefault(prepos, []).append(l)
                 script.append(l[4:])
             elif l.startswith(b'  '):  # results
                 # Queue up a list of expected results.
                 expected.setdefault(pos, []).append(l[2:])
             else:
                 if inpython:
                     script.append(b'EOF\n')
                     inpython = False
                 # Non-command/result. Queue up for merged output.
                 after.setdefault(pos, []).append(l)
 
         if inpython:
             script.append(b'EOF\n')
         if skipping is not None:
             after.setdefault(pos, []).append('  !!! missing #endif\n')
         addsalt(n + 1, False)
         # Need to end any current per-command trace
         if activetrace:
             toggletrace()
         return salt, script, after, expected
 
     def _processoutput(self, exitcode, output, salt, after, expected):
         # Merge the script output back into a unified test.
         warnonly = WARN_UNDEFINED  # 1: not yet; 2: yes; 3: for sure not
         if exitcode != 0:
             warnonly = WARN_NO
 
         pos = -1
         postout = []
         for out_rawline in output:
             out_line, cmd_line = out_rawline, None
             if salt in out_rawline:
                 out_line, cmd_line = out_rawline.split(salt, 1)
 
             pos, postout, warnonly = self._process_out_line(
                 out_line, pos, postout, expected, warnonly
             )
             pos, postout = self._process_cmd_line(cmd_line, pos, postout, after)
 
         if pos in after:
             postout += after.pop(pos)
 
         if warnonly == WARN_YES:
             exitcode = False  # Set exitcode to warned.
 
         return exitcode, postout
 
     def _process_out_line(self, out_line, pos, postout, expected, warnonly):
         while out_line:
             if not out_line.endswith(b'\n'):
                 out_line += b' (no-eol)\n'
 
             # Find the expected output at the current position.
             els = [None]
             if expected.get(pos, None):
                 els = expected[pos]
 
             optional = []
             for i, el in enumerate(els):
                 r = False
                 if el:
                     r, exact = self.linematch(el, out_line)
                 if isinstance(r, str):
                     if r == '-glob':
                         out_line = ''.join(el.rsplit(' (glob)', 1))
                         r = ''  # Warn only this line.
                     elif r == "retry":
                         postout.append(b'  ' + el)
                     else:
                         log('\ninfo, unknown linematch result: %r\n' % r)
                         r = False
                 if r:
                     els.pop(i)
                     break
                 if el:
                     if isoptional(el):
                         optional.append(i)
                     else:
                         m = optline.match(el)
                         if m:
                             conditions = [c for c in m.group(2).split(b' ')]
 
                             if not self._iftest(conditions):
                                 optional.append(i)
                     if exact:
                         # Don't allow line to be matches against a later
                         # line in the output
                         els.pop(i)
                         break
 
             if r:
                 if r == "retry":
                     continue
                 # clean up any optional leftovers
                 for i in optional:
                     postout.append(b'  ' + els[i])
                 for i in reversed(optional):
                     del els[i]
                 postout.append(b'  ' + el)
             else:
                 if self.NEEDESCAPE(out_line):
                     out_line = TTest._stringescape(
                         b'%s (esc)\n' % out_line.rstrip(b'\n')
                     )
                 postout.append(b'  ' + out_line)  # Let diff deal with it.
                 if r != '':  # If line failed.
                     warnonly = WARN_NO
                 elif warnonly == WARN_UNDEFINED:
                     warnonly = WARN_YES
             break
         else:
             # clean up any optional leftovers
             while expected.get(pos, None):
                 el = expected[pos].pop(0)
                 if el:
                     if not isoptional(el):
                         m = optline.match(el)
                         if m:
                             conditions = [c for c in m.group(2).split(b' ')]
 
                             if self._iftest(conditions):
                                 # Don't append as optional line
                                 continue
                         else:
                             continue
                 postout.append(b'  ' + el)
         return pos, postout, warnonly
 
     def _process_cmd_line(self, cmd_line, pos, postout, after):
         """process a "command" part of a line from unified test output"""
         if cmd_line:
             # Add on last return code.
             ret = int(cmd_line.split()[1])
             if ret != 0:
                 postout.append(b'  [%d]\n' % ret)
             if pos in after:
                 # Merge in non-active test bits.
                 postout += after.pop(pos)
             pos = int(cmd_line.split()[0])
         return pos, postout
 
     @staticmethod
     def rematch(el, l):
         try:
             el = b'(?:' + el + b')'
             # use \Z to ensure that the regex matches to the end of the string
             if os.name == 'nt':
                 return re.match(el + br'\r?\n\Z', l)
             return re.match(el + br'\n\Z', l)
         except re.error:
             # el is an invalid regex
             return False
 
     @staticmethod
     def globmatch(el, l):
         # The only supported special characters are * and ? plus / which also
         # matches \ on windows. Escaping of these characters is supported.
         if el + b'\n' == l:
             if os.altsep:
                 # matching on "/" is not needed for this line
                 for pat in checkcodeglobpats:
                     if pat.match(el):
                         return True
                 return b'-glob'
             return True
         el = el.replace(b'$LOCALIP', b'*')
         i, n = 0, len(el)
         res = b''
         while i < n:
             c = el[i : i + 1]
             i += 1
             if c == b'\\' and i < n and el[i : i + 1] in b'*?\\/':
                 res += el[i - 1 : i + 1]
                 i += 1
             elif c == b'*':
                 res += b'.*'
             elif c == b'?':
                 res += b'.'
             elif c == b'/' and os.altsep:
                 res += b'[/\\\\]'
             else:
                 res += re.escape(c)
         return TTest.rematch(res, l)
 
     def linematch(self, el, l):
         if el == l:  # perfect match (fast)
             return True, True
         retry = False
         if isoptional(el):
             retry = "retry"
             el = el[: -len(MARK_OPTIONAL)] + b"\n"
         else:
             m = optline.match(el)
             if m:
                 conditions = [c for c in m.group(2).split(b' ')]
 
                 el = m.group(1) + b"\n"
                 if not self._iftest(conditions):
                     # listed feature missing, should not match
                     return "retry", False
 
         if el.endswith(b" (esc)\n"):
             if PYTHON3:
                 el = el[:-7].decode('unicode_escape') + '\n'
                 el = el.encode('utf-8')
             else:
                 el = el[:-7].decode('string-escape') + '\n'
         if el == l or os.name == 'nt' and el[:-1] + b'\r\n' == l:
             return True, True
         if el.endswith(b" (re)\n"):
             return (TTest.rematch(el[:-6], l) or retry), False
         if el.endswith(b" (glob)\n"):
             # ignore '(glob)' added to l by 'replacements'
             if l.endswith(b" (glob)\n"):
                 l = l[:-8] + b"\n"
             return (TTest.globmatch(el[:-8], l) or retry), False
         if os.altsep:
             _l = l.replace(b'\\', b'/')
             if el == _l or os.name == 'nt' and el[:-1] + b'\r\n' == _l:
                 return True, True
         return retry, True
 
     @staticmethod
     def parsehghaveoutput(lines):
         '''Parse hghave log lines.
 
         Return tuple of lists (missing, failed):
           * the missing/unknown features
           * the features for which existence check failed'''
         missing = []
         failed = []
         for line in lines:
             if line.startswith(TTest.SKIPPED_PREFIX):
                 line = line.splitlines()[0]
                 missing.append(
                     line[len(TTest.SKIPPED_PREFIX) :].decode('utf-8')
                 )
             elif line.startswith(TTest.FAILED_PREFIX):
                 line = line.splitlines()[0]
                 failed.append(line[len(TTest.FAILED_PREFIX) :].decode('utf-8'))
 
         return missing, failed
 
     @staticmethod
     def _escapef(m):
         return TTest.ESCAPEMAP[m.group(0)]
 
     @staticmethod
     def _stringescape(s):
         return TTest.ESCAPESUB(TTest._escapef, s)
 
 
 iolock = threading.RLock()
 firstlock = threading.RLock()
 firsterror = False
 
 
 class TestResult(unittest._TextTestResult):
     """Holds results when executing via unittest."""
 
     # Don't worry too much about accessing the non-public _TextTestResult.
     # It is relatively common in Python testing tools.
     def __init__(self, options, *args, **kwargs):
         super(TestResult, self).__init__(*args, **kwargs)
 
         self._options = options
 
         # unittest.TestResult didn't have skipped until 2.7. We need to
         # polyfill it.
         self.skipped = []
 
         # We have a custom "ignored" result that isn't present in any Python
         # unittest implementation. It is very similar to skipped. It may make
         # sense to map it into skip some day.
         self.ignored = []
 
         self.times = []
         self._firststarttime = None
         # Data stored for the benefit of generating xunit reports.
         self.successes = []
         self.faildata = {}
 
         if options.color == 'auto':
             self.color = pygmentspresent and self.stream.isatty()
         elif options.color == 'never':
             self.color = False
         else:  # 'always', for testing purposes
             self.color = pygmentspresent
 
     def onStart(self, test):
         """ Can be overriden by custom TestResult
         """
 
     def onEnd(self):
         """ Can be overriden by custom TestResult
         """
 
     def addFailure(self, test, reason):
         self.failures.append((test, reason))
 
         if self._options.first:
             self.stop()
         else:
             with iolock:
                 if reason == "timed out":
                     self.stream.write('t')
                 else:
                     if not self._options.nodiff:
                         self.stream.write('\n')
                         # Exclude the '\n' from highlighting to lex correctly
                         formatted = 'ERROR: %s output changed\n' % test
                         self.stream.write(highlightmsg(formatted, self.color))
                     self.stream.write('!')
 
                 self.stream.flush()
 
     def addSuccess(self, test):
         with iolock:
             super(TestResult, self).addSuccess(test)
         self.successes.append(test)
 
     def addError(self, test, err):
         super(TestResult, self).addError(test, err)
         if self._options.first:
             self.stop()
 
     # Polyfill.
     def addSkip(self, test, reason):
         self.skipped.append((test, reason))
         with iolock:
             if self.showAll:
                 self.stream.writeln('skipped %s' % reason)
             else:
                 self.stream.write('s')
                 self.stream.flush()
 
     def addIgnore(self, test, reason):
         self.ignored.append((test, reason))
         with iolock:
             if self.showAll:
                 self.stream.writeln('ignored %s' % reason)
             else:
                 if reason not in ('not retesting', "doesn't match keyword"):
                     self.stream.write('i')
                 else:
                     self.testsRun += 1
                 self.stream.flush()
 
     def addOutputMismatch(self, test, ret, got, expected):
         """Record a mismatch in test output for a particular test."""
         if self.shouldStop or firsterror:
             # don't print, some other test case already failed and
             # printed, we're just stale and probably failed due to our
             # temp dir getting cleaned up.
             return
 
         accepted = False
         lines = []
 
         with iolock:
             if self._options.nodiff:
                 pass
             elif self._options.view:
                 v = self._options.view
                 subprocess.call(
                     r'"%s" "%s" "%s"'
                     % (v, _strpath(test.refpath), _strpath(test.errpath)),
                     shell=True,
                 )
             else:
                 servefail, lines = getdiff(
                     expected, got, test.refpath, test.errpath
                 )
                 self.stream.write('\n')
                 for line in lines:
                     line = highlightdiff(line, self.color)
                     if PYTHON3:
                         self.stream.flush()
                         self.stream.buffer.write(line)
                         self.stream.buffer.flush()
                     else:
                         self.stream.write(line)
                         self.stream.flush()
 
                 if servefail:
                     raise test.failureException(
                         'server failed to start (HGPORT=%s)' % test._startport
                     )
 
             # handle interactive prompt without releasing iolock
             if self._options.interactive:
                 if test.readrefout() != expected:
                     self.stream.write(
                         'Reference output has changed (run again to prompt '
                         'changes)'
                     )
                 else:
                     self.stream.write('Accept this change? [n] ')
                     self.stream.flush()
                     answer = sys.stdin.readline().strip()
                     if answer.lower() in ('y', 'yes'):
                         if test.path.endswith(b'.t'):
                             rename(test.errpath, test.path)
                         else:
                             rename(test.errpath, '%s.out' % test.path)
                         accepted = True
             if not accepted:
                 self.faildata[test.name] = b''.join(lines)
 
         return accepted
 
     def startTest(self, test):
         super(TestResult, self).startTest(test)
 
         # os.times module computes the user time and system time spent by
         # child's processes along with real elapsed time taken by a process.
         # This module has one limitation. It can only work for Linux user
         # and not for Windows.
         test.started = os.times()
         if self._firststarttime is None:  # thread racy but irrelevant
             self._firststarttime = test.started[4]
 
     def stopTest(self, test, interrupted=False):
         super(TestResult, self).stopTest(test)
 
         test.stopped = os.times()
 
         starttime = test.started
         endtime = test.stopped
         origin = self._firststarttime
         self.times.append(
             (
                 test.name,
                 endtime[2] - starttime[2],  # user space CPU time
                 endtime[3] - starttime[3],  # sys  space CPU time
                 endtime[4] - starttime[4],  # real time
                 starttime[4] - origin,  # start date in run context
                 endtime[4] - origin,  # end date in run context
             )
         )
 
         if interrupted:
             with iolock:
                 self.stream.writeln(
                     'INTERRUPTED: %s (after %d seconds)'
                     % (test.name, self.times[-1][3])
                 )
 
 
 def getTestResult():
     """
     Returns the relevant test result
     """
     if "CUSTOM_TEST_RESULT" in os.environ:
         testresultmodule = __import__(os.environ["CUSTOM_TEST_RESULT"])
         return testresultmodule.TestResult
     else:
         return TestResult
 
 
 class TestSuite(unittest.TestSuite):
     """Custom unittest TestSuite that knows how to execute Mercurial tests."""
 
     def __init__(
         self,
         testdir,
         jobs=1,
         whitelist=None,
         blacklist=None,
         retest=False,
         keywords=None,
         loop=False,
         runs_per_test=1,
         loadtest=None,
         showchannels=False,
         *args,
         **kwargs
     ):
         """Create a new instance that can run tests with a configuration.
 
         testdir specifies the directory where tests are executed from. This
         is typically the ``tests`` directory from Mercurial's source
         repository.
 
         jobs specifies the number of jobs to run concurrently. Each test
         executes on its own thread. Tests actually spawn new processes, so
         state mutation should not be an issue.
 
         If there is only one job, it will use the main thread.
 
         whitelist and blacklist denote tests that have been whitelisted and
         blacklisted, respectively. These arguments don't belong in TestSuite.
         Instead, whitelist and blacklist should be handled by the thing that
         populates the TestSuite with tests. They are present to preserve
         backwards compatible behavior which reports skipped tests as part
         of the results.
 
         retest denotes whether to retest failed tests. This arguably belongs
         outside of TestSuite.
 
         keywords denotes key words that will be used to filter which tests
         to execute. This arguably belongs outside of TestSuite.
 
         loop denotes whether to loop over tests forever.
         """
         super(TestSuite, self).__init__(*args, **kwargs)
 
         self._jobs = jobs
         self._whitelist = whitelist
         self._blacklist = blacklist
         self._retest = retest
         self._keywords = keywords
         self._loop = loop
         self._runs_per_test = runs_per_test
         self._loadtest = loadtest
         self._showchannels = showchannels
 
     def run(self, result):
         # We have a number of filters that need to be applied. We do this
         # here instead of inside Test because it makes the running logic for
         # Test simpler.
         tests = []
         num_tests = [0]
         for test in self._tests:
 
             def get():
                 num_tests[0] += 1
                 if getattr(test, 'should_reload', False):
                     return self._loadtest(test, num_tests[0])
                 return test
 
             if not os.path.exists(test.path):
                 result.addSkip(test, "Doesn't exist")
                 continue
 
             if not (self._whitelist and test.bname in self._whitelist):
                 if self._blacklist and test.bname in self._blacklist:
                     result.addSkip(test, 'blacklisted')
                     continue
 
                 if self._retest and not os.path.exists(test.errpath):
                     result.addIgnore(test, 'not retesting')
                     continue
 
                 if self._keywords:
                     with open(test.path, 'rb') as f:
                         t = f.read().lower() + test.bname.lower()
                     ignored = False
                     for k in self._keywords.lower().split():
                         if k not in t:
                             result.addIgnore(test, "doesn't match keyword")
                             ignored = True
                             break
 
                     if ignored:
                         continue
             for _ in xrange(self._runs_per_test):
                 tests.append(get())
 
         runtests = list(tests)
         done = queue.Queue()
         running = 0
 
         channels = [""] * self._jobs
 
         def job(test, result):
             for n, v in enumerate(channels):
                 if not v:
                     channel = n
                     break
             else:
                 raise ValueError('Could not find output channel')
             channels[channel] = "=" + test.name[5:].split(".")[0]
             try:
                 test(result)
                 done.put(None)
             except KeyboardInterrupt:
                 pass
             except:  # re-raises
                 done.put(('!', test, 'run-test raised an error, see traceback'))
                 raise
             finally:
                 try:
                     channels[channel] = ''
                 except IndexError:
                     pass
 
         def stat():
             count = 0
             while channels:
                 d = '\n%03s  ' % count
                 for n, v in enumerate(channels):
                     if v:
                         d += v[0]
                         channels[n] = v[1:] or '.'
                     else:
                         d += ' '
                     d += ' '
                 with iolock:
                     sys.stdout.write(d + '  ')
                     sys.stdout.flush()
                 for x in xrange(10):
                     if channels:
                         time.sleep(0.1)
                 count += 1
 
         stoppedearly = False
 
         if self._showchannels:
             statthread = threading.Thread(target=stat, name="stat")
             statthread.start()
 
         try:
             while tests or running:
                 if not done.empty() or running == self._jobs or not tests:
                     try:
                         done.get(True, 1)
                         running -= 1
                         if result and result.shouldStop:
                             stoppedearly = True
                             break
                     except queue.Empty:
                         continue
                 if tests and not running == self._jobs:
                     test = tests.pop(0)
                     if self._loop:
                         if getattr(test, 'should_reload', False):
                             num_tests[0] += 1
                             tests.append(self._loadtest(test, num_tests[0]))
                         else:
                             tests.append(test)
                     if self._jobs == 1:
                         job(test, result)
                     else:
                         t = threading.Thread(
                             target=job, name=test.name, args=(test, result)
                         )
                         t.start()
                     running += 1
 
             # If we stop early we still need to wait on started tests to
             # finish. Otherwise, there is a race between the test completing
             # and the test's cleanup code running. This could result in the
             # test reporting incorrect.
             if stoppedearly:
                 while running:
                     try:
                         done.get(True, 1)
                         running -= 1
                     except queue.Empty:
                         continue
         except KeyboardInterrupt:
             for test in runtests:
                 test.abort()
 
         channels = []
 
         return result
 
 
 # Save the most recent 5 wall-clock runtimes of each test to a
 # human-readable text file named .testtimes. Tests are sorted
 # alphabetically, while times for each test are listed from oldest to
 # newest.
 
 
 def loadtimes(outputdir):
     times = []
     try:
         with open(os.path.join(outputdir, b'.testtimes')) as fp:
             for line in fp:
                 m = re.match('(.*?) ([0-9. ]+)', line)
                 times.append(
                     (m.group(1), [float(t) for t in m.group(2).split()])
                 )
     except IOError as err:
         if err.errno != errno.ENOENT:
             raise
     return times
 
 
 def savetimes(outputdir, result):
     saved = dict(loadtimes(outputdir))
     maxruns = 5
     skipped = set([str(t[0]) for t in result.skipped])
     for tdata in result.times:
         test, real = tdata[0], tdata[3]
         if test not in skipped:
             ts = saved.setdefault(test, [])
             ts.append(real)
             ts[:] = ts[-maxruns:]
 
     fd, tmpname = tempfile.mkstemp(
         prefix=b'.testtimes', dir=outputdir, text=True
     )
     with os.fdopen(fd, 'w') as fp:
         for name, ts in sorted(saved.items()):
             fp.write('%s %s\n' % (name, ' '.join(['%.3f' % (t,) for t in ts])))
     timepath = os.path.join(outputdir, b'.testtimes')
     try:
         os.unlink(timepath)
     except OSError:
         pass
     try:
         os.rename(tmpname, timepath)
     except OSError:
         pass
 
 
 class TextTestRunner(unittest.TextTestRunner):
     """Custom unittest test runner that uses appropriate settings."""
 
     def __init__(self, runner, *args, **kwargs):
         super(TextTestRunner, self).__init__(*args, **kwargs)
 
         self._runner = runner
 
         self._result = getTestResult()(
             self._runner.options, self.stream, self.descriptions, self.verbosity
         )
 
     def listtests(self, test):
         test = sorted(test, key=lambda t: t.name)
 
         self._result.onStart(test)
 
         for t in test:
             print(t.name)
             self._result.addSuccess(t)
 
         if self._runner.options.xunit:
             with open(self._runner.options.xunit, "wb") as xuf:
                 self._writexunit(self._result, xuf)
 
         if self._runner.options.json:
             jsonpath = os.path.join(self._runner._outputdir, b'report.json')
             with open(jsonpath, 'w') as fp:
                 self._writejson(self._result, fp)
 
         return self._result
 
     def run(self, test):
         self._result.onStart(test)
         test(self._result)
 
         failed = len(self._result.failures)
         skipped = len(self._result.skipped)
         ignored = len(self._result.ignored)
 
         with iolock:
             self.stream.writeln('')
 
             if not self._runner.options.noskips:
                 for test, msg in sorted(
                     self._result.skipped, key=lambda s: s[0].name
                 ):
                     formatted = 'Skipped %s: %s\n' % (test.name, msg)
                     msg = highlightmsg(formatted, self._result.color)
                     self.stream.write(msg)
             for test, msg in sorted(
                 self._result.failures, key=lambda f: f[0].name
             ):
                 formatted = 'Failed %s: %s\n' % (test.name, msg)
                 self.stream.write(highlightmsg(formatted, self._result.color))
             for test, msg in sorted(
                 self._result.errors, key=lambda e: e[0].name
             ):
                 self.stream.writeln('Errored %s: %s' % (test.name, msg))
 
             if self._runner.options.xunit:
                 with open(self._runner.options.xunit, "wb") as xuf:
                     self._writexunit(self._result, xuf)
 
             if self._runner.options.json:
                 jsonpath = os.path.join(self._runner._outputdir, b'report.json')
                 with open(jsonpath, 'w') as fp:
                     self._writejson(self._result, fp)
 
             self._runner._checkhglib('Tested')
 
             savetimes(self._runner._outputdir, self._result)
 
             if failed and self._runner.options.known_good_rev:
                 self._bisecttests(t for t, m in self._result.failures)
             self.stream.writeln(
                 '# Ran %d tests, %d skipped, %d failed.'
                 % (self._result.testsRun, skipped + ignored, failed)
             )
             if failed:
                 self.stream.writeln(
                     'python hash seed: %s' % os.environ['PYTHONHASHSEED']
                 )
             if self._runner.options.time:
                 self.printtimes(self._result.times)
 
             if self._runner.options.exceptions:
                 exceptions = aggregateexceptions(
                     os.path.join(self._runner._outputdir, b'exceptions')
                 )
 
                 self.stream.writeln('Exceptions Report:')
                 self.stream.writeln(
                     '%d total from %d frames'
                     % (exceptions['total'], len(exceptions['exceptioncounts']))
                 )
                 combined = exceptions['combined']
                 for key in sorted(combined, key=combined.get, reverse=True):
                     frame, line, exc = key
                     totalcount, testcount, leastcount, leasttest = combined[key]
 
                     self.stream.writeln(
                         '%d (%d tests)\t%s: %s (%s - %d total)'
                         % (
                             totalcount,
                             testcount,
                             frame,
                             exc,
                             leasttest,
                             leastcount,
                         )
                     )
 
             self.stream.flush()
 
         return self._result
 
     def _bisecttests(self, tests):
         bisectcmd = ['hg', 'bisect']
         bisectrepo = self._runner.options.bisect_repo
         if bisectrepo:
             bisectcmd.extend(['-R', os.path.abspath(bisectrepo)])
 
         def pread(args):
             env = os.environ.copy()
             env['HGPLAIN'] = '1'
             p = subprocess.Popen(
                 args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env
             )
             data = p.stdout.read()
             p.wait()
             return data
 
         for test in tests:
             pread(bisectcmd + ['--reset']),
             pread(bisectcmd + ['--bad', '.'])
             pread(bisectcmd + ['--good', self._runner.options.known_good_rev])
             # TODO: we probably need to forward more options
             # that alter hg's behavior inside the tests.
             opts = ''
             withhg = self._runner.options.with_hg
             if withhg:
                 opts += ' --with-hg=%s ' % shellquote(_strpath(withhg))
             rtc = '%s %s %s %s' % (sysexecutable, sys.argv[0], opts, test)
             data = pread(bisectcmd + ['--command', rtc])
             m = re.search(
                 (
                     br'\nThe first (?P<goodbad>bad|good) revision '
                     br'is:\nchangeset: +\d+:(?P<node>[a-f0-9]+)\n.*\n'
                     br'summary: +(?P<summary>[^\n]+)\n'
                 ),
                 data,
                 (re.MULTILINE | re.DOTALL),
             )
             if m is None:
                 self.stream.writeln(
                     'Failed to identify failure point for %s' % test
                 )
                 continue
             dat = m.groupdict()
             verb = 'broken' if dat['goodbad'] == b'bad' else 'fixed'
             self.stream.writeln(
                 '%s %s by %s (%s)'
                 % (
                     test,
                     verb,
                     dat['node'].decode('ascii'),
                     dat['summary'].decode('utf8', 'ignore'),
                 )
             )
 
     def printtimes(self, times):
         # iolock held by run
         self.stream.writeln('# Producing time report')
         times.sort(key=lambda t: (t[3]))
         cols = '%7.3f %7.3f %7.3f %7.3f %7.3f   %s'
         self.stream.writeln(
             '%-7s %-7s %-7s %-7s %-7s   %s'
             % ('start', 'end', 'cuser', 'csys', 'real', 'Test')
         )
         for tdata in times:
             test = tdata[0]
             cuser, csys, real, start, end = tdata[1:6]
             self.stream.writeln(cols % (start, end, cuser, csys, real, test))
 
     @staticmethod
     def _writexunit(result, outf):
         # See http://llg.cubic.org/docs/junit/ for a reference.
         timesd = dict((t[0], t[3]) for t in result.times)
         doc = minidom.Document()
         s = doc.createElement('testsuite')
         s.setAttribute('errors', "0")  # TODO
         s.setAttribute('failures', str(len(result.failures)))
         s.setAttribute('name', 'run-tests')
         s.setAttribute(
             'skipped', str(len(result.skipped) + len(result.ignored))
         )
         s.setAttribute('tests', str(result.testsRun))
         doc.appendChild(s)
         for tc in result.successes:
             t = doc.createElement('testcase')
             t.setAttribute('name', tc.name)
             tctime = timesd.get(tc.name)
             if tctime is not None:
                 t.setAttribute('time', '%.3f' % tctime)
             s.appendChild(t)
         for tc, err in sorted(result.faildata.items()):
             t = doc.createElement('testcase')
             t.setAttribute('name', tc)
             tctime = timesd.get(tc)
             if tctime is not None:
                 t.setAttribute('time', '%.3f' % tctime)
             # createCDATASection expects a unicode or it will
             # convert using default conversion rules, which will
             # fail if string isn't ASCII.
             err = cdatasafe(err).decode('utf-8', 'replace')
             cd = doc.createCDATASection(err)
             # Use 'failure' here instead of 'error' to match errors = 0,
             # failures = len(result.failures) in the testsuite element.
             failelem = doc.createElement('failure')
             failelem.setAttribute('message', 'output changed')
             failelem.setAttribute('type', 'output-mismatch')
             failelem.appendChild(cd)
             t.appendChild(failelem)
             s.appendChild(t)
         for tc, message in result.skipped:
             # According to the schema, 'skipped' has no attributes. So store
             # the skip message as a text node instead.
             t = doc.createElement('testcase')
             t.setAttribute('name', tc.name)
             binmessage = message.encode('utf-8')
             message = cdatasafe(binmessage).decode('utf-8', 'replace')
             cd = doc.createCDATASection(message)
             skipelem = doc.createElement('skipped')
             skipelem.appendChild(cd)
             t.appendChild(skipelem)
             s.appendChild(t)
         outf.write(doc.toprettyxml(indent='  ', encoding='utf-8'))
 
     @staticmethod
     def _writejson(result, outf):
         timesd = {}
         for tdata in result.times:
             test = tdata[0]
             timesd[test] = tdata[1:]
 
         outcome = {}
         groups = [
             ('success', ((tc, None) for tc in result.successes)),
             ('failure', result.failures),
             ('skip', result.skipped),
         ]
         for res, testcases in groups:
             for tc, __ in testcases:
                 if tc.name in timesd:
                     diff = result.faildata.get(tc.name, b'')
                     try:
                         diff = diff.decode('unicode_escape')
                     except UnicodeDecodeError as e:
                         diff = '%r decoding diff, sorry' % e
                     tres = {
                         'result': res,
                         'time': ('%0.3f' % timesd[tc.name][2]),
                         'cuser': ('%0.3f' % timesd[tc.name][0]),
                         'csys': ('%0.3f' % timesd[tc.name][1]),
                         'start': ('%0.3f' % timesd[tc.name][3]),
                         'end': ('%0.3f' % timesd[tc.name][4]),
                         'diff': diff,
                     }
                 else:
                     # blacklisted test
                     tres = {'result': res}
 
                 outcome[tc.name] = tres
         jsonout = json.dumps(
             outcome, sort_keys=True, indent=4, separators=(',', ': ')
         )
         outf.writelines(("testreport =", jsonout))
 
 
 def sorttests(testdescs, previoustimes, shuffle=False):
     """Do an in-place sort of tests."""
     if shuffle:
         random.shuffle(testdescs)
         return
 
     if previoustimes:
 
         def sortkey(f):
             f = f['path']
             if f in previoustimes:
                 # Use most recent time as estimate
                 return -(previoustimes[f][-1])
             else:
                 # Default to a rather arbitrary value of 1 second for new tests
                 return -1.0
 
     else:
         # keywords for slow tests
         slow = {
             b'svn': 10,
             b'cvs': 10,
             b'hghave': 10,
             b'largefiles-update': 10,
             b'run-tests': 10,
             b'corruption': 10,
             b'race': 10,
             b'i18n': 10,
             b'check': 100,
             b'gendoc': 100,
             b'contrib-perf': 200,
             b'merge-combination': 100,
         }
         perf = {}
 
         def sortkey(f):
             # run largest tests first, as they tend to take the longest
             f = f['path']
             try:
                 return perf[f]
             except KeyError:
                 try:
                     val = -os.stat(f).st_size
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                         raise
                     perf[f] = -1e9  # file does not exist, tell early
                     return -1e9
                 for kw, mul in slow.items():
                     if kw in f:
                         val *= mul
                 if f.endswith(b'.py'):
                     val /= 10.0
                 perf[f] = val / 1000.0
                 return perf[f]
 
     testdescs.sort(key=sortkey)
 
 
 class TestRunner(object):
     """Holds context for executing tests.
 
     Tests rely on a lot of state. This object holds it for them.
     """
 
     # Programs required to run tests.
     REQUIREDTOOLS = [
         b'diff',
         b'grep',
         b'unzip',
         b'gunzip',
         b'bunzip2',
         b'sed',
     ]
 
     # Maps file extensions to test class.
     TESTTYPES = [
         (b'.py', PythonTest),
         (b'.t', TTest),
     ]
 
     def __init__(self):
         self.options = None
         self._hgroot = None
         self._testdir = None
         self._outputdir = None
         self._hgtmp = None
         self._installdir = None
         self._bindir = None
         self._tmpbinddir = None
         self._pythondir = None
         self._coveragefile = None
         self._createdfiles = []
         self._hgcommand = None
         self._hgpath = None
         self._portoffset = 0
         self._ports = {}
 
     def run(self, args, parser=None):
         """Run the test suite."""
         oldmask = os.umask(0o22)
         try:
             parser = parser or getparser()
             options = parseargs(args, parser)
             tests = [_bytespath(a) for a in options.tests]
             if options.test_list is not None:
                 for listfile in options.test_list:
                     with open(listfile, 'rb') as f:
                         tests.extend(t for t in f.read().splitlines() if t)
             self.options = options
 
             self._checktools()
             testdescs = self.findtests(tests)
             if options.profile_runner:
                 import statprof
 
                 statprof.start()
             result = self._run(testdescs)
             if options.profile_runner:
                 statprof.stop()
                 statprof.display()
             return result
 
         finally:
             os.umask(oldmask)
 
     def _run(self, testdescs):
         testdir = getcwdb()
         self._testdir = osenvironb[b'TESTDIR'] = getcwdb()
         # assume all tests in same folder for now
         if testdescs:
             pathname = os.path.dirname(testdescs[0]['path'])
             if pathname:
                 testdir = os.path.join(testdir, pathname)
         self._testdir = osenvironb[b'TESTDIR'] = testdir
         if self.options.outputdir:
             self._outputdir = canonpath(_bytespath(self.options.outputdir))
         else:
             self._outputdir = getcwdb()
             if testdescs and pathname:
                 self._outputdir = os.path.join(self._outputdir, pathname)
         previoustimes = {}
         if self.options.order_by_runtime:
             previoustimes = dict(loadtimes(self._outputdir))
         sorttests(testdescs, previoustimes, shuffle=self.options.random)
 
         if 'PYTHONHASHSEED' not in os.environ:
             # use a random python hash seed all the time
             # we do the randomness ourself to know what seed is used
             os.environ['PYTHONHASHSEED'] = str(random.getrandbits(32))
 
         if self.options.tmpdir:
             self.options.keep_tmpdir = True
             tmpdir = _bytespath(self.options.tmpdir)
             if os.path.exists(tmpdir):
                 # Meaning of tmpdir has changed since 1.3: we used to create
                 # HGTMP inside tmpdir; now HGTMP is tmpdir.  So fail if
                 # tmpdir already exists.
                 print("error: temp dir %r already exists" % tmpdir)
                 return 1
 
             os.makedirs(tmpdir)
         else:
             d = None
             if os.name == 'nt':
                 # without this, we get the default temp dir location, but
                 # in all lowercase, which causes troubles with paths (issue3490)
                 d = osenvironb.get(b'TMP', None)
             tmpdir = tempfile.mkdtemp(b'', b'hgtests.', d)
 
         self._hgtmp = osenvironb[b'HGTMP'] = os.path.realpath(tmpdir)
 
         if self.options.with_hg:
             self._installdir = None
             whg = self.options.with_hg
             self._bindir = os.path.dirname(os.path.realpath(whg))
             assert isinstance(self._bindir, bytes)
             self._hgcommand = os.path.basename(whg)
             self._tmpbindir = os.path.join(self._hgtmp, b'install', b'bin')
             os.makedirs(self._tmpbindir)
 
             normbin = os.path.normpath(os.path.abspath(whg))
             normbin = normbin.replace(os.sep.encode('ascii'), b'/')
 
             # Other Python scripts in the test harness need to
             # `import mercurial`. If `hg` is a Python script, we assume
             # the Mercurial modules are relative to its path and tell the tests
             # to load Python modules from its directory.
             with open(whg, 'rb') as fh:
                 initial = fh.read(1024)
 
             if re.match(b'#!.*python', initial):
                 self._pythondir = self._bindir
             # If it looks like our in-repo Rust binary, use the source root.
             # This is a bit hacky. But rhg is still not supported outside the
             # source directory. So until it is, do the simple thing.
             elif re.search(b'/rust/target/[^/]+/hg', normbin):
                 self._pythondir = os.path.dirname(self._testdir)
             # Fall back to the legacy behavior.
             else:
                 self._pythondir = self._bindir
 
         else:
             self._installdir = os.path.join(self._hgtmp, b"install")
             self._bindir = os.path.join(self._installdir, b"bin")
             self._hgcommand = b'hg'
             self._tmpbindir = self._bindir
             self._pythondir = os.path.join(self._installdir, b"lib", b"python")
 
         # Force the use of hg.exe instead of relying on MSYS to recognize hg is
         # a python script and feed it to python.exe.  Legacy stdio is force
         # enabled by hg.exe, and this is a more realistic way to launch hg
         # anyway.
         if os.name == 'nt' and not self._hgcommand.endswith(b'.exe'):
             self._hgcommand += b'.exe'
 
         # set CHGHG, then replace "hg" command by "chg"
         chgbindir = self._bindir
         if self.options.chg or self.options.with_chg:
             osenvironb[b'CHGHG'] = os.path.join(self._bindir, self._hgcommand)
         else:
             osenvironb.pop(b'CHGHG', None)  # drop flag for hghave
         if self.options.chg:
             self._hgcommand = b'chg'
         elif self.options.with_chg:
             chgbindir = os.path.dirname(os.path.realpath(self.options.with_chg))
             self._hgcommand = os.path.basename(self.options.with_chg)
 
         osenvironb[b"BINDIR"] = self._bindir
         osenvironb[b"PYTHON"] = PYTHON
 
         fileb = _bytespath(__file__)
         runtestdir = os.path.abspath(os.path.dirname(fileb))
         osenvironb[b'RUNTESTDIR'] = runtestdir
         if PYTHON3:
             sepb = _bytespath(os.pathsep)
         else:
             sepb = os.pathsep
         path = [self._bindir, runtestdir] + osenvironb[b"PATH"].split(sepb)
         if os.path.islink(__file__):
             # test helper will likely be at the end of the symlink
             realfile = os.path.realpath(fileb)
             realdir = os.path.abspath(os.path.dirname(realfile))
             path.insert(2, realdir)
         if chgbindir != self._bindir:
             path.insert(1, chgbindir)
         if self._testdir != runtestdir:
             path = [self._testdir] + path
         if self._tmpbindir != self._bindir:
             path = [self._tmpbindir] + path
         osenvironb[b"PATH"] = sepb.join(path)
 
         # Include TESTDIR in PYTHONPATH so that out-of-tree extensions
         # can run .../tests/run-tests.py test-foo where test-foo
         # adds an extension to HGRC. Also include run-test.py directory to
         # import modules like heredoctest.
         pypath = [self._pythondir, self._testdir, runtestdir]
         # We have to augment PYTHONPATH, rather than simply replacing
         # it, in case external libraries are only available via current
         # PYTHONPATH.  (In particular, the Subversion bindings on OS X
         # are in /opt/subversion.)
         oldpypath = osenvironb.get(IMPL_PATH)
         if oldpypath:
             pypath.append(oldpypath)
         osenvironb[IMPL_PATH] = sepb.join(pypath)
 
         if self.options.pure:
             os.environ["HGTEST_RUN_TESTS_PURE"] = "--pure"
             os.environ["HGMODULEPOLICY"] = "py"
 
         if self.options.allow_slow_tests:
             os.environ["HGTEST_SLOW"] = "slow"
         elif 'HGTEST_SLOW' in os.environ:
             del os.environ['HGTEST_SLOW']
 
         self._coveragefile = os.path.join(self._testdir, b'.coverage')
 
         if self.options.exceptions:
             exceptionsdir = os.path.join(self._outputdir, b'exceptions')
             try:
                 os.makedirs(exceptionsdir)
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
             # Remove all existing exception reports.
             for f in os.listdir(exceptionsdir):
                 os.unlink(os.path.join(exceptionsdir, f))
 
             osenvironb[b'HGEXCEPTIONSDIR'] = exceptionsdir
             logexceptions = os.path.join(self._testdir, b'logexceptions.py')
             self.options.extra_config_opt.append(
                 'extensions.logexceptions=%s' % logexceptions.decode('utf-8')
             )
 
         vlog("# Using TESTDIR", _strpath(self._testdir))
         vlog("# Using RUNTESTDIR", _strpath(osenvironb[b'RUNTESTDIR']))
         vlog("# Using HGTMP", _strpath(self._hgtmp))
         vlog("# Using PATH", os.environ["PATH"])
         vlog(
-            "# Using",
-            _strpath(IMPL_PATH),
-            _strpath(osenvironb[IMPL_PATH]),
+            "# Using", _strpath(IMPL_PATH), _strpath(osenvironb[IMPL_PATH]),
         )
         vlog("# Writing to directory", _strpath(self._outputdir))
 
         try:
             return self._runtests(testdescs) or 0
         finally:
             time.sleep(0.1)
             self._cleanup()
 
     def findtests(self, args):
         """Finds possible test files from arguments.
 
         If you wish to inject custom tests into the test harness, this would
         be a good function to monkeypatch or override in a derived class.
         """
         if not args:
             if self.options.changed:
                 proc = Popen4(
                     b'hg st --rev "%s" -man0 .'
                     % _bytespath(self.options.changed),
                     None,
                     0,
                 )
                 stdout, stderr = proc.communicate()
                 args = stdout.strip(b'\0').split(b'\0')
             else:
                 args = os.listdir(b'.')
 
         expanded_args = []
         for arg in args:
             if os.path.isdir(arg):
                 if not arg.endswith(b'/'):
                     arg += b'/'
                 expanded_args.extend([arg + a for a in os.listdir(arg)])
             else:
                 expanded_args.append(arg)
         args = expanded_args
 
         testcasepattern = re.compile(
             br'([\w-]+\.t|py)(?:#([a-zA-Z0-9_\-\.#]+))'
         )
         tests = []
         for t in args:
             case = []
 
             if not (
                 os.path.basename(t).startswith(b'test-')
                 and (t.endswith(b'.py') or t.endswith(b'.t'))
             ):
 
                 m = testcasepattern.match(os.path.basename(t))
                 if m is not None:
                     t_basename, casestr = m.groups()
                     t = os.path.join(os.path.dirname(t), t_basename)
                     if casestr:
                         case = casestr.split(b'#')
                 else:
                     continue
 
             if t.endswith(b'.t'):
                 # .t file may contain multiple test cases
                 casedimensions = parsettestcases(t)
                 if casedimensions:
                     cases = []
 
                     def addcases(case, casedimensions):
                         if not casedimensions:
                             cases.append(case)
                         else:
                             for c in casedimensions[0]:
                                 addcases(case + [c], casedimensions[1:])
 
                     addcases([], casedimensions)
                     if case and case in cases:
                         cases = [case]
                     elif case:
                         # Ignore invalid cases
                         cases = []
                     else:
                         pass
                     tests += [{'path': t, 'case': c} for c in sorted(cases)]
                 else:
                     tests.append({'path': t})
             else:
                 tests.append({'path': t})
         return tests
 
     def _runtests(self, testdescs):
         def _reloadtest(test, i):
             # convert a test back to its description dict
             desc = {'path': test.path}
             case = getattr(test, '_case', [])
             if case:
                 desc['case'] = case
             return self._gettest(desc, i)
 
         try:
             if self.options.restart:
                 orig = list(testdescs)
                 while testdescs:
                     desc = testdescs[0]
                     # desc['path'] is a relative path
                     if 'case' in desc:
                         casestr = b'#'.join(desc['case'])
                         errpath = b'%s#%s.err' % (desc['path'], casestr)
                     else:
                         errpath = b'%s.err' % desc['path']
                     errpath = os.path.join(self._outputdir, errpath)
                     if os.path.exists(errpath):
                         break
                     testdescs.pop(0)
                 if not testdescs:
                     print("running all tests")
                     testdescs = orig
 
             tests = [self._gettest(d, i) for i, d in enumerate(testdescs)]
             num_tests = len(tests) * self.options.runs_per_test
 
             jobs = min(num_tests, self.options.jobs)
 
             failed = False
             kws = self.options.keywords
             if kws is not None and PYTHON3:
                 kws = kws.encode('utf-8')
 
             suite = TestSuite(
                 self._testdir,
                 jobs=jobs,
                 whitelist=self.options.whitelisted,
                 blacklist=self.options.blacklist,
                 retest=self.options.retest,
                 keywords=kws,
                 loop=self.options.loop,
                 runs_per_test=self.options.runs_per_test,
                 showchannels=self.options.showchannels,
                 tests=tests,
                 loadtest=_reloadtest,
             )
             verbosity = 1
             if self.options.list_tests:
                 verbosity = 0
             elif self.options.verbose:
                 verbosity = 2
             runner = TextTestRunner(self, verbosity=verbosity)
 
             if self.options.list_tests:
                 result = runner.listtests(suite)
             else:
                 if self._installdir:
                     self._installhg()
                     self._checkhglib("Testing")
                 else:
                     self._usecorrectpython()
                 if self.options.chg:
                     assert self._installdir
                     self._installchg()
 
                 log(
                     'running %d tests using %d parallel processes'
                     % (num_tests, jobs)
                 )
 
                 result = runner.run(suite)
 
             if result.failures or result.errors:
                 failed = True
 
             result.onEnd()
 
             if self.options.anycoverage:
                 self._outputcoverage()
         except KeyboardInterrupt:
             failed = True
             print("\ninterrupted!")
 
         if failed:
             return 1
 
     def _getport(self, count):
         port = self._ports.get(count)  # do we have a cached entry?
         if port is None:
             portneeded = 3
             # above 100 tries we just give up and let test reports failure
             for tries in xrange(100):
                 allfree = True
                 port = self.options.port + self._portoffset
                 for idx in xrange(portneeded):
                     if not checkportisavailable(port + idx):
                         allfree = False
                         break
                 self._portoffset += portneeded
                 if allfree:
                     break
             self._ports[count] = port
         return port
 
     def _gettest(self, testdesc, count):
         """Obtain a Test by looking at its filename.
 
         Returns a Test instance. The Test may not be runnable if it doesn't
         map to a known type.
         """
         path = testdesc['path']
         lctest = path.lower()
         testcls = Test
 
         for ext, cls in self.TESTTYPES:
             if lctest.endswith(ext):
                 testcls = cls
                 break
 
         refpath = os.path.join(getcwdb(), path)
         tmpdir = os.path.join(self._hgtmp, b'child%d' % count)
 
         # extra keyword parameters. 'case' is used by .t tests
         kwds = dict((k, testdesc[k]) for k in ['case'] if k in testdesc)
 
         t = testcls(
             refpath,
             self._outputdir,
             tmpdir,
             keeptmpdir=self.options.keep_tmpdir,
             debug=self.options.debug,
             first=self.options.first,
             timeout=self.options.timeout,
             startport=self._getport(count),
             extraconfigopts=self.options.extra_config_opt,
             py3warnings=self.options.py3_warnings,
             shell=self.options.shell,
             hgcommand=self._hgcommand,
             usechg=bool(self.options.with_chg or self.options.chg),
             useipv6=useipv6,
             **kwds
         )
         t.should_reload = True
         return t
 
     def _cleanup(self):
         """Clean up state from this test invocation."""
         if self.options.keep_tmpdir:
             return
 
         vlog("# Cleaning up HGTMP", _strpath(self._hgtmp))
         shutil.rmtree(self._hgtmp, True)
         for f in self._createdfiles:
             try:
                 os.remove(f)
             except OSError:
                 pass
 
     def _usecorrectpython(self):
         """Configure the environment to use the appropriate Python in tests."""
         # Tests must use the same interpreter as us or bad things will happen.
         pyexename = sys.platform == 'win32' and b'python.exe' or b'python'
 
         # os.symlink() is a thing with py3 on Windows, but it requires
         # Administrator rights.
         if getattr(os, 'symlink', None) and os.name != 'nt':
             vlog(
                 "# Making python executable in test path a symlink to '%s'"
                 % sysexecutable
             )
             mypython = os.path.join(self._tmpbindir, pyexename)
             try:
                 if os.readlink(mypython) == sysexecutable:
                     return
                 os.unlink(mypython)
             except OSError as err:
                 if err.errno != errno.ENOENT:
                     raise
             if self._findprogram(pyexename) != sysexecutable:
                 try:
                     os.symlink(sysexecutable, mypython)
                     self._createdfiles.append(mypython)
                 except OSError as err:
                     # child processes may race, which is harmless
                     if err.errno != errno.EEXIST:
                         raise
         else:
             exedir, exename = os.path.split(sysexecutable)
             vlog(
                 "# Modifying search path to find %s as %s in '%s'"
                 % (exename, pyexename, exedir)
             )
             path = os.environ['PATH'].split(os.pathsep)
             while exedir in path:
                 path.remove(exedir)
             os.environ['PATH'] = os.pathsep.join([exedir] + path)
             if not self._findprogram(pyexename):
                 print("WARNING: Cannot find %s in search path" % pyexename)
 
     def _installhg(self):
         """Install hg into the test environment.
 
         This will also configure hg with the appropriate testing settings.
         """
         vlog("# Performing temporary installation of HG")
         installerrs = os.path.join(self._hgtmp, b"install.err")
         compiler = ''
         if self.options.compiler:
             compiler = '--compiler ' + self.options.compiler
         if self.options.pure:
             pure = b"--pure"
         else:
             pure = b""
 
         # Run installer in hg root
         script = os.path.realpath(sys.argv[0])
         exe = sysexecutable
         if PYTHON3:
             compiler = _bytespath(compiler)
             script = _bytespath(script)
             exe = _bytespath(exe)
         hgroot = os.path.dirname(os.path.dirname(script))
         self._hgroot = hgroot
         os.chdir(hgroot)
         nohome = b'--home=""'
         if os.name == 'nt':
             # The --home="" trick works only on OS where os.sep == '/'
             # because of a distutils convert_path() fast-path. Avoid it at
             # least on Windows for now, deal with .pydistutils.cfg bugs
             # when they happen.
             nohome = b''
         cmd = (
             b'"%(exe)s" setup.py %(pure)s clean --all'
             b' build %(compiler)s --build-base="%(base)s"'
             b' install --force --prefix="%(prefix)s"'
             b' --install-lib="%(libdir)s"'
             b' --install-scripts="%(bindir)s" %(nohome)s >%(logfile)s 2>&1'
             % {
                 b'exe': exe,
                 b'pure': pure,
                 b'compiler': compiler,
                 b'base': os.path.join(self._hgtmp, b"build"),
                 b'prefix': self._installdir,
                 b'libdir': self._pythondir,
                 b'bindir': self._bindir,
                 b'nohome': nohome,
                 b'logfile': installerrs,
             }
         )
 
         # setuptools requires install directories to exist.
         def makedirs(p):
             try:
                 os.makedirs(p)
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
         makedirs(self._pythondir)
         makedirs(self._bindir)
 
         vlog("# Running", cmd.decode("utf-8"))
         if subprocess.call(_strpath(cmd), shell=True) == 0:
             if not self.options.verbose:
                 try:
                     os.remove(installerrs)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                         raise
         else:
             with open(installerrs, 'rb') as f:
                 for line in f:
                     if PYTHON3:
                         sys.stdout.buffer.write(line)
                     else:
                         sys.stdout.write(line)
             sys.exit(1)
         os.chdir(self._testdir)
 
         self._usecorrectpython()
 
         if self.options.py3_warnings and not self.options.anycoverage:
             vlog("# Updating hg command to enable Py3k Warnings switch")
             with open(os.path.join(self._bindir, 'hg'), 'rb') as f:
                 lines = [line.rstrip() for line in f]
                 lines[0] += ' -3'
             with open(os.path.join(self._bindir, 'hg'), 'wb') as f:
                 for line in lines:
                     f.write(line + '\n')
 
         hgbat = os.path.join(self._bindir, b'hg.bat')
         if os.path.isfile(hgbat):
             # hg.bat expects to be put in bin/scripts while run-tests.py
             # installation layout put it in bin/ directly. Fix it
             with open(hgbat, 'rb') as f:
                 data = f.read()
             if br'"%~dp0..\python" "%~dp0hg" %*' in data:
                 data = data.replace(
                     br'"%~dp0..\python" "%~dp0hg" %*',
                     b'"%~dp0python" "%~dp0hg" %*',
                 )
                 with open(hgbat, 'wb') as f:
                     f.write(data)
             else:
                 print('WARNING: cannot fix hg.bat reference to python.exe')
 
         if self.options.anycoverage:
             custom = os.path.join(
                 osenvironb[b'RUNTESTDIR'], b'sitecustomize.py'
             )
             target = os.path.join(self._pythondir, b'sitecustomize.py')
             vlog('# Installing coverage trigger to %s' % target)
             shutil.copyfile(custom, target)
             rc = os.path.join(self._testdir, b'.coveragerc')
             vlog('# Installing coverage rc to %s' % rc)
             osenvironb[b'COVERAGE_PROCESS_START'] = rc
             covdir = os.path.join(self._installdir, b'..', b'coverage')
             try:
                 os.mkdir(covdir)
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
             osenvironb[b'COVERAGE_DIR'] = covdir
 
     def _checkhglib(self, verb):
         """Ensure that the 'mercurial' package imported by python is
         the one we expect it to be.  If not, print a warning to stderr."""
         if (self._bindir == self._pythondir) and (
             self._bindir != self._tmpbindir
         ):
             # The pythondir has been inferred from --with-hg flag.
             # We cannot expect anything sensible here.
             return
         expecthg = os.path.join(self._pythondir, b'mercurial')
         actualhg = self._gethgpath()
         if os.path.abspath(actualhg) != os.path.abspath(expecthg):
             sys.stderr.write(
                 'warning: %s with unexpected mercurial lib: %s\n'
                 '         (expected %s)\n' % (verb, actualhg, expecthg)
             )
 
     def _gethgpath(self):
         """Return the path to the mercurial package that is actually found by
         the current Python interpreter."""
         if self._hgpath is not None:
             return self._hgpath
 
         cmd = b'"%s" -c "import mercurial; print (mercurial.__path__[0])"'
         cmd = cmd % PYTHON
         if PYTHON3:
             cmd = _strpath(cmd)
 
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
         out, err = p.communicate()
 
         self._hgpath = out.strip()
 
         return self._hgpath
 
     def _installchg(self):
         """Install chg into the test environment"""
         vlog('# Performing temporary installation of CHG')
         assert os.path.dirname(self._bindir) == self._installdir
         assert self._hgroot, 'must be called after _installhg()'
         cmd = b'"%(make)s" clean install PREFIX="%(prefix)s"' % {
             b'make': b'make',  # TODO: switch by option or environment?
             b'prefix': self._installdir,
         }
         cwd = os.path.join(self._hgroot, b'contrib', b'chg')
         vlog("# Running", cmd)
         proc = subprocess.Popen(
             cmd,
             shell=True,
             cwd=cwd,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
         )
         out, _err = proc.communicate()
         if proc.returncode != 0:
             if PYTHON3:
                 sys.stdout.buffer.write(out)
             else:
                 sys.stdout.write(out)
             sys.exit(1)
 
     def _outputcoverage(self):
         """Produce code coverage output."""
         import coverage
 
         coverage = coverage.coverage
 
         vlog('# Producing coverage report')
         # chdir is the easiest way to get short, relative paths in the
         # output.
         os.chdir(self._hgroot)
         covdir = os.path.join(_strpath(self._installdir), '..', 'coverage')
         cov = coverage(data_file=os.path.join(covdir, 'cov'))
 
         # Map install directory paths back to source directory.
         cov.config.paths['srcdir'] = ['.', _strpath(self._pythondir)]
 
         cov.combine()
 
         omit = [
             _strpath(os.path.join(x, b'*'))
             for x in [self._bindir, self._testdir]
         ]
         cov.report(ignore_errors=True, omit=omit)
 
         if self.options.htmlcov:
             htmldir = os.path.join(_strpath(self._outputdir), 'htmlcov')
             cov.html_report(directory=htmldir, omit=omit)
         if self.options.annotate:
             adir = os.path.join(_strpath(self._outputdir), 'annotated')
             if not os.path.isdir(adir):
                 os.mkdir(adir)
             cov.annotate(directory=adir, omit=omit)
 
     def _findprogram(self, program):
         """Search PATH for a executable program"""
         dpb = _bytespath(os.defpath)
         sepb = _bytespath(os.pathsep)
         for p in osenvironb.get(b'PATH', dpb).split(sepb):
             name = os.path.join(p, program)
             if os.name == 'nt' or os.access(name, os.X_OK):
                 return name
         return None
 
     def _checktools(self):
         """Ensure tools required to run tests are present."""
         for p in self.REQUIREDTOOLS:
             if os.name == 'nt' and not p.endswith(b'.exe'):
                 p += b'.exe'
             found = self._findprogram(p)
             p = p.decode("utf-8")
             if found:
                 vlog("# Found prerequisite", p, "at", _strpath(found))
             else:
                 print("WARNING: Did not find prerequisite tool: %s " % p)
 
 
 def aggregateexceptions(path):
     exceptioncounts = collections.Counter()
     testsbyfailure = collections.defaultdict(set)
     failuresbytest = collections.defaultdict(set)
 
     for f in os.listdir(path):
         with open(os.path.join(path, f), 'rb') as fh:
             data = fh.read().split(b'\0')
             if len(data) != 5:
                 continue
 
             exc, mainframe, hgframe, hgline, testname = data
             exc = exc.decode('utf-8')
             mainframe = mainframe.decode('utf-8')
             hgframe = hgframe.decode('utf-8')
             hgline = hgline.decode('utf-8')
             testname = testname.decode('utf-8')
 
             key = (hgframe, hgline, exc)
             exceptioncounts[key] += 1
             testsbyfailure[key].add(testname)
             failuresbytest[testname].add(key)
 
     # Find test having fewest failures for each failure.
     leastfailing = {}
     for key, tests in testsbyfailure.items():
         fewesttest = None
         fewestcount = 99999999
         for test in sorted(tests):
             if len(failuresbytest[test]) < fewestcount:
                 fewesttest = test
                 fewestcount = len(failuresbytest[test])
 
         leastfailing[key] = (fewestcount, fewesttest)
 
     # Create a combined counter so we can sort by total occurrences and
     # impacted tests.
     combined = {}
     for key in exceptioncounts:
         combined[key] = (
             exceptioncounts[key],
             len(testsbyfailure[key]),
             leastfailing[key][0],
             leastfailing[key][1],
         )
 
     return {
         'exceptioncounts': exceptioncounts,
         'total': sum(exceptioncounts.values()),
         'combined': combined,
         'leastfailing': leastfailing,
         'byfailure': testsbyfailure,
         'bytest': failuresbytest,
     }
 
 
 if __name__ == '__main__':
     runner = TestRunner()
 
     try:
         import msvcrt
 
         msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
         msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
         msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
     except ImportError:
         pass
 
     sys.exit(runner.run(sys.argv[1:]))