branching: merge stable into default

mercurial/bundlecaches.py

@@ -1,559 +1,562 b''
 # bundlecaches.py - utility to deal with pre-computed bundle for servers
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import collections
 
 from typing import (
     Dict,
     Union,
     cast,
 )
 
 from .i18n import _
 
 from .thirdparty import attr
 
 from . import (
     error,
     requirements as requirementsmod,
     sslutil,
     util,
 )
 from .utils import stringutil
 
 urlreq = util.urlreq
 
 BUNDLE_CACHE_DIR = b'bundle-cache'
 CB_MANIFEST_FILE = b'clonebundles.manifest'
 CLONEBUNDLESCHEME = b"peer-bundle-cache://"
 
 
 def get_manifest(repo):
     """get the bundle manifest to be served to a client from a server"""
     raw_text = repo.vfs.tryread(CB_MANIFEST_FILE)
     entries = [e.split(b' ', 1) for e in raw_text.splitlines()]
 
     new_lines = []
     for e in entries:
         url = alter_bundle_url(repo, e[0])
         if len(e) == 1:
             line = url + b'\n'
         else:
             line = b"%s %s\n" % (url, e[1])
         new_lines.append(line)
     return b''.join(new_lines)
 
 
 def alter_bundle_url(repo, url):
     """a function that exist to help extension and hosting to alter the url
 
     This will typically be used to inject authentication information in the url
     of cached bundles."""
     return url
 
 
 SUPPORTED_CLONEBUNDLE_SCHEMES = [
     b"http://",
     b"https://",
     b"largefile://",
     CLONEBUNDLESCHEME,
 ]
 
 
 @attr.s
 class bundlespec:
     compression = attr.ib()
     wirecompression = attr.ib()
     version = attr.ib()
     wireversion = attr.ib()
     # parameters explicitly overwritten by the config or the specification
     _explicit_params = attr.ib()
     # default parameter for the version
     #
     # Keeping it separated is useful to check what was actually overwritten.
     _default_opts = attr.ib()
 
     @property
     def params(self):
         return collections.ChainMap(self._explicit_params, self._default_opts)
 
     @property
     def contentopts(self):
         # kept for Backward Compatibility concerns.
         return self.params
 
     def set_param(self, key, value, overwrite=True):
         """Set a bundle parameter value.
 
         Will only overwrite if overwrite is true"""
         if overwrite or key not in self._explicit_params:
             self._explicit_params[key] = value
 
     def as_spec(self):
         parts = [b"%s-%s" % (self.compression, self.version)]
         for param in sorted(self._explicit_params.items()):
             parts.append(b'%s=%s' % param)
         return b';'.join(parts)
 
 
 # Maps bundle version human names to changegroup versions.
 _bundlespeccgversions = {
     b'v1': b'01',
     b'v2': b'02',
     b'v3': b'03',
     b'packed1': b's1',
     b'bundle2': b'02',  # legacy
 }
 
 # Maps bundle version with content opts to choose which part to bundle
 _bundlespeccontentopts: Dict[bytes, Dict[bytes, Union[bool, bytes]]] = {
     b'v1': {
         b'changegroup': True,
         b'cg.version': b'01',
         b'obsolescence': False,
         b'phases': False,
         b'tagsfnodescache': False,
         b'revbranchcache': False,
     },
     b'v2': {
         b'changegroup': True,
         b'cg.version': b'02',
         b'obsolescence': False,
         b'phases': False,
         b'tagsfnodescache': True,
         b'revbranchcache': True,
     },
     b'v3': {
         b'changegroup': True,
         b'cg.version': b'03',
         b'obsolescence': False,
         b'phases': True,
         b'tagsfnodescache': True,
         b'revbranchcache': True,
     },
     b'streamv2': {
         b'changegroup': False,
         b'cg.version': b'02',
         b'obsolescence': False,
         b'phases': False,
         b"stream": b"v2",
         b'tagsfnodescache': False,
         b'revbranchcache': False,
     },
     b'streamv3-exp': {
         b'changegroup': False,
         b'cg.version': b'03',
         b'obsolescence': False,
         b'phases': False,
         b"stream": b"v3-exp",
         b'tagsfnodescache': False,
         b'revbranchcache': False,
     },
     b'packed1': {
         b'cg.version': b's1',
     },
     b'bundle2': {  # legacy
         b'cg.version': b'02',
     },
 }
 _bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2']
 
 # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
 _bundlespecv1compengines = {b'gzip', b'bzip2', b'none'}
 
 
 def param_bool(key, value):
     """make a boolean out of a parameter value"""
     b = stringutil.parsebool(value)
     if b is None:
         msg = _(b"parameter %s should be a boolean ('%s')")
         msg %= (key, value)
         raise error.InvalidBundleSpecification(msg)
     return b
 
 
 # mapping of known parameter name need their value processed
 bundle_spec_param_processing = {
     b"obsolescence": param_bool,
     b"obsolescence-mandatory": param_bool,
     b"phases": param_bool,
+    b"changegroup": param_bool,
+    b"tagsfnodescache": param_bool,
+    b"revbranchcache": param_bool,
 }
 
 
 def _parseparams(s):
     """parse bundlespec parameter section
 
     input: "comp-version;params" string
 
     return: (spec; {param_key: param_value})
     """
     if b';' not in s:
         return s, {}
 
     params = {}
     version, paramstr = s.split(b';', 1)
 
     err = _(b'invalid bundle specification: missing "=" in parameter: %s')
     for p in paramstr.split(b';'):
         if b'=' not in p:
             msg = err % p
             raise error.InvalidBundleSpecification(msg)
 
         key, value = p.split(b'=', 1)
         key = urlreq.unquote(key)
         value = urlreq.unquote(value)
         process = bundle_spec_param_processing.get(key)
         if process is not None:
             value = process(key, value)
         params[key] = value
 
     return version, params
 
 
 def parsebundlespec(repo, spec, strict=True):
     """Parse a bundle string specification into parts.
 
     Bundle specifications denote a well-defined bundle/exchange format.
     The content of a given specification should not change over time in
     order to ensure that bundles produced by a newer version of Mercurial are
     readable from an older version.
 
     The string currently has the form:
 
        <compression>-<type>[;<parameter0>[;<parameter1>]]
 
     Where <compression> is one of the supported compression formats
     and <type> is (currently) a version string. A ";" can follow the type and
     all text afterwards is interpreted as URI encoded, ";" delimited key=value
     pairs.
 
     If ``strict`` is True (the default) <compression> is required. Otherwise,
     it is optional.
 
     Returns a bundlespec object of (compression, version, parameters).
     Compression will be ``None`` if not in strict mode and a compression isn't
     defined.
 
     An ``InvalidBundleSpecification`` is raised when the specification is
     not syntactically well formed.
 
     An ``UnsupportedBundleSpecification`` is raised when the compression or
     bundle type/version is not recognized.
 
     Note: this function will likely eventually return a more complex data
     structure, including bundle2 part information.
     """
     if strict and b'-' not in spec:
         raise error.InvalidBundleSpecification(
             _(
                 b'invalid bundle specification; '
                 b'must be prefixed with compression: %s'
             )
             % spec
         )
 
     pre_args = spec.split(b';', 1)[0]
     if b'-' in pre_args:
         compression, version = spec.split(b'-', 1)
 
         if compression not in util.compengines.supportedbundlenames:
             raise error.UnsupportedBundleSpecification(
                 _(b'%s compression is not supported') % compression
             )
 
         version, params = _parseparams(version)
 
         if version not in _bundlespeccontentopts:
             raise error.UnsupportedBundleSpecification(
                 _(b'%s is not a recognized bundle version') % version
             )
     else:
         # Value could be just the compression or just the version, in which
         # case some defaults are assumed (but only when not in strict mode).
         assert not strict
 
         spec, params = _parseparams(spec)
 
         if spec in util.compengines.supportedbundlenames:
             compression = spec
             version = b'v1'
             # Generaldelta repos require v2.
             if requirementsmod.GENERALDELTA_REQUIREMENT in repo.requirements:
                 version = b'v2'
             elif requirementsmod.REVLOGV2_REQUIREMENT in repo.requirements:
                 version = b'v2'
             # Modern compression engines require v2.
             if compression not in _bundlespecv1compengines:
                 version = b'v2'
         elif spec in _bundlespeccontentopts:
             if spec == b'packed1':
                 compression = b'none'
             else:
                 compression = b'bzip2'
             version = spec
         else:
             raise error.UnsupportedBundleSpecification(
                 _(b'%s is not a recognized bundle specification') % spec
             )
 
     # Bundle version 1 only supports a known set of compression engines.
     if version == b'v1' and compression not in _bundlespecv1compengines:
         raise error.UnsupportedBundleSpecification(
             _(b'compression engine %s is not supported on v1 bundles')
             % compression
         )
 
     # The specification for packed1 can optionally declare the data formats
     # required to apply it. If we see this metadata, compare against what the
     # repo supports and error if the bundle isn't compatible.
     if version == b'packed1' and b'requirements' in params:
         requirements = set(cast(bytes, params[b'requirements']).split(b','))
         missingreqs = requirements - requirementsmod.STREAM_FIXED_REQUIREMENTS
         if missingreqs:
             raise error.UnsupportedBundleSpecification(
                 _(b'missing support for repository features: %s')
                 % b', '.join(sorted(missingreqs))
             )
 
     # Compute contentopts based on the version
     if b"stream" in params:
         # This case is fishy as this mostly derails the version selection
         # mechanism. `stream` bundles are quite specific and used differently
         # as "normal" bundles.
         #
         # (we should probably define a cleaner way to do this and raise a
         # warning when the old way is encountered)
         if params[b"stream"] == b"v2":
             version = b"streamv2"
         if params[b"stream"] == b"v3-exp":
             version = b"streamv3-exp"
     contentopts = _bundlespeccontentopts.get(version, {}).copy()
     if version == b"streamv2" or version == b"streamv3-exp":
         # streamv2 have been reported as "v2" for a while.
         version = b"v2"
 
     engine = util.compengines.forbundlename(compression)
     compression, wirecompression = engine.bundletype()
     wireversion = _bundlespeccontentopts[version][b'cg.version']
 
     return bundlespec(
         compression, wirecompression, version, wireversion, params, contentopts
     )
 
 
 def parseclonebundlesmanifest(repo, s):
     """Parses the raw text of a clone bundles manifest.
 
     Returns a list of dicts. The dicts have a ``URL`` key corresponding
     to the URL and other keys are the attributes for the entry.
     """
     m = []
     for line in s.splitlines():
         fields = line.split()
         if not fields:
             continue
         attrs = {b'URL': fields[0]}
         for rawattr in fields[1:]:
             key, value = rawattr.split(b'=', 1)
             key = util.urlreq.unquote(key)
             value = util.urlreq.unquote(value)
             attrs[key] = value
 
             # Parse BUNDLESPEC into components. This makes client-side
             # preferences easier to specify since you can prefer a single
             # component of the BUNDLESPEC.
             if key == b'BUNDLESPEC':
                 try:
                     bundlespec = parsebundlespec(repo, value)
                     attrs[b'COMPRESSION'] = bundlespec.compression
                     attrs[b'VERSION'] = bundlespec.version
                 except error.InvalidBundleSpecification:
                     pass
                 except error.UnsupportedBundleSpecification:
                     pass
 
         m.append(attrs)
 
     return m
 
 
 def isstreamclonespec(bundlespec):
     # Stream clone v1
     if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1':
         return True
 
     # Stream clone v2
     if (
         bundlespec.wirecompression == b'UN'
         and bundlespec.wireversion == b'02'
         and bundlespec.contentopts.get(b'stream', None) in (b"v2", b"v3-exp")
     ):
         return True
 
     return False
 
 
 def filterclonebundleentries(
     repo, entries, streamclonerequested=False, pullbundles=False
 ):
     """Remove incompatible clone bundle manifest entries.
 
     Accepts a list of entries parsed with ``parseclonebundlesmanifest``
     and returns a new list consisting of only the entries that this client
     should be able to apply.
 
     There is no guarantee we'll be able to apply all returned entries because
     the metadata we use to filter on may be missing or wrong.
     """
     newentries = []
     for entry in entries:
         url = entry.get(b'URL')
         if not pullbundles and not any(
             [url.startswith(scheme) for scheme in SUPPORTED_CLONEBUNDLE_SCHEMES]
         ):
             repo.ui.debug(
                 b'filtering %s because not a supported clonebundle scheme\n'
                 % url
             )
             continue
 
         spec = entry.get(b'BUNDLESPEC')
         if spec:
             try:
                 bundlespec = parsebundlespec(repo, spec, strict=True)
 
                 # If a stream clone was requested, filter out non-streamclone
                 # entries.
                 if streamclonerequested and not isstreamclonespec(bundlespec):
                     repo.ui.debug(
                         b'filtering %s because not a stream clone\n' % url
                     )
                     continue
 
             except error.InvalidBundleSpecification as e:
                 repo.ui.debug(stringutil.forcebytestr(e) + b'\n')
                 continue
             except error.UnsupportedBundleSpecification as e:
                 repo.ui.debug(
                     b'filtering %s because unsupported bundle '
                     b'spec: %s\n' % (url, stringutil.forcebytestr(e))
                 )
                 continue
         # If we don't have a spec and requested a stream clone, we don't know
         # what the entry is so don't attempt to apply it.
         elif streamclonerequested:
             repo.ui.debug(
                 b'filtering %s because cannot determine if a stream '
                 b'clone bundle\n' % url
             )
             continue
 
         if b'REQUIRESNI' in entry and not sslutil.hassni:
             repo.ui.debug(b'filtering %s because SNI not supported\n' % url)
             continue
 
         if b'REQUIREDRAM' in entry:
             try:
                 requiredram = util.sizetoint(entry[b'REQUIREDRAM'])
             except error.ParseError:
                 repo.ui.debug(
                     b'filtering %s due to a bad REQUIREDRAM attribute\n' % url
                 )
                 continue
             actualram = repo.ui.estimatememory()
             if actualram is not None and actualram * 0.66 < requiredram:
                 repo.ui.debug(
                     b'filtering %s as it needs more than 2/3 of system memory\n'
                     % url
                 )
                 continue
 
         newentries.append(entry)
 
     return newentries
 
 
 class clonebundleentry:
     """Represents an item in a clone bundles manifest.
 
     This rich class is needed to support sorting since sorted() in Python 3
     doesn't support ``cmp`` and our comparison is complex enough that ``key=``
     won't work.
     """
 
     def __init__(self, value, prefers):
         self.value = value
         self.prefers = prefers
 
     def _cmp(self, other):
         for prefkey, prefvalue in self.prefers:
             avalue = self.value.get(prefkey)
             bvalue = other.value.get(prefkey)
 
             # Special case for b missing attribute and a matches exactly.
             if avalue is not None and bvalue is None and avalue == prefvalue:
                 return -1
 
             # Special case for a missing attribute and b matches exactly.
             if bvalue is not None and avalue is None and bvalue == prefvalue:
                 return 1
 
             # We can't compare unless attribute present on both.
             if avalue is None or bvalue is None:
                 continue
 
             # Same values should fall back to next attribute.
             if avalue == bvalue:
                 continue
 
             # Exact matches come first.
             if avalue == prefvalue:
                 return -1
             if bvalue == prefvalue:
                 return 1
 
             # Fall back to next attribute.
             continue
 
         # If we got here we couldn't sort by attributes and prefers. Fall
         # back to index order.
         return 0
 
     def __lt__(self, other):
         return self._cmp(other) < 0
 
     def __gt__(self, other):
         return self._cmp(other) > 0
 
     def __eq__(self, other):
         return self._cmp(other) == 0
 
     def __le__(self, other):
         return self._cmp(other) <= 0
 
     def __ge__(self, other):
         return self._cmp(other) >= 0
 
     def __ne__(self, other):
         return self._cmp(other) != 0
 
 
 def sortclonebundleentries(ui, entries):
     prefers = ui.configlist(b'ui', b'clonebundleprefers')
     if not prefers:
         return list(entries)
 
     def _split(p):
         if b'=' not in p:
             hint = _(b"each comma separated item should be key=value pairs")
             raise error.Abort(
                 _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint
             )
         return p.split(b'=', 1)
 
     prefers = [_split(p) for p in prefers]
 
     items = sorted(clonebundleentry(v, prefers) for v in entries)
     return [i.value for i in items]

mercurial/exchange.py

@@ -1,2944 +1,2943 b''
 # exchange.py - utility to exchange data between repos.
 #
 # Copyright 2005-2007 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import collections
 import weakref
 
 from .i18n import _
 from .node import (
     hex,
     nullrev,
 )
 from . import (
     bookmarks as bookmod,
     bundle2,
     bundlecaches,
     changegroup,
     discovery,
     error,
     lock as lockmod,
     logexchange,
     narrowspec,
     obsolete,
     obsutil,
     phases,
     pushkey,
     pycompat,
     requirements,
     scmutil,
     streamclone,
     url as urlmod,
     util,
     wireprototypes,
 )
 from .utils import (
     hashutil,
     stringutil,
     urlutil,
 )
 from .interfaces import repository
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 _NARROWACL_SECTION = b'narrowacl'
 
 
 def readbundle(ui, fh, fname, vfs=None):
     header = changegroup.readexactly(fh, 4)
 
     alg = None
     if not fname:
         fname = b"stream"
         if not header.startswith(b'HG') and header.startswith(b'\0'):
             fh = changegroup.headerlessfixup(fh, header)
             header = b"HG10"
             alg = b'UN'
     elif vfs:
         fname = vfs.join(fname)
 
     magic, version = header[0:2], header[2:4]
 
     if magic != b'HG':
         raise error.Abort(_(b'%s: not a Mercurial bundle') % fname)
     if version == b'10':
         if alg is None:
             alg = changegroup.readexactly(fh, 2)
         return changegroup.cg1unpacker(fh, alg)
     elif version.startswith(b'2'):
         return bundle2.getunbundler(ui, fh, magicstring=magic + version)
     elif version == b'S1':
         return streamclone.streamcloneapplier(fh)
     else:
         raise error.Abort(
             _(b'%s: unknown bundle version %s') % (fname, version)
         )
 
 
 def _format_params(params):
     parts = []
     for key, value in sorted(params.items()):
         value = urlreq.quote(value)
         parts.append(b"%s=%s" % (key, value))
     return b';'.join(parts)
 
 
 def getbundlespec(ui, fh):
     """Infer the bundlespec from a bundle file handle.
 
     The input file handle is seeked and the original seek position is not
     restored.
     """
 
     def speccompression(alg):
         try:
             return util.compengines.forbundletype(alg).bundletype()[0]
         except KeyError:
             return None
 
     params = {}
 
     b = readbundle(ui, fh, None)
     if isinstance(b, changegroup.cg1unpacker):
         alg = b._type
         if alg == b'_truncatedBZ':
             alg = b'BZ'
         comp = speccompression(alg)
         if not comp:
             raise error.Abort(_(b'unknown compression algorithm: %s') % alg)
         return b'%s-v1' % comp
     elif isinstance(b, bundle2.unbundle20):
         if b'Compression' in b.params:
             comp = speccompression(b.params[b'Compression'])
             if not comp:
                 raise error.Abort(
                     _(b'unknown compression algorithm: %s') % comp
                 )
         else:
             comp = b'none'
 
         version = None
         for part in b.iterparts():
             if part.type == b'changegroup':
                 cgversion = part.params[b'version']
                 if cgversion in (b'01', b'02'):
                     version = b'v2'
                 elif cgversion in (b'03',):
                     version = b'v2'
                     params[b'cg.version'] = cgversion
                 else:
                     raise error.Abort(
                         _(
                             b'changegroup version %s does not have '
                             b'a known bundlespec'
                         )
                         % version,
                         hint=_(b'try upgrading your Mercurial client'),
                     )
             elif part.type == b'stream2' and version is None:
                 # A stream2 part requires to be part of a v2 bundle
                 requirements = urlreq.unquote(part.params[b'requirements'])
                 splitted = requirements.split()
                 params = bundle2._formatrequirementsparams(splitted)
                 return b'none-v2;stream=v2;%s' % params
             elif part.type == b'stream3-exp' and version is None:
                 # A stream3 part requires to be part of a v2 bundle
                 requirements = urlreq.unquote(part.params[b'requirements'])
                 splitted = requirements.split()
                 params = bundle2._formatrequirementsparams(splitted)
                 return b'none-v2;stream=v3-exp;%s' % params
             elif part.type == b'obsmarkers':
                 params[b'obsolescence'] = b'yes'
                 if not part.mandatory:
                     params[b'obsolescence-mandatory'] = b'no'
 
         if not version:
-            raise error.Abort(
-                _(b'could not identify changegroup version in bundle')
-            )
+            params[b'changegroup'] = b'no'
+            version = b'v2'
         spec = b'%s-%s' % (comp, version)
         if params:
             spec += b';'
             spec += _format_params(params)
         return spec
 
     elif isinstance(b, streamclone.streamcloneapplier):
         requirements = streamclone.readbundle1header(fh)[2]
         formatted = bundle2._formatrequirementsparams(requirements)
         return b'none-packed1;%s' % formatted
     else:
         raise error.Abort(_(b'unknown bundle type: %s') % b)
 
 
 def _computeoutgoing(repo, heads, common):
     """Computes which revs are outgoing given a set of common
     and a set of heads.
 
     This is a separate function so extensions can have access to
     the logic.
 
     Returns a discovery.outgoing object.
     """
     cl = repo.changelog
     if common:
         hasnode = cl.hasnode
         common = [n for n in common if hasnode(n)]
     else:
         common = [repo.nullid]
     if not heads:
         heads = cl.heads()
     return discovery.outgoing(repo, common, heads)
 
 
 def _checkpublish(pushop):
     repo = pushop.repo
     ui = repo.ui
     behavior = ui.config(b'experimental', b'auto-publish')
     if pushop.publish or behavior not in (b'warn', b'confirm', b'abort'):
         return
     remotephases = listkeys(pushop.remote, b'phases')
     if not remotephases.get(b'publishing', False):
         return
 
     if pushop.revs is None:
         published = repo.filtered(b'served').revs(b'not public()')
     else:
         published = repo.revs(b'::%ln - public()', pushop.revs)
         # we want to use pushop.revs in the revset even if they themselves are
         # secret, but we don't want to have anything that the server won't see
         # in the result of this expression
         published &= repo.filtered(b'served')
     if published:
         if behavior == b'warn':
             ui.warn(
                 _(b'%i changesets about to be published\n') % len(published)
             )
         elif behavior == b'confirm':
             if ui.promptchoice(
                 _(b'push and publish %i changesets (yn)?$$ &Yes $$ &No')
                 % len(published)
             ):
                 raise error.CanceledError(_(b'user quit'))
         elif behavior == b'abort':
             msg = _(b'push would publish %i changesets') % len(published)
             hint = _(
                 b"use --publish or adjust 'experimental.auto-publish'"
                 b" config"
             )
             raise error.Abort(msg, hint=hint)
 
 
 def _forcebundle1(op):
     """return true if a pull/push must use bundle1
 
     This function is used to allow testing of the older bundle version"""
     ui = op.repo.ui
     # The goal is this config is to allow developer to choose the bundle
     # version used during exchanged. This is especially handy during test.
     # Value is a list of bundle version to be picked from, highest version
     # should be used.
     #
     # developer config: devel.legacy.exchange
     exchange = ui.configlist(b'devel', b'legacy.exchange')
     forcebundle1 = b'bundle2' not in exchange and b'bundle1' in exchange
     return forcebundle1 or not op.remote.capable(b'bundle2')
 
 
 class pushoperation:
     """A object that represent a single push operation
 
     Its purpose is to carry push related state and very common operations.
 
     A new pushoperation should be created at the beginning of each push and
     discarded afterward.
     """
 
     def __init__(
         self,
         repo,
         remote,
         force=False,
         revs=None,
         newbranch=False,
         bookmarks=(),
         publish=False,
         pushvars=None,
     ):
         # repo we push from
         self.repo = repo
         self.ui = repo.ui
         # repo we push to
         self.remote = remote
         # force option provided
         self.force = force
         # revs to be pushed (None is "all")
         self.revs = revs
         # bookmark explicitly pushed
         self.bookmarks = bookmarks
         # allow push of new branch
         self.newbranch = newbranch
         # step already performed
         # (used to check what steps have been already performed through bundle2)
         self.stepsdone = set()
         # Integer version of the changegroup push result
         # - None means nothing to push
         # - 0 means HTTP error
         # - 1 means we pushed and remote head count is unchanged *or*
         #   we have outgoing changesets but refused to push
         # - other values as described by addchangegroup()
         self.cgresult = None
         # Boolean value for the bookmark push
         self.bkresult = None
         # discover.outgoing object (contains common and outgoing data)
         self.outgoing = None
         # all remote topological heads before the push
         self.remoteheads = None
         # Details of the remote branch pre and post push
         #
         # mapping: {'branch': ([remoteheads],
         #                      [newheads],
         #                      [unsyncedheads],
         #                      [discardedheads])}
         # - branch: the branch name
         # - remoteheads: the list of remote heads known locally
         #                None if the branch is new
         # - newheads: the new remote heads (known locally) with outgoing pushed
         # - unsyncedheads: the list of remote heads unknown locally.
         # - discardedheads: the list of remote heads made obsolete by the push
         self.pushbranchmap = None
         # testable as a boolean indicating if any nodes are missing locally.
         self.incoming = None
         # summary of the remote phase situation
         self.remotephases = None
         # phases changes that must be pushed along side the changesets
         self.outdatedphases = None
         # phases changes that must be pushed if changeset push fails
         self.fallbackoutdatedphases = None
         # outgoing obsmarkers
         self.outobsmarkers = set()
         # outgoing bookmarks, list of (bm, oldnode | '', newnode | '')
         self.outbookmarks = []
         # transaction manager
         self.trmanager = None
         # map { pushkey partid -> callback handling failure}
         # used to handle exception from mandatory pushkey part failure
         self.pkfailcb = {}
         # an iterable of pushvars or None
         self.pushvars = pushvars
         # publish pushed changesets
         self.publish = publish
 
     @util.propertycache
     def futureheads(self):
         """future remote heads if the changeset push succeeds"""
         return self.outgoing.ancestorsof
 
     @util.propertycache
     def fallbackheads(self):
         """future remote heads if the changeset push fails"""
         if self.revs is None:
             # not target to push, all common are relevant
             return self.outgoing.commonheads
         unfi = self.repo.unfiltered()
         # I want cheads = heads(::push_heads and ::commonheads)
         #
         # To push, we already computed
         #     common = (::commonheads)
         #     missing = ((commonheads::push_heads) - commonheads)
         #
         # So we basically search
         #
         #     almost_heads = heads((parents(missing) + push_heads) & common)
         #
         # We use "almost" here as this can return revision that are ancestors
         # of other in the set and we need to explicitly turn it into an
         # antichain later. We can do so using:
         #
         #     cheads = heads(almost_heads::almost_heads)
         #
         # In pratice the code is a bit more convulted to avoid some extra
         # computation. It aims at doing the same computation as highlighted
         # above however.
         common = self.outgoing.common
         unfi = self.repo.unfiltered()
         cl = unfi.changelog
         to_rev = cl.index.rev
         to_node = cl.node
         parent_revs = cl.parentrevs
         unselected = []
         cheads = set()
         # XXX-perf: `self.revs` and `outgoing.missing` could hold revs directly
         for n in self.revs:
             r = to_rev(n)
             if r in common:
                 cheads.add(r)
             else:
                 unselected.append(r)
         known_non_heads = cl.ancestors(cheads, inclusive=True)
         if unselected:
             missing_revs = {to_rev(n) for n in self.outgoing.missing}
             missing_revs.add(nullrev)
             root_points = set()
             for r in missing_revs:
                 p1, p2 = parent_revs(r)
                 if p1 not in missing_revs and p1 not in known_non_heads:
                     root_points.add(p1)
                 if p2 not in missing_revs and p2 not in known_non_heads:
                     root_points.add(p2)
             if root_points:
                 heads = unfi.revs('heads(%ld::%ld)', root_points, root_points)
                 cheads.update(heads)
         # XXX-perf: could this be a set of revision?
         return [to_node(r) for r in sorted(cheads)]
 
     @property
     def commonheads(self):
         """set of all common heads after changeset bundle push"""
         if self.cgresult:
             return self.futureheads
         else:
             return self.fallbackheads
 
 
 # mapping of message used when pushing bookmark
 bookmsgmap = {
     b'update': (
         _(b"updating bookmark %s\n"),
         _(b'updating bookmark %s failed\n'),
     ),
     b'export': (
         _(b"exporting bookmark %s\n"),
         _(b'exporting bookmark %s failed\n'),
     ),
     b'delete': (
         _(b"deleting remote bookmark %s\n"),
         _(b'deleting remote bookmark %s failed\n'),
     ),
 }
 
 
 def push(
     repo,
     remote,
     force=False,
     revs=None,
     newbranch=False,
     bookmarks=(),
     publish=False,
     opargs=None,
 ):
     """Push outgoing changesets (limited by revs) from a local
     repository to remote. Return an integer:
       - None means nothing to push
       - 0 means HTTP error
       - 1 means we pushed and remote head count is unchanged *or*
         we have outgoing changesets but refused to push
       - other values as described by addchangegroup()
     """
     if opargs is None:
         opargs = {}
     pushop = pushoperation(
         repo,
         remote,
         force,
         revs,
         newbranch,
         bookmarks,
         publish,
         **pycompat.strkwargs(opargs)
     )
     if pushop.remote.local():
         missing = (
             set(pushop.repo.requirements) - pushop.remote.local().supported
         )
         if missing:
             msg = _(
                 b"required features are not"
                 b" supported in the destination:"
                 b" %s"
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     if not pushop.remote.canpush():
         raise error.Abort(_(b"destination does not support push"))
 
     if not pushop.remote.capable(b'unbundle'):
         raise error.Abort(
             _(
                 b'cannot push: destination does not support the '
                 b'unbundle wire protocol command'
             )
         )
     for category in sorted(bundle2.read_remote_wanted_sidedata(pushop.remote)):
         # Check that a computer is registered for that category for at least
         # one revlog kind.
         for kind, computers in repo._sidedata_computers.items():
             if computers.get(category):
                 break
         else:
             raise error.Abort(
                 _(
                     b'cannot push: required sidedata category not supported'
                     b" by this client: '%s'"
                 )
                 % pycompat.bytestr(category)
             )
     # get lock as we might write phase data
     wlock = lock = None
     try:
         # bundle2 push may receive a reply bundle touching bookmarks
         # requiring the wlock. Take it now to ensure proper ordering.
         maypushback = pushop.ui.configbool(b'experimental', b'bundle2.pushback')
         if (
             (not _forcebundle1(pushop))
             and maypushback
             and not bookmod.bookmarksinstore(repo)
         ):
             wlock = pushop.repo.wlock()
         lock = pushop.repo.lock()
         pushop.trmanager = transactionmanager(
             pushop.repo, b'push-response', pushop.remote.url()
         )
     except error.LockUnavailable as err:
         # source repo cannot be locked.
         # We do not abort the push, but just disable the local phase
         # synchronisation.
         msg = b'cannot lock source repository: %s\n' % stringutil.forcebytestr(
             err
         )
         pushop.ui.debug(msg)
 
     with wlock or util.nullcontextmanager():
         with lock or util.nullcontextmanager():
             with pushop.trmanager or util.nullcontextmanager():
                 pushop.repo.checkpush(pushop)
                 _checkpublish(pushop)
                 _pushdiscovery(pushop)
                 if not pushop.force:
                     _checksubrepostate(pushop)
                 if not _forcebundle1(pushop):
                     _pushbundle2(pushop)
                 _pushchangeset(pushop)
                 _pushsyncphase(pushop)
                 _pushobsolete(pushop)
                 _pushbookmark(pushop)
 
     if repo.ui.configbool(b'experimental', b'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pushop
 
 
 # list of steps to perform discovery before push
 pushdiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pushdiscoverymapping = {}
 
 
 def pushdiscovery(stepname):
     """decorator for function performing discovery before push
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pushdiscovery dictionary directly."""
 
     def dec(func):
         assert stepname not in pushdiscoverymapping
         pushdiscoverymapping[stepname] = func
         pushdiscoveryorder.append(stepname)
         return func
 
     return dec
 
 
 def _pushdiscovery(pushop):
     """Run all discovery steps"""
     for stepname in pushdiscoveryorder:
         step = pushdiscoverymapping[stepname]
         step(pushop)
 
 
 def _checksubrepostate(pushop):
     """Ensure all outgoing referenced subrepo revisions are present locally"""
 
     repo = pushop.repo
 
     # If the repository does not use subrepos, skip the expensive
     # manifest checks.
     if not len(repo.file(b'.hgsub')) or not len(repo.file(b'.hgsubstate')):
         return
 
     for n in pushop.outgoing.missing:
         ctx = repo[n]
 
         if b'.hgsub' in ctx.manifest() and b'.hgsubstate' in ctx.files():
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 sub.verify(onpush=True)
 
 
 @pushdiscovery(b'changeset')
 def _pushdiscoverychangeset(pushop):
     """discover the changeset that need to be pushed"""
     fci = discovery.findcommonincoming
     if pushop.revs:
         commoninc = fci(
             pushop.repo,
             pushop.remote,
             force=pushop.force,
             ancestorsof=pushop.revs,
         )
     else:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
     common, inc, remoteheads = commoninc
     fco = discovery.findcommonoutgoing
     outgoing = fco(
         pushop.repo,
         pushop.remote,
         onlyheads=pushop.revs,
         commoninc=commoninc,
         force=pushop.force,
     )
     pushop.outgoing = outgoing
     pushop.remoteheads = remoteheads
     pushop.incoming = inc
 
 
 @pushdiscovery(b'phase')
 def _pushdiscoveryphase(pushop):
     """discover the phase that needs to be pushed
 
     (computed for both success and failure case for changesets push)"""
     outgoing = pushop.outgoing
     repo = pushop.repo
     unfi = repo.unfiltered()
     cl = unfi.changelog
     to_rev = cl.index.rev
     remotephases = listkeys(pushop.remote, b'phases')
 
     if (
         pushop.ui.configbool(b'ui', b'_usedassubrepo')
         and remotephases  # server supports phases
         and not pushop.outgoing.missing  # no changesets to be pushed
         and remotephases.get(b'publishing', False)
     ):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset are to be pushed
         # - and remote is publishing
         # We may be in issue 3781 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         pushop.outdatedphases = []
         pushop.fallbackoutdatedphases = []
         return
 
     fallbackheads_rev = {to_rev(n) for n in pushop.fallbackheads}
     pushop.remotephases = phases.RemotePhasesSummary(
         pushop.repo,
         fallbackheads_rev,
         remotephases,
     )
     droots = set(pushop.remotephases.draft_roots)
 
     fallback_publishing = pushop.remotephases.publishing
     push_publishing = pushop.remotephases.publishing or pushop.publish
     missing_revs = {to_rev(n) for n in outgoing.missing}
     drafts = unfi._phasecache.get_raw_set(unfi, phases.draft)
 
     if fallback_publishing:
         fallback_roots = droots - missing_revs
         revset = b'heads(%ld::%ld)'
     else:
         fallback_roots = droots - drafts
         fallback_roots -= missing_revs
         # Get the list of all revs draft on remote but public here.
         revset = b'heads((%ld::%ld) and public())'
     if not fallback_roots:
         fallback = fallback_rev = []
     else:
         fallback_rev = unfi.revs(revset, fallback_roots, fallbackheads_rev)
         fallback = [repo[r] for r in fallback_rev]
 
     if push_publishing:
         published = missing_revs.copy()
     else:
         published = missing_revs - drafts
     if pushop.publish:
         published.update(fallbackheads_rev & drafts)
     elif fallback:
         published.update(fallback_rev)
 
     pushop.outdatedphases = [repo[r] for r in cl.headrevs(published)]
     pushop.fallbackoutdatedphases = fallback
 
 
 @pushdiscovery(b'obsmarker')
 def _pushdiscoveryobsmarkers(pushop):
     if not obsolete.isenabled(pushop.repo, obsolete.exchangeopt):
         return
 
     if not pushop.repo.obsstore:
         return
 
     if b'obsolete' not in listkeys(pushop.remote, b'namespaces'):
         return
 
     repo = pushop.repo
     # very naive computation, that can be quite expensive on big repo.
     # However: evolution is currently slow on them anyway.
     nodes = (c.node() for c in repo.set(b'::%ln', pushop.futureheads))
     pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
 
 
 @pushdiscovery(b'bookmarks')
 def _pushdiscoverybookmarks(pushop):
     ui = pushop.ui
     repo = pushop.repo.unfiltered()
     remote = pushop.remote
     ui.debug(b"checking for updated bookmarks\n")
     ancestors = ()
     if pushop.revs:
         revnums = pycompat.maplist(repo.changelog.rev, pushop.revs)
         ancestors = repo.changelog.ancestors(revnums, inclusive=True)
 
     remotebookmark = bookmod.unhexlifybookmarks(listkeys(remote, b'bookmarks'))
 
     explicit = {
         repo._bookmarks.expandname(bookmark) for bookmark in pushop.bookmarks
     }
 
     comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
     return _processcompared(pushop, ancestors, explicit, remotebookmark, comp)
 
 
 def _processcompared(pushop, pushed, explicit, remotebms, comp):
     """take decision on bookmarks to push to the remote repo
 
     Exists to help extensions alter this behavior.
     """
     addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
 
     repo = pushop.repo
 
     for b, scid, dcid in advsrc:
         if b in explicit:
             explicit.remove(b)
         if not pushed or repo[scid].rev() in pushed:
             pushop.outbookmarks.append((b, dcid, scid))
     # search added bookmark
     for b, scid, dcid in addsrc:
         if b in explicit:
             explicit.remove(b)
             if bookmod.isdivergent(b):
                 pushop.ui.warn(_(b'cannot push divergent bookmark %s!\n') % b)
                 pushop.bkresult = 2
             else:
                 pushop.outbookmarks.append((b, b'', scid))
     # search for overwritten bookmark
     for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, dcid, scid))
     # search for bookmark to delete
     for b, scid, dcid in adddst:
         if b in explicit:
             explicit.remove(b)
             # treat as "deleted locally"
             pushop.outbookmarks.append((b, dcid, b''))
     # identical bookmarks shouldn't get reported
     for b, scid, dcid in same:
         if b in explicit:
             explicit.remove(b)
 
     if explicit:
         explicit = sorted(explicit)
         # we should probably list all of them
         pushop.ui.warn(
             _(
                 b'bookmark %s does not exist on the local '
                 b'or remote repository!\n'
             )
             % explicit[0]
         )
         pushop.bkresult = 2
 
     pushop.outbookmarks.sort()
 
 
 def _pushcheckoutgoing(pushop):
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     if not outgoing.missing:
         # nothing to push
         scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
         return False
     # something to push
     if not pushop.force:
         # if repo.obsstore == False --> no obsolete
         # then, save the iteration
         if unfi.obsstore:
             # this message are here for 80 char limit reason
             mso = _(b"push includes obsolete changeset: %s!")
             mspd = _(b"push includes phase-divergent changeset: %s!")
             mscd = _(b"push includes content-divergent changeset: %s!")
             mst = {
                 b"orphan": _(b"push includes orphan changeset: %s!"),
                 b"phase-divergent": mspd,
                 b"content-divergent": mscd,
             }
             # If we are to push if there is at least one
             # obsolete or unstable changeset in missing, at
             # least one of the missinghead will be obsolete or
             # unstable. So checking heads only is ok
             for node in outgoing.ancestorsof:
                 ctx = unfi[node]
                 if ctx.obsolete():
                     raise error.Abort(mso % ctx)
                 elif ctx.isunstable():
                     # TODO print more than one instability in the abort
                     # message
                     raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
 
         discovery.checkheads(pushop)
     return True
 
 
 # List of names of steps to perform for an outgoing bundle2, order matters.
 b2partsgenorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 b2partsgenmapping = {}
 
 
 def b2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the b2partsgenmapping dictionary directly."""
 
     def dec(func):
         assert stepname not in b2partsgenmapping
         b2partsgenmapping[stepname] = func
         if idx is None:
             b2partsgenorder.append(stepname)
         else:
             b2partsgenorder.insert(idx, stepname)
         return func
 
     return dec
 
 
 def _pushb2ctxcheckheads(pushop, bundler):
     """Generate race condition checking parts
 
     Exists as an independent function to aid extensions
     """
     # * 'force' do not check for push race,
     # * if we don't push anything, there are nothing to check.
     if not pushop.force and pushop.outgoing.ancestorsof:
         allowunrelated = b'related' in bundler.capabilities.get(
             b'checkheads', ()
         )
         emptyremote = pushop.pushbranchmap is None
         if not allowunrelated or emptyremote:
             bundler.newpart(b'check:heads', data=iter(pushop.remoteheads))
         else:
             affected = set()
             for branch, heads in pushop.pushbranchmap.items():
                 remoteheads, newheads, unsyncedheads, discardedheads = heads
                 if remoteheads is not None:
                     remote = set(remoteheads)
                     affected |= set(discardedheads) & remote
                     affected |= remote - set(newheads)
             if affected:
                 data = iter(sorted(affected))
                 bundler.newpart(b'check:updated-heads', data=data)
 
 
 def _pushing(pushop):
     """return True if we are pushing anything"""
     return bool(
         pushop.outgoing.missing
         or pushop.outdatedphases
         or pushop.outobsmarkers
         or pushop.outbookmarks
     )
 
 
 @b2partsgenerator(b'check-bookmarks')
 def _pushb2checkbookmarks(pushop, bundler):
     """insert bookmark move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasbookmarkcheck = b'bookmarks' in b2caps
     if not (pushop.outbookmarks and hasbookmarkcheck):
         return
     data = []
     for book, old, new in pushop.outbookmarks:
         data.append((book, old))
     checkdata = bookmod.binaryencode(pushop.repo, data)
     bundler.newpart(b'check:bookmarks', data=checkdata)
 
 
 @b2partsgenerator(b'check-phases')
 def _pushb2checkphases(pushop, bundler):
     """insert phase move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasphaseheads = b'heads' in b2caps.get(b'phases', ())
     if pushop.remotephases is not None and hasphaseheads:
         # check that the remote phase has not changed
         checks = {p: [] for p in phases.allphases}
         to_node = pushop.repo.unfiltered().changelog.node
         checks[phases.public].extend(
             to_node(r) for r in pushop.remotephases.public_heads
         )
         checks[phases.draft].extend(
             to_node(r) for r in pushop.remotephases.draft_roots
         )
         if any(checks.values()):
             for phase in checks:
                 checks[phase].sort()
             checkdata = phases.binaryencode(checks)
             bundler.newpart(b'check:phases', data=checkdata)
 
 
 @b2partsgenerator(b'changeset')
 def _pushb2ctx(pushop, bundler):
     """handle changegroup push through bundle2
 
     addchangegroup result is stored in the ``pushop.cgresult`` attribute.
     """
     if b'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'changesets')
     # Send known heads to the server for race detection.
     if not _pushcheckoutgoing(pushop):
         return
     pushop.repo.prepushoutgoinghooks(pushop)
 
     _pushb2ctxcheckheads(pushop, bundler)
 
     b2caps = bundle2.bundle2caps(pushop.remote)
     version = b'01'
     cgversions = b2caps.get(b'changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [
             v
             for v in cgversions
             if v in changegroup.supportedoutgoingversions(pushop.repo)
         ]
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
 
     remote_sidedata = bundle2.read_remote_wanted_sidedata(pushop.remote)
     cgstream = changegroup.makestream(
         pushop.repo,
         pushop.outgoing,
         version,
         b'push',
         bundlecaps=b2caps,
         remote_sidedata=remote_sidedata,
     )
     cgpart = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
         cgpart.addparam(b'version', version)
     if scmutil.istreemanifest(pushop.repo):
         cgpart.addparam(b'treemanifest', b'1')
     if repository.REPO_FEATURE_SIDE_DATA in pushop.repo.features:
         cgpart.addparam(b'exp-sidedata', b'1')
 
     def handlereply(op):
         """extract addchangegroup returns from server reply"""
         cgreplies = op.records.getreplies(cgpart.id)
         assert len(cgreplies[b'changegroup']) == 1
         pushop.cgresult = cgreplies[b'changegroup'][0][b'return']
 
     return handlereply
 
 
 @b2partsgenerator(b'phase')
 def _pushb2phases(pushop, bundler):
     """handle phase push through bundle2"""
     if b'phases' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     ui = pushop.repo.ui
 
     legacyphase = b'phases' in ui.configlist(b'devel', b'legacy.exchange')
     haspushkey = b'pushkey' in b2caps
     hasphaseheads = b'heads' in b2caps.get(b'phases', ())
 
     if hasphaseheads and not legacyphase:
         return _pushb2phaseheads(pushop, bundler)
     elif haspushkey:
         return _pushb2phasespushkey(pushop, bundler)
 
 
 def _pushb2phaseheads(pushop, bundler):
     """push phase information through a bundle2 - binary part"""
     pushop.stepsdone.add(b'phases')
     if pushop.outdatedphases:
         updates = {p: [] for p in phases.allphases}
         updates[0].extend(h.node() for h in pushop.outdatedphases)
         phasedata = phases.binaryencode(updates)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 def _pushb2phasespushkey(pushop, bundler):
     """push phase information through a bundle2 - pushkey part"""
     pushop.stepsdone.add(b'phases')
     part2node = []
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, node in part2node:
             if partid == targetid:
                 raise error.Abort(_(b'updating %s to public failed') % node)
 
     enc = pushkey.encode
     for newremotehead in pushop.outdatedphases:
         part = bundler.newpart(b'pushkey')
         part.addparam(b'namespace', enc(b'phases'))
         part.addparam(b'key', enc(newremotehead.hex()))
         part.addparam(b'old', enc(b'%d' % phases.draft))
         part.addparam(b'new', enc(b'%d' % phases.public))
         part2node.append((part.id, newremotehead))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         for partid, node in part2node:
             partrep = op.records.getreplies(partid)
             results = partrep[b'pushkey']
             assert len(results) <= 1
             msg = None
             if not results:
                 msg = _(b'server ignored update of %s to public!\n') % node
             elif not int(results[0][b'return']):
                 msg = _(b'updating %s to public failed!\n') % node
             if msg is not None:
                 pushop.ui.warn(msg)
 
     return handlereply
 
 
 @b2partsgenerator(b'obsmarkers')
 def _pushb2obsmarkers(pushop, bundler):
     if b'obsmarkers' in pushop.stepsdone:
         return
     remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
     if obsolete.commonversion(remoteversions) is None:
         return
     pushop.stepsdone.add(b'obsmarkers')
     if pushop.outobsmarkers:
         markers = obsutil.sortedmarkers(pushop.outobsmarkers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 
 @b2partsgenerator(b'bookmarks')
 def _pushb2bookmarks(pushop, bundler):
     """handle bookmark push through bundle2"""
     if b'bookmarks' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
 
     legacy = pushop.repo.ui.configlist(b'devel', b'legacy.exchange')
     legacybooks = b'bookmarks' in legacy
 
     if not legacybooks and b'bookmarks' in b2caps:
         return _pushb2bookmarkspart(pushop, bundler)
     elif b'pushkey' in b2caps:
         return _pushb2bookmarkspushkey(pushop, bundler)
 
 
 def _bmaction(old, new):
     """small utility for bookmark pushing"""
     if not old:
         return b'export'
     elif not new:
         return b'delete'
     return b'update'
 
 
 def _abortonsecretctx(pushop, node, b):
     """abort if a given bookmark points to a secret changeset"""
     if node and pushop.repo[node].phase() == phases.secret:
         raise error.Abort(
             _(b'cannot push bookmark %s as it points to a secret changeset') % b
         )
 
 
 def _pushb2bookmarkspart(pushop, bundler):
     pushop.stepsdone.add(b'bookmarks')
     if not pushop.outbookmarks:
         return
 
     allactions = []
     data = []
     for book, old, new in pushop.outbookmarks:
         _abortonsecretctx(pushop, new, book)
         data.append((book, new))
         allactions.append((book, _bmaction(old, new)))
     checkdata = bookmod.binaryencode(pushop.repo, data)
     bundler.newpart(b'bookmarks', data=checkdata)
 
     def handlereply(op):
         ui = pushop.ui
         # if success
         for book, action in allactions:
             ui.status(bookmsgmap[action][0] % book)
 
     return handlereply
 
 
 def _pushb2bookmarkspushkey(pushop, bundler):
     pushop.stepsdone.add(b'bookmarks')
     part2book = []
     enc = pushkey.encode
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, book, action in part2book:
             if partid == targetid:
                 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
         # we should not be called for part we did not generated
         assert False
 
     for book, old, new in pushop.outbookmarks:
         _abortonsecretctx(pushop, new, book)
         part = bundler.newpart(b'pushkey')
         part.addparam(b'namespace', enc(b'bookmarks'))
         part.addparam(b'key', enc(book))
         part.addparam(b'old', enc(hex(old)))
         part.addparam(b'new', enc(hex(new)))
         action = b'update'
         if not old:
             action = b'export'
         elif not new:
             action = b'delete'
         part2book.append((part.id, book, action))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         ui = pushop.ui
         for partid, book, action in part2book:
             partrep = op.records.getreplies(partid)
             results = partrep[b'pushkey']
             assert len(results) <= 1
             if not results:
                 pushop.ui.warn(_(b'server ignored bookmark %s update\n') % book)
             else:
                 ret = int(results[0][b'return'])
                 if ret:
                     ui.status(bookmsgmap[action][0] % book)
                 else:
                     ui.warn(bookmsgmap[action][1] % book)
                     if pushop.bkresult is not None:
                         pushop.bkresult = 1
 
     return handlereply
 
 
 @b2partsgenerator(b'pushvars', idx=0)
 def _getbundlesendvars(pushop, bundler):
     '''send shellvars via bundle2'''
     pushvars = pushop.pushvars
     if pushvars:
         shellvars = {}
         for raw in pushvars:
             if b'=' not in raw:
                 msg = (
                     b"unable to parse variable '%s', should follow "
                     b"'KEY=VALUE' or 'KEY=' format"
                 )
                 raise error.Abort(msg % raw)
             k, v = raw.split(b'=', 1)
             shellvars[k] = v
 
         part = bundler.newpart(b'pushvars')
 
         for key, value in shellvars.items():
             part.addparam(key, value, mandatory=False)
 
 
 def _pushbundle2(pushop):
     """push data to the remote using bundle2
 
     The only currently supported type of data is changegroup but this will
     evolve in the future."""
     bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
     pushback = pushop.trmanager and pushop.ui.configbool(
         b'experimental', b'bundle2.pushback'
     )
 
     # create reply capability
     capsblob = bundle2.encodecaps(
         bundle2.getrepocaps(pushop.repo, allowpushback=pushback, role=b'client')
     )
     bundler.newpart(b'replycaps', data=capsblob)
     replyhandlers = []
     for partgenname in b2partsgenorder:
         partgen = b2partsgenmapping[partgenname]
         ret = partgen(pushop, bundler)
         if callable(ret):
             replyhandlers.append(ret)
     # do not push if nothing to push
     if bundler.nbparts <= 1:
         return
     stream = util.chunkbuffer(bundler.getchunks())
     try:
         try:
             with pushop.remote.commandexecutor() as e:
                 reply = e.callcommand(
                     b'unbundle',
                     {
                         b'bundle': stream,
                         b'heads': [b'force'],
                         b'url': pushop.remote.url(),
                     },
                 ).result()
         except error.BundleValueError as exc:
             raise error.RemoteError(_(b'missing support for %s') % exc)
         try:
             trgetter = None
             if pushback:
                 trgetter = pushop.trmanager.transaction
             op = bundle2.processbundle(
                 pushop.repo,
                 reply,
                 trgetter,
                 remote=pushop.remote,
             )
         except error.BundleValueError as exc:
             raise error.RemoteError(_(b'missing support for %s') % exc)
         except bundle2.AbortFromPart as exc:
             pushop.ui.error(_(b'remote: %s\n') % exc)
             if exc.hint is not None:
                 pushop.ui.error(_(b'remote: %s\n') % (b'(%s)' % exc.hint))
             raise error.RemoteError(_(b'push failed on remote'))
     except error.PushkeyFailed as exc:
         partid = int(exc.partid)
         if partid not in pushop.pkfailcb:
             raise
         pushop.pkfailcb[partid](pushop, exc)
     for rephand in replyhandlers:
         rephand(op)
 
 
 def _pushchangeset(pushop):
     """Make the actual push of changeset bundle to remote repo"""
     if b'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'changesets')
     if not _pushcheckoutgoing(pushop):
         return
 
     # Should have verified this in push().
     assert pushop.remote.capable(b'unbundle')
 
     pushop.repo.prepushoutgoinghooks(pushop)
     outgoing = pushop.outgoing
     # TODO: get bundlecaps from remote
     bundlecaps = None
     # create a changegroup from local
     if pushop.revs is None and not (
         outgoing.excluded or pushop.repo.changelog.filteredrevs
     ):
         # push everything,
         # use the fast path, no race possible on push
         cg = changegroup.makechangegroup(
             pushop.repo,
             outgoing,
             b'01',
             b'push',
             fastpath=True,
             bundlecaps=bundlecaps,
         )
     else:
         cg = changegroup.makechangegroup(
             pushop.repo, outgoing, b'01', b'push', bundlecaps=bundlecaps
         )
 
     # apply changegroup to remote
     # local repo finds heads on server, finds out what
     # revs it must push. once revs transferred, if server
     # finds it has different heads (someone else won
     # commit/push race), server aborts.
     if pushop.force:
         remoteheads = [b'force']
     else:
         remoteheads = pushop.remoteheads
     # ssh: return remote's addchangegroup()
     # http: return remote's addchangegroup() or 0 for error
     pushop.cgresult = pushop.remote.unbundle(cg, remoteheads, pushop.repo.url())
 
 
 def _pushsyncphase(pushop):
     """synchronise phase information locally and remotely"""
     cheads = pushop.commonheads
     # even when we don't push, exchanging phase data is useful
     remotephases = listkeys(pushop.remote, b'phases')
     if (
         pushop.ui.configbool(b'ui', b'_usedassubrepo')
         and remotephases  # server supports phases
         and pushop.cgresult is None  # nothing was pushed
         and remotephases.get(b'publishing', False)
     ):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset was pushed
         # - and remote is publishing
         # We may be in issue 3871 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         remotephases = {b'publishing': b'True'}
     if not remotephases:  # old server or public only reply from non-publishing
         _localphasemove(pushop, cheads)
         # don't push any phase data as there is nothing to push
     else:
         unfi = pushop.repo.unfiltered()
         to_rev = unfi.changelog.index.rev
         to_node = unfi.changelog.node
         cheads_revs = [to_rev(n) for n in cheads]
         pheads_revs, _dr = phases.analyze_remote_phases(
             pushop.repo,
             cheads_revs,
             remotephases,
         )
         pheads = [to_node(r) for r in pheads_revs]
         ### Apply remote phase on local
         if remotephases.get(b'publishing', False):
             _localphasemove(pushop, cheads)
         else:  # publish = False
             _localphasemove(pushop, pheads)
             _localphasemove(pushop, cheads, phases.draft)
         ### Apply local phase on remote
 
         if pushop.cgresult:
             if b'phases' in pushop.stepsdone:
                 # phases already pushed though bundle2
                 return
             outdated = pushop.outdatedphases
         else:
             outdated = pushop.fallbackoutdatedphases
 
         pushop.stepsdone.add(b'phases')
 
         # filter heads already turned public by the push
         outdated = [c for c in outdated if c.node() not in pheads]
         # fallback to independent pushkey command
         for newremotehead in outdated:
             with pushop.remote.commandexecutor() as e:
                 r = e.callcommand(
                     b'pushkey',
                     {
                         b'namespace': b'phases',
                         b'key': newremotehead.hex(),
                         b'old': b'%d' % phases.draft,
                         b'new': b'%d' % phases.public,
                     },
                 ).result()
 
             if not r:
                 pushop.ui.warn(
                     _(b'updating %s to public failed!\n') % newremotehead
                 )
 
 
 def _localphasemove(pushop, nodes, phase=phases.public):
     """move <nodes> to <phase> in the local source repo"""
     if pushop.trmanager:
         phases.advanceboundary(
             pushop.repo, pushop.trmanager.transaction(), phase, nodes
         )
     else:
         # repo is not locked, do not change any phases!
         # Informs the user that phases should have been moved when
         # applicable.
         actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
         phasestr = phases.phasenames[phase]
         if actualmoves:
             pushop.ui.status(
                 _(
                     b'cannot lock source repo, skipping '
                     b'local %s phase update\n'
                 )
                 % phasestr
             )
 
 
 def _pushobsolete(pushop):
     """utility function to push obsolete markers to a remote"""
     if b'obsmarkers' in pushop.stepsdone:
         return
     repo = pushop.repo
     remote = pushop.remote
     pushop.stepsdone.add(b'obsmarkers')
     if pushop.outobsmarkers:
         pushop.ui.debug(b'try to push obsolete markers to remote\n')
         rslts = []
         markers = obsutil.sortedmarkers(pushop.outobsmarkers)
         remotedata = obsolete._pushkeyescape(markers)
         for key in sorted(remotedata, reverse=True):
             # reverse sort to ensure we end with dump0
             data = remotedata[key]
             rslts.append(remote.pushkey(b'obsolete', key, b'', data))
         if [r for r in rslts if not r]:
             msg = _(b'failed to push some obsolete markers!\n')
             repo.ui.warn(msg)
 
 
 def _pushbookmark(pushop):
     """Update bookmark position on remote"""
     if pushop.cgresult == 0 or b'bookmarks' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'bookmarks')
     ui = pushop.ui
     remote = pushop.remote
 
     for b, old, new in pushop.outbookmarks:
         action = b'update'
         if not old:
             action = b'export'
         elif not new:
             action = b'delete'
 
         with remote.commandexecutor() as e:
             r = e.callcommand(
                 b'pushkey',
                 {
                     b'namespace': b'bookmarks',
                     b'key': b,
                     b'old': hex(old),
                     b'new': hex(new),
                 },
             ).result()
 
         if r:
             ui.status(bookmsgmap[action][0] % b)
         else:
             ui.warn(bookmsgmap[action][1] % b)
             # discovery can have set the value form invalid entry
             if pushop.bkresult is not None:
                 pushop.bkresult = 1
 
 
 class pulloperation:
     """A object that represent a single pull operation
 
     It purpose is to carry pull related state and very common operation.
 
     A new should be created at the beginning of each pull and discarded
     afterward.
     """
 
     def __init__(
         self,
         repo,
         remote,
         heads=None,
         force=False,
         bookmarks=(),
         remotebookmarks=None,
         streamclonerequested=None,
         includepats=None,
         excludepats=None,
         depth=None,
         path=None,
     ):
         # repo we pull into
         self.repo = repo
         # repo we pull from
         self.remote = remote
         # path object used to build this remote
         #
         # Ideally, the remote peer would carry that directly.
         self.remote_path = path
         # revision we try to pull (None is "all")
         self.heads = heads
         # bookmark pulled explicitly
         self.explicitbookmarks = [
             repo._bookmarks.expandname(bookmark) for bookmark in bookmarks
         ]
         # do we force pull?
         self.force = force
         # whether a streaming clone was requested
         self.streamclonerequested = streamclonerequested
         # transaction manager
         self.trmanager = None
         # set of common changeset between local and remote before pull
         self.common = None
         # set of pulled head
         self.rheads = None
         # list of missing changeset to fetch remotely
         self.fetch = None
         # remote bookmarks data
         self.remotebookmarks = remotebookmarks
         # result of changegroup pulling (used as return code by pull)
         self.cgresult = None
         # list of step already done
         self.stepsdone = set()
         # Whether we attempted a clone from pre-generated bundles.
         self.clonebundleattempted = False
         # Set of file patterns to include.
         self.includepats = includepats
         # Set of file patterns to exclude.
         self.excludepats = excludepats
         # Number of ancestor changesets to pull from each pulled head.
         self.depth = depth
 
     @util.propertycache
     def pulledsubset(self):
         """heads of the set of changeset target by the pull"""
         # compute target subset
         if self.heads is None:
             # We pulled every thing possible
             # sync on everything common
             c = set(self.common)
             ret = list(self.common)
             for n in self.rheads:
                 if n not in c:
                     ret.append(n)
             return ret
         else:
             # We pulled a specific subset
             # sync on this subset
             return self.heads
 
     @util.propertycache
     def canusebundle2(self):
         return not _forcebundle1(self)
 
     @util.propertycache
     def remotebundle2caps(self):
         return bundle2.bundle2caps(self.remote)
 
     def gettransaction(self):
         # deprecated; talk to trmanager directly
         return self.trmanager.transaction()
 
 
 class transactionmanager(util.transactional):
     """An object to manage the life cycle of a transaction
 
     It creates the transaction on demand and calls the appropriate hooks when
     closing the transaction."""
 
     def __init__(self, repo, source, url):
         self.repo = repo
         self.source = source
         self.url = url
         self._tr = None
 
     def transaction(self):
         """Return an open transaction object, constructing if necessary"""
         if not self._tr:
             trname = b'%s\n%s' % (self.source, urlutil.hidepassword(self.url))
             self._tr = self.repo.transaction(trname)
             self._tr.hookargs[b'source'] = self.source
             self._tr.hookargs[b'url'] = self.url
         return self._tr
 
     def close(self):
         """close transaction if created"""
         if self._tr is not None:
             self._tr.close()
 
     def release(self):
         """release transaction if created"""
         if self._tr is not None:
             self._tr.release()
 
 
 def listkeys(remote, namespace):
     with remote.commandexecutor() as e:
         return e.callcommand(b'listkeys', {b'namespace': namespace}).result()
 
 
 def _fullpullbundle2(repo, pullop):
     # The server may send a partial reply, i.e. when inlining
     # pre-computed bundles. In that case, update the common
     # set based on the results and pull another bundle.
     #
     # There are two indicators that the process is finished:
     # - no changeset has been added, or
     # - all remote heads are known locally.
     # The head check must use the unfiltered view as obsoletion
     # markers can hide heads.
     unfi = repo.unfiltered()
     unficl = unfi.changelog
 
     def headsofdiff(h1, h2):
         """Returns heads(h1 % h2)"""
         res = unfi.set(b'heads(%ln %% %ln)', h1, h2)
         return {ctx.node() for ctx in res}
 
     def headsofunion(h1, h2):
         """Returns heads((h1 + h2) - null)"""
         res = unfi.set(b'heads((%ln + %ln - null))', h1, h2)
         return {ctx.node() for ctx in res}
 
     while True:
         old_heads = unficl.heads()
         clstart = len(unficl)
         _pullbundle2(pullop)
         if requirements.NARROW_REQUIREMENT in repo.requirements:
             # XXX narrow clones filter the heads on the server side during
             # XXX getbundle and result in partial replies as well.
             # XXX Disable pull bundles in this case as band aid to avoid
             # XXX extra round trips.
             break
         if clstart == len(unficl):
             break
         if all(unficl.hasnode(n) for n in pullop.rheads):
             break
         new_heads = headsofdiff(unficl.heads(), old_heads)
         pullop.common = headsofunion(new_heads, pullop.common)
         pullop.rheads = set(pullop.rheads) - pullop.common
 
 
 def add_confirm_callback(repo, pullop):
     """adds a finalize callback to transaction which can be used to show stats
     to user and confirm the pull before committing transaction"""
 
     tr = pullop.trmanager.transaction()
     scmutil.registersummarycallback(
         repo, tr, txnname=b'pull', as_validator=True
     )
     reporef = weakref.ref(repo.unfiltered())
 
     def prompt(tr):
         repo = reporef()
         cm = _(b'accept incoming changes (yn)?$$ &Yes $$ &No')
         if repo.ui.promptchoice(cm):
             raise error.Abort(b"user aborted")
 
     tr.addvalidator(b'900-pull-prompt', prompt)
 
 
 def pull(
     repo,
     remote,
     path=None,
     heads=None,
     force=False,
     bookmarks=(),
     opargs=None,
     streamclonerequested=None,
     includepats=None,
     excludepats=None,
     depth=None,
     confirm=None,
 ):
     """Fetch repository data from a remote.
 
     This is the main function used to retrieve data from a remote repository.
 
     ``repo`` is the local repository to clone into.
     ``remote`` is a peer instance.
     ``heads`` is an iterable of revisions we want to pull. ``None`` (the
     default) means to pull everything from the remote.
     ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
     default, all remote bookmarks are pulled.
     ``opargs`` are additional keyword arguments to pass to ``pulloperation``
     initialization.
     ``streamclonerequested`` is a boolean indicating whether a "streaming
     clone" is requested. A "streaming clone" is essentially a raw file copy
     of revlogs from the server. This only works when the local repository is
     empty. The default value of ``None`` means to respect the server
     configuration for preferring stream clones.
     ``includepats`` and ``excludepats`` define explicit file patterns to
     include and exclude in storage, respectively. If not defined, narrow
     patterns from the repo instance are used, if available.
     ``depth`` is an integer indicating the DAG depth of history we're
     interested in. If defined, for each revision specified in ``heads``, we
     will fetch up to this many of its ancestors and data associated with them.
     ``confirm`` is a boolean indicating whether the pull should be confirmed
     before committing the transaction. This overrides HGPLAIN.
 
     Returns the ``pulloperation`` created for this pull.
     """
     if opargs is None:
         opargs = {}
 
     # We allow the narrow patterns to be passed in explicitly to provide more
     # flexibility for API consumers.
     if includepats is not None or excludepats is not None:
         includepats = includepats or set()
         excludepats = excludepats or set()
     else:
         includepats, excludepats = repo.narrowpats
 
     narrowspec.validatepatterns(includepats)
     narrowspec.validatepatterns(excludepats)
 
     pullop = pulloperation(
         repo,
         remote,
         path=path,
         heads=heads,
         force=force,
         bookmarks=bookmarks,
         streamclonerequested=streamclonerequested,
         includepats=includepats,
         excludepats=excludepats,
         depth=depth,
         **pycompat.strkwargs(opargs)
     )
 
     peerlocal = pullop.remote.local()
     if peerlocal:
         missing = set(peerlocal.requirements) - pullop.repo.supported
         if missing:
             msg = _(
                 b"required features are not"
                 b" supported in the destination:"
                 b" %s"
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     for category in repo._wanted_sidedata:
         # Check that a computer is registered for that category for at least
         # one revlog kind.
         for kind, computers in repo._sidedata_computers.items():
             if computers.get(category):
                 break
         else:
             # This should never happen since repos are supposed to be able to
             # generate the sidedata they require.
             raise error.ProgrammingError(
                 _(
                     b'sidedata category requested by local side without local'
                     b"support: '%s'"
                 )
                 % pycompat.bytestr(category)
             )
 
     pullop.trmanager = transactionmanager(repo, b'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
         wlock = repo.wlock()
     with wlock, repo.lock(), pullop.trmanager:
         if confirm or (
             repo.ui.configbool(b"pull", b"confirm") and not repo.ui.plain()
         ):
             add_confirm_callback(repo, pullop)
 
         # This should ideally be in _pullbundle2(). However, it needs to run
         # before discovery to avoid extra work.
         _maybeapplyclonebundle(pullop)
         streamclone.maybeperformlegacystreamclone(pullop)
         _pulldiscovery(pullop)
         if pullop.canusebundle2:
             _fullpullbundle2(repo, pullop)
         _pullchangeset(pullop)
         _pullphase(pullop)
         _pullbookmarks(pullop)
         _pullobsolete(pullop)
 
     # storing remotenames
     if repo.ui.configbool(b'experimental', b'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pullop
 
 
 # list of steps to perform discovery before pull
 pulldiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pulldiscoverymapping = {}
 
 
 def pulldiscovery(stepname):
     """decorator for function performing discovery before pull
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pulldiscovery dictionary directly."""
 
     def dec(func):
         assert stepname not in pulldiscoverymapping
         pulldiscoverymapping[stepname] = func
         pulldiscoveryorder.append(stepname)
         return func
 
     return dec
 
 
 def _pulldiscovery(pullop):
     """Run all discovery steps"""
     for stepname in pulldiscoveryorder:
         step = pulldiscoverymapping[stepname]
         step(pullop)
 
 
 @pulldiscovery(b'b1:bookmarks')
 def _pullbookmarkbundle1(pullop):
     """fetch bookmark data in bundle1 case
 
     If not using bundle2, we have to fetch bookmarks before changeset
     discovery to reduce the chance and impact of race conditions."""
     if pullop.remotebookmarks is not None:
         return
     if pullop.canusebundle2 and b'listkeys' in pullop.remotebundle2caps:
         # all known bundle2 servers now support listkeys, but lets be nice with
         # new implementation.
         return
     books = listkeys(pullop.remote, b'bookmarks')
     pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
 
 
 @pulldiscovery(b'changegroup')
 def _pulldiscoverychangegroup(pullop):
     """discovery phase for the pull
 
     Current handle changeset discovery only, will change handle all discovery
     at some point."""
     tmp = discovery.findcommonincoming(
         pullop.repo, pullop.remote, heads=pullop.heads, force=pullop.force
     )
     common, fetch, rheads = tmp
     has_node = pullop.repo.unfiltered().changelog.index.has_node
     if fetch and rheads:
         # If a remote heads is filtered locally, put in back in common.
         #
         # This is a hackish solution to catch most of "common but locally
         # hidden situation".  We do not performs discovery on unfiltered
         # repository because it end up doing a pathological amount of round
         # trip for w huge amount of changeset we do not care about.
         #
         # If a set of such "common but filtered" changeset exist on the server
         # but are not including a remote heads, we'll not be able to detect it,
         scommon = set(common)
         for n in rheads:
             if has_node(n):
                 if n not in scommon:
                     common.append(n)
         if set(rheads).issubset(set(common)):
             fetch = []
     pullop.common = common
     pullop.fetch = fetch
     pullop.rheads = rheads
 
 
 def _pullbundle2(pullop):
     """pull data using bundle2
 
     For now, the only supported data are changegroup."""
     kwargs = {b'bundlecaps': caps20to10(pullop.repo, role=b'client')}
 
     # make ui easier to access
     ui = pullop.repo.ui
 
     # At the moment we don't do stream clones over bundle2. If that is
     # implemented then here's where the check for that will go.
     streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
 
     # declare pull perimeters
     kwargs[b'common'] = pullop.common
     kwargs[b'heads'] = pullop.heads or pullop.rheads
 
     # check server supports narrow and then adding includepats and excludepats
     servernarrow = pullop.remote.capable(wireprototypes.NARROWCAP)
     if servernarrow and pullop.includepats:
         kwargs[b'includepats'] = pullop.includepats
     if servernarrow and pullop.excludepats:
         kwargs[b'excludepats'] = pullop.excludepats
 
     if streaming:
         kwargs[b'cg'] = False
         kwargs[b'stream'] = True
         pullop.stepsdone.add(b'changegroup')
         pullop.stepsdone.add(b'phases')
 
     else:
         # pulling changegroup
         pullop.stepsdone.add(b'changegroup')
 
         kwargs[b'cg'] = pullop.fetch
 
         legacyphase = b'phases' in ui.configlist(b'devel', b'legacy.exchange')
         hasbinaryphase = b'heads' in pullop.remotebundle2caps.get(b'phases', ())
         if not legacyphase and hasbinaryphase:
             kwargs[b'phases'] = True
             pullop.stepsdone.add(b'phases')
 
         if b'listkeys' in pullop.remotebundle2caps:
             if b'phases' not in pullop.stepsdone:
                 kwargs[b'listkeys'] = [b'phases']
 
     bookmarksrequested = False
     legacybookmark = b'bookmarks' in ui.configlist(b'devel', b'legacy.exchange')
     hasbinarybook = b'bookmarks' in pullop.remotebundle2caps
 
     if pullop.remotebookmarks is not None:
         pullop.stepsdone.add(b'request-bookmarks')
 
     if (
         b'request-bookmarks' not in pullop.stepsdone
         and pullop.remotebookmarks is None
         and not legacybookmark
         and hasbinarybook
     ):
         kwargs[b'bookmarks'] = True
         bookmarksrequested = True
 
     if b'listkeys' in pullop.remotebundle2caps:
         if b'request-bookmarks' not in pullop.stepsdone:
             # make sure to always includes bookmark data when migrating
             # `hg incoming --bundle` to using this function.
             pullop.stepsdone.add(b'request-bookmarks')
             kwargs.setdefault(b'listkeys', []).append(b'bookmarks')
 
     # If this is a full pull / clone and the server supports the clone bundles
     # feature, tell the server whether we attempted a clone bundle. The
     # presence of this flag indicates the client supports clone bundles. This
     # will enable the server to treat clients that support clone bundles
     # differently from those that don't.
     if (
         pullop.remote.capable(b'clonebundles')
         and pullop.heads is None
         and list(pullop.common) == [pullop.repo.nullid]
     ):
         kwargs[b'cbattempted'] = pullop.clonebundleattempted
 
     if streaming:
         pullop.repo.ui.status(_(b'streaming all changes\n'))
     elif not pullop.fetch:
         pullop.repo.ui.status(_(b"no changes found\n"))
         pullop.cgresult = 0
     else:
         if pullop.heads is None and list(pullop.common) == [pullop.repo.nullid]:
             pullop.repo.ui.status(_(b"requesting all changes\n"))
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
         if obsolete.commonversion(remoteversions) is not None:
             kwargs[b'obsmarkers'] = True
             pullop.stepsdone.add(b'obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
     remote_sidedata = bundle2.read_remote_wanted_sidedata(pullop.remote)
     if remote_sidedata:
         kwargs[b'remote_sidedata'] = remote_sidedata
 
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args[b'source'] = b'pull'
         bundle = e.callcommand(b'getbundle', args).result()
 
         try:
             op = bundle2.bundleoperation(
                 pullop.repo,
                 pullop.gettransaction,
                 source=b'pull',
                 remote=pullop.remote,
             )
             op.modes[b'bookmarks'] = b'records'
             bundle2.processbundle(
                 pullop.repo,
                 bundle,
                 op=op,
                 remote=pullop.remote,
             )
         except bundle2.AbortFromPart as exc:
             pullop.repo.ui.error(_(b'remote: abort: %s\n') % exc)
             raise error.RemoteError(_(b'pull failed on remote'), hint=exc.hint)
         except error.BundleValueError as exc:
             raise error.RemoteError(_(b'missing support for %s') % exc)
 
     if pullop.fetch:
         pullop.cgresult = bundle2.combinechangegroupresults(op)
 
     # processing phases change
     for namespace, value in op.records[b'listkeys']:
         if namespace == b'phases':
             _pullapplyphases(pullop, value)
 
     # processing bookmark update
     if bookmarksrequested:
         books = {}
         for record in op.records[b'bookmarks']:
             books[record[b'bookmark']] = record[b"node"]
         pullop.remotebookmarks = books
     else:
         for namespace, value in op.records[b'listkeys']:
             if namespace == b'bookmarks':
                 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
 
     # bookmark data were either already there or pulled in the bundle
     if pullop.remotebookmarks is not None:
         _pullbookmarks(pullop)
 
 
 def _pullbundle2extraprepare(pullop, kwargs):
     """hook function so that extensions can extend the getbundle call"""
 
 
 def _pullchangeset(pullop):
     """pull changeset from unbundle into the local repo"""
     # We delay the open of the transaction as late as possible so we
     # don't open transaction for nothing or you break future useful
     # rollback call
     if b'changegroup' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'changegroup')
     if not pullop.fetch:
         pullop.repo.ui.status(_(b"no changes found\n"))
         pullop.cgresult = 0
         return
     tr = pullop.gettransaction()
     if pullop.heads is None and list(pullop.common) == [pullop.repo.nullid]:
         pullop.repo.ui.status(_(b"requesting all changes\n"))
     elif pullop.heads is None and pullop.remote.capable(b'changegroupsubset'):
         # issue1320, avoid a race if remote changed after discovery
         pullop.heads = pullop.rheads
 
     if pullop.remote.capable(b'getbundle'):
         # TODO: get bundlecaps from remote
         cg = pullop.remote.getbundle(
             b'pull', common=pullop.common, heads=pullop.heads or pullop.rheads
         )
     elif pullop.heads is None:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand(
                 b'changegroup',
                 {
                     b'nodes': pullop.fetch,
                     b'source': b'pull',
                 },
             ).result()
 
     elif not pullop.remote.capable(b'changegroupsubset'):
         raise error.Abort(
             _(
                 b"partial pull cannot be done because "
                 b"other repository doesn't support "
                 b"changegroupsubset."
             )
         )
     else:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand(
                 b'changegroupsubset',
                 {
                     b'bases': pullop.fetch,
                     b'heads': pullop.heads,
                     b'source': b'pull',
                 },
             ).result()
 
     bundleop = bundle2.applybundle(
         pullop.repo,
         cg,
         tr,
         b'pull',
         pullop.remote.url(),
         remote=pullop.remote,
     )
     pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
 
 
 def _pullphase(pullop):
     # Get remote phases data from remote
     if b'phases' in pullop.stepsdone:
         return
     remotephases = listkeys(pullop.remote, b'phases')
     _pullapplyphases(pullop, remotephases)
 
 
 def _pullapplyphases(pullop, remotephases):
     """apply phase movement from observed remote state"""
     if b'phases' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'phases')
     publishing = bool(remotephases.get(b'publishing', False))
     if remotephases and not publishing:
         unfi = pullop.repo.unfiltered()
         to_rev = unfi.changelog.index.rev
         to_node = unfi.changelog.node
         pulledsubset_revs = [to_rev(n) for n in pullop.pulledsubset]
         # remote is new and non-publishing
         pheads_revs, _dr = phases.analyze_remote_phases(
             pullop.repo,
             pulledsubset_revs,
             remotephases,
         )
         pheads = [to_node(r) for r in pheads_revs]
         dheads = pullop.pulledsubset
     else:
         # Remote is old or publishing all common changesets
         # should be seen as public
         pheads = pullop.pulledsubset
         dheads = []
     unfi = pullop.repo.unfiltered()
     phase = unfi._phasecache.phase
     rev = unfi.changelog.index.get_rev
     public = phases.public
     draft = phases.draft
 
     # exclude changesets already public locally and update the others
     pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
     if pheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, public, pheads)
 
     # exclude changesets already draft locally and update the others
     dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
     if dheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, draft, dheads)
 
 
 def _pullbookmarks(pullop):
     """process the remote bookmark information to update the local one"""
     if b'bookmarks' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'bookmarks')
     repo = pullop.repo
     remotebookmarks = pullop.remotebookmarks
     bookmarks_mode = None
     if pullop.remote_path is not None:
         bookmarks_mode = pullop.remote_path.bookmarks_mode
     bookmod.updatefromremote(
         repo.ui,
         repo,
         remotebookmarks,
         pullop.remote.url(),
         pullop.gettransaction,
         explicit=pullop.explicitbookmarks,
         mode=bookmarks_mode,
     )
 
 
 def _pullobsolete(pullop):
     """utility function to pull obsolete markers from a remote
 
     The `gettransaction` is function that return the pull transaction, creating
     one if necessary. We return the transaction to inform the calling code that
     a new transaction have been created (when applicable).
 
     Exists mostly to allow overriding for experimentation purpose"""
     if b'obsmarkers' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'obsmarkers')
     tr = None
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         pullop.repo.ui.debug(b'fetching remote obsolete markers\n')
         remoteobs = listkeys(pullop.remote, b'obsolete')
         if b'dump0' in remoteobs:
             tr = pullop.gettransaction()
             markers = []
             for key in sorted(remoteobs, reverse=True):
                 if key.startswith(b'dump'):
                     data = util.b85decode(remoteobs[key])
                     version, newmarks = obsolete._readmarkers(data)
                     markers += newmarks
             if markers:
                 pullop.repo.obsstore.add(tr, markers)
             pullop.repo.invalidatevolatilesets()
     return tr
 
 
 def applynarrowacl(repo, kwargs):
     """Apply narrow fetch access control.
 
     This massages the named arguments for getbundle wire protocol commands
     so requested data is filtered through access control rules.
     """
     ui = repo.ui
     # TODO this assumes existence of HTTP and is a layering violation.
     username = ui.shortuser(ui.environ.get(b'REMOTE_USER') or ui.username())
     user_includes = ui.configlist(
         _NARROWACL_SECTION,
         username + b'.includes',
         ui.configlist(_NARROWACL_SECTION, b'default.includes'),
     )
     user_excludes = ui.configlist(
         _NARROWACL_SECTION,
         username + b'.excludes',
         ui.configlist(_NARROWACL_SECTION, b'default.excludes'),
     )
     if not user_includes:
         raise error.Abort(
             _(b"%s configuration for user %s is empty")
             % (_NARROWACL_SECTION, username)
         )
 
     user_includes = [
         b'path:.' if p == b'*' else b'path:' + p for p in user_includes
     ]
     user_excludes = [
         b'path:.' if p == b'*' else b'path:' + p for p in user_excludes
     ]
 
     req_includes = set(kwargs.get('includepats', []))
     req_excludes = set(kwargs.get('excludepats', []))
 
     req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
         req_includes, req_excludes, user_includes, user_excludes
     )
 
     if invalid_includes:
         raise error.Abort(
             _(b"The following includes are not accessible for %s: %s")
             % (username, stringutil.pprint(invalid_includes))
         )
 
     new_args = {}
     new_args.update(kwargs)
     new_args['narrow'] = True
     new_args['narrow_acl'] = True
     new_args['includepats'] = req_includes
     if req_excludes:
         new_args['excludepats'] = req_excludes
 
     return new_args
 
 
 def _computeellipsis(repo, common, heads, known, match, depth=None):
     """Compute the shape of a narrowed DAG.
 
     Args:
       repo: The repository we're transferring.
       common: The roots of the DAG range we're transferring.
               May be just [nullid], which means all ancestors of heads.
       heads: The heads of the DAG range we're transferring.
       match: The narrowmatcher that allows us to identify relevant changes.
       depth: If not None, only consider nodes to be full nodes if they are at
              most depth changesets away from one of heads.
 
     Returns:
       A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
 
         visitnodes: The list of nodes (either full or ellipsis) which
                     need to be sent to the client.
         relevant_nodes: The set of changelog nodes which change a file inside
                  the narrowspec. The client needs these as non-ellipsis nodes.
         ellipsisroots: A dict of {rev: parents} that is used in
                        narrowchangegroup to produce ellipsis nodes with the
                        correct parents.
     """
     cl = repo.changelog
     mfl = repo.manifestlog
 
     clrev = cl.rev
 
     commonrevs = {clrev(n) for n in common} | {nullrev}
     headsrevs = {clrev(n) for n in heads}
 
     if depth:
         revdepth = {h: 0 for h in headsrevs}
 
     ellipsisheads = collections.defaultdict(set)
     ellipsisroots = collections.defaultdict(set)
 
     def addroot(head, curchange):
         """Add a root to an ellipsis head, splitting heads with 3 roots."""
         ellipsisroots[head].add(curchange)
         # Recursively split ellipsis heads with 3 roots by finding the
         # roots' youngest common descendant which is an elided merge commit.
         # That descendant takes 2 of the 3 roots as its own, and becomes a
         # root of the head.
         while len(ellipsisroots[head]) > 2:
             child, roots = splithead(head)
             splitroots(head, child, roots)
             head = child  # Recurse in case we just added a 3rd root
 
     def splitroots(head, child, roots):
         ellipsisroots[head].difference_update(roots)
         ellipsisroots[head].add(child)
         ellipsisroots[child].update(roots)
         ellipsisroots[child].discard(child)
 
     def splithead(head):
         r1, r2, r3 = sorted(ellipsisroots[head])
         for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
             mid = repo.revs(
                 b'sort(merge() & %d::%d & %d::%d, -rev)', nr1, head, nr2, head
             )
             for j in mid:
                 if j == nr2:
                     return nr2, (nr1, nr2)
                 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
                     return j, (nr1, nr2)
         raise error.Abort(
             _(
                 b'Failed to split up ellipsis node! head: %d, '
                 b'roots: %d %d %d'
             )
             % (head, r1, r2, r3)
         )
 
     missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
     visit = reversed(missing)
     relevant_nodes = set()
     visitnodes = [cl.node(m) for m in missing]
     required = set(headsrevs) | known
     for rev in visit:
         clrev = cl.changelogrevision(rev)
         ps = [prev for prev in cl.parentrevs(rev) if prev != nullrev]
         if depth is not None:
             curdepth = revdepth[rev]
             for p in ps:
                 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
         needed = False
         shallow_enough = depth is None or revdepth[rev] <= depth
         if shallow_enough:
             curmf = mfl[clrev.manifest].read()
             if ps:
                 # We choose to not trust the changed files list in
                 # changesets because it's not always correct. TODO: could
                 # we trust it for the non-merge case?
                 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
                 needed = bool(curmf.diff(p1mf, match))
                 if not needed and len(ps) > 1:
                     # For merge changes, the list of changed files is not
                     # helpful, since we need to emit the merge if a file
                     # in the narrow spec has changed on either side of the
                     # merge. As a result, we do a manifest diff to check.
                     p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
                     needed = bool(curmf.diff(p2mf, match))
             else:
                 # For a root node, we need to include the node if any
                 # files in the node match the narrowspec.
                 needed = any(curmf.walk(match))
 
         if needed:
             for head in ellipsisheads[rev]:
                 addroot(head, rev)
             for p in ps:
                 required.add(p)
             relevant_nodes.add(cl.node(rev))
         else:
             if not ps:
                 ps = [nullrev]
             if rev in required:
                 for head in ellipsisheads[rev]:
                     addroot(head, rev)
                 for p in ps:
                     ellipsisheads[p].add(rev)
             else:
                 for p in ps:
                     ellipsisheads[p] |= ellipsisheads[rev]
 
     # add common changesets as roots of their reachable ellipsis heads
     for c in commonrevs:
         for head in ellipsisheads[c]:
             addroot(head, c)
     return visitnodes, relevant_nodes, ellipsisroots
 
 
 def caps20to10(repo, role):
     """return a set with appropriate options to use bundle20 during getbundle"""
     caps = {b'HG20'}
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
     caps.add(b'bundle2=' + urlreq.quote(capsblob))
     return caps
 
 
 # List of names of steps to perform for a bundle2 for getbundle, order matters.
 getbundle2partsorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 getbundle2partsmapping = {}
 
 
 def getbundle2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part for getbundle
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the getbundle2partsmapping dictionary directly."""
 
     def dec(func):
         assert stepname not in getbundle2partsmapping
         getbundle2partsmapping[stepname] = func
         if idx is None:
             getbundle2partsorder.append(stepname)
         else:
             getbundle2partsorder.insert(idx, stepname)
         return func
 
     return dec
 
 
 def bundle2requested(bundlecaps):
     if bundlecaps is not None:
         return any(cap.startswith(b'HG2') for cap in bundlecaps)
     return False
 
 
 def getbundlechunks(
     repo,
     source,
     heads=None,
     common=None,
     bundlecaps=None,
     remote_sidedata=None,
     **kwargs
 ):
     """Return chunks constituting a bundle's raw data.
 
     Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
     passed.
 
     Returns a 2-tuple of a dict with metadata about the generated bundle
     and an iterator over raw chunks (of varying sizes).
     """
     kwargs = pycompat.byteskwargs(kwargs)
     info = {}
     usebundle2 = bundle2requested(bundlecaps)
     # bundle10 case
     if not usebundle2:
         if bundlecaps and not kwargs.get(b'cg', True):
             raise ValueError(
                 _(b'request for bundle10 must include changegroup')
             )
 
         if kwargs:
             raise ValueError(
                 _(b'unsupported getbundle arguments: %s')
                 % b', '.join(sorted(kwargs.keys()))
             )
         outgoing = _computeoutgoing(repo, heads, common)
         info[b'bundleversion'] = 1
         return (
             info,
             changegroup.makestream(
                 repo,
                 outgoing,
                 b'01',
                 source,
                 bundlecaps=bundlecaps,
                 remote_sidedata=remote_sidedata,
             ),
         )
 
     # bundle20 case
     info[b'bundleversion'] = 2
     b2caps = {}
     for bcaps in bundlecaps:
         if bcaps.startswith(b'bundle2='):
             blob = urlreq.unquote(bcaps[len(b'bundle2=') :])
             b2caps.update(bundle2.decodecaps(blob))
     bundler = bundle2.bundle20(repo.ui, b2caps)
 
     kwargs[b'heads'] = heads
     kwargs[b'common'] = common
 
     for name in getbundle2partsorder:
         func = getbundle2partsmapping[name]
         func(
             bundler,
             repo,
             source,
             bundlecaps=bundlecaps,
             b2caps=b2caps,
             remote_sidedata=remote_sidedata,
             **pycompat.strkwargs(kwargs)
         )
 
     info[b'prefercompressed'] = bundler.prefercompressed
 
     return info, bundler.getchunks()
 
 
 @getbundle2partsgenerator(b'stream')
 def _getbundlestream2(bundler, repo, *args, **kwargs):
     return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
 
 
 @getbundle2partsgenerator(b'changegroup')
 def _getbundlechangegrouppart(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     remote_sidedata=None,
     **kwargs
 ):
     """add a changegroup part to the requested bundle"""
     if not kwargs.get('cg', True) or not b2caps:
         return
 
     version = b'01'
     cgversions = b2caps.get(b'changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [
             v
             for v in cgversions
             if v in changegroup.supportedoutgoingversions(repo)
         ]
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
 
     outgoing = _computeoutgoing(repo, heads, common)
     if not outgoing.missing:
         return
 
     if kwargs.get('narrow', False):
         include = sorted(filter(bool, kwargs.get('includepats', [])))
         exclude = sorted(filter(bool, kwargs.get('excludepats', [])))
         matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
     else:
         matcher = None
 
     cgstream = changegroup.makestream(
         repo,
         outgoing,
         version,
         source,
         bundlecaps=bundlecaps,
         matcher=matcher,
         remote_sidedata=remote_sidedata,
     )
 
     part = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
         part.addparam(b'version', version)
 
     part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False)
 
     if scmutil.istreemanifest(repo):
         part.addparam(b'treemanifest', b'1')
 
     if repository.REPO_FEATURE_SIDE_DATA in repo.features:
         part.addparam(b'exp-sidedata', b'1')
         sidedata = bundle2.format_remote_wanted_sidedata(repo)
         part.addparam(b'exp-wanted-sidedata', sidedata)
 
     if (
         kwargs.get('narrow', False)
         and kwargs.get('narrow_acl', False)
         and (include or exclude)
     ):
         # this is mandatory because otherwise ACL clients won't work
         narrowspecpart = bundler.newpart(b'Narrow:responsespec')
         narrowspecpart.data = b'%s\0%s' % (
             b'\n'.join(include),
             b'\n'.join(exclude),
         )
 
 
 @getbundle2partsgenerator(b'bookmarks')
 def _getbundlebookmarkpart(
     bundler, repo, source, bundlecaps=None, b2caps=None, **kwargs
 ):
     """add a bookmark part to the requested bundle"""
     if not kwargs.get('bookmarks', False):
         return
     if not b2caps or b'bookmarks' not in b2caps:
         raise error.Abort(_(b'no common bookmarks exchange method'))
     books = bookmod.listbinbookmarks(repo)
     data = bookmod.binaryencode(repo, books)
     if data:
         bundler.newpart(b'bookmarks', data=data)
 
 
 @getbundle2partsgenerator(b'listkeys')
 def _getbundlelistkeysparts(
     bundler, repo, source, bundlecaps=None, b2caps=None, **kwargs
 ):
     """add parts containing listkeys namespaces to the requested bundle"""
     listkeys = kwargs.get('listkeys', ())
     for namespace in listkeys:
         part = bundler.newpart(b'listkeys')
         part.addparam(b'namespace', namespace)
         keys = repo.listkeys(namespace).items()
         part.data = pushkey.encodekeys(keys)
 
 
 @getbundle2partsgenerator(b'obsmarkers')
 def _getbundleobsmarkerpart(
     bundler, repo, source, bundlecaps=None, b2caps=None, heads=None, **kwargs
 ):
     """add an obsolescence markers part to the requested bundle"""
     if kwargs.get('obsmarkers', False):
         if heads is None:
             heads = repo.heads()
         subset = [c.node() for c in repo.set(b'::%ln', heads)]
         markers = repo.obsstore.relevantmarkers(subset)
         markers = obsutil.sortedmarkers(markers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 
 @getbundle2partsgenerator(b'phases')
 def _getbundlephasespart(
     bundler, repo, source, bundlecaps=None, b2caps=None, heads=None, **kwargs
 ):
     """add phase heads part to the requested bundle"""
     if kwargs.get('phases', False):
         if not b2caps or b'heads' not in b2caps.get(b'phases'):
             raise error.Abort(_(b'no common phases exchange method'))
         if heads is None:
             heads = repo.heads()
 
         headsbyphase = collections.defaultdict(set)
         if repo.publishing():
             headsbyphase[phases.public] = heads
         else:
             # find the appropriate heads to move
 
             phase = repo._phasecache.phase
             node = repo.changelog.node
             rev = repo.changelog.rev
             for h in heads:
                 headsbyphase[phase(repo, rev(h))].add(h)
             seenphases = list(headsbyphase.keys())
 
             # We do not handle anything but public and draft phase for now)
             if seenphases:
                 assert max(seenphases) <= phases.draft
 
             # if client is pulling non-public changesets, we need to find
             # intermediate public heads.
             draftheads = headsbyphase.get(phases.draft, set())
             if draftheads:
                 publicheads = headsbyphase.get(phases.public, set())
 
                 revset = b'heads(only(%ln, %ln) and public())'
                 extraheads = repo.revs(revset, draftheads, publicheads)
                 for r in extraheads:
                     headsbyphase[phases.public].add(node(r))
 
         # transform data in a format used by the encoding function
         phasemapping = {
             phase: sorted(headsbyphase[phase]) for phase in phases.allphases
         }
 
         # generate the actual part
         phasedata = phases.binaryencode(phasemapping)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 @getbundle2partsgenerator(b'hgtagsfnodes')
 def _getbundletagsfnodes(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """Transfer the .hgtags filenodes mapping.
 
     Only values for heads in this bundle will be transferred.
 
     The part data consists of pairs of 20 byte changeset node and .hgtags
     filenodes raw values.
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     if not b2caps or not (kwargs.get('cg', True) and b'hgtagsfnodes' in b2caps):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addparttagsfnodescache(repo, bundler, outgoing)
 
 
 @getbundle2partsgenerator(b'cache:rev-branch-cache')
 def _getbundlerevbranchcache(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """Transfer the rev-branch-cache mapping
 
     The payload is a series of data related to each branch
 
     1) branch name length
     2) number of open heads
     3) number of closed heads
     4) open heads nodes
     5) closed heads nodes
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     # - narrow bundle isn't in play (not currently compatible).
     if (
         not kwargs.get('cg', True)
         or not b2caps
         or b'rev-branch-cache' not in b2caps
         or kwargs.get('narrow', False)
         or repo.ui.has_section(_NARROWACL_SECTION)
     ):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addpartrevbranchcache(repo, bundler, outgoing)
 
 
 def check_heads(repo, their_heads, context):
     """check if the heads of a repo have been modified
 
     Used by peer for unbundling.
     """
     heads = repo.heads()
     heads_hash = hashutil.sha1(b''.join(sorted(heads))).digest()
     if not (
         their_heads == [b'force']
         or their_heads == heads
         or their_heads == [b'hashed', heads_hash]
     ):
         # someone else committed/pushed/unbundled while we
         # were transferring data
         raise error.PushRaced(
             b'repository changed while %s - please try again' % context
         )
 
 
 def unbundle(repo, cg, heads, source, url):
     """Apply a bundle to a repo.
 
     this function makes sure the repo is locked during the application and have
     mechanism to check that no push race occurred between the creation of the
     bundle and its application.
 
     If the push was raced as PushRaced exception is raised."""
     r = 0
     # need a transaction when processing a bundle2 stream
     # [wlock, lock, tr] - needs to be an array so nested functions can modify it
     lockandtr = [None, None, None]
     recordout = None
     # quick fix for output mismatch with bundle2 in 3.4
     captureoutput = repo.ui.configbool(
         b'experimental', b'bundle2-output-capture'
     )
     if url.startswith(b'remote:http:') or url.startswith(b'remote:https:'):
         captureoutput = True
     try:
         # note: outside bundle1, 'heads' is expected to be empty and this
         # 'check_heads' call wil be a no-op
         check_heads(repo, heads, b'uploading changes')
         # push can proceed
         if not isinstance(cg, bundle2.unbundle20):
             # legacy case: bundle1 (changegroup 01)
             txnname = b"\n".join([source, urlutil.hidepassword(url)])
             with repo.lock(), repo.transaction(txnname) as tr:
                 op = bundle2.applybundle(repo, cg, tr, source, url)
                 r = bundle2.combinechangegroupresults(op)
         else:
             r = None
             try:
 
                 def gettransaction():
                     if not lockandtr[2]:
                         if not bookmod.bookmarksinstore(repo):
                             lockandtr[0] = repo.wlock()
                         lockandtr[1] = repo.lock()
                         lockandtr[2] = repo.transaction(source)
                         lockandtr[2].hookargs[b'source'] = source
                         lockandtr[2].hookargs[b'url'] = url
                         lockandtr[2].hookargs[b'bundle2'] = b'1'
                     return lockandtr[2]
 
                 # Do greedy locking by default until we're satisfied with lazy
                 # locking.
                 if not repo.ui.configbool(
                     b'experimental', b'bundle2lazylocking'
                 ):
                     gettransaction()
 
                 op = bundle2.bundleoperation(
                     repo,
                     gettransaction,
                     captureoutput=captureoutput,
                     source=b'push',
                 )
                 try:
                     op = bundle2.processbundle(repo, cg, op=op)
                 finally:
                     r = op.reply
                     if captureoutput and r is not None:
                         repo.ui.pushbuffer(error=True, subproc=True)
 
                         def recordout(output):
                             r.newpart(b'output', data=output, mandatory=False)
 
                 if lockandtr[2] is not None:
                     lockandtr[2].close()
             except BaseException as exc:
                 exc.duringunbundle2 = True
                 if captureoutput and r is not None:
                     parts = exc._bundle2salvagedoutput = r.salvageoutput()
 
                     def recordout(output):
                         part = bundle2.bundlepart(
                             b'output', data=output, mandatory=False
                         )
                         parts.append(part)
 
                 raise
     finally:
         lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
         if recordout is not None:
             recordout(repo.ui.popbuffer())
     return r
 
 
 def _maybeapplyclonebundle(pullop):
     """Apply a clone bundle from a remote, if possible."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     if not repo.ui.configbool(b'ui', b'clonebundles'):
         return
 
     # Only run if local repo is empty.
     if len(repo):
         return
 
     if pullop.heads:
         return
 
     if not remote.capable(b'clonebundles'):
         return
 
     with remote.commandexecutor() as e:
         res = e.callcommand(b'clonebundles', {}).result()
 
     # If we call the wire protocol command, that's good enough to record the
     # attempt.
     pullop.clonebundleattempted = True
 
     entries = bundlecaches.parseclonebundlesmanifest(repo, res)
     if not entries:
         repo.ui.note(
             _(
                 b'no clone bundles available on remote; '
                 b'falling back to regular clone\n'
             )
         )
         return
 
     entries = bundlecaches.filterclonebundleentries(
         repo, entries, streamclonerequested=pullop.streamclonerequested
     )
 
     if not entries:
         # There is a thundering herd concern here. However, if a server
         # operator doesn't advertise bundles appropriate for its clients,
         # they deserve what's coming. Furthermore, from a client's
         # perspective, no automatic fallback would mean not being able to
         # clone!
         repo.ui.warn(
             _(
                 b'no compatible clone bundles available on server; '
                 b'falling back to regular clone\n'
             )
         )
         repo.ui.warn(
             _(b'(you may want to report this to the server operator)\n')
         )
         return
 
     entries = bundlecaches.sortclonebundleentries(repo.ui, entries)
 
     url = entries[0][b'URL']
     repo.ui.status(_(b'applying clone bundle from %s\n') % url)
     if trypullbundlefromurl(repo.ui, repo, url, remote):
         repo.ui.status(_(b'finished applying clone bundle\n'))
     # Bundle failed.
     #
     # We abort by default to avoid the thundering herd of
     # clients flooding a server that was expecting expensive
     # clone load to be offloaded.
     elif repo.ui.configbool(b'ui', b'clonebundlefallback'):
         repo.ui.warn(_(b'falling back to normal clone\n'))
     else:
         raise error.Abort(
             _(b'error applying bundle'),
             hint=_(
                 b'if this error persists, consider contacting '
                 b'the server operator or disable clone '
                 b'bundles via '
                 b'"--config ui.clonebundles=false"'
             ),
         )
 
 
 def inline_clone_bundle_open(ui, url, peer):
     if not peer:
         raise error.Abort(_(b'no remote repository supplied for %s' % url))
     clonebundleid = url[len(bundlecaches.CLONEBUNDLESCHEME) :]
     peerclonebundle = peer.get_cached_bundle_inline(clonebundleid)
     return util.chunkbuffer(peerclonebundle)
 
 
 def trypullbundlefromurl(ui, repo, url, peer):
     """Attempt to apply a bundle from a URL."""
     with repo.lock(), repo.transaction(b'bundleurl') as tr:
         try:
             if url.startswith(bundlecaches.CLONEBUNDLESCHEME):
                 fh = inline_clone_bundle_open(ui, url, peer)
             else:
                 fh = urlmod.open(ui, url)
             cg = readbundle(ui, fh, b'stream')
 
             if isinstance(cg, streamclone.streamcloneapplier):
                 cg.apply(repo)
             else:
                 bundle2.applybundle(repo, cg, tr, b'clonebundles', url)
             return True
         except urlerr.httperror as e:
             ui.warn(
                 _(b'HTTP error fetching bundle: %s\n')
                 % stringutil.forcebytestr(e)
             )
         except urlerr.urlerror as e:
             ui.warn(
                 _(b'error fetching bundle: %s\n')
                 % stringutil.forcebytestr(e.reason)
             )
 
         return False

mercurial/match.py

@@ -1,1703 +1,1706 b''
 # match.py - filename matching
 #
 #  Copyright 2008, 2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import bisect
 import copy
 import itertools
 import os
 import re
 
 from .i18n import _
 from .pycompat import open
 from . import (
     encoding,
     error,
     pathutil,
     policy,
     pycompat,
     util,
 )
 from .utils import stringutil
 
 rustmod = policy.importrust('dirstate')
 
 allpatternkinds = (
     b're',
     b'glob',
     b'path',
     b'filepath',
     b'relglob',
     b'relpath',
     b'relre',
     b'rootglob',
     b'listfile',
     b'listfile0',
     b'set',
     b'include',
     b'subinclude',
     b'rootfilesin',
 )
 cwdrelativepatternkinds = (b'relpath', b'glob')
 
 propertycache = util.propertycache
 
 
 def _rematcher(regex):
     """compile the regexp with the best available regexp engine and return a
     matcher function"""
     m = util.re.compile(regex)
     try:
         # slightly faster, provided by facebook's re2 bindings
         return m.test_match
     except AttributeError:
         return m.match
 
 
 def _expandsets(cwd, kindpats, ctx=None, listsubrepos=False, badfn=None):
     '''Returns the kindpats list with the 'set' patterns expanded to matchers'''
     matchers = []
     other = []
 
     for kind, pat, source in kindpats:
         if kind == b'set':
             if ctx is None:
                 raise error.ProgrammingError(
                     b"fileset expression with no context"
                 )
             matchers.append(ctx.matchfileset(cwd, pat, badfn=badfn))
 
             if listsubrepos:
                 for subpath in ctx.substate:
                     sm = ctx.sub(subpath).matchfileset(cwd, pat, badfn=badfn)
                     pm = prefixdirmatcher(subpath, sm, badfn=badfn)
                     matchers.append(pm)
 
             continue
         other.append((kind, pat, source))
     return matchers, other
 
 
 def _expandsubinclude(kindpats, root):
     """Returns the list of subinclude matcher args and the kindpats without the
     subincludes in it."""
     relmatchers = []
     other = []
 
     for kind, pat, source in kindpats:
         if kind == b'subinclude':
             sourceroot = pathutil.dirname(util.normpath(source))
             pat = util.pconvert(pat)
             path = pathutil.join(sourceroot, pat)
 
             newroot = pathutil.dirname(path)
             matcherargs = (newroot, b'', [], [b'include:%s' % path])
 
             prefix = pathutil.canonpath(root, root, newroot)
             if prefix:
                 prefix += b'/'
             relmatchers.append((prefix, matcherargs))
         else:
             other.append((kind, pat, source))
 
     return relmatchers, other
 
 
 def _kindpatsalwaysmatch(kindpats):
     """Checks whether the kindspats match everything, as e.g.
     'relpath:.' does.
     """
     for kind, pat, source in kindpats:
         if pat != b'' or kind not in [b'relpath', b'glob']:
             return False
     return True
 
 
 def _buildkindpatsmatcher(
     matchercls,
     root,
     cwd,
     kindpats,
     ctx=None,
     listsubrepos=False,
     badfn=None,
 ):
     matchers = []
     fms, kindpats = _expandsets(
         cwd,
         kindpats,
         ctx=ctx,
         listsubrepos=listsubrepos,
         badfn=badfn,
     )
     if kindpats:
         m = matchercls(root, kindpats, badfn=badfn)
         matchers.append(m)
     if fms:
         matchers.extend(fms)
     if not matchers:
         return nevermatcher(badfn=badfn)
     if len(matchers) == 1:
         return matchers[0]
     return unionmatcher(matchers)
 
 
 def match(
     root,
     cwd,
     patterns=None,
     include=None,
     exclude=None,
     default=b'glob',
     auditor=None,
     ctx=None,
     listsubrepos=False,
     warn=None,
     badfn=None,
     icasefs=False,
 ):
     r"""build an object to match a set of file patterns
 
     arguments:
     root - the canonical root of the tree you're matching against
     cwd - the current working directory, if relevant
     patterns - patterns to find
     include - patterns to include (unless they are excluded)
     exclude - patterns to exclude (even if they are included)
     default - if a pattern in patterns has no explicit type, assume this one
     auditor - optional path auditor
     ctx - optional changecontext
     listsubrepos - if True, recurse into subrepositories
     warn - optional function used for printing warnings
     badfn - optional bad() callback for this matcher instead of the default
     icasefs - make a matcher for wdir on case insensitive filesystems, which
         normalizes the given patterns to the case in the filesystem
 
     a pattern is one of:
     'glob:<glob>' - a glob relative to cwd
     're:<regexp>' - a regular expression
     'path:<path>' - a path relative to repository root, which is matched
                     recursively
     'filepath:<path>' - an exact path to a single file, relative to the
                         repository root
     'rootfilesin:<path>' - a path relative to repository root, which is
                     matched non-recursively (will not match subdirectories)
     'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs)
     'relpath:<path>' - a path relative to cwd
     'relre:<regexp>' - a regexp that needn't match the start of a name
     'set:<fileset>' - a fileset expression
     'include:<path>' - a file of patterns to read and include
     'subinclude:<path>' - a file of patterns to match against files under
                           the same directory
     '<something>' - a pattern of the specified default type
 
     >>> def _match(root, *args, **kwargs):
     ...     return match(util.localpath(root), *args, **kwargs)
 
     Usually a patternmatcher is returned:
     >>> _match(b'/foo', b'.', [br're:.*\.c$', b'path:foo/a', b'*.py'])
     <patternmatcher patterns='[^/]*\\.py$|foo/a(?:/|$)|.*\\.c$'>
 
     Combining 'patterns' with 'include' (resp. 'exclude') gives an
     intersectionmatcher (resp. a differencematcher):
     >>> type(_match(b'/foo', b'.', [br're:.*\.c$'], include=[b'path:lib']))
     <class 'mercurial.match.intersectionmatcher'>
     >>> type(_match(b'/foo', b'.', [br're:.*\.c$'], exclude=[b'path:build']))
     <class 'mercurial.match.differencematcher'>
 
     Notice that, if 'patterns' is empty, an alwaysmatcher is returned:
     >>> _match(b'/foo', b'.', [])
     <alwaysmatcher>
 
     The 'default' argument determines which kind of pattern is assumed if a
     pattern has no prefix:
     >>> _match(b'/foo', b'.', [br'.*\.c$'], default=b're')
     <patternmatcher patterns='.*\\.c$'>
     >>> _match(b'/foo', b'.', [b'main.py'], default=b'relpath')
     <patternmatcher patterns='main\\.py(?:/|$)'>
     >>> _match(b'/foo', b'.', [b'main.py'], default=b're')
     <patternmatcher patterns='main.py'>
 
     The primary use of matchers is to check whether a value (usually a file
     name) matches againset one of the patterns given at initialization. There
     are two ways of doing this check.
 
     >>> m = _match(b'/foo', b'', [br're:.*\.c$', b'relpath:a'])
 
     1. Calling the matcher with a file name returns True if any pattern
     matches that file name:
     >>> m(b'a')
     True
     >>> m(b'main.c')
     True
     >>> m(b'test.py')
     False
 
     2. Using the exact() method only returns True if the file name matches one
     of the exact patterns (i.e. not re: or glob: patterns):
     >>> m.exact(b'a')
     True
     >>> m.exact(b'main.c')
     False
     """
     assert os.path.isabs(root)
     cwd = os.path.join(root, util.localpath(cwd))
     normalize = _donormalize
     if icasefs:
         dirstate = ctx.repo().dirstate
         dsnormalize = dirstate.normalize
 
         def normalize(patterns, default, root, cwd, auditor, warn):
             kp = _donormalize(patterns, default, root, cwd, auditor, warn)
             kindpats = []
             for kind, pats, source in kp:
                 if kind not in (b're', b'relre'):  # regex can't be normalized
                     p = pats
                     pats = dsnormalize(pats)
 
                     # Preserve the original to handle a case only rename.
                     if p != pats and p in dirstate:
                         kindpats.append((kind, p, source))
 
                 kindpats.append((kind, pats, source))
             return kindpats
 
     if patterns:
         kindpats = normalize(patterns, default, root, cwd, auditor, warn)
         if _kindpatsalwaysmatch(kindpats):
             m = alwaysmatcher(badfn)
         else:
             m = _buildkindpatsmatcher(
                 patternmatcher,
                 root,
                 cwd,
                 kindpats,
                 ctx=ctx,
                 listsubrepos=listsubrepos,
                 badfn=badfn,
             )
     else:
         # It's a little strange that no patterns means to match everything.
         # Consider changing this to match nothing (probably using nevermatcher).
         m = alwaysmatcher(badfn)
 
     if include:
         kindpats = normalize(include, b'glob', root, cwd, auditor, warn)
         im = _buildkindpatsmatcher(
             includematcher,
             root,
             cwd,
             kindpats,
             ctx=ctx,
             listsubrepos=listsubrepos,
             badfn=None,
         )
         m = intersectmatchers(m, im)
     if exclude:
         kindpats = normalize(exclude, b'glob', root, cwd, auditor, warn)
         em = _buildkindpatsmatcher(
             includematcher,
             root,
             cwd,
             kindpats,
             ctx=ctx,
             listsubrepos=listsubrepos,
             badfn=None,
         )
         m = differencematcher(m, em)
     return m
 
 
 def exact(files, badfn=None):
     return exactmatcher(files, badfn=badfn)
 
 
 def always(badfn=None):
     return alwaysmatcher(badfn)
 
 
 def never(badfn=None):
     return nevermatcher(badfn)
 
 
 def badmatch(match, badfn):
     """Make a copy of the given matcher, replacing its bad method with the given
     one.
     """
     m = copy.copy(match)
     m.bad = badfn
     return m
 
 
 def _donormalize(patterns, default, root, cwd, auditor=None, warn=None):
     """Convert 'kind:pat' from the patterns list to tuples with kind and
     normalized and rooted patterns and with listfiles expanded."""
     kindpats = []
     kinds_to_normalize = (
         b'relglob',
         b'path',
         b'filepath',
         b'rootfilesin',
         b'rootglob',
     )
 
     for kind, pat in [_patsplit(p, default) for p in patterns]:
         if kind in cwdrelativepatternkinds:
             pat = pathutil.canonpath(root, cwd, pat, auditor=auditor)
         elif kind in kinds_to_normalize:
             pat = util.normpath(pat)
         elif kind in (b'listfile', b'listfile0'):
             try:
                 files = util.readfile(pat)
                 if kind == b'listfile0':
                     files = files.split(b'\0')
                 else:
                     files = files.splitlines()
                 files = [f for f in files if f]
             except EnvironmentError:
                 raise error.Abort(_(b"unable to read file list (%s)") % pat)
             for k, p, source in _donormalize(
                 files, default, root, cwd, auditor, warn
             ):
                 kindpats.append((k, p, pat))
             continue
         elif kind == b'include':
             try:
                 fullpath = os.path.join(root, util.localpath(pat))
                 includepats = readpatternfile(fullpath, warn)
                 for k, p, source in _donormalize(
                     includepats, default, root, cwd, auditor, warn
                 ):
                     kindpats.append((k, p, source or pat))
             except error.Abort as inst:
                 raise error.Abort(
                     b'%s: %s'
                     % (
                         pat,
                         inst.message,
                     )
                 )
             except IOError as inst:
                 if warn:
                     warn(
                         _(b"skipping unreadable pattern file '%s': %s\n")
                         % (pat, stringutil.forcebytestr(inst.strerror))
                     )
             continue
         # else: re or relre - which cannot be normalized
         kindpats.append((kind, pat, b''))
     return kindpats
 
 
 class basematcher:
     def __init__(self, badfn=None):
         if badfn is not None:
             self.bad = badfn
 
     def __call__(self, fn):
         return self.matchfn(fn)
 
     # Callbacks related to how the matcher is used by dirstate.walk.
     # Subscribers to these events must monkeypatch the matcher object.
     def bad(self, f, msg):
         """Callback from dirstate.walk for each explicit file that can't be
         found/accessed, with an error message."""
 
     # If an traversedir is set, it will be called when a directory discovered
     # by recursive traversal is visited.
     traversedir = None
 
     @propertycache
     def _files(self):
         return []
 
     def files(self):
         """Explicitly listed files or patterns or roots:
         if no patterns or .always(): empty list,
         if exact: list exact files,
         if not .anypats(): list all files and dirs,
         else: optimal roots"""
         return self._files
 
     @propertycache
     def _fileset(self):
         return set(self._files)
 
     def exact(self, f):
         '''Returns True if f is in .files().'''
         return f in self._fileset
 
     def matchfn(self, f):
         return False
 
     def visitdir(self, dir):
         """Decides whether a directory should be visited based on whether it
         has potential matches in it or one of its subdirectories. This is
         based on the match's primary, included, and excluded patterns.
 
         Returns the string 'all' if the given directory and all subdirectories
         should be visited. Otherwise returns True or False indicating whether
         the given directory should be visited.
         """
         return True
 
     def visitchildrenset(self, dir):
         """Decides whether a directory should be visited based on whether it
         has potential matches in it or one of its subdirectories, and
         potentially lists which subdirectories of that directory should be
         visited. This is based on the match's primary, included, and excluded
         patterns.
 
         This function is very similar to 'visitdir', and the following mapping
         can be applied:
 
              visitdir | visitchildrenlist
             ----------+-------------------
              False    | set()
              'all'    | 'all'
              True     | 'this' OR non-empty set of subdirs -or files- to visit
 
         Example:
           Assume matchers ['path:foo/bar', 'rootfilesin:qux'], we would return
           the following values (assuming the implementation of visitchildrenset
           is capable of recognizing this; some implementations are not).
 
           '' -> {'foo', 'qux'}
           'baz' -> set()
           'foo' -> {'bar'}
           # Ideally this would be 'all', but since the prefix nature of matchers
           # is applied to the entire matcher, we have to downgrade this to
           # 'this' due to the non-prefix 'rootfilesin'-kind matcher being mixed
           # in.
           'foo/bar' -> 'this'
           'qux' -> 'this'
 
         Important:
           Most matchers do not know if they're representing files or
           directories. They see ['path:dir/f'] and don't know whether 'f' is a
           file or a directory, so visitchildrenset('dir') for most matchers will
           return {'f'}, but if the matcher knows it's a file (like exactmatcher
           does), it may return 'this'. Do not rely on the return being a set
           indicating that there are no files in this dir to investigate (or
           equivalently that if there are files to investigate in 'dir' that it
           will always return 'this').
         """
         return b'this'
 
     def always(self):
         """Matcher will match everything and .files() will be empty --
         optimization might be possible."""
         return False
 
     def isexact(self):
         """Matcher will match exactly the list of files in .files() --
         optimization might be possible."""
         return False
 
     def prefix(self):
         """Matcher will match the paths in .files() recursively --
         optimization might be possible."""
         return False
 
     def anypats(self):
         """None of .always(), .isexact(), and .prefix() is true --
         optimizations will be difficult."""
         return not self.always() and not self.isexact() and not self.prefix()
 
 
 class alwaysmatcher(basematcher):
     '''Matches everything.'''
 
     def __init__(self, badfn=None):
         super(alwaysmatcher, self).__init__(badfn)
 
     def always(self):
         return True
 
     def matchfn(self, f):
         return True
 
     def visitdir(self, dir):
         return b'all'
 
     def visitchildrenset(self, dir):
         return b'all'
 
     def __repr__(self):
         return r'<alwaysmatcher>'
 
 
 class nevermatcher(basematcher):
     '''Matches nothing.'''
 
     def __init__(self, badfn=None):
         super(nevermatcher, self).__init__(badfn)
 
     # It's a little weird to say that the nevermatcher is an exact matcher
     # or a prefix matcher, but it seems to make sense to let callers take
     # fast paths based on either. There will be no exact matches, nor any
     # prefixes (files() returns []), so fast paths iterating over them should
     # be efficient (and correct).
     def isexact(self):
         return True
 
     def prefix(self):
         return True
 
     def visitdir(self, dir):
         return False
 
     def visitchildrenset(self, dir):
         return set()
 
     def __repr__(self):
         return r'<nevermatcher>'
 
 
 class predicatematcher(basematcher):
     """A matcher adapter for a simple boolean function"""
 
     def __init__(self, predfn, predrepr=None, badfn=None):
         super(predicatematcher, self).__init__(badfn)
         self.matchfn = predfn
         self._predrepr = predrepr
 
     @encoding.strmethod
     def __repr__(self):
         s = stringutil.buildrepr(self._predrepr) or pycompat.byterepr(
             self.matchfn
         )
         return b'<predicatenmatcher pred=%s>' % s
 
 
 def path_or_parents_in_set(path, prefix_set):
     """Returns True if `path` (or any parent of `path`) is in `prefix_set`."""
     l = len(prefix_set)
     if l == 0:
         return False
     if path in prefix_set:
         return True
     # If there's more than 5 paths in prefix_set, it's *probably* quicker to
     # "walk up" the directory hierarchy instead, with the assumption that most
     # directory hierarchies are relatively shallow and hash lookup is cheap.
     if l > 5:
         return any(
             parentdir in prefix_set for parentdir in pathutil.finddirs(path)
         )
 
     # FIXME: Ideally we'd never get to this point if this is the case - we'd
     # recognize ourselves as an 'always' matcher and skip this.
     if b'' in prefix_set:
         return True
 
     sl = ord(b'/')
 
     # We already checked that path isn't in prefix_set exactly, so
     # `path[len(pf)] should never raise IndexError.
     return any(path.startswith(pf) and path[len(pf)] == sl for pf in prefix_set)
 
 
 class patternmatcher(basematcher):
     r"""Matches a set of (kind, pat, source) against a 'root' directory.
 
     >>> kindpats = [
     ...     (b're', br'.*\.c$', b''),
     ...     (b'path', b'foo/a', b''),
     ...     (b'relpath', b'b', b''),
     ...     (b'glob', b'*.h', b''),
     ... ]
     >>> m = patternmatcher(b'foo', kindpats)
     >>> m(b'main.c')  # matches re:.*\.c$
     True
     >>> m(b'b.txt')
     False
     >>> m(b'foo/a')  # matches path:foo/a
     True
     >>> m(b'a')  # does not match path:b, since 'root' is 'foo'
     False
     >>> m(b'b')  # matches relpath:b, since 'root' is 'foo'
     True
     >>> m(b'lib.h')  # matches glob:*.h
     True
 
     >>> m.files()
     [b'', b'foo/a', b'', b'b']
     >>> m.exact(b'foo/a')
     True
     >>> m.exact(b'b')
     True
     >>> m.exact(b'lib.h')  # exact matches are for (rel)path kinds
     False
     """
 
     def __init__(self, root, kindpats, badfn=None):
         super(patternmatcher, self).__init__(badfn)
         kindpats.sort()
 
+        roots, dirs, parents = _rootsdirsandparents(kindpats)
         self._files = _explicitfiles(kindpats)
+        self._dirs_explicit = set(dirs)
+        self._dirs = parents
         self._prefix = _prefix(kindpats)
         self._pats, self._matchfn = _buildmatch(kindpats, b'$', root)
 
     def matchfn(self, fn):
         if fn in self._fileset:
             return True
         return self._matchfn(fn)
 
-    @propertycache
-    def _dirs(self):
-        return set(pathutil.dirs(self._fileset))
-
     def visitdir(self, dir):
         if self._prefix and dir in self._fileset:
             return b'all'
-        return dir in self._dirs or path_or_parents_in_set(dir, self._fileset)
+        return (
+            dir in self._dirs
+            or path_or_parents_in_set(dir, self._fileset)
+            or path_or_parents_in_set(dir, self._dirs_explicit)
+        )
 
     def visitchildrenset(self, dir):
         ret = self.visitdir(dir)
         if ret is True:
             return b'this'
         elif not ret:
             return set()
         assert ret == b'all'
         return b'all'
 
     def prefix(self):
         return self._prefix
 
     @encoding.strmethod
     def __repr__(self):
         return b'<patternmatcher patterns=%r>' % pycompat.bytestr(self._pats)
 
 
 # This is basically a reimplementation of pathutil.dirs that stores the
 # children instead of just a count of them, plus a small optional optimization
 # to avoid some directories we don't need.
 class _dirchildren:
     def __init__(self, paths, onlyinclude=None):
         self._dirs = {}
         self._onlyinclude = onlyinclude or []
         addpath = self.addpath
         for f in paths:
             addpath(f)
 
     def addpath(self, path):
         if path == b'':
             return
         dirs = self._dirs
         findsplitdirs = _dirchildren._findsplitdirs
         for d, b in findsplitdirs(path):
             if d not in self._onlyinclude:
                 continue
             dirs.setdefault(d, set()).add(b)
 
     @staticmethod
     def _findsplitdirs(path):
         # yields (dirname, basename) tuples, walking back to the root.  This is
         # very similar to pathutil.finddirs, except:
         #  - produces a (dirname, basename) tuple, not just 'dirname'
         # Unlike manifest._splittopdir, this does not suffix `dirname` with a
         # slash.
         oldpos = len(path)
         pos = path.rfind(b'/')
         while pos != -1:
             yield path[:pos], path[pos + 1 : oldpos]
             oldpos = pos
             pos = path.rfind(b'/', 0, pos)
         yield b'', path[:oldpos]
 
     def get(self, path):
         return self._dirs.get(path, set())
 
 
 class includematcher(basematcher):
     def __init__(self, root, kindpats, badfn=None):
         super(includematcher, self).__init__(badfn)
         if rustmod is not None:
             # We need to pass the patterns to Rust because they can contain
             # patterns from the user interface
             self._kindpats = kindpats
         self._pats, self.matchfn = _buildmatch(kindpats, b'(?:/|$)', root)
         self._prefix = _prefix(kindpats)
         roots, dirs, parents = _rootsdirsandparents(kindpats)
         # roots are directories which are recursively included.
         self._roots = set(roots)
         # dirs are directories which are non-recursively included.
         self._dirs = set(dirs)
         # parents are directories which are non-recursively included because
         # they are needed to get to items in _dirs or _roots.
         self._parents = parents
 
     def visitdir(self, dir):
         if self._prefix and dir in self._roots:
             return b'all'
         return (
             dir in self._dirs
             or dir in self._parents
             or path_or_parents_in_set(dir, self._roots)
         )
 
     @propertycache
     def _allparentschildren(self):
         # It may seem odd that we add dirs, roots, and parents, and then
         # restrict to only parents. This is to catch the case of:
         #   dirs = ['foo/bar']
         #   parents = ['foo']
         # if we asked for the children of 'foo', but had only added
         # self._parents, we wouldn't be able to respond ['bar'].
         return _dirchildren(
             itertools.chain(self._dirs, self._roots, self._parents),
             onlyinclude=self._parents,
         )
 
     def visitchildrenset(self, dir):
         if self._prefix and dir in self._roots:
             return b'all'
         # Note: this does *not* include the 'dir in self._parents' case from
         # visitdir, that's handled below.
         if (
             b'' in self._roots
             or dir in self._dirs
             or path_or_parents_in_set(dir, self._roots)
         ):
             return b'this'
 
         if dir in self._parents:
             return self._allparentschildren.get(dir) or set()
         return set()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<includematcher includes=%r>' % pycompat.bytestr(self._pats)
 
 
 class exactmatcher(basematcher):
     r"""Matches the input files exactly. They are interpreted as paths, not
     patterns (so no kind-prefixes).
 
     >>> m = exactmatcher([b'a.txt', br're:.*\.c$'])
     >>> m(b'a.txt')
     True
     >>> m(b'b.txt')
     False
 
     Input files that would be matched are exactly those returned by .files()
     >>> m.files()
     ['a.txt', 're:.*\\.c$']
 
     So pattern 're:.*\.c$' is not considered as a regex, but as a file name
     >>> m(b'main.c')
     False
     >>> m(br're:.*\.c$')
     True
     """
 
     def __init__(self, files, badfn=None):
         super(exactmatcher, self).__init__(badfn)
 
         if isinstance(files, list):
             self._files = files
         else:
             self._files = list(files)
 
     matchfn = basematcher.exact
 
     @propertycache
     def _dirs(self):
         return set(pathutil.dirs(self._fileset))
 
     def visitdir(self, dir):
         return dir in self._dirs
 
     @propertycache
     def _visitchildrenset_candidates(self):
         """A memoized set of candidates for visitchildrenset."""
         return self._fileset | self._dirs - {b''}
 
     @propertycache
     def _sorted_visitchildrenset_candidates(self):
         """A memoized sorted list of candidates for visitchildrenset."""
         return sorted(self._visitchildrenset_candidates)
 
     def visitchildrenset(self, dir):
         if not self._fileset or dir not in self._dirs:
             return set()
 
         if dir == b'':
             candidates = self._visitchildrenset_candidates
         else:
             candidates = self._sorted_visitchildrenset_candidates
             d = dir + b'/'
             # Use bisect to find the first element potentially starting with d
             # (i.e. >= d). This should always find at least one element (we'll
             # assert later if this is not the case).
             first = bisect.bisect_left(candidates, d)
             # We need a representation of the first element that is > d that
             # does not start with d, so since we added a `/` on the end of dir,
             # we'll add whatever comes after slash (we could probably assume
             # that `0` is after `/`, but let's not) to the end of dir instead.
             dnext = dir + encoding.strtolocal(chr(ord(b'/') + 1))
             # Use bisect to find the first element >= d_next
             last = bisect.bisect_left(candidates, dnext, lo=first)
             dlen = len(d)
             candidates = {c[dlen:] for c in candidates[first:last]}
         # self._dirs includes all of the directories, recursively, so if
         # we're attempting to match foo/bar/baz.txt, it'll have '', 'foo',
         # 'foo/bar' in it. Thus we can safely ignore a candidate that has a
         # '/' in it, indicating a it's for a subdir-of-a-subdir; the
         # immediate subdir will be in there without a slash.
         ret = {c for c in candidates if b'/' not in c}
         # We really do not expect ret to be empty, since that would imply that
         # there's something in _dirs that didn't have a file in _fileset.
         assert ret
         return ret
 
     def isexact(self):
         return True
 
     @encoding.strmethod
     def __repr__(self):
         return b'<exactmatcher files=%r>' % self._files
 
 
 class differencematcher(basematcher):
     """Composes two matchers by matching if the first matches and the second
     does not.
 
     The second matcher's non-matching-attributes (bad, traversedir) are ignored.
     """
 
     def __init__(self, m1, m2):
         super(differencematcher, self).__init__()
         self._m1 = m1
         self._m2 = m2
         self.bad = m1.bad
         self.traversedir = m1.traversedir
 
     def matchfn(self, f):
         return self._m1(f) and not self._m2(f)
 
     @propertycache
     def _files(self):
         if self.isexact():
             return [f for f in self._m1.files() if self(f)]
         # If m1 is not an exact matcher, we can't easily figure out the set of
         # files, because its files() are not always files. For example, if
         # m1 is "path:dir" and m2 is "rootfileins:.", we don't
         # want to remove "dir" from the set even though it would match m2,
         # because the "dir" in m1 may not be a file.
         return self._m1.files()
 
     def visitdir(self, dir):
         if self._m2.visitdir(dir) == b'all':
             return False
         elif not self._m2.visitdir(dir):
             # m2 does not match dir, we can return 'all' here if possible
             return self._m1.visitdir(dir)
         return bool(self._m1.visitdir(dir))
 
     def visitchildrenset(self, dir):
         m2_set = self._m2.visitchildrenset(dir)
         if m2_set == b'all':
             return set()
         m1_set = self._m1.visitchildrenset(dir)
         # Possible values for m1: 'all', 'this', set(...), set()
         # Possible values for m2:        'this', set(...), set()
         # If m2 has nothing under here that we care about, return m1, even if
         # it's 'all'. This is a change in behavior from visitdir, which would
         # return True, not 'all', for some reason.
         if not m2_set:
             return m1_set
         if m1_set in [b'all', b'this']:
             # Never return 'all' here if m2_set is any kind of non-empty (either
             # 'this' or set(foo)), since m2 might return set() for a
             # subdirectory.
             return b'this'
         # Possible values for m1:         set(...), set()
         # Possible values for m2: 'this', set(...)
         # We ignore m2's set results. They're possibly incorrect:
         #  m1 = path:dir/subdir, m2=rootfilesin:dir, visitchildrenset(''):
         #    m1 returns {'dir'}, m2 returns {'dir'}, if we subtracted we'd
         #    return set(), which is *not* correct, we still need to visit 'dir'!
         return m1_set
 
     def isexact(self):
         return self._m1.isexact()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<differencematcher m1=%r, m2=%r>' % (self._m1, self._m2)
 
 
 def intersectmatchers(m1, m2):
     """Composes two matchers by matching if both of them match.
 
     The second matcher's non-matching-attributes (bad, traversedir) are ignored.
     """
     if m1 is None or m2 is None:
         return m1 or m2
     if m1.always():
         m = copy.copy(m2)
         # TODO: Consider encapsulating these things in a class so there's only
         # one thing to copy from m1.
         m.bad = m1.bad
         m.traversedir = m1.traversedir
         return m
     if m2.always():
         m = copy.copy(m1)
         return m
     return intersectionmatcher(m1, m2)
 
 
 class intersectionmatcher(basematcher):
     def __init__(self, m1, m2):
         super(intersectionmatcher, self).__init__()
         self._m1 = m1
         self._m2 = m2
         self.bad = m1.bad
         self.traversedir = m1.traversedir
 
     @propertycache
     def _files(self):
         if self.isexact():
             m1, m2 = self._m1, self._m2
             if not m1.isexact():
                 m1, m2 = m2, m1
             return [f for f in m1.files() if m2(f)]
         # It neither m1 nor m2 is an exact matcher, we can't easily intersect
         # the set of files, because their files() are not always files. For
         # example, if intersecting a matcher "-I glob:foo.txt" with matcher of
         # "path:dir2", we don't want to remove "dir2" from the set.
         return self._m1.files() + self._m2.files()
 
     def matchfn(self, f):
         return self._m1(f) and self._m2(f)
 
     def visitdir(self, dir):
         visit1 = self._m1.visitdir(dir)
         if visit1 == b'all':
             return self._m2.visitdir(dir)
         # bool() because visit1=True + visit2='all' should not be 'all'
         return bool(visit1 and self._m2.visitdir(dir))
 
     def visitchildrenset(self, dir):
         m1_set = self._m1.visitchildrenset(dir)
         if not m1_set:
             return set()
         m2_set = self._m2.visitchildrenset(dir)
         if not m2_set:
             return set()
 
         if m1_set == b'all':
             return m2_set
         elif m2_set == b'all':
             return m1_set
 
         if m1_set == b'this' or m2_set == b'this':
             return b'this'
 
         assert isinstance(m1_set, set) and isinstance(m2_set, set)
         return m1_set.intersection(m2_set)
 
     def always(self):
         return self._m1.always() and self._m2.always()
 
     def isexact(self):
         return self._m1.isexact() or self._m2.isexact()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<intersectionmatcher m1=%r, m2=%r>' % (self._m1, self._m2)
 
 
 class subdirmatcher(basematcher):
     """Adapt a matcher to work on a subdirectory only.
 
     The paths are remapped to remove/insert the path as needed:
 
     >>> from . import pycompat
     >>> m1 = match(util.localpath(b'/root'), b'', [b'a.txt', b'sub/b.txt'], auditor=lambda name: None)
     >>> m2 = subdirmatcher(b'sub', m1)
     >>> m2(b'a.txt')
     False
     >>> m2(b'b.txt')
     True
     >>> m2.matchfn(b'a.txt')
     False
     >>> m2.matchfn(b'b.txt')
     True
     >>> m2.files()
     ['b.txt']
     >>> m2.exact(b'b.txt')
     True
     >>> def bad(f, msg):
     ...     print(pycompat.sysstr(b"%s: %s" % (f, msg)))
     >>> m1.bad = bad
     >>> m2.bad(b'x.txt', b'No such file')
     sub/x.txt: No such file
     """
 
     def __init__(self, path, matcher):
         super(subdirmatcher, self).__init__()
         self._path = path
         self._matcher = matcher
         self._always = matcher.always()
 
         self._files = [
             f[len(path) + 1 :]
             for f in matcher._files
             if f.startswith(path + b"/")
         ]
 
         # If the parent repo had a path to this subrepo and the matcher is
         # a prefix matcher, this submatcher always matches.
         if matcher.prefix():
             self._always = any(f == path for f in matcher._files)
 
     def bad(self, f, msg):
         self._matcher.bad(self._path + b"/" + f, msg)
 
     def matchfn(self, f):
         # Some information is lost in the superclass's constructor, so we
         # can not accurately create the matching function for the subdirectory
         # from the inputs. Instead, we override matchfn() and visitdir() to
         # call the original matcher with the subdirectory path prepended.
         return self._matcher.matchfn(self._path + b"/" + f)
 
     def visitdir(self, dir):
         if dir == b'':
             dir = self._path
         else:
             dir = self._path + b"/" + dir
         return self._matcher.visitdir(dir)
 
     def visitchildrenset(self, dir):
         if dir == b'':
             dir = self._path
         else:
             dir = self._path + b"/" + dir
         return self._matcher.visitchildrenset(dir)
 
     def always(self):
         return self._always
 
     def prefix(self):
         return self._matcher.prefix() and not self._always
 
     @encoding.strmethod
     def __repr__(self):
         return b'<subdirmatcher path=%r, matcher=%r>' % (
             self._path,
             self._matcher,
         )
 
 
 class prefixdirmatcher(basematcher):
     """Adapt a matcher to work on a parent directory.
 
     The matcher's non-matching-attributes (bad, traversedir) are ignored.
 
     The prefix path should usually be the relative path from the root of
     this matcher to the root of the wrapped matcher.
 
     >>> m1 = match(util.localpath(b'/root/d/e'), b'f', [b'../a.txt', b'b.txt'], auditor=lambda name: None)
     >>> m2 = prefixdirmatcher(b'd/e', m1)
     >>> m2(b'a.txt')
     False
     >>> m2(b'd/e/a.txt')
     True
     >>> m2(b'd/e/b.txt')
     False
     >>> m2.files()
     ['d/e/a.txt', 'd/e/f/b.txt']
     >>> m2.exact(b'd/e/a.txt')
     True
     >>> m2.visitdir(b'd')
     True
     >>> m2.visitdir(b'd/e')
     True
     >>> m2.visitdir(b'd/e/f')
     True
     >>> m2.visitdir(b'd/e/g')
     False
     >>> m2.visitdir(b'd/ef')
     False
     """
 
     def __init__(self, path, matcher, badfn=None):
         super(prefixdirmatcher, self).__init__(badfn)
         if not path:
             raise error.ProgrammingError(b'prefix path must not be empty')
         self._path = path
         self._pathprefix = path + b'/'
         self._matcher = matcher
 
     @propertycache
     def _files(self):
         return [self._pathprefix + f for f in self._matcher._files]
 
     def matchfn(self, f):
         if not f.startswith(self._pathprefix):
             return False
         return self._matcher.matchfn(f[len(self._pathprefix) :])
 
     @propertycache
     def _pathdirs(self):
         return set(pathutil.finddirs(self._path))
 
     def visitdir(self, dir):
         if dir == self._path:
             return self._matcher.visitdir(b'')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitdir(dir[len(self._pathprefix) :])
         return dir in self._pathdirs
 
     def visitchildrenset(self, dir):
         if dir == self._path:
             return self._matcher.visitchildrenset(b'')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitchildrenset(dir[len(self._pathprefix) :])
         if dir in self._pathdirs:
             return b'this'
         return set()
 
     def isexact(self):
         return self._matcher.isexact()
 
     def prefix(self):
         return self._matcher.prefix()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<prefixdirmatcher path=%r, matcher=%r>' % (
             pycompat.bytestr(self._path),
             self._matcher,
         )
 
 
 class unionmatcher(basematcher):
     """A matcher that is the union of several matchers.
 
     The non-matching-attributes (bad, traversedir) are taken from the first
     matcher.
     """
 
     def __init__(self, matchers):
         m1 = matchers[0]
         super(unionmatcher, self).__init__()
         self.traversedir = m1.traversedir
         self._matchers = matchers
 
     def matchfn(self, f):
         for match in self._matchers:
             if match(f):
                 return True
         return False
 
     def visitdir(self, dir):
         r = False
         for m in self._matchers:
             v = m.visitdir(dir)
             if v == b'all':
                 return v
             r |= v
         return r
 
     def visitchildrenset(self, dir):
         r = set()
         this = False
         for m in self._matchers:
             v = m.visitchildrenset(dir)
             if not v:
                 continue
             if v == b'all':
                 return v
             if this or v == b'this':
                 this = True
                 # don't break, we might have an 'all' in here.
                 continue
             assert isinstance(v, set)
             r = r.union(v)
         if this:
             return b'this'
         return r
 
     @encoding.strmethod
     def __repr__(self):
         return b'<unionmatcher matchers=%r>' % self._matchers
 
 
 def patkind(pattern, default=None):
     r"""If pattern is 'kind:pat' with a known kind, return kind.
 
     >>> patkind(br're:.*\.c$')
     're'
     >>> patkind(b'glob:*.c')
     'glob'
     >>> patkind(b'relpath:test.py')
     'relpath'
     >>> patkind(b'main.py')
     >>> patkind(b'main.py', default=b're')
     're'
     """
     return _patsplit(pattern, default)[0]
 
 
 def _patsplit(pattern, default):
     """Split a string into the optional pattern kind prefix and the actual
     pattern."""
     if b':' in pattern:
         kind, pat = pattern.split(b':', 1)
         if kind in allpatternkinds:
             return kind, pat
     return default, pattern
 
 
 def _globre(pat):
     r"""Convert an extended glob string to a regexp string.
 
     >>> from . import pycompat
     >>> def bprint(s):
     ...     print(pycompat.sysstr(s))
     >>> bprint(_globre(br'?'))
     .
     >>> bprint(_globre(br'*'))
     [^/]*
     >>> bprint(_globre(br'**'))
     .*
     >>> bprint(_globre(br'**/a'))
     (?:.*/)?a
     >>> bprint(_globre(br'a/**/b'))
     a/(?:.*/)?b
     >>> bprint(_globre(br'[a*?!^][^b][!c]'))
     [a*?!^][\^b][^c]
     >>> bprint(_globre(br'{a,b}'))
     (?:a|b)
     >>> bprint(_globre(br'.\*\?'))
     \.\*\?
     """
     i, n = 0, len(pat)
     res = b''
     group = 0
     escape = util.stringutil.regexbytesescapemap.get
 
     def peek():
         return i < n and pat[i : i + 1]
 
     while i < n:
         c = pat[i : i + 1]
         i += 1
         if c not in b'*?[{},\\':
             res += escape(c, c)
         elif c == b'*':
             if peek() == b'*':
                 i += 1
                 if peek() == b'/':
                     i += 1
                     res += b'(?:.*/)?'
                 else:
                     res += b'.*'
             else:
                 res += b'[^/]*'
         elif c == b'?':
             res += b'.'
         elif c == b'[':
             j = i
             if j < n and pat[j : j + 1] in b'!]':
                 j += 1
             while j < n and pat[j : j + 1] != b']':
                 j += 1
             if j >= n:
                 res += b'\\['
             else:
                 stuff = pat[i:j].replace(b'\\', b'\\\\')
                 i = j + 1
                 if stuff[0:1] == b'!':
                     stuff = b'^' + stuff[1:]
                 elif stuff[0:1] == b'^':
                     stuff = b'\\' + stuff
                 res = b'%s[%s]' % (res, stuff)
         elif c == b'{':
             group += 1
             res += b'(?:'
         elif c == b'}' and group:
             res += b')'
             group -= 1
         elif c == b',' and group:
             res += b'|'
         elif c == b'\\':
             p = peek()
             if p:
                 i += 1
                 res += escape(p, p)
             else:
                 res += escape(c, c)
         else:
             res += escape(c, c)
     return res
 
 
 FLAG_RE = util.re.compile(br'^\(\?([aiLmsux]+)\)(.*)')
 
 
 def _regex(kind, pat, globsuffix):
     """Convert a (normalized) pattern of any kind into a
     regular expression.
     globsuffix is appended to the regexp of globs."""
     if not pat and kind in (b'glob', b'relpath'):
         return b''
     if kind == b're':
         return pat
     if kind == b'filepath':
         raise error.ProgrammingError(
             "'filepath:' patterns should not be converted to a regex"
         )
     if kind in (b'path', b'relpath'):
         if pat == b'.':
             return b''
         return util.stringutil.reescape(pat) + b'(?:/|$)'
     if kind == b'rootfilesin':
         if pat == b'.':
             escaped = b''
         else:
             # Pattern is a directory name.
             escaped = util.stringutil.reescape(pat) + b'/'
         # Anything after the pattern must be a non-directory.
         return escaped + b'[^/]+$'
     if kind == b'relglob':
         globre = _globre(pat)
         if globre.startswith(b'[^/]*'):
             # When pat has the form *XYZ (common), make the returned regex more
             # legible by returning the regex for **XYZ instead of **/*XYZ.
             return b'.*' + globre[len(b'[^/]*') :] + globsuffix
         return b'(?:|.*/)' + globre + globsuffix
     if kind == b'relre':
         flag = None
         m = FLAG_RE.match(pat)
         if m:
             flag, pat = m.groups()
         if not pat.startswith(b'^'):
             pat = b'.*' + pat
         if flag is not None:
             pat = br'(?%s:%s)' % (flag, pat)
         return pat
     if kind in (b'glob', b'rootglob'):
         return _globre(pat) + globsuffix
     raise error.ProgrammingError(b'not a regex pattern: %s:%s' % (kind, pat))
 
 
 def _buildmatch(kindpats, globsuffix, root):
     """Return regexp string and a matcher function for kindpats.
     globsuffix is appended to the regexp of globs."""
     matchfuncs = []
 
     subincludes, kindpats = _expandsubinclude(kindpats, root)
     if subincludes:
         submatchers = {}
 
         def matchsubinclude(f):
             for prefix, matcherargs in subincludes:
                 if f.startswith(prefix):
                     mf = submatchers.get(prefix)
                     if mf is None:
                         mf = match(*matcherargs)
                         submatchers[prefix] = mf
 
                     if mf(f[len(prefix) :]):
                         return True
             return False
 
         matchfuncs.append(matchsubinclude)
 
     regex = b''
     if kindpats:
         if all(k == b'rootfilesin' for k, p, s in kindpats):
             dirs = {p for k, p, s in kindpats}
 
             def mf(f):
                 i = f.rfind(b'/')
                 if i >= 0:
                     dir = f[:i]
                 else:
                     dir = b'.'
                 return dir in dirs
 
             regex = b'rootfilesin: %s' % stringutil.pprint(list(sorted(dirs)))
             matchfuncs.append(mf)
         else:
             regex, mf = _buildregexmatch(kindpats, globsuffix)
             matchfuncs.append(mf)
 
     if len(matchfuncs) == 1:
         return regex, matchfuncs[0]
     else:
         return regex, lambda f: any(mf(f) for mf in matchfuncs)
 
 
 MAX_RE_SIZE = 20000
 
 
 def _joinregexes(regexps):
     """gather multiple regular expressions into a single one"""
     return b'|'.join(regexps)
 
 
 def _buildregexmatch(kindpats, globsuffix):
     """Build a match function from a list of kinds and kindpats,
     return regexp string and a matcher function.
 
     Test too large input
     >>> _buildregexmatch([
     ...     (b'relglob', b'?' * MAX_RE_SIZE, b'')
     ... ], b'$')
     Traceback (most recent call last):
     ...
     Abort: matcher pattern is too long (20009 bytes)
     """
     try:
         allgroups = []
         regexps = []
         exact = set()
-        for (kind, pattern, _source) in kindpats:
+        for kind, pattern, _source in kindpats:
             if kind == b'filepath':
                 exact.add(pattern)
                 continue
             regexps.append(_regex(kind, pattern, globsuffix))
 
         fullregexp = _joinregexes(regexps)
 
         startidx = 0
         groupsize = 0
         for idx, r in enumerate(regexps):
             piecesize = len(r)
             if piecesize > MAX_RE_SIZE:
                 msg = _(b"matcher pattern is too long (%d bytes)") % piecesize
                 raise error.Abort(msg)
             elif (groupsize + piecesize) > MAX_RE_SIZE:
                 group = regexps[startidx:idx]
                 allgroups.append(_joinregexes(group))
                 startidx = idx
                 groupsize = 0
             groupsize += piecesize + 1
 
         if startidx == 0:
             matcher = _rematcher(fullregexp)
             func = lambda s: bool(matcher(s))
         else:
             group = regexps[startidx:]
             allgroups.append(_joinregexes(group))
             allmatchers = [_rematcher(g) for g in allgroups]
             func = lambda s: any(m(s) for m in allmatchers)
 
         actualfunc = func
         if exact:
             # An empty regex will always match, so only call the regex if
             # there were any actual patterns to match.
             if not regexps:
                 actualfunc = lambda s: s in exact
             else:
                 actualfunc = lambda s: s in exact or func(s)
         return fullregexp, actualfunc
     except re.error:
         for k, p, s in kindpats:
             if k == b'filepath':
                 continue
             try:
                 _rematcher(_regex(k, p, globsuffix))
             except re.error:
                 if s:
                     raise error.Abort(
                         _(b"%s: invalid pattern (%s): %s") % (s, k, p)
                     )
                 else:
                     raise error.Abort(_(b"invalid pattern (%s): %s") % (k, p))
         raise error.Abort(_(b"invalid pattern"))
 
 
 def _patternrootsanddirs(kindpats):
     """Returns roots and directories corresponding to each pattern.
 
     This calculates the roots and directories exactly matching the patterns and
     returns a tuple of (roots, dirs) for each. It does not return other
     directories which may also need to be considered, like the parent
     directories.
     """
     r = []
     d = []
     for kind, pat, source in kindpats:
         if kind in (b'glob', b'rootglob'):  # find the non-glob prefix
             root = []
             for p in pat.split(b'/'):
                 if b'[' in p or b'{' in p or b'*' in p or b'?' in p:
                     break
                 root.append(p)
             r.append(b'/'.join(root))
         elif kind in (b'relpath', b'path', b'filepath'):
             if pat == b'.':
                 pat = b''
             r.append(pat)
         elif kind in (b'rootfilesin',):
             if pat == b'.':
                 pat = b''
             d.append(pat)
         else:  # relglob, re, relre
             r.append(b'')
     return r, d
 
 
 def _roots(kindpats):
     '''Returns root directories to match recursively from the given patterns.'''
     roots, dirs = _patternrootsanddirs(kindpats)
     return roots
 
 
 def _rootsdirsandparents(kindpats):
     """Returns roots and exact directories from patterns.
 
     `roots` are directories to match recursively, `dirs` should
     be matched non-recursively, and `parents` are the implicitly required
     directories to walk to items in either roots or dirs.
 
     Returns a tuple of (roots, dirs, parents).
 
     >>> r = _rootsdirsandparents(
     ...     [(b'glob', b'g/h/*', b''), (b'glob', b'g/h', b''),
     ...      (b'glob', b'g*', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['g/h', 'g/h', ''], []) ['', 'g']
     >>> r = _rootsdirsandparents(
     ...     [(b'rootfilesin', b'g/h', b''), (b'rootfilesin', b'', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     ([], ['g/h', '']) ['', 'g']
     >>> r = _rootsdirsandparents(
     ...     [(b'relpath', b'r', b''), (b'path', b'p/p', b''),
     ...      (b'path', b'', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['r', 'p/p', ''], []) ['', 'p']
     >>> r = _rootsdirsandparents(
     ...     [(b'relglob', b'rg*', b''), (b're', b're/', b''),
     ...      (b'relre', b'rr', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['', '', ''], []) ['']
     """
     r, d = _patternrootsanddirs(kindpats)
 
     p = set()
     # Add the parents as non-recursive/exact directories, since they must be
     # scanned to get to either the roots or the other exact directories.
     p.update(pathutil.dirs(d))
     p.update(pathutil.dirs(r))
 
     # FIXME: all uses of this function convert these to sets, do so before
     # returning.
     # FIXME: all uses of this function do not need anything in 'roots' and
     # 'dirs' to also be in 'parents', consider removing them before returning.
     return r, d, p
 
 
 def _explicitfiles(kindpats):
     """Returns the potential explicit filenames from the patterns.
 
     >>> _explicitfiles([(b'path', b'foo/bar', b'')])
     ['foo/bar']
     >>> _explicitfiles([(b'rootfilesin', b'foo/bar', b'')])
     []
     """
     # Keep only the pattern kinds where one can specify filenames (vs only
     # directory names).
     filable = [kp for kp in kindpats if kp[0] not in (b'rootfilesin',)]
     return _roots(filable)
 
 
 def _prefix(kindpats):
     '''Whether all the patterns match a prefix (i.e. recursively)'''
     for kind, pat, source in kindpats:
         if kind not in (b'path', b'relpath'):
             return False
     return True
 
 
 _commentre = None
 
 
 def readpatternfile(filepath, warn, sourceinfo=False):
     """parse a pattern file, returning a list of
     patterns. These patterns should be given to compile()
     to be validated and converted into a match function.
 
     trailing white space is dropped.
     the escape character is backslash.
     comments start with #.
     empty lines are skipped.
 
     lines can be of the following formats:
 
     syntax: regexp # defaults following lines to non-rooted regexps
     syntax: glob   # defaults following lines to non-rooted globs
     re:pattern     # non-rooted regular expression
     glob:pattern   # non-rooted glob
     rootglob:pat   # rooted glob (same root as ^ in regexps)
     pattern        # pattern of the current default type
 
     if sourceinfo is set, returns a list of tuples:
     (pattern, lineno, originalline).
     This is useful to debug ignore patterns.
     """
 
     syntaxes = {
         b're': b'relre:',
         b'regexp': b'relre:',
         b'glob': b'relglob:',
         b'rootglob': b'rootglob:',
         b'include': b'include',
         b'subinclude': b'subinclude',
     }
     syntax = b'relre:'
     patterns = []
 
     fp = open(filepath, b'rb')
     for lineno, line in enumerate(fp, start=1):
         if b"#" in line:
             global _commentre
             if not _commentre:
                 _commentre = util.re.compile(br'((?:^|[^\\])(?:\\\\)*)#.*')
             # remove comments prefixed by an even number of escapes
             m = _commentre.search(line)
             if m:
                 line = line[: m.end(1)]
             # fixup properly escaped comments that survived the above
             line = line.replace(b"\\#", b"#")
         line = line.rstrip()
         if not line:
             continue
 
         if line.startswith(b'syntax:'):
             s = line[7:].strip()
             try:
                 syntax = syntaxes[s]
             except KeyError:
                 if warn:
                     warn(
                         _(b"%s: ignoring invalid syntax '%s'\n") % (filepath, s)
                     )
             continue
 
         linesyntax = syntax
         for s, rels in syntaxes.items():
             if line.startswith(rels):
                 linesyntax = rels
                 line = line[len(rels) :]
                 break
             elif line.startswith(s + b':'):
                 linesyntax = rels
                 line = line[len(s) + 1 :]
                 break
         if sourceinfo:
             patterns.append((linesyntax + line, lineno, line))
         else:
             patterns.append(linesyntax + line)
     fp.close()
     return patterns

mercurial/wireprotoserver.py

@@ -1,550 +1,560 b''
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import contextlib
 import struct
 import threading
 
 from .i18n import _
 from . import (
     encoding,
     error,
     pycompat,
     util,
     wireprototypes,
     wireprotov1server,
 )
 from .interfaces import util as interfaceutil
 from .utils import (
     compression,
     stringutil,
 )
 
 stringio = util.stringio
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 HTTP_OK = 200
 
 HGTYPE = b'application/mercurial-0.1'
 HGTYPE2 = b'application/mercurial-0.2'
 HGERRTYPE = b'application/hg-error'
 
 SSHV1 = wireprototypes.SSHV1
 
 
 def decodevaluefromheaders(req, headerprefix):
     """Decode a long value from multiple HTTP request headers.
 
     Returns the value as a bytes, not a str.
     """
     chunks = []
     i = 1
     while True:
         v = req.headers.get(b'%s-%d' % (headerprefix, i))
         if v is None:
             break
         chunks.append(pycompat.bytesurl(v))
         i += 1
 
     return b''.join(chunks)
 
 
 @interfaceutil.implementer(wireprototypes.baseprotocolhandler)
 class httpv1protocolhandler:
     def __init__(self, req, ui, checkperm):
         self._req = req
         self._ui = ui
         self._checkperm = checkperm
         self._protocaps = None
 
     @property
     def name(self):
         return b'http-v1'
 
     def getargs(self, args):
         knownargs = self._args()
         data = {}
         keys = args.split()
         for k in keys:
             if k == b'*':
                 star = {}
                 for key in knownargs.keys():
                     if key != b'cmd' and key not in keys:
                         star[key] = knownargs[key][0]
                 data[b'*'] = star
             else:
                 data[k] = knownargs[k][0]
         return [data[k] for k in keys]
 
     def _args(self):
         args = self._req.qsparams.asdictoflists()
         postlen = int(self._req.headers.get(b'X-HgArgs-Post', 0))
         if postlen:
             args.update(
                 urlreq.parseqs(
                     self._req.bodyfh.read(postlen), keep_blank_values=True
                 )
             )
             return args
 
         argvalue = decodevaluefromheaders(self._req, b'X-HgArg')
         args.update(urlreq.parseqs(argvalue, keep_blank_values=True))
         return args
 
     def getprotocaps(self):
         if self._protocaps is None:
             value = decodevaluefromheaders(self._req, b'X-HgProto')
             self._protocaps = set(value.split(b' '))
         return self._protocaps
 
     def getpayload(self):
         # Existing clients *always* send Content-Length.
         length = int(self._req.headers[b'Content-Length'])
 
         # If httppostargs is used, we need to read Content-Length
         # minus the amount that was consumed by args.
         length -= int(self._req.headers.get(b'X-HgArgs-Post', 0))
         return util.filechunkiter(self._req.bodyfh, limit=length)
 
     @contextlib.contextmanager
     def mayberedirectstdio(self):
         oldout = self._ui.fout
         olderr = self._ui.ferr
 
         out = util.stringio()
 
         try:
             self._ui.fout = out
             self._ui.ferr = out
             yield out
         finally:
             self._ui.fout = oldout
             self._ui.ferr = olderr
 
     def client(self):
         return b'remote:%s:%s:%s' % (
             self._req.urlscheme,
             urlreq.quote(self._req.remotehost or b''),
             urlreq.quote(self._req.remoteuser or b''),
         )
 
     def addcapabilities(self, repo, caps):
         caps.append(b'batch')
 
         caps.append(
             b'httpheader=%d' % repo.ui.configint(b'server', b'maxhttpheaderlen')
         )
         if repo.ui.configbool(b'experimental', b'httppostargs'):
             caps.append(b'httppostargs')
 
         # FUTURE advertise 0.2rx once support is implemented
         # FUTURE advertise minrx and mintx after consulting config option
         caps.append(b'httpmediatype=0.1rx,0.1tx,0.2tx')
 
         compengines = wireprototypes.supportedcompengines(
             repo.ui, compression.SERVERROLE
         )
         if compengines:
             comptypes = b','.join(
                 urlreq.quote(e.wireprotosupport().name) for e in compengines
             )
             caps.append(b'compression=%s' % comptypes)
 
         return caps
 
     def checkperm(self, perm):
         return self._checkperm(perm)
 
 
 # This method exists mostly so that extensions like remotefilelog can
 # disable a kludgey legacy method only over http. As of early 2018,
 # there are no other known users, so with any luck we can discard this
 # hook if remotefilelog becomes a first-party extension.
 def iscmd(cmd):
     return cmd in wireprotov1server.commands
 
 
 def handlewsgirequest(rctx, req, res, checkperm):
     """Possibly process a wire protocol request.
 
     If the current request is a wire protocol request, the request is
     processed by this function.
 
     ``req`` is a ``parsedrequest`` instance.
     ``res`` is a ``wsgiresponse`` instance.
 
     Returns a bool indicating if the request was serviced. If set, the caller
     should stop processing the request, as a response has already been issued.
     """
     # Avoid cycle involving hg module.
     from .hgweb import common as hgwebcommon
 
     repo = rctx.repo
 
     # HTTP version 1 wire protocol requests are denoted by a "cmd" query
     # string parameter. If it isn't present, this isn't a wire protocol
     # request.
     if b'cmd' not in req.qsparams:
         return False
 
     cmd = req.qsparams[b'cmd']
 
     # The "cmd" request parameter is used by both the wire protocol and hgweb.
     # While not all wire protocol commands are available for all transports,
     # if we see a "cmd" value that resembles a known wire protocol command, we
     # route it to a protocol handler. This is better than routing possible
     # wire protocol requests to hgweb because it prevents hgweb from using
     # known wire protocol commands and it is less confusing for machine
     # clients.
     if not iscmd(cmd):
         return False
 
     # The "cmd" query string argument is only valid on the root path of the
     # repo. e.g. ``/?cmd=foo``, ``/repo?cmd=foo``. URL paths within the repo
     # like ``/blah?cmd=foo`` are not allowed. So don't recognize the request
     # in this case. We send an HTTP 404 for backwards compatibility reasons.
     if req.dispatchpath:
         res.status = hgwebcommon.statusmessage(404)
         res.headers[b'Content-Type'] = HGTYPE
         # TODO This is not a good response to issue for this request. This
         # is mostly for BC for now.
         res.setbodybytes(b'0\n%s\n' % b'Not Found')
         return True
 
     proto = httpv1protocolhandler(
         req, repo.ui, lambda perm: checkperm(rctx, req, perm)
     )
 
     # The permissions checker should be the only thing that can raise an
     # ErrorResponse. It is kind of a layer violation to catch an hgweb
     # exception here. So consider refactoring into a exception type that
     # is associated with the wire protocol.
     try:
         _callhttp(repo, req, res, proto, cmd)
     except hgwebcommon.ErrorResponse as e:
         for k, v in e.headers:
             res.headers[k] = v
         res.status = hgwebcommon.statusmessage(
             e.code, stringutil.forcebytestr(e)
         )
         # TODO This response body assumes the failed command was
         # "unbundle." That assumption is not always valid.
         res.setbodybytes(b'0\n%s\n' % stringutil.forcebytestr(e))
 
     return True
 
 
 def _httpresponsetype(ui, proto, prefer_uncompressed):
     """Determine the appropriate response type and compression settings.
 
     Returns a tuple of (mediatype, compengine, engineopts).
     """
     # Determine the response media type and compression engine based
     # on the request parameters.
 
     if b'0.2' in proto.getprotocaps():
         # All clients are expected to support uncompressed data.
         if prefer_uncompressed:
             return HGTYPE2, compression._noopengine(), {}
 
         # Now find an agreed upon compression format.
         compformats = wireprotov1server.clientcompressionsupport(proto)
         for engine in wireprototypes.supportedcompengines(
             ui, compression.SERVERROLE
         ):
             if engine.wireprotosupport().name in compformats:
                 opts = {}
                 level = ui.configint(b'server', b'%slevel' % engine.name())
                 if level is not None:
                     opts[b'level'] = level
 
                 return HGTYPE2, engine, opts
 
         # No mutually supported compression format. Fall back to the
         # legacy protocol.
 
     # Don't allow untrusted settings because disabling compression or
     # setting a very high compression level could lead to flooding
     # the server's network or CPU.
     opts = {b'level': ui.configint(b'server', b'zliblevel')}
     return HGTYPE, util.compengines[b'zlib'], opts
 
 
 def _callhttp(repo, req, res, proto, cmd):
     # Avoid cycle involving hg module.
     from .hgweb import common as hgwebcommon
 
     def genversion2(gen, engine, engineopts):
         # application/mercurial-0.2 always sends a payload header
         # identifying the compression engine.
         name = engine.wireprotosupport().name
         assert 0 < len(name) < 256
         yield struct.pack(b'B', len(name))
         yield name
 
         for chunk in gen:
             yield chunk
 
     def setresponse(code, contenttype, bodybytes=None, bodygen=None):
         if code == HTTP_OK:
             res.status = b'200 Script output follows'
         else:
             res.status = hgwebcommon.statusmessage(code)
 
         res.headers[b'Content-Type'] = contenttype
 
         if bodybytes is not None:
             res.setbodybytes(bodybytes)
         if bodygen is not None:
             res.setbodygen(bodygen)
 
     if not wireprotov1server.commands.commandavailable(cmd, proto):
         setresponse(
             HTTP_OK,
             HGERRTYPE,
             _(
                 b'requested wire protocol command is not available over '
                 b'HTTP'
             ),
         )
         return
 
     proto.checkperm(wireprotov1server.commands[cmd].permission)
 
     accesshidden = hgwebcommon.hashiddenaccess(repo, req)
     rsp = wireprotov1server.dispatch(repo, proto, cmd, accesshidden)
 
     if isinstance(rsp, bytes):
         setresponse(HTTP_OK, HGTYPE, bodybytes=rsp)
     elif isinstance(rsp, wireprototypes.bytesresponse):
         setresponse(HTTP_OK, HGTYPE, bodybytes=rsp.data)
     elif isinstance(rsp, wireprototypes.streamreslegacy):
         setresponse(HTTP_OK, HGTYPE, bodygen=rsp.gen)
     elif isinstance(rsp, wireprototypes.streamres):
         gen = rsp.gen
 
         # This code for compression should not be streamres specific. It
         # is here because we only compress streamres at the moment.
         mediatype, engine, engineopts = _httpresponsetype(
             repo.ui, proto, rsp.prefer_uncompressed
         )
         gen = engine.compressstream(gen, engineopts)
 
         if mediatype == HGTYPE2:
             gen = genversion2(gen, engine, engineopts)
 
         setresponse(HTTP_OK, mediatype, bodygen=gen)
     elif isinstance(rsp, wireprototypes.pushres):
         rsp = b'%d\n%s' % (rsp.res, rsp.output)
         setresponse(HTTP_OK, HGTYPE, bodybytes=rsp)
     elif isinstance(rsp, wireprototypes.pusherr):
         rsp = b'0\n%s\n' % rsp.res
         res.drain = True
         setresponse(HTTP_OK, HGTYPE, bodybytes=rsp)
     elif isinstance(rsp, wireprototypes.ooberror):
         setresponse(HTTP_OK, HGERRTYPE, bodybytes=rsp.message)
     else:
         raise error.ProgrammingError(b'hgweb.protocol internal failure', rsp)
 
 
 def _sshv1respondbytes(fout, value):
     """Send a bytes response for protocol version 1."""
     fout.write(b'%d\n' % len(value))
     fout.write(value)
     fout.flush()
 
 
 def _sshv1respondstream(fout, source):
     write = fout.write
     for chunk in source.gen:
         write(chunk)
     fout.flush()
 
 
 def _sshv1respondooberror(fout, ferr, rsp):
     ferr.write(b'%s\n-\n' % rsp)
     ferr.flush()
     fout.write(b'\n')
     fout.flush()
 
 
 @interfaceutil.implementer(wireprototypes.baseprotocolhandler)
 class sshv1protocolhandler:
     """Handler for requests services via version 1 of SSH protocol."""
 
     def __init__(self, ui, fin, fout):
         self._ui = ui
         self._fin = fin
         self._fout = fout
         self._protocaps = set()
 
     @property
     def name(self):
         return wireprototypes.SSHV1
 
     def getargs(self, args):
         data = {}
         keys = args.split()
         for n in range(len(keys)):
             argline = self._fin.readline()[:-1]
             arg, l = argline.split()
             if arg not in keys:
                 raise error.Abort(_(b"unexpected parameter %r") % arg)
             if arg == b'*':
                 star = {}
                 for k in range(int(l)):
                     argline = self._fin.readline()[:-1]
                     arg, l = argline.split()
                     val = self._fin.read(int(l))
                     star[arg] = val
                 data[b'*'] = star
             else:
                 val = self._fin.read(int(l))
                 data[arg] = val
         return [data[k] for k in keys]
 
     def getprotocaps(self):
         return self._protocaps
 
     def getpayload(self):
         # We initially send an empty response. This tells the client it is
         # OK to start sending data. If a client sees any other response, it
         # interprets it as an error.
         _sshv1respondbytes(self._fout, b'')
 
         # The file is in the form:
         #
         # <chunk size>\n<chunk>
         # ...
         # 0\n
         count = int(self._fin.readline())
         while count:
             yield self._fin.read(count)
             count = int(self._fin.readline())
 
     @contextlib.contextmanager
     def mayberedirectstdio(self):
         yield None
 
     def client(self):
         client = encoding.environ.get(b'SSH_CLIENT', b'').split(b' ', 1)[0]
         return b'remote:ssh:' + client
 
     def addcapabilities(self, repo, caps):
         if self.name == wireprototypes.SSHV1:
             caps.append(b'protocaps')
         caps.append(b'batch')
         return caps
 
     def checkperm(self, perm):
         pass
 
 
 def _runsshserver(ui, repo, fin, fout, ev, accesshidden=False):
     # This function operates like a state machine of sorts. The following
     # states are defined:
     #
     # protov1-serving
     #    Server is in protocol version 1 serving mode. Commands arrive on
     #    new lines. These commands are processed in this state, one command
     #    after the other.
     #
     # shutdown
     #    The server is shutting down, possibly in reaction to a client event.
     #
     # And here are their transitions:
     #
     # protov1-serving -> shutdown
     #    When server receives an empty request or encounters another
     #    error.
 
     state = b'protov1-serving'
     proto = sshv1protocolhandler(ui, fin, fout)
 
     while not ev.is_set():
         if state == b'protov1-serving':
             # Commands are issued on new lines.
             request = fin.readline()[:-1]
 
             # Empty lines signal to terminate the connection.
             if not request:
                 state = b'shutdown'
                 continue
 
             available = wireprotov1server.commands.commandavailable(
                 request, proto
             )
 
             # This command isn't available. Send an empty response and go
             # back to waiting for a new command.
             if not available:
                 _sshv1respondbytes(fout, b'')
                 continue
 
             rsp = wireprotov1server.dispatch(
                 repo, proto, request, accesshidden=accesshidden
             )
             repo.ui.fout.flush()
             repo.ui.ferr.flush()
 
             if isinstance(rsp, bytes):
                 _sshv1respondbytes(fout, rsp)
             elif isinstance(rsp, wireprototypes.bytesresponse):
                 _sshv1respondbytes(fout, rsp.data)
             elif isinstance(rsp, wireprototypes.streamres):
                 _sshv1respondstream(fout, rsp)
             elif isinstance(rsp, wireprototypes.streamreslegacy):
                 _sshv1respondstream(fout, rsp)
             elif isinstance(rsp, wireprototypes.pushres):
                 _sshv1respondbytes(fout, b'')
                 _sshv1respondbytes(fout, b'%d' % rsp.res)
             elif isinstance(rsp, wireprototypes.pusherr):
                 _sshv1respondbytes(fout, rsp.res)
             elif isinstance(rsp, wireprototypes.ooberror):
                 _sshv1respondooberror(fout, ui.ferr, rsp.message)
             else:
                 raise error.ProgrammingError(
                     b'unhandled response type from '
                     b'wire protocol command: %s' % rsp
                 )
 
         elif state == b'shutdown':
             break
 
         else:
             raise error.ProgrammingError(
                 b'unhandled ssh server state: %s' % state
             )
 
 
 class sshserver:
     def __init__(self, ui, repo, logfh=None, accesshidden=False):
         self._ui = ui
         self._repo = repo
-        self._fin, self._fout = ui.protectfinout()
         self._accesshidden = accesshidden
-
-        # Log write I/O to stdout and stderr if configured.
-        if logfh:
-            self._fout = util.makeloggingfileobject(
-                logfh, self._fout, b'o', logdata=True
-            )
-            ui.ferr = util.makeloggingfileobject(
-                logfh, ui.ferr, b'e', logdata=True
-            )
+        self._logfh = logfh
 
     def serve_forever(self):
         self.serveuntil(threading.Event())
-        self._ui.restorefinout(self._fin, self._fout)
 
     def serveuntil(self, ev):
         """Serve until a threading.Event is set."""
-        _runsshserver(
-            self._ui, self._repo, self._fin, self._fout, ev, self._accesshidden
-        )
+        with self._ui.protectedfinout() as (fin, fout):
+            if self._logfh:
+                # Log write I/O to stdout and stderr if configured.
+                fout = util.makeloggingfileobject(
+                    self._logfh,
+                    fout,
+                    b'o',
+                    logdata=True,
+                )
+                self._ui.ferr = util.makeloggingfileobject(
+                    self._logfh,
+                    self._ui.ferr,
+                    b'e',
+                    logdata=True,
+                )
+            _runsshserver(
+                self._ui,
+                self._repo,
+                fin,
+                fout,
+                ev,
+                self._accesshidden,
+            )

rust/hg-core/src/dirstate/dirs_multiset.rs

@@ -1,416 +1,415 b''
 // dirs_multiset.rs
 //
 // Copyright 2019 Raphaël Gomès <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! A multiset of directory names.
 //!
 //! Used to counts the references to directories in a manifest or dirstate.
 use crate::dirstate_tree::on_disk::DirstateV2ParseError;
 use crate::{
     utils::{
         files,
         hg_path::{HgPath, HgPathBuf, HgPathError},
     },
     DirstateEntry, DirstateError, DirstateMapError, FastHashMap,
 };
 use std::collections::{hash_map, hash_map::Entry, HashMap, HashSet};
 
 // could be encapsulated if we care API stability more seriously
 pub type DirsMultisetIter<'a> = hash_map::Keys<'a, HgPathBuf, u32>;
 
 #[derive(PartialEq, Debug)]
 pub struct DirsMultiset {
     inner: FastHashMap<HgPathBuf, u32>,
 }
 
 impl DirsMultiset {
     /// Initializes the multiset from a dirstate.
     ///
     /// If `skip_state` is provided, skips dirstate entries with equal state.
     pub fn from_dirstate<I, P>(
         dirstate: I,
         only_tracked: bool,
     ) -> Result<Self, DirstateError>
     where
         I: IntoIterator<
             Item = Result<(P, DirstateEntry), DirstateV2ParseError>,
         >,
         P: AsRef<HgPath>,
     {
         let mut multiset = DirsMultiset {
             inner: FastHashMap::default(),
         };
         for item in dirstate {
             let (filename, entry) = item?;
             let filename = filename.as_ref();
             // This `if` is optimized out of the loop
             if only_tracked {
                 if !entry.removed() {
                     multiset.add_path(filename)?;
                 }
             } else {
                 multiset.add_path(filename)?;
             }
         }
 
         Ok(multiset)
     }
 
     /// Initializes the multiset from a manifest.
     pub fn from_manifest(
         manifest: &[impl AsRef<HgPath>],
     ) -> Result<Self, HgPathError> {
         let mut multiset = DirsMultiset {
             inner: FastHashMap::default(),
         };
 
         for filename in manifest {
             multiset.add_path(filename.as_ref())?;
         }
 
         Ok(multiset)
     }
 
     /// Increases the count of deepest directory contained in the path.
     ///
     /// If the directory is not yet in the map, adds its parents.
     pub fn add_path(
         &mut self,
         path: impl AsRef<HgPath>,
     ) -> Result<(), HgPathError> {
         for subpath in files::find_dirs(path.as_ref()) {
             if subpath.as_bytes().last() == Some(&b'/') {
                 // TODO Remove this once PathAuditor is certified
                 // as the only entrypoint for path data
                 let second_slash_index = subpath.len() - 1;
 
                 return Err(HgPathError::ConsecutiveSlashes {
                     bytes: path.as_ref().as_bytes().to_owned(),
                     second_slash_index,
                 });
             }
             if let Some(val) = self.inner.get_mut(subpath) {
                 *val += 1;
                 break;
             }
             self.inner.insert(subpath.to_owned(), 1);
         }
         Ok(())
     }
 
     /// Decreases the count of deepest directory contained in the path.
     ///
     /// If it is the only reference, decreases all parents until one is
     /// removed.
     /// If the directory is not in the map, something horrible has happened.
     pub fn delete_path(
         &mut self,
         path: impl AsRef<HgPath>,
     ) -> Result<(), DirstateMapError> {
         for subpath in files::find_dirs(path.as_ref()) {
             match self.inner.entry(subpath.to_owned()) {
                 Entry::Occupied(mut entry) => {
                     let val = *entry.get();
                     if val > 1 {
                         entry.insert(val - 1);
                         break;
                     }
                     entry.remove();
                 }
                 Entry::Vacant(_) => {
                     return Err(DirstateMapError::PathNotFound(
                         path.as_ref().to_owned(),
                     ))
                 }
             };
         }
 
         Ok(())
     }
 
     pub fn contains(&self, key: impl AsRef<HgPath>) -> bool {
         self.inner.contains_key(key.as_ref())
     }
 
     pub fn iter(&self) -> DirsMultisetIter {
         self.inner.keys()
     }
 
     pub fn len(&self) -> usize {
         self.inner.len()
     }
 
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
 }
 
 /// This is basically a reimplementation of `DirsMultiset` that stores the
 /// children instead of just a count of them, plus a small optional
 /// optimization to avoid some directories we don't need.
 #[derive(PartialEq, Debug)]
 pub struct DirsChildrenMultiset<'a> {
     inner: FastHashMap<&'a HgPath, HashSet<&'a HgPath>>,
     only_include: Option<HashSet<&'a HgPath>>,
 }
 
 impl<'a> DirsChildrenMultiset<'a> {
-    pub fn new(
+    pub fn new<I: Iterator<Item = &'a HgPathBuf>>(
         paths: impl Iterator<Item = &'a HgPathBuf>,
-        only_include: Option<&'a HashSet<impl AsRef<HgPath> + 'a>>,
+        only_include: Option<I>,
     ) -> Self {
         let mut new = Self {
             inner: HashMap::default(),
-            only_include: only_include
-                .map(|s| s.iter().map(AsRef::as_ref).collect()),
+            only_include: only_include.map(|s| s.map(AsRef::as_ref).collect()),
         };
 
         for path in paths {
             new.add_path(path)
         }
 
         new
     }
     fn add_path(&mut self, path: &'a (impl AsRef<HgPath> + 'a)) {
         if path.as_ref().is_empty() {
             return;
         }
         for (directory, basename) in files::find_dirs_with_base(path.as_ref())
         {
             if !self.is_dir_included(directory) {
                 continue;
             }
             self.inner
                 .entry(directory)
                 .and_modify(|e| {
                     e.insert(basename);
                 })
                 .or_insert_with(|| {
                     let mut set = HashSet::new();
                     set.insert(basename);
                     set
                 });
         }
     }
     fn is_dir_included(&self, dir: impl AsRef<HgPath>) -> bool {
         match &self.only_include {
             None => false,
             Some(i) => i.contains(dir.as_ref()),
         }
     }
 
     pub fn get(
         &self,
         path: impl AsRef<HgPath>,
     ) -> Option<&HashSet<&'a HgPath>> {
         self.inner.get(path.as_ref())
     }
 }
 
 #[cfg(test)]
 mod tests {
     use crate::EntryState;
 
     use super::*;
 
     #[test]
     fn test_delete_path_path_not_found() {
         let manifest: Vec<HgPathBuf> = vec![];
         let mut map = DirsMultiset::from_manifest(&manifest).unwrap();
         let path = HgPathBuf::from_bytes(b"doesnotexist/");
         assert_eq!(
             Err(DirstateMapError::PathNotFound(path.to_owned())),
             map.delete_path(&path)
         );
     }
 
     #[test]
     fn test_delete_path_empty_path() {
         let mut map =
             DirsMultiset::from_manifest(&[HgPathBuf::new()]).unwrap();
         let path = HgPath::new(b"");
         assert_eq!(Ok(()), map.delete_path(path));
         assert_eq!(
             Err(DirstateMapError::PathNotFound(path.to_owned())),
             map.delete_path(path)
         );
     }
 
     #[test]
     fn test_delete_path_successful() {
         let mut map = DirsMultiset {
             inner: [("", 5), ("a", 3), ("a/b", 2), ("a/c", 1)]
                 .iter()
                 .map(|(k, v)| (HgPathBuf::from_bytes(k.as_bytes()), *v))
                 .collect(),
         };
 
         assert_eq!(Ok(()), map.delete_path(HgPath::new(b"a/b/")));
         eprintln!("{:?}", map);
         assert_eq!(Ok(()), map.delete_path(HgPath::new(b"a/b/")));
         eprintln!("{:?}", map);
         assert_eq!(
             Err(DirstateMapError::PathNotFound(HgPathBuf::from_bytes(
                 b"a/b/"
             ))),
             map.delete_path(HgPath::new(b"a/b/"))
         );
 
         assert_eq!(2, *map.inner.get(HgPath::new(b"a")).unwrap());
         assert_eq!(1, *map.inner.get(HgPath::new(b"a/c")).unwrap());
         eprintln!("{:?}", map);
         assert_eq!(Ok(()), map.delete_path(HgPath::new(b"a/")));
         eprintln!("{:?}", map);
 
         assert_eq!(Ok(()), map.delete_path(HgPath::new(b"a/c/")));
         assert_eq!(
             Err(DirstateMapError::PathNotFound(HgPathBuf::from_bytes(
                 b"a/c/"
             ))),
             map.delete_path(HgPath::new(b"a/c/"))
         );
     }
 
     #[test]
     fn test_add_path_empty_path() {
         let manifest: Vec<HgPathBuf> = vec![];
         let mut map = DirsMultiset::from_manifest(&manifest).unwrap();
         let path = HgPath::new(b"");
         map.add_path(path).unwrap();
 
         assert_eq!(1, map.len());
     }
 
     #[test]
     fn test_add_path_successful() {
         let manifest: Vec<HgPathBuf> = vec![];
         let mut map = DirsMultiset::from_manifest(&manifest).unwrap();
 
         map.add_path(HgPath::new(b"a/")).unwrap();
         assert_eq!(1, *map.inner.get(HgPath::new(b"a")).unwrap());
         assert_eq!(1, *map.inner.get(HgPath::new(b"")).unwrap());
         assert_eq!(2, map.len());
 
         // Non directory should be ignored
         map.add_path(HgPath::new(b"a")).unwrap();
         assert_eq!(1, *map.inner.get(HgPath::new(b"a")).unwrap());
         assert_eq!(2, map.len());
 
         // Non directory will still add its base
         map.add_path(HgPath::new(b"a/b")).unwrap();
         assert_eq!(2, *map.inner.get(HgPath::new(b"a")).unwrap());
         assert_eq!(2, map.len());
 
         // Duplicate path works
         map.add_path(HgPath::new(b"a/")).unwrap();
         assert_eq!(3, *map.inner.get(HgPath::new(b"a")).unwrap());
 
         // Nested dir adds to its base
         map.add_path(HgPath::new(b"a/b/")).unwrap();
         assert_eq!(4, *map.inner.get(HgPath::new(b"a")).unwrap());
         assert_eq!(1, *map.inner.get(HgPath::new(b"a/b")).unwrap());
 
         // but not its base's base, because it already existed
         map.add_path(HgPath::new(b"a/b/c/")).unwrap();
         assert_eq!(4, *map.inner.get(HgPath::new(b"a")).unwrap());
         assert_eq!(2, *map.inner.get(HgPath::new(b"a/b")).unwrap());
 
         map.add_path(HgPath::new(b"a/c/")).unwrap();
         assert_eq!(1, *map.inner.get(HgPath::new(b"a/c")).unwrap());
 
         let expected = DirsMultiset {
             inner: [("", 2), ("a", 5), ("a/b", 2), ("a/b/c", 1), ("a/c", 1)]
                 .iter()
                 .map(|(k, v)| (HgPathBuf::from_bytes(k.as_bytes()), *v))
                 .collect(),
         };
         assert_eq!(map, expected);
     }
 
     #[test]
     fn test_dirsmultiset_new_empty() {
         let manifest: Vec<HgPathBuf> = vec![];
         let new = DirsMultiset::from_manifest(&manifest).unwrap();
         let expected = DirsMultiset {
             inner: FastHashMap::default(),
         };
         assert_eq!(expected, new);
 
         let new = DirsMultiset::from_dirstate::<_, HgPathBuf>(
             std::iter::empty(),
             false,
         )
         .unwrap();
         let expected = DirsMultiset {
             inner: FastHashMap::default(),
         };
         assert_eq!(expected, new);
     }
 
     #[test]
     fn test_dirsmultiset_new_no_skip() {
         let input_vec: Vec<HgPathBuf> = ["a/", "b/", "a/c", "a/d/"]
             .iter()
             .map(|e| HgPathBuf::from_bytes(e.as_bytes()))
             .collect();
         let expected_inner = [("", 2), ("a", 3), ("b", 1), ("a/d", 1)]
             .iter()
             .map(|(k, v)| (HgPathBuf::from_bytes(k.as_bytes()), *v))
             .collect();
 
         let new = DirsMultiset::from_manifest(&input_vec).unwrap();
         let expected = DirsMultiset {
             inner: expected_inner,
         };
         assert_eq!(expected, new);
 
         let input_map = ["b/x", "a/c", "a/d/x"].iter().map(|f| {
             Ok((
                 HgPathBuf::from_bytes(f.as_bytes()),
                 DirstateEntry::from_v1_data(EntryState::Normal, 0, 0, 0),
             ))
         });
         let expected_inner = [("", 2), ("a", 2), ("b", 1), ("a/d", 1)]
             .iter()
             .map(|(k, v)| (HgPathBuf::from_bytes(k.as_bytes()), *v))
             .collect();
 
         let new = DirsMultiset::from_dirstate(input_map, false).unwrap();
         let expected = DirsMultiset {
             inner: expected_inner,
         };
         assert_eq!(expected, new);
     }
 
     #[test]
     fn test_dirsmultiset_new_skip() {
         let input_map = [
             ("a/", EntryState::Normal),
             ("a/b", EntryState::Normal),
             ("a/c", EntryState::Removed),
             ("a/d", EntryState::Merged),
         ]
         .iter()
         .map(|(f, state)| {
             Ok((
                 HgPathBuf::from_bytes(f.as_bytes()),
                 DirstateEntry::from_v1_data(*state, 0, 0, 0),
             ))
         });
 
         // "a" incremented with "a/c" and "a/d/"
         let expected_inner = [("", 1), ("a", 3)]
             .iter()
             .map(|(k, v)| (HgPathBuf::from_bytes(k.as_bytes()), *v))
             .collect();
 
         let new = DirsMultiset::from_dirstate(input_map, true).unwrap();
         let expected = DirsMultiset {
             inner: expected_inner,
         };
         assert_eq!(expected, new);
     }
 }

rust/hg-core/src/filepatterns.rs

@@ -1,874 +1,874 b''
 // filepatterns.rs
 //
 // Copyright 2019 Raphaël Gomès <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Handling of Mercurial-specific patterns.
 
 use crate::{
     utils::{
         files::{canonical_path, get_bytes_from_path, get_path_from_bytes},
         hg_path::{path_to_hg_path_buf, HgPathBuf, HgPathError},
         SliceExt,
     },
     FastHashMap, PatternError,
 };
 use lazy_static::lazy_static;
 use regex::bytes::{NoExpand, Regex};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::vec::Vec;
 
 lazy_static! {
     static ref RE_ESCAPE: Vec<Vec<u8>> = {
         let mut v: Vec<Vec<u8>> = (0..=255).map(|byte| vec![byte]).collect();
         let to_escape = b"()[]{}?*+-|^$\\.&~#\t\n\r\x0b\x0c";
         for byte in to_escape {
             v[*byte as usize].insert(0, b'\\');
         }
         v
     };
 }
 
 /// These are matched in order
 const GLOB_REPLACEMENTS: &[(&[u8], &[u8])] =
     &[(b"*/", b"(?:.*/)?"), (b"*", b".*"), (b"", b"[^/]*")];
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum PatternSyntax {
     /// A regular expression
     Regexp,
     /// Glob that matches at the front of the path
     RootGlob,
     /// Glob that matches at any suffix of the path (still anchored at
     /// slashes)
     Glob,
     /// a path relative to repository root, which is matched recursively
     Path,
     /// a single exact path relative to repository root
     FilePath,
     /// A path relative to cwd
     RelPath,
     /// an unrooted glob (*.rs matches Rust files in all dirs)
     RelGlob,
     /// A regexp that needn't match the start of a name
     RelRegexp,
     /// A path relative to repository root, which is matched non-recursively
     /// (will not match subdirectories)
-    RootFiles,
+    RootFilesIn,
     /// A file of patterns to read and include
     Include,
     /// A file of patterns to match against files under the same directory
     SubInclude,
     /// SubInclude with the result of parsing the included file
     ///
     /// Note: there is no ExpandedInclude because that expansion can be done
     /// in place by replacing the Include pattern by the included patterns.
     /// SubInclude requires more handling.
     ///
     /// Note: `Box` is used to minimize size impact on other enum variants
     ExpandedSubInclude(Box<SubInclude>),
 }
 
 /// Transforms a glob pattern into a regex
 pub fn glob_to_re(pat: &[u8]) -> Vec<u8> {
     let mut input = pat;
     let mut res: Vec<u8> = vec![];
     let mut group_depth = 0;
 
     while let Some((c, rest)) = input.split_first() {
         input = rest;
 
         match c {
             b'*' => {
                 for (source, repl) in GLOB_REPLACEMENTS {
                     if let Some(rest) = input.drop_prefix(source) {
                         input = rest;
                         res.extend(*repl);
                         break;
                     }
                 }
             }
             b'?' => res.extend(b"."),
             b'[' => {
                 match input.iter().skip(1).position(|b| *b == b']') {
                     None => res.extend(b"\\["),
                     Some(end) => {
                         // Account for the one we skipped
                         let end = end + 1;
 
                         res.extend(b"[");
 
                         for (i, b) in input[..end].iter().enumerate() {
                             if *b == b'!' && i == 0 {
                                 res.extend(b"^")
                             } else if *b == b'^' && i == 0 {
                                 res.extend(b"\\^")
                             } else if *b == b'\\' {
                                 res.extend(b"\\\\")
                             } else {
                                 res.push(*b)
                             }
                         }
                         res.extend(b"]");
                         input = &input[end + 1..];
                     }
                 }
             }
             b'{' => {
                 group_depth += 1;
                 res.extend(b"(?:")
             }
             b'}' if group_depth > 0 => {
                 group_depth -= 1;
                 res.extend(b")");
             }
             b',' if group_depth > 0 => res.extend(b"|"),
             b'\\' => {
                 let c = {
                     if let Some((c, rest)) = input.split_first() {
                         input = rest;
                         c
                     } else {
                         c
                     }
                 };
                 res.extend(&RE_ESCAPE[*c as usize])
             }
             _ => res.extend(&RE_ESCAPE[*c as usize]),
         }
     }
     res
 }
 
 fn escape_pattern(pattern: &[u8]) -> Vec<u8> {
     pattern
         .iter()
         .flat_map(|c| RE_ESCAPE[*c as usize].clone())
         .collect()
 }
 
 pub fn parse_pattern_syntax(
     kind: &[u8],
 ) -> Result<PatternSyntax, PatternError> {
     match kind {
         b"re:" => Ok(PatternSyntax::Regexp),
         b"path:" => Ok(PatternSyntax::Path),
         b"filepath:" => Ok(PatternSyntax::FilePath),
         b"relpath:" => Ok(PatternSyntax::RelPath),
-        b"rootfilesin:" => Ok(PatternSyntax::RootFiles),
+        b"rootfilesin:" => Ok(PatternSyntax::RootFilesIn),
         b"relglob:" => Ok(PatternSyntax::RelGlob),
         b"relre:" => Ok(PatternSyntax::RelRegexp),
         b"glob:" => Ok(PatternSyntax::Glob),
         b"rootglob:" => Ok(PatternSyntax::RootGlob),
         b"include:" => Ok(PatternSyntax::Include),
         b"subinclude:" => Ok(PatternSyntax::SubInclude),
         _ => Err(PatternError::UnsupportedSyntax(
             String::from_utf8_lossy(kind).to_string(),
         )),
     }
 }
 
 lazy_static! {
     static ref FLAG_RE: Regex = Regex::new(r"^\(\?[aiLmsux]+\)").unwrap();
 }
 
 /// Builds the regex that corresponds to the given pattern.
 /// If within a `syntax: regexp` context, returns the pattern,
 /// otherwise, returns the corresponding regex.
 fn _build_single_regex(entry: &IgnorePattern, glob_suffix: &[u8]) -> Vec<u8> {
     let IgnorePattern {
         syntax, pattern, ..
     } = entry;
     if pattern.is_empty() {
         return vec![];
     }
     match syntax {
         PatternSyntax::Regexp => pattern.to_owned(),
         PatternSyntax::RelRegexp => {
             // The `regex` crate accepts `**` while `re2` and Python's `re`
             // do not. Checking for `*` correctly triggers the same error all
             // engines.
             if pattern[0] == b'^'
                 || pattern[0] == b'*'
                 || pattern.starts_with(b".*")
             {
                 return pattern.to_owned();
             }
             match FLAG_RE.find(pattern) {
                 Some(mat) => {
                     let s = mat.start();
                     let e = mat.end();
                     [
                         &b"(?"[..],
                         &pattern[s + 2..e - 1],
                         &b":"[..],
                         if pattern[e] == b'^'
                             || pattern[e] == b'*'
                             || pattern[e..].starts_with(b".*")
                         {
                             &b""[..]
                         } else {
                             &b".*"[..]
                         },
                         &pattern[e..],
                         &b")"[..],
                     ]
                     .concat()
                 }
                 None => [&b".*"[..], pattern].concat(),
             }
         }
         PatternSyntax::Path | PatternSyntax::RelPath => {
             if pattern == b"." {
                 return vec![];
             }
             [escape_pattern(pattern).as_slice(), b"(?:/|$)"].concat()
         }
-        PatternSyntax::RootFiles => {
+        PatternSyntax::RootFilesIn => {
             let mut res = if pattern == b"." {
                 vec![]
             } else {
                 // Pattern is a directory name.
                 [escape_pattern(pattern).as_slice(), b"/"].concat()
             };
 
             // Anything after the pattern must be a non-directory.
             res.extend(b"[^/]+$");
             res
         }
         PatternSyntax::RelGlob => {
             let glob_re = glob_to_re(pattern);
             if let Some(rest) = glob_re.drop_prefix(b"[^/]*") {
                 [b".*", rest, glob_suffix].concat()
             } else {
                 [b"(?:.*/)?", glob_re.as_slice(), glob_suffix].concat()
             }
         }
         PatternSyntax::Glob | PatternSyntax::RootGlob => {
             [glob_to_re(pattern).as_slice(), glob_suffix].concat()
         }
         PatternSyntax::Include
         | PatternSyntax::SubInclude
         | PatternSyntax::ExpandedSubInclude(_)
         | PatternSyntax::FilePath => unreachable!(),
     }
 }
 
 const GLOB_SPECIAL_CHARACTERS: [u8; 7] =
     [b'*', b'?', b'[', b']', b'{', b'}', b'\\'];
 
 /// TODO support other platforms
 #[cfg(unix)]
 pub fn normalize_path_bytes(bytes: &[u8]) -> Vec<u8> {
     if bytes.is_empty() {
         return b".".to_vec();
     }
     let sep = b'/';
 
     let mut initial_slashes = bytes.iter().take_while(|b| **b == sep).count();
     if initial_slashes > 2 {
         // POSIX allows one or two initial slashes, but treats three or more
         // as single slash.
         initial_slashes = 1;
     }
     let components = bytes
         .split(|b| *b == sep)
         .filter(|c| !(c.is_empty() || c == b"."))
         .fold(vec![], |mut acc, component| {
             if component != b".."
                 || (initial_slashes == 0 && acc.is_empty())
                 || (!acc.is_empty() && acc[acc.len() - 1] == b"..")
             {
                 acc.push(component)
             } else if !acc.is_empty() {
                 acc.pop();
             }
             acc
         });
     let mut new_bytes = components.join(&sep);
 
     if initial_slashes > 0 {
         let mut buf: Vec<_> = (0..initial_slashes).map(|_| sep).collect();
         buf.extend(new_bytes);
         new_bytes = buf;
     }
     if new_bytes.is_empty() {
         b".".to_vec()
     } else {
         new_bytes
     }
 }
 
 /// Wrapper function to `_build_single_regex` that short-circuits 'exact' globs
 /// that don't need to be transformed into a regex.
 pub fn build_single_regex(
     entry: &IgnorePattern,
     glob_suffix: &[u8],
 ) -> Result<Option<Vec<u8>>, PatternError> {
     let IgnorePattern {
         pattern, syntax, ..
     } = entry;
     let pattern = match syntax {
         PatternSyntax::RootGlob
         | PatternSyntax::Path
         | PatternSyntax::RelGlob
         | PatternSyntax::RelPath
-        | PatternSyntax::RootFiles => normalize_path_bytes(pattern),
+        | PatternSyntax::RootFilesIn => normalize_path_bytes(pattern),
         PatternSyntax::Include | PatternSyntax::SubInclude => {
             return Err(PatternError::NonRegexPattern(entry.clone()))
         }
         _ => pattern.to_owned(),
     };
     let is_simple_rootglob = *syntax == PatternSyntax::RootGlob
         && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b));
     if is_simple_rootglob || syntax == &PatternSyntax::FilePath {
         Ok(None)
     } else {
         let mut entry = entry.clone();
         entry.pattern = pattern;
         Ok(Some(_build_single_regex(&entry, glob_suffix)))
     }
 }
 
 lazy_static! {
     static ref SYNTAXES: FastHashMap<&'static [u8], PatternSyntax> = {
         let mut m = FastHashMap::default();
 
         m.insert(b"re:".as_ref(), PatternSyntax::Regexp);
         m.insert(b"regexp:".as_ref(), PatternSyntax::Regexp);
         m.insert(b"path:".as_ref(), PatternSyntax::Path);
         m.insert(b"filepath:".as_ref(), PatternSyntax::FilePath);
         m.insert(b"relpath:".as_ref(), PatternSyntax::RelPath);
-        m.insert(b"rootfilesin:".as_ref(), PatternSyntax::RootFiles);
+        m.insert(b"rootfilesin:".as_ref(), PatternSyntax::RootFilesIn);
         m.insert(b"relglob:".as_ref(), PatternSyntax::RelGlob);
         m.insert(b"relre:".as_ref(), PatternSyntax::RelRegexp);
         m.insert(b"glob:".as_ref(), PatternSyntax::Glob);
         m.insert(b"rootglob:".as_ref(), PatternSyntax::RootGlob);
         m.insert(b"include:".as_ref(), PatternSyntax::Include);
         m.insert(b"subinclude:".as_ref(), PatternSyntax::SubInclude);
 
         m
     };
 }
 
 #[derive(Debug)]
 pub enum PatternFileWarning {
     /// (file path, syntax bytes)
     InvalidSyntax(PathBuf, Vec<u8>),
     /// File path
     NoSuchFile(PathBuf),
 }
 
 pub fn parse_one_pattern(
     pattern: &[u8],
     source: &Path,
     default: PatternSyntax,
     normalize: bool,
 ) -> IgnorePattern {
     let mut pattern_bytes: &[u8] = pattern;
     let mut syntax = default;
 
     for (s, val) in SYNTAXES.iter() {
         if let Some(rest) = pattern_bytes.drop_prefix(s) {
             syntax = val.clone();
             pattern_bytes = rest;
             break;
         }
     }
 
     let pattern = match syntax {
         PatternSyntax::RootGlob
         | PatternSyntax::Path
         | PatternSyntax::Glob
         | PatternSyntax::RelGlob
         | PatternSyntax::RelPath
-        | PatternSyntax::RootFiles
+        | PatternSyntax::RootFilesIn
             if normalize =>
         {
             normalize_path_bytes(pattern_bytes)
         }
         _ => pattern_bytes.to_vec(),
     };
 
     IgnorePattern {
         syntax,
         pattern,
         source: source.to_owned(),
     }
 }
 
 pub fn parse_pattern_file_contents(
     lines: &[u8],
     file_path: &Path,
     default_syntax_override: Option<PatternSyntax>,
     warn: bool,
     relativize: bool,
 ) -> Result<(Vec<IgnorePattern>, Vec<PatternFileWarning>), PatternError> {
     let comment_regex = Regex::new(r"((?:^|[^\\])(?:\\\\)*)#.*").unwrap();
 
     #[allow(clippy::trivial_regex)]
     let comment_escape_regex = Regex::new(r"\\#").unwrap();
     let mut inputs: Vec<IgnorePattern> = vec![];
     let mut warnings: Vec<PatternFileWarning> = vec![];
 
     let mut current_syntax =
         default_syntax_override.unwrap_or(PatternSyntax::RelRegexp);
 
     for mut line in lines.split(|c| *c == b'\n') {
         let line_buf;
         if line.contains(&b'#') {
             if let Some(cap) = comment_regex.captures(line) {
                 line = &line[..cap.get(1).unwrap().end()]
             }
             line_buf = comment_escape_regex.replace_all(line, NoExpand(b"#"));
             line = &line_buf;
         }
 
         let line = line.trim_end();
 
         if line.is_empty() {
             continue;
         }
 
         if let Some(syntax) = line.drop_prefix(b"syntax:") {
             let syntax = syntax.trim();
 
             if let Some(parsed) =
                 SYNTAXES.get([syntax, &b":"[..]].concat().as_slice())
             {
                 current_syntax = parsed.clone();
             } else if warn {
                 warnings.push(PatternFileWarning::InvalidSyntax(
                     file_path.to_owned(),
                     syntax.to_owned(),
                 ));
             }
         } else {
             let pattern = parse_one_pattern(
                 line,
                 file_path,
                 current_syntax.clone(),
                 false,
             );
             inputs.push(if relativize {
                 pattern.to_relative()
             } else {
                 pattern
             })
         }
     }
     Ok((inputs, warnings))
 }
 
 pub fn parse_pattern_args(
     patterns: Vec<Vec<u8>>,
     cwd: &Path,
     root: &Path,
 ) -> Result<Vec<IgnorePattern>, HgPathError> {
     let mut ignore_patterns: Vec<IgnorePattern> = Vec::new();
     for pattern in patterns {
         let pattern = parse_one_pattern(
             &pattern,
             Path::new("<args>"),
             PatternSyntax::RelPath,
             true,
         );
         match pattern.syntax {
             PatternSyntax::RelGlob | PatternSyntax::RelPath => {
                 let name = get_path_from_bytes(&pattern.pattern);
                 let canon = canonical_path(root, cwd, name)?;
                 ignore_patterns.push(IgnorePattern {
                     syntax: pattern.syntax,
                     pattern: get_bytes_from_path(canon),
                     source: pattern.source,
                 })
             }
             _ => ignore_patterns.push(pattern.to_owned()),
         };
     }
     Ok(ignore_patterns)
 }
 
 pub fn read_pattern_file(
     file_path: &Path,
     warn: bool,
     inspect_pattern_bytes: &mut impl FnMut(&Path, &[u8]),
 ) -> Result<(Vec<IgnorePattern>, Vec<PatternFileWarning>), PatternError> {
     match std::fs::read(file_path) {
         Ok(contents) => {
             inspect_pattern_bytes(file_path, &contents);
             parse_pattern_file_contents(&contents, file_path, None, warn, true)
         }
         Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok((
             vec![],
             vec![PatternFileWarning::NoSuchFile(file_path.to_owned())],
         )),
         Err(e) => Err(e.into()),
     }
 }
 
 /// Represents an entry in an "ignore" file.
 #[derive(Debug, Eq, PartialEq, Clone)]
 pub struct IgnorePattern {
     pub syntax: PatternSyntax,
     pub pattern: Vec<u8>,
     pub source: PathBuf,
 }
 
 impl IgnorePattern {
     pub fn new(syntax: PatternSyntax, pattern: &[u8], source: &Path) -> Self {
         Self {
             syntax,
             pattern: pattern.to_owned(),
             source: source.to_owned(),
         }
     }
 
     pub fn to_relative(self) -> Self {
         let Self {
             syntax,
             pattern,
             source,
         } = self;
         Self {
             syntax: match syntax {
                 PatternSyntax::Regexp => PatternSyntax::RelRegexp,
                 PatternSyntax::Glob => PatternSyntax::RelGlob,
                 x => x,
             },
             pattern,
             source,
         }
     }
 }
 
 pub type PatternResult<T> = Result<T, PatternError>;
 
 /// Wrapper for `read_pattern_file` that also recursively expands `include:`
 /// and `subinclude:` patterns.
 ///
 /// The former are expanded in place, while `PatternSyntax::ExpandedSubInclude`
 /// is used for the latter to form a tree of patterns.
 pub fn get_patterns_from_file(
     pattern_file: &Path,
     root_dir: &Path,
     inspect_pattern_bytes: &mut impl FnMut(&Path, &[u8]),
 ) -> PatternResult<(Vec<IgnorePattern>, Vec<PatternFileWarning>)> {
     let (patterns, mut warnings) =
         read_pattern_file(pattern_file, true, inspect_pattern_bytes)?;
     let patterns = patterns
         .into_iter()
         .flat_map(|entry| -> PatternResult<_> {
             Ok(match &entry.syntax {
                 PatternSyntax::Include => {
                     let inner_include =
                         root_dir.join(get_path_from_bytes(&entry.pattern));
                     let (inner_pats, inner_warnings) = get_patterns_from_file(
                         &inner_include,
                         root_dir,
                         inspect_pattern_bytes,
                     )?;
                     warnings.extend(inner_warnings);
                     inner_pats
                 }
                 PatternSyntax::SubInclude => {
                     let mut sub_include = SubInclude::new(
                         root_dir,
                         &entry.pattern,
                         &entry.source,
                     )?;
                     let (inner_patterns, inner_warnings) =
                         get_patterns_from_file(
                             &sub_include.path,
                             &sub_include.root,
                             inspect_pattern_bytes,
                         )?;
                     sub_include.included_patterns = inner_patterns;
                     warnings.extend(inner_warnings);
                     vec![IgnorePattern {
                         syntax: PatternSyntax::ExpandedSubInclude(Box::new(
                             sub_include,
                         )),
                         ..entry
                     }]
                 }
                 _ => vec![entry],
             })
         })
         .flatten()
         .collect();
 
     Ok((patterns, warnings))
 }
 
 /// Holds all the information needed to handle a `subinclude:` pattern.
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct SubInclude {
     /// Will be used for repository (hg) paths that start with this prefix.
     /// It is relative to the current working directory, so comparing against
     /// repository paths is painless.
     pub prefix: HgPathBuf,
     /// The file itself, containing the patterns
     pub path: PathBuf,
     /// Folder in the filesystem where this it applies
     pub root: PathBuf,
 
     pub included_patterns: Vec<IgnorePattern>,
 }
 
 impl SubInclude {
     pub fn new(
         root_dir: &Path,
         pattern: &[u8],
         source: &Path,
     ) -> Result<SubInclude, HgPathError> {
         let normalized_source =
             normalize_path_bytes(&get_bytes_from_path(source));
 
         let source_root = get_path_from_bytes(&normalized_source);
         let source_root = source_root.parent().unwrap_or(source_root);
 
         let path = source_root.join(get_path_from_bytes(pattern));
         let new_root = path.parent().unwrap_or_else(|| path.deref());
 
         let prefix = canonical_path(root_dir, root_dir, new_root)?;
 
         Ok(Self {
             prefix: path_to_hg_path_buf(prefix).map(|mut p| {
                 if !p.is_empty() {
                     p.push_byte(b'/');
                 }
                 p
             })?,
             path: path.to_owned(),
             root: new_root.to_owned(),
             included_patterns: Vec::new(),
         })
     }
 }
 
 /// Separate and pre-process subincludes from other patterns for the "ignore"
 /// phase.
 pub fn filter_subincludes(
     ignore_patterns: Vec<IgnorePattern>,
 ) -> Result<(Vec<SubInclude>, Vec<IgnorePattern>), HgPathError> {
     let mut subincludes = vec![];
     let mut others = vec![];
 
     for pattern in ignore_patterns {
         if let PatternSyntax::ExpandedSubInclude(sub_include) = pattern.syntax
         {
             subincludes.push(*sub_include);
         } else {
             others.push(pattern)
         }
     }
     Ok((subincludes, others))
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
 
     #[test]
     fn escape_pattern_test() {
         let untouched =
             br#"!"%',/0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ_`abcdefghijklmnopqrstuvwxyz"#;
         assert_eq!(escape_pattern(untouched), untouched.to_vec());
         // All escape codes
         assert_eq!(
             escape_pattern(br"()[]{}?*+-|^$\\.&~#\t\n\r\v\f"),
             br"\(\)\[\]\{\}\?\*\+\-\|\^\$\\\\\.\&\~\#\\t\\n\\r\\v\\f".to_vec()
         );
     }
 
     #[test]
     fn glob_test() {
         assert_eq!(glob_to_re(br"?"), br".");
         assert_eq!(glob_to_re(br"*"), br"[^/]*");
         assert_eq!(glob_to_re(br"**"), br".*");
         assert_eq!(glob_to_re(br"**/a"), br"(?:.*/)?a");
         assert_eq!(glob_to_re(br"a/**/b"), br"a/(?:.*/)?b");
         assert_eq!(glob_to_re(br"[a*?!^][^b][!c]"), br"[a*?!^][\^b][^c]");
         assert_eq!(glob_to_re(br"{a,b}"), br"(?:a|b)");
         assert_eq!(glob_to_re(br".\*\?"), br"\.\*\?");
     }
 
     #[test]
     fn test_parse_pattern_file_contents() {
         let lines = b"syntax: glob\n*.elc";
 
         assert_eq!(
             parse_pattern_file_contents(
                 lines,
                 Path::new("file_path"),
                 None,
                 false,
                 true,
             )
             .unwrap()
             .0,
             vec![IgnorePattern::new(
                 PatternSyntax::RelGlob,
                 b"*.elc",
                 Path::new("file_path")
             )],
         );
 
         let lines = b"syntax: include\nsyntax: glob";
 
         assert_eq!(
             parse_pattern_file_contents(
                 lines,
                 Path::new("file_path"),
                 None,
                 false,
                 true,
             )
             .unwrap()
             .0,
             vec![]
         );
         let lines = b"glob:**.o";
         assert_eq!(
             parse_pattern_file_contents(
                 lines,
                 Path::new("file_path"),
                 None,
                 false,
                 true,
             )
             .unwrap()
             .0,
             vec![IgnorePattern::new(
                 PatternSyntax::RelGlob,
                 b"**.o",
                 Path::new("file_path")
             )]
         );
     }
 
     #[test]
     fn test_build_single_regex() {
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RelGlob,
                     b"rust/target/",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(br"(?:.*/)?rust/target(?:/|$)".to_vec()),
         );
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::Regexp,
                     br"rust/target/\d+",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(br"rust/target/\d+".to_vec()),
         );
     }
 
     #[test]
     fn test_build_single_regex_shortcut() {
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RootGlob,
                     b"",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             None,
         );
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RootGlob,
                     b"whatever",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             None,
         );
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RootGlob,
                     b"*.o",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(br"[^/]*\.o(?:/|$)".to_vec()),
         );
     }
 
     #[test]
     fn test_build_single_relregex() {
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RelRegexp,
                     b"^ba{2}r",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(b"^ba{2}r".to_vec()),
         );
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RelRegexp,
                     b"ba{2}r",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(b".*ba{2}r".to_vec()),
         );
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RelRegexp,
                     b"(?ia)ba{2}r",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(b"(?ia:.*ba{2}r)".to_vec()),
         );
         assert_eq!(
             build_single_regex(
                 &IgnorePattern::new(
                     PatternSyntax::RelRegexp,
                     b"(?ia)^ba{2}r",
                     Path::new("")
                 ),
                 b"(?:/|$)"
             )
             .unwrap(),
             Some(b"(?ia:^ba{2}r)".to_vec()),
         );
     }
 }

rust/hg-core/src/matchers.rs

@@ -1,2122 +1,2450 b''
 // matchers.rs
 //
 // Copyright 2019 Raphaël Gomès <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Structs and types for matching files and directories.
 
 use format_bytes::format_bytes;
 use once_cell::sync::OnceCell;
 
 use crate::{
     dirstate::dirs_multiset::DirsChildrenMultiset,
     filepatterns::{
         build_single_regex, filter_subincludes, get_patterns_from_file,
         PatternFileWarning, PatternResult,
     },
     utils::{
-        files::find_dirs,
+        files::{dir_ancestors, find_dirs},
         hg_path::{HgPath, HgPathBuf, HgPathError},
         Escaped,
     },
     DirsMultiset, FastHashMap, IgnorePattern, PatternError, PatternSyntax,
 };
 
 use crate::dirstate::status::IgnoreFnType;
 use crate::filepatterns::normalize_path_bytes;
 use std::collections::HashSet;
 use std::fmt::{Display, Error, Formatter};
 use std::path::{Path, PathBuf};
 use std::{borrow::ToOwned, collections::BTreeSet};
 
 #[derive(Debug, PartialEq)]
 pub enum VisitChildrenSet {
     /// Don't visit anything
     Empty,
-    /// Only visit this directory
+    /// Visit this directory and probably its children
     This,
-    /// Visit this directory and these subdirectories
+    /// Only visit the children (both files and directories) if they
+    /// are mentioned in this set. (empty set corresponds to [Empty])
     /// TODO Should we implement a `NonEmptyHashSet`?
     Set(HashSet<HgPathBuf>),
     /// Visit this directory and all subdirectories
+    /// (you can stop asking about the children set)
     Recursive,
 }
 
 pub trait Matcher: core::fmt::Debug {
     /// Explicitly listed files
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>>;
     /// Returns whether `filename` is in `file_set`
     fn exact_match(&self, filename: &HgPath) -> bool;
     /// Returns whether `filename` is matched by this matcher
     fn matches(&self, filename: &HgPath) -> bool;
     /// Decides whether a directory should be visited based on whether it
     /// has potential matches in it or one of its subdirectories, and
     /// potentially lists which subdirectories of that directory should be
     /// visited. This is based on the match's primary, included, and excluded
     /// patterns.
     ///
     /// # Example
     ///
     /// Assume matchers `['path:foo/bar', 'rootfilesin:qux']`, we would
     /// return the following values (assuming the implementation of
     /// visit_children_set is capable of recognizing this; some implementations
     /// are not).
     ///
     /// ```text
     /// ```ignore
     /// '' -> {'foo', 'qux'}
     /// 'baz' -> set()
     /// 'foo' -> {'bar'}
     /// // Ideally this would be `Recursive`, but since the prefix nature of
     /// // matchers is applied to the entire matcher, we have to downgrade this
     /// // to `This` due to the (yet to be implemented in Rust) non-prefix
     /// // `RootFilesIn'-kind matcher being mixed in.
     /// 'foo/bar' -> 'this'
     /// 'qux' -> 'this'
     /// ```
     /// # Important
     ///
     /// Most matchers do not know if they're representing files or
     /// directories. They see `['path:dir/f']` and don't know whether `f` is a
     /// file or a directory, so `visit_children_set('dir')` for most matchers
     /// will return `HashSet{ HgPath { "f" } }`, but if the matcher knows it's
     /// a file (like the yet to be implemented in Rust `ExactMatcher` does),
     /// it may return `VisitChildrenSet::This`.
     /// Do not rely on the return being a `HashSet` indicating that there are
     /// no files in this dir to investigate (or equivalently that if there are
     /// files to investigate in 'dir' that it will always return
     /// `VisitChildrenSet::This`).
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet;
     /// Matcher will match everything and `files_set()` will be empty:
     /// optimization might be possible.
     fn matches_everything(&self) -> bool;
     /// Matcher will match exactly the files in `files_set()`: optimization
     /// might be possible.
     fn is_exact(&self) -> bool;
 }
 
 /// Matches everything.
 ///```
 /// use hg::{ matchers::{Matcher, AlwaysMatcher}, utils::hg_path::HgPath };
 ///
 /// let matcher = AlwaysMatcher;
 ///
 /// assert_eq!(matcher.matches(HgPath::new(b"whatever")), true);
 /// assert_eq!(matcher.matches(HgPath::new(b"b.txt")), true);
 /// assert_eq!(matcher.matches(HgPath::new(b"main.c")), true);
 /// assert_eq!(matcher.matches(HgPath::new(br"re:.*\.c$")), true);
 /// ```
 #[derive(Debug)]
 pub struct AlwaysMatcher;
 
 impl Matcher for AlwaysMatcher {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         None
     }
     fn exact_match(&self, _filename: &HgPath) -> bool {
         false
     }
     fn matches(&self, _filename: &HgPath) -> bool {
         true
     }
     fn visit_children_set(&self, _directory: &HgPath) -> VisitChildrenSet {
         VisitChildrenSet::Recursive
     }
     fn matches_everything(&self) -> bool {
         true
     }
     fn is_exact(&self) -> bool {
         false
     }
 }
 
 /// Matches nothing.
 #[derive(Debug)]
 pub struct NeverMatcher;
 
 impl Matcher for NeverMatcher {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         None
     }
     fn exact_match(&self, _filename: &HgPath) -> bool {
         false
     }
     fn matches(&self, _filename: &HgPath) -> bool {
         false
     }
     fn visit_children_set(&self, _directory: &HgPath) -> VisitChildrenSet {
         VisitChildrenSet::Empty
     }
     fn matches_everything(&self) -> bool {
         false
     }
     fn is_exact(&self) -> bool {
         true
     }
 }
 
 /// Matches the input files exactly. They are interpreted as paths, not
 /// patterns.
 ///
 ///```
 /// use hg::{ matchers::{Matcher, FileMatcher}, utils::hg_path::{HgPath, HgPathBuf} };
 ///
 /// let files = vec![HgPathBuf::from_bytes(b"a.txt"), HgPathBuf::from_bytes(br"re:.*\.c$")];
 /// let matcher = FileMatcher::new(files).unwrap();
 ///
 /// assert_eq!(matcher.matches(HgPath::new(b"a.txt")), true);
 /// assert_eq!(matcher.matches(HgPath::new(b"b.txt")), false);
 /// assert_eq!(matcher.matches(HgPath::new(b"main.c")), false);
 /// assert_eq!(matcher.matches(HgPath::new(br"re:.*\.c$")), true);
 /// ```
 #[derive(Debug)]
 pub struct FileMatcher {
     files: HashSet<HgPathBuf>,
     dirs: DirsMultiset,
     sorted_visitchildrenset_candidates: OnceCell<BTreeSet<HgPathBuf>>,
 }
 
 impl FileMatcher {
     pub fn new(files: Vec<HgPathBuf>) -> Result<Self, HgPathError> {
         let dirs = DirsMultiset::from_manifest(&files)?;
         Ok(Self {
             files: HashSet::from_iter(files),
             dirs,
             sorted_visitchildrenset_candidates: OnceCell::new(),
         })
     }
     fn inner_matches(&self, filename: &HgPath) -> bool {
         self.files.contains(filename.as_ref())
     }
 }
 
 impl Matcher for FileMatcher {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         Some(&self.files)
     }
     fn exact_match(&self, filename: &HgPath) -> bool {
         self.inner_matches(filename)
     }
     fn matches(&self, filename: &HgPath) -> bool {
         self.inner_matches(filename)
     }
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet {
         if self.files.is_empty() || !self.dirs.contains(directory) {
             return VisitChildrenSet::Empty;
         }
 
         let compute_candidates = || -> BTreeSet<HgPathBuf> {
             let mut candidates: BTreeSet<HgPathBuf> =
                 self.dirs.iter().cloned().collect();
             candidates.extend(self.files.iter().cloned());
             candidates.remove(HgPath::new(b""));
             candidates
         };
         let candidates =
             if directory.as_ref().is_empty() {
                 compute_candidates()
             } else {
                 let sorted_candidates = self
                     .sorted_visitchildrenset_candidates
                     .get_or_init(compute_candidates);
                 let directory_bytes = directory.as_ref().as_bytes();
                 let start: HgPathBuf =
                     format_bytes!(b"{}/", directory_bytes).into();
                 let start_len = start.len();
                 // `0` sorts after `/`
                 let end = format_bytes!(b"{}0", directory_bytes).into();
                 BTreeSet::from_iter(sorted_candidates.range(start..end).map(
                     |c| HgPathBuf::from_bytes(&c.as_bytes()[start_len..]),
                 ))
             };
 
         // `self.dirs` includes all of the directories, recursively, so if
         // we're attempting to match 'foo/bar/baz.txt', it'll have '', 'foo',
         // 'foo/bar' in it. Thus we can safely ignore a candidate that has a
         // '/' in it, indicating it's for a subdir-of-a-subdir; the immediate
         // subdir will be in there without a slash.
         VisitChildrenSet::Set(
             candidates
                 .into_iter()
                 .filter_map(|c| {
                     if c.bytes().all(|b| *b != b'/') {
                         Some(c)
                     } else {
                         None
                     }
                 })
                 .collect(),
         )
     }
     fn matches_everything(&self) -> bool {
         false
     }
     fn is_exact(&self) -> bool {
         true
     }
 }
 
 /// Matches a set of (kind, pat, source) against a 'root' directory.
 /// (Currently the 'root' directory is effectively always empty)
 /// ```
 /// use hg::{
 ///     matchers::{PatternMatcher, Matcher},
 ///     IgnorePattern,
 ///     PatternSyntax,
 ///     utils::hg_path::{HgPath, HgPathBuf}
 /// };
 /// use std::collections::HashSet;
 /// use std::path::Path;
 /// ///
 /// let ignore_patterns : Vec<IgnorePattern> =
 ///     vec![IgnorePattern::new(PatternSyntax::Regexp, br".*\.c$", Path::new("")),
 ///          IgnorePattern::new(PatternSyntax::Path, b"foo/a", Path::new("")),
 ///          IgnorePattern::new(PatternSyntax::RelPath, b"b", Path::new("")),
 ///          IgnorePattern::new(PatternSyntax::Glob, b"*.h", Path::new("")),
 ///     ];
 /// let matcher = PatternMatcher::new(ignore_patterns).unwrap();
 /// ///
 /// assert_eq!(matcher.matches(HgPath::new(b"main.c")), true); // matches re:.*\.c$
 /// assert_eq!(matcher.matches(HgPath::new(b"b.txt")), false);
 /// assert_eq!(matcher.matches(HgPath::new(b"foo/a")), true); // matches path:foo/a
 /// assert_eq!(matcher.matches(HgPath::new(b"a")), false); // does not match path:b, since 'root' is 'foo'
 /// assert_eq!(matcher.matches(HgPath::new(b"b")), true); // matches relpath:b, since 'root' is 'foo'
 /// assert_eq!(matcher.matches(HgPath::new(b"lib.h")), true); // matches glob:*.h
 /// assert_eq!(matcher.file_set().unwrap(),
 ///            &HashSet::from([HgPathBuf::from_bytes(b""), HgPathBuf::from_bytes(b"foo/a"),
 ///                            HgPathBuf::from_bytes(b""), HgPathBuf::from_bytes(b"b")]));
 /// assert_eq!(matcher.exact_match(HgPath::new(b"foo/a")), true);
 /// assert_eq!(matcher.exact_match(HgPath::new(b"b")), true);
 /// assert_eq!(matcher.exact_match(HgPath::new(b"lib.h")), false); // exact matches are for (rel)path kinds
 /// ```
 pub struct PatternMatcher<'a> {
     patterns: Vec<u8>,
     match_fn: IgnoreFnType<'a>,
     /// Whether all the patterns match a prefix (i.e. recursively)
     prefix: bool,
     files: HashSet<HgPathBuf>,
+    dirs_explicit: HashSet<HgPathBuf>,
     dirs: DirsMultiset,
 }
 
 impl core::fmt::Debug for PatternMatcher<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("PatternMatcher")
             .field("patterns", &String::from_utf8_lossy(&self.patterns))
             .field("prefix", &self.prefix)
             .field("files", &self.files)
             .field("dirs", &self.dirs)
             .finish()
     }
 }
 
 impl<'a> PatternMatcher<'a> {
     pub fn new(ignore_patterns: Vec<IgnorePattern>) -> PatternResult<Self> {
-        let (files, _) = roots_and_dirs(&ignore_patterns);
-        let dirs = DirsMultiset::from_manifest(&files)?;
+        let RootsDirsAndParents {
+            roots,
+            dirs: dirs_explicit,
+            parents,
+        } = roots_dirs_and_parents(&ignore_patterns)?;
+        let files = roots;
+        let dirs = parents;
         let files: HashSet<HgPathBuf> = HashSet::from_iter(files);
 
         let prefix = ignore_patterns.iter().all(|k| {
             matches!(k.syntax, PatternSyntax::Path | PatternSyntax::RelPath)
         });
         let (patterns, match_fn) = build_match(ignore_patterns, b"$")?;
 
         Ok(Self {
             patterns,
             match_fn,
             prefix,
             files,
             dirs,
+            dirs_explicit,
         })
     }
 }
 
 impl<'a> Matcher for PatternMatcher<'a> {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         Some(&self.files)
     }
 
     fn exact_match(&self, filename: &HgPath) -> bool {
         self.files.contains(filename)
     }
 
     fn matches(&self, filename: &HgPath) -> bool {
         if self.files.contains(filename) {
             return true;
         }
         (self.match_fn)(filename)
     }
 
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet {
         if self.prefix && self.files.contains(directory) {
             return VisitChildrenSet::Recursive;
         }
-        let path_or_parents_in_set = find_dirs(directory)
-            .any(|parent_dir| self.files.contains(parent_dir));
-        if self.dirs.contains(directory) || path_or_parents_in_set {
+        if self.dirs.contains(directory) {
+            return VisitChildrenSet::This;
+        }
+        if dir_ancestors(directory).any(|parent_dir| {
+            self.files.contains(parent_dir)
+                || self.dirs_explicit.contains(parent_dir)
+        }) {
             VisitChildrenSet::This
         } else {
             VisitChildrenSet::Empty
         }
     }
 
     fn matches_everything(&self) -> bool {
         false
     }
 
     fn is_exact(&self) -> bool {
         false
     }
 }
 
 /// Matches files that are included in the ignore rules.
 /// ```
 /// use hg::{
 ///     matchers::{IncludeMatcher, Matcher},
 ///     IgnorePattern,
 ///     PatternSyntax,
 ///     utils::hg_path::HgPath
 /// };
 /// use std::path::Path;
 /// ///
 /// let ignore_patterns =
 /// vec![IgnorePattern::new(PatternSyntax::RootGlob, b"this*", Path::new(""))];
 /// let matcher = IncludeMatcher::new(ignore_patterns).unwrap();
 /// ///
 /// assert_eq!(matcher.matches(HgPath::new(b"testing")), false);
 /// assert_eq!(matcher.matches(HgPath::new(b"this should work")), true);
 /// assert_eq!(matcher.matches(HgPath::new(b"this also")), true);
 /// assert_eq!(matcher.matches(HgPath::new(b"but not this")), false);
 /// ///
 /// let ignore_patterns =
-/// vec![IgnorePattern::new(PatternSyntax::RootFiles, b"dir/subdir", Path::new(""))];
+/// vec![IgnorePattern::new(PatternSyntax::RootFilesIn, b"dir/subdir", Path::new(""))];
 /// let matcher = IncludeMatcher::new(ignore_patterns).unwrap();
 /// ///
 /// assert!(!matcher.matches(HgPath::new(b"file")));
 /// assert!(!matcher.matches(HgPath::new(b"dir/file")));
 /// assert!(matcher.matches(HgPath::new(b"dir/subdir/file")));
 /// assert!(!matcher.matches(HgPath::new(b"dir/subdir/subsubdir/file")));
 /// ```
 pub struct IncludeMatcher<'a> {
     patterns: Vec<u8>,
     match_fn: IgnoreFnType<'a>,
     /// Whether all the patterns match a prefix (i.e. recursively)
     prefix: bool,
     roots: HashSet<HgPathBuf>,
     dirs: HashSet<HgPathBuf>,
-    parents: HashSet<HgPathBuf>,
+    parents: DirsMultiset,
 }
 
 impl core::fmt::Debug for IncludeMatcher<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("IncludeMatcher")
             .field("patterns", &String::from_utf8_lossy(&self.patterns))
             .field("prefix", &self.prefix)
             .field("roots", &self.roots)
             .field("dirs", &self.dirs)
             .field("parents", &self.parents)
             .finish()
     }
 }
 
 impl<'a> Matcher for IncludeMatcher<'a> {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         None
     }
 
     fn exact_match(&self, _filename: &HgPath) -> bool {
         false
     }
 
     fn matches(&self, filename: &HgPath) -> bool {
         (self.match_fn)(filename)
     }
 
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet {
         let dir = directory;
         if self.prefix && self.roots.contains(dir) {
             return VisitChildrenSet::Recursive;
         }
         if self.roots.contains(HgPath::new(b""))
             || self.roots.contains(dir)
             || self.dirs.contains(dir)
             || find_dirs(dir).any(|parent_dir| self.roots.contains(parent_dir))
         {
             return VisitChildrenSet::This;
         }
 
         if self.parents.contains(dir.as_ref()) {
             let multiset = self.get_all_parents_children();
             if let Some(children) = multiset.get(dir) {
                 return VisitChildrenSet::Set(
                     children.iter().map(HgPathBuf::from).collect(),
                 );
             }
         }
         VisitChildrenSet::Empty
     }
 
     fn matches_everything(&self) -> bool {
         false
     }
 
     fn is_exact(&self) -> bool {
         false
     }
 }
 
 /// The union of multiple matchers. Will match if any of the matchers match.
 #[derive(Debug)]
 pub struct UnionMatcher {
     matchers: Vec<Box<dyn Matcher + Sync>>,
 }
 
 impl Matcher for UnionMatcher {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         None
     }
 
     fn exact_match(&self, _filename: &HgPath) -> bool {
         false
     }
 
     fn matches(&self, filename: &HgPath) -> bool {
         self.matchers.iter().any(|m| m.matches(filename))
     }
 
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet {
         let mut result = HashSet::new();
         let mut this = false;
         for matcher in self.matchers.iter() {
             let visit = matcher.visit_children_set(directory);
             match visit {
                 VisitChildrenSet::Empty => continue,
                 VisitChildrenSet::This => {
                     this = true;
                     // Don't break, we might have an 'all' in here.
                     continue;
                 }
                 VisitChildrenSet::Set(set) => {
                     result.extend(set);
                 }
                 VisitChildrenSet::Recursive => {
                     return visit;
                 }
             }
         }
         if this {
             return VisitChildrenSet::This;
         }
         if result.is_empty() {
             VisitChildrenSet::Empty
         } else {
             VisitChildrenSet::Set(result)
         }
     }
 
     fn matches_everything(&self) -> bool {
         // TODO Maybe if all are AlwaysMatcher?
         false
     }
 
     fn is_exact(&self) -> bool {
         false
     }
 }
 
 impl UnionMatcher {
     pub fn new(matchers: Vec<Box<dyn Matcher + Sync>>) -> Self {
         Self { matchers }
     }
 }
 
 #[derive(Debug)]
 pub struct IntersectionMatcher {
     m1: Box<dyn Matcher + Sync>,
     m2: Box<dyn Matcher + Sync>,
     files: Option<HashSet<HgPathBuf>>,
 }
 
 impl Matcher for IntersectionMatcher {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         self.files.as_ref()
     }
 
     fn exact_match(&self, filename: &HgPath) -> bool {
         self.files.as_ref().map_or(false, |f| f.contains(filename))
     }
 
     fn matches(&self, filename: &HgPath) -> bool {
         self.m1.matches(filename) && self.m2.matches(filename)
     }
 
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet {
         let m1_set = self.m1.visit_children_set(directory);
         if m1_set == VisitChildrenSet::Empty {
             return VisitChildrenSet::Empty;
         }
         let m2_set = self.m2.visit_children_set(directory);
         if m2_set == VisitChildrenSet::Empty {
             return VisitChildrenSet::Empty;
         }
 
         if m1_set == VisitChildrenSet::Recursive {
             return m2_set;
         } else if m2_set == VisitChildrenSet::Recursive {
             return m1_set;
         }
 
         match (&m1_set, &m2_set) {
             (VisitChildrenSet::Recursive, _) => m2_set,
             (_, VisitChildrenSet::Recursive) => m1_set,
             (VisitChildrenSet::This, _) | (_, VisitChildrenSet::This) => {
                 VisitChildrenSet::This
             }
             (VisitChildrenSet::Set(m1), VisitChildrenSet::Set(m2)) => {
                 let set: HashSet<_> = m1.intersection(m2).cloned().collect();
                 if set.is_empty() {
                     VisitChildrenSet::Empty
                 } else {
                     VisitChildrenSet::Set(set)
                 }
             }
             _ => unreachable!(),
         }
     }
 
     fn matches_everything(&self) -> bool {
         self.m1.matches_everything() && self.m2.matches_everything()
     }
 
     fn is_exact(&self) -> bool {
         self.m1.is_exact() || self.m2.is_exact()
     }
 }
 
 impl IntersectionMatcher {
     pub fn new(
         mut m1: Box<dyn Matcher + Sync>,
         mut m2: Box<dyn Matcher + Sync>,
     ) -> Self {
         let files = if m1.is_exact() || m2.is_exact() {
             if !m1.is_exact() {
                 std::mem::swap(&mut m1, &mut m2);
             }
             m1.file_set().map(|m1_files| {
                 m1_files.iter().cloned().filter(|f| m2.matches(f)).collect()
             })
         } else {
             // without exact input file sets, we can't do an exact
             // intersection, so we must over-approximate by
             // unioning instead
             m1.file_set().map(|m1_files| match m2.file_set() {
                 Some(m2_files) => m1_files.union(m2_files).cloned().collect(),
                 None => m1_files.iter().cloned().collect(),
             })
         };
         Self { m1, m2, files }
     }
 }
 
 #[derive(Debug)]
 pub struct DifferenceMatcher {
     base: Box<dyn Matcher + Sync>,
     excluded: Box<dyn Matcher + Sync>,
     files: Option<HashSet<HgPathBuf>>,
 }
 
 impl Matcher for DifferenceMatcher {
     fn file_set(&self) -> Option<&HashSet<HgPathBuf>> {
         self.files.as_ref()
     }
 
     fn exact_match(&self, filename: &HgPath) -> bool {
         self.files.as_ref().map_or(false, |f| f.contains(filename))
     }
 
     fn matches(&self, filename: &HgPath) -> bool {
         self.base.matches(filename) && !self.excluded.matches(filename)
     }
 
     fn visit_children_set(&self, directory: &HgPath) -> VisitChildrenSet {
         let excluded_set = self.excluded.visit_children_set(directory);
         if excluded_set == VisitChildrenSet::Recursive {
             return VisitChildrenSet::Empty;
         }
         let base_set = self.base.visit_children_set(directory);
         // Possible values for base: 'recursive', 'this', set(...), set()
         // Possible values for excluded:          'this', set(...), set()
         // If excluded has nothing under here that we care about, return base,
         // even if it's 'recursive'.
         if excluded_set == VisitChildrenSet::Empty {
             return base_set;
         }
         match base_set {
             VisitChildrenSet::This | VisitChildrenSet::Recursive => {
                 // Never return 'recursive' here if excluded_set is any kind of
                 // non-empty (either 'this' or set(foo)), since excluded might
                 // return set() for a subdirectory.
                 VisitChildrenSet::This
             }
             set => {
                 // Possible values for base:         set(...), set()
                 // Possible values for excluded: 'this', set(...)
                 // We ignore excluded set results. They're possibly incorrect:
                 //  base = path:dir/subdir
                 //  excluded=rootfilesin:dir,
                 //  visit_children_set(''):
                 //   base returns {'dir'}, excluded returns {'dir'}, if we
                 //   subtracted we'd return set(), which is *not* correct, we
                 //   still need to visit 'dir'!
                 set
             }
         }
     }
 
     fn matches_everything(&self) -> bool {
         false
     }
 
     fn is_exact(&self) -> bool {
         self.base.is_exact()
     }
 }
 
 impl DifferenceMatcher {
     pub fn new(
         base: Box<dyn Matcher + Sync>,
         excluded: Box<dyn Matcher + Sync>,
     ) -> Self {
         let base_is_exact = base.is_exact();
         let base_files = base.file_set().map(ToOwned::to_owned);
         let mut new = Self {
             base,
             excluded,
             files: None,
         };
         if base_is_exact {
             new.files = base_files.map(|files| {
                 files.iter().cloned().filter(|f| new.matches(f)).collect()
             });
         }
         new
     }
 }
 
 /// Wraps [`regex::bytes::Regex`] to improve performance in multithreaded
 /// contexts.
 ///
 /// The `status` algorithm makes heavy use of threads, and calling `is_match`
 /// from many threads at once is prone to contention, probably within the
 /// scratch space needed as the regex DFA is built lazily.
 ///
 /// We are in the process of raising the issue upstream, but for now
 /// the workaround used here is to store the `Regex` in a lazily populated
 /// thread-local variable, sharing the initial read-only compilation, but
 /// not the lazy dfa scratch space mentioned above.
 ///
 /// This reduces the contention observed with 16+ threads, but does not
 /// completely remove it. Hopefully this can be addressed upstream.
 struct RegexMatcher {
     /// Compiled at the start of the status algorithm, used as a base for
     /// cloning in each thread-local `self.local`, thus sharing the expensive
     /// first compilation.
     base: regex::bytes::Regex,
     /// Thread-local variable that holds the `Regex` that is actually queried
     /// from each thread.
     local: thread_local::ThreadLocal<regex::bytes::Regex>,
 }
 
 impl RegexMatcher {
     /// Returns whether the path matches the stored `Regex`.
     pub fn is_match(&self, path: &HgPath) -> bool {
         self.local
             .get_or(|| self.base.clone())
             .is_match(path.as_bytes())
     }
 }
 
 /// Return a `RegexBuilder` from a bytes pattern
 ///
 /// This works around the fact that even if it works on byte haysacks,
 /// [`regex::bytes::Regex`] still uses UTF-8 patterns.
 pub fn re_bytes_builder(pattern: &[u8]) -> regex::bytes::RegexBuilder {
     use std::io::Write;
 
     // The `regex` crate adds `.*` to the start and end of expressions if there
     // are no anchors, so add the start anchor.
     let mut escaped_bytes = vec![b'^', b'(', b'?', b':'];
     for byte in pattern {
         if *byte > 127 {
             write!(escaped_bytes, "\\x{:x}", *byte).unwrap();
         } else {
             escaped_bytes.push(*byte);
         }
     }
     escaped_bytes.push(b')');
 
     // Avoid the cost of UTF8 checking
     //
     // # Safety
     // This is safe because we escaped all non-ASCII bytes.
     let pattern_string = unsafe { String::from_utf8_unchecked(escaped_bytes) };
     regex::bytes::RegexBuilder::new(&pattern_string)
 }
 
 /// Returns a function that matches an `HgPath` against the given regex
 /// pattern.
 ///
 /// This can fail when the pattern is invalid or not supported by the
 /// underlying engine (the `regex` crate), for instance anything with
 /// back-references.
 #[logging_timer::time("trace")]
 fn re_matcher(pattern: &[u8]) -> PatternResult<RegexMatcher> {
     let re = re_bytes_builder(pattern)
         .unicode(false)
         // Big repos with big `.hgignore` will hit the default limit and
         // incur a significant performance hit. One repo's `hg status` hit
         // multiple *minutes*.
         .dfa_size_limit(50 * (1 << 20))
         .build()
         .map_err(|e| PatternError::UnsupportedSyntax(e.to_string()))?;
 
     Ok(RegexMatcher {
         base: re,
         local: Default::default(),
     })
 }
 
 /// Returns the regex pattern and a function that matches an `HgPath` against
 /// said regex formed by the given ignore patterns.
 fn build_regex_match<'a>(
     ignore_patterns: &[IgnorePattern],
     glob_suffix: &[u8],
 ) -> PatternResult<(Vec<u8>, IgnoreFnType<'a>)> {
     let mut regexps = vec![];
     let mut exact_set = HashSet::new();
 
     for pattern in ignore_patterns {
         if let Some(re) = build_single_regex(pattern, glob_suffix)? {
             regexps.push(re);
         } else {
             let exact = normalize_path_bytes(&pattern.pattern);
             exact_set.insert(HgPathBuf::from_bytes(&exact));
         }
     }
 
     let full_regex = regexps.join(&b'|');
 
     // An empty pattern would cause the regex engine to incorrectly match the
     // (empty) root directory
     let func = if !(regexps.is_empty()) {
         let matcher = re_matcher(&full_regex)?;
         let func = move |filename: &HgPath| {
             exact_set.contains(filename) || matcher.is_match(filename)
         };
         Box::new(func) as IgnoreFnType
     } else {
         let func = move |filename: &HgPath| exact_set.contains(filename);
         Box::new(func) as IgnoreFnType
     };
 
     Ok((full_regex, func))
 }
 
 /// Returns roots and directories corresponding to each pattern.
 ///
 /// This calculates the roots and directories exactly matching the patterns and
 /// returns a tuple of (roots, dirs). It does not return other directories
 /// which may also need to be considered, like the parent directories.
 fn roots_and_dirs(
     ignore_patterns: &[IgnorePattern],
 ) -> (Vec<HgPathBuf>, Vec<HgPathBuf>) {
     let mut roots = Vec::new();
     let mut dirs = Vec::new();
 
     for ignore_pattern in ignore_patterns {
         let IgnorePattern {
             syntax, pattern, ..
         } = ignore_pattern;
         match syntax {
             PatternSyntax::RootGlob | PatternSyntax::Glob => {
                 let mut root = HgPathBuf::new();
                 for p in pattern.split(|c| *c == b'/') {
                     if p.iter()
                         .any(|c| matches!(*c, b'[' | b'{' | b'*' | b'?'))
                     {
                         break;
                     }
                     root.push(HgPathBuf::from_bytes(p).as_ref());
                 }
                 roots.push(root);
             }
             PatternSyntax::Path
             | PatternSyntax::RelPath
             | PatternSyntax::FilePath => {
                 let pat = HgPath::new(if pattern == b"." {
                     &[] as &[u8]
                 } else {
                     pattern
                 });
                 roots.push(pat.to_owned());
             }
-            PatternSyntax::RootFiles => {
+            PatternSyntax::RootFilesIn => {
                 let pat = if pattern == b"." {
                     &[] as &[u8]
                 } else {
                     pattern
                 };
                 dirs.push(HgPathBuf::from_bytes(pat));
             }
             _ => {
                 roots.push(HgPathBuf::new());
             }
         }
     }
     (roots, dirs)
 }
 
 /// Paths extracted from patterns
 #[derive(Debug, PartialEq)]
 struct RootsDirsAndParents {
     /// Directories to match recursively
     pub roots: HashSet<HgPathBuf>,
     /// Directories to match non-recursively
     pub dirs: HashSet<HgPathBuf>,
     /// Implicitly required directories to go to items in either roots or dirs
-    pub parents: HashSet<HgPathBuf>,
+    pub parents: DirsMultiset,
 }
 
 /// Extract roots, dirs and parents from patterns.
 fn roots_dirs_and_parents(
     ignore_patterns: &[IgnorePattern],
 ) -> PatternResult<RootsDirsAndParents> {
     let (roots, dirs) = roots_and_dirs(ignore_patterns);
 
-    let mut parents = HashSet::new();
+    let mut parents = DirsMultiset::from_manifest(&dirs)?;
 
-    parents.extend(
-        DirsMultiset::from_manifest(&dirs)?
-            .iter()
-            .map(ToOwned::to_owned),
-    );
-    parents.extend(
-        DirsMultiset::from_manifest(&roots)?
-            .iter()
-            .map(ToOwned::to_owned),
-    );
+    for path in &roots {
+        parents.add_path(path)?
+    }
 
     Ok(RootsDirsAndParents {
         roots: HashSet::from_iter(roots),
         dirs: HashSet::from_iter(dirs),
         parents,
     })
 }
 
 /// Returns a function that checks whether a given file (in the general sense)
 /// should be matched.
 fn build_match<'a>(
     ignore_patterns: Vec<IgnorePattern>,
     glob_suffix: &[u8],
 ) -> PatternResult<(Vec<u8>, IgnoreFnType<'a>)> {
     let mut match_funcs: Vec<IgnoreFnType<'a>> = vec![];
     // For debugging and printing
     let mut patterns = vec![];
 
     let (subincludes, ignore_patterns) = filter_subincludes(ignore_patterns)?;
 
     if !subincludes.is_empty() {
         // Build prefix-based matcher functions for subincludes
         let mut submatchers = FastHashMap::default();
         let mut prefixes = vec![];
 
         for sub_include in subincludes {
             let matcher = IncludeMatcher::new(sub_include.included_patterns)?;
             let match_fn =
                 Box::new(move |path: &HgPath| matcher.matches(path));
             prefixes.push(sub_include.prefix.clone());
             submatchers.insert(sub_include.prefix.clone(), match_fn);
         }
 
         let match_subinclude = move |filename: &HgPath| {
             for prefix in prefixes.iter() {
                 if let Some(rel) = filename.relative_to(prefix) {
                     if (submatchers[prefix])(rel) {
                         return true;
                     }
                 }
             }
             false
         };
 
         match_funcs.push(Box::new(match_subinclude));
     }
 
     if !ignore_patterns.is_empty() {
         // Either do dumb matching if all patterns are rootfiles, or match
         // with a regex.
         if ignore_patterns
             .iter()
-            .all(|k| k.syntax == PatternSyntax::RootFiles)
+            .all(|k| k.syntax == PatternSyntax::RootFilesIn)
         {
             let dirs: HashSet<_> = ignore_patterns
                 .iter()
                 .map(|k| k.pattern.to_owned())
                 .collect();
             let mut dirs_vec: Vec<_> = dirs.iter().cloned().collect();
 
             let match_func = move |path: &HgPath| -> bool {
                 let path = path.as_bytes();
                 let i = path.iter().rposition(|a| *a == b'/');
                 let dir = if let Some(i) = i { &path[..i] } else { b"." };
                 dirs.contains(dir)
             };
             match_funcs.push(Box::new(match_func));
 
             patterns.extend(b"rootfilesin: ");
             dirs_vec.sort();
             patterns.extend(dirs_vec.escaped_bytes());
         } else {
             let (new_re, match_func) =
                 build_regex_match(&ignore_patterns, glob_suffix)?;
             patterns = new_re;
             match_funcs.push(match_func)
         }
     }
 
     Ok(if match_funcs.len() == 1 {
         (patterns, match_funcs.remove(0))
     } else {
         (
             patterns,
             Box::new(move |f: &HgPath| -> bool {
                 match_funcs.iter().any(|match_func| match_func(f))
             }),
         )
     })
 }
 
 /// Parses all "ignore" files with their recursive includes and returns a
 /// function that checks whether a given file (in the general sense) should be
 /// ignored.
 pub fn get_ignore_matcher<'a>(
     mut all_pattern_files: Vec<PathBuf>,
     root_dir: &Path,
     inspect_pattern_bytes: &mut impl FnMut(&Path, &[u8]),
 ) -> PatternResult<(IncludeMatcher<'a>, Vec<PatternFileWarning>)> {
     let mut all_patterns = vec![];
     let mut all_warnings = vec![];
 
     // Sort to make the ordering of calls to `inspect_pattern_bytes`
     // deterministic even if the ordering of `all_pattern_files` is not (such
     // as when a iteration order of a Python dict or Rust HashMap is involved).
     // Sort by "string" representation instead of the default by component
     // (with a Rust-specific definition of a component)
     all_pattern_files
         .sort_unstable_by(|a, b| a.as_os_str().cmp(b.as_os_str()));
 
     for pattern_file in &all_pattern_files {
         let (patterns, warnings) = get_patterns_from_file(
             pattern_file,
             root_dir,
             inspect_pattern_bytes,
         )?;
 
         all_patterns.extend(patterns.to_owned());
         all_warnings.extend(warnings);
     }
     let matcher = IncludeMatcher::new(all_patterns)?;
     Ok((matcher, all_warnings))
 }
 
 /// Parses all "ignore" files with their recursive includes and returns a
 /// function that checks whether a given file (in the general sense) should be
 /// ignored.
 pub fn get_ignore_function<'a>(
     all_pattern_files: Vec<PathBuf>,
     root_dir: &Path,
     inspect_pattern_bytes: &mut impl FnMut(&Path, &[u8]),
 ) -> PatternResult<(IgnoreFnType<'a>, Vec<PatternFileWarning>)> {
     let res =
         get_ignore_matcher(all_pattern_files, root_dir, inspect_pattern_bytes);
     res.map(|(matcher, all_warnings)| {
         let res: IgnoreFnType<'a> =
             Box::new(move |path: &HgPath| matcher.matches(path));
 
         (res, all_warnings)
     })
 }
 
 impl<'a> IncludeMatcher<'a> {
     pub fn new(ignore_patterns: Vec<IgnorePattern>) -> PatternResult<Self> {
         let RootsDirsAndParents {
             roots,
             dirs,
             parents,
         } = roots_dirs_and_parents(&ignore_patterns)?;
         let prefix = ignore_patterns.iter().all(|k| {
             matches!(k.syntax, PatternSyntax::Path | PatternSyntax::RelPath)
         });
         let (patterns, match_fn) = build_match(ignore_patterns, b"(?:/|$)")?;
 
         Ok(Self {
             patterns,
             match_fn,
             prefix,
             roots,
             dirs,
             parents,
         })
     }
 
     fn get_all_parents_children(&self) -> DirsChildrenMultiset {
         // TODO cache
         let thing = self
             .dirs
             .iter()
             .chain(self.roots.iter())
             .chain(self.parents.iter());
-        DirsChildrenMultiset::new(thing, Some(&self.parents))
+        DirsChildrenMultiset::new(thing, Some(self.parents.iter()))
     }
 
     pub fn debug_get_patterns(&self) -> &[u8] {
         self.patterns.as_ref()
     }
 }
 
 impl<'a> Display for IncludeMatcher<'a> {
     fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
         // XXX What about exact matches?
         // I'm not sure it's worth it to clone the HashSet and keep it
         // around just in case someone wants to display the matcher, plus
         // it's going to be unreadable after a few entries, but we need to
         // inform in this display that exact matches are being used and are
         // (on purpose) missing from the `includes`.
         write!(
             f,
             "IncludeMatcher(includes='{}')",
             String::from_utf8_lossy(&self.patterns.escaped_bytes())
         )
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
+    use std::collections::BTreeMap;
+    use std::collections::BTreeSet;
+    use std::fmt::Debug;
     use std::path::Path;
 
     #[test]
     fn test_roots_and_dirs() {
         let pats = vec![
             IgnorePattern::new(PatternSyntax::Glob, b"g/h/*", Path::new("")),
             IgnorePattern::new(PatternSyntax::Glob, b"g/h", Path::new("")),
             IgnorePattern::new(PatternSyntax::Glob, b"g*", Path::new("")),
         ];
         let (roots, dirs) = roots_and_dirs(&pats);
 
         assert_eq!(
             roots,
             vec!(
                 HgPathBuf::from_bytes(b"g/h"),
                 HgPathBuf::from_bytes(b"g/h"),
                 HgPathBuf::new()
             ),
         );
         assert_eq!(dirs, vec!());
     }
 
     #[test]
     fn test_roots_dirs_and_parents() {
         let pats = vec![
             IgnorePattern::new(PatternSyntax::Glob, b"g/h/*", Path::new("")),
             IgnorePattern::new(PatternSyntax::Glob, b"g/h", Path::new("")),
             IgnorePattern::new(PatternSyntax::Glob, b"g*", Path::new("")),
         ];
 
         let mut roots = HashSet::new();
         roots.insert(HgPathBuf::from_bytes(b"g/h"));
         roots.insert(HgPathBuf::new());
 
         let dirs = HashSet::new();
 
-        let mut parents = HashSet::new();
-        parents.insert(HgPathBuf::new());
-        parents.insert(HgPathBuf::from_bytes(b"g"));
+        let parents = DirsMultiset::from_manifest(&[
+            HgPathBuf::from_bytes(b"x"),
+            HgPathBuf::from_bytes(b"g/x"),
+            HgPathBuf::from_bytes(b"g/y"),
+        ])
+        .unwrap();
 
         assert_eq!(
             roots_dirs_and_parents(&pats).unwrap(),
             RootsDirsAndParents {
                 roots,
                 dirs,
                 parents
             }
         );
     }
 
     #[test]
     fn test_filematcher_visit_children_set() {
         // Visitchildrenset
         let files = vec![HgPathBuf::from_bytes(b"dir/subdir/foo.txt")];
         let matcher = FileMatcher::new(files).unwrap();
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"foo.txt"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Set(set)
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/foo.txt")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
     }
 
     #[test]
     fn test_filematcher_visit_children_set_files_and_dirs() {
         let files = vec![
             HgPathBuf::from_bytes(b"rootfile.txt"),
             HgPathBuf::from_bytes(b"a/file1.txt"),
             HgPathBuf::from_bytes(b"a/b/file2.txt"),
             // No file in a/b/c
             HgPathBuf::from_bytes(b"a/b/c/d/file4.txt"),
         ];
         let matcher = FileMatcher::new(files).unwrap();
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"a"));
         set.insert(HgPathBuf::from_bytes(b"rootfile.txt"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"b"));
         set.insert(HgPathBuf::from_bytes(b"file1.txt"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"a")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"c"));
         set.insert(HgPathBuf::from_bytes(b"file2.txt"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"a/b")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"d"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"a/b/c")),
             VisitChildrenSet::Set(set)
         );
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"file4.txt"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"a/b/c/d")),
             VisitChildrenSet::Set(set)
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"a/b/c/d/e")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
     }
 
     #[test]
     fn test_patternmatcher() {
         // VisitdirPrefix
         let m = PatternMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::Path,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         // OPT: This should probably be Recursive if its parent is?
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
 
         // VisitchildrensetPrefix
         let m = PatternMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::Path,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         // OPT: This should probably be Recursive if its parent is?
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
 
         // VisitdirRootfilesin
         let m = PatternMatcher::new(vec![IgnorePattern::new(
-            PatternSyntax::RootFiles,
+            PatternSyntax::RootFilesIn,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
-        // FIXME: These should probably be This.
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
 
         // VisitchildrensetRootfilesin
         let m = PatternMatcher::new(vec![IgnorePattern::new(
-            PatternSyntax::RootFiles,
+            PatternSyntax::RootFilesIn,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         // FIXME: These should probably be {'dir'}, {'subdir'} and This,
-        // respectively, or at least This for all three.
+        // respectively
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
 
         // VisitdirGlob
         let m = PatternMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::Glob,
             b"dir/z*",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
-        // FIXME: This probably should be This
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         // OPT: these should probably be False.
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
 
         // VisitchildrensetGlob
         let m = PatternMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::Glob,
             b"dir/z*",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
-        // FIXME: This probably should be This
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
-            VisitChildrenSet::Empty
+            VisitChildrenSet::This
         );
         // OPT: these should probably be Empty
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
 
         // VisitdirFilepath
         let m = PatternMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::FilePath,
             b"dir/z",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
 
         // VisitchildrensetFilepath
         let m = PatternMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::FilePath,
             b"dir/z",
             Path::new(""),
         )])
         .unwrap();
         assert_eq!(
             m.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             m.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
     }
 
     #[test]
     fn test_includematcher() {
         // VisitchildrensetPrefix
         let matcher = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::RelPath,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         // OPT: This should probably be 'all' if its parent is?
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
 
         // VisitchildrensetRootfilesin
         let matcher = IncludeMatcher::new(vec![IgnorePattern::new(
-            PatternSyntax::RootFiles,
+            PatternSyntax::RootFilesIn,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
 
         // VisitchildrensetGlob
         let matcher = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::Glob,
             b"dir/z*",
             Path::new(""),
         )])
         .unwrap();
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         // OPT: these should probably be set().
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
 
         // VisitchildrensetFilePath
         let matcher = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::FilePath,
             b"dir/z",
             Path::new(""),
         )])
         .unwrap();
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"z"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
         // OPT: these should probably be set().
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
 
         // Test multiple patterns
         let matcher = IncludeMatcher::new(vec![
             IgnorePattern::new(PatternSyntax::RelPath, b"foo", Path::new("")),
             IgnorePattern::new(PatternSyntax::Glob, b"g*", Path::new("")),
         ])
         .unwrap();
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
 
         // Test multiple patterns
         let matcher = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::Glob,
             b"**/*.exe",
             Path::new(""),
         )])
         .unwrap();
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::This
         );
     }
 
     #[test]
     fn test_unionmatcher() {
         // Path + Rootfiles
         let m1 = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::RelPath,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         let m2 = IncludeMatcher::new(vec![IgnorePattern::new(
-            PatternSyntax::RootFiles,
+            PatternSyntax::RootFilesIn,
             b"dir",
             Path::new(""),
         )])
         .unwrap();
         let matcher = UnionMatcher::new(vec![Box::new(m1), Box::new(m2)]);
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
 
         // OPT: These next two could be 'all' instead of 'this'.
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
 
         // Path + unrelated Path
         let m1 = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::RelPath,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         let m2 = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::RelPath,
             b"folder",
             Path::new(""),
         )])
         .unwrap();
         let matcher = UnionMatcher::new(vec![Box::new(m1), Box::new(m2)]);
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"folder"));
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Recursive
         );
         // OPT: These next two could be 'all' instead of 'this'.
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
 
         // Path + subpath
         let m1 = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::RelPath,
             b"dir/subdir/x",
             Path::new(""),
         )])
         .unwrap();
         let m2 = IncludeMatcher::new(vec![IgnorePattern::new(
             PatternSyntax::RelPath,
             b"dir/subdir",
             Path::new(""),
         )])
         .unwrap();
         let matcher = UnionMatcher::new(vec![Box::new(m1), Box::new(m2)]);
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Recursive
         );
         // OPT: this should probably be 'all' not 'this'.
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::This
         );
     }
 
     #[test]
     fn test_intersectionmatcher() {
         // Include path + Include rootfiles
         let m1 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let m2 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
-                PatternSyntax::RootFiles,
+                PatternSyntax::RootFilesIn,
                 b"dir",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let matcher = IntersectionMatcher::new(m1, m2);
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
 
         // Non intersecting paths
         let m1 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let m2 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"folder",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let matcher = IntersectionMatcher::new(m1, m2);
 
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
 
         // Nested paths
         let m1 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir/x",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let m2 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let matcher = IntersectionMatcher::new(m1, m2);
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"x"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::Empty
         );
         // OPT: this should probably be 'all' not 'this'.
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
 
         // Diverging paths
         let m1 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir/x",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let m2 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir/z",
                 Path::new(""),
             )])
             .unwrap(),
         );
         let matcher = IntersectionMatcher::new(m1, m2);
 
         // OPT: these next two could probably be Empty as well.
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
         // OPT: these next two could probably be Empty as well.
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::Empty
         );
     }
 
     #[test]
     fn test_differencematcher() {
         // Two alwaysmatchers should function like a nevermatcher
         let m1 = AlwaysMatcher;
         let m2 = AlwaysMatcher;
         let matcher = DifferenceMatcher::new(Box::new(m1), Box::new(m2));
 
         for case in &[
             &b""[..],
             b"dir",
             b"dir/subdir",
             b"dir/subdir/z",
             b"dir/foo",
             b"dir/subdir/x",
             b"folder",
         ] {
             assert_eq!(
                 matcher.visit_children_set(HgPath::new(case)),
                 VisitChildrenSet::Empty
             );
         }
 
         // One always and one never should behave the same as an always
         let m1 = AlwaysMatcher;
         let m2 = NeverMatcher;
         let matcher = DifferenceMatcher::new(Box::new(m1), Box::new(m2));
 
         for case in &[
             &b""[..],
             b"dir",
             b"dir/subdir",
             b"dir/subdir/z",
             b"dir/foo",
             b"dir/subdir/x",
             b"folder",
         ] {
             assert_eq!(
                 matcher.visit_children_set(HgPath::new(case)),
                 VisitChildrenSet::Recursive
             );
         }
 
         // Two include matchers
         let m1 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
                 PatternSyntax::RelPath,
                 b"dir/subdir",
                 Path::new("/repo"),
             )])
             .unwrap(),
         );
         let m2 = Box::new(
             IncludeMatcher::new(vec![IgnorePattern::new(
-                PatternSyntax::RootFiles,
+                PatternSyntax::RootFilesIn,
                 b"dir",
                 Path::new("/repo"),
             )])
             .unwrap(),
         );
 
         let matcher = DifferenceMatcher::new(m1, m2);
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"dir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"")),
             VisitChildrenSet::Set(set)
         );
 
         let mut set = HashSet::new();
         set.insert(HgPathBuf::from_bytes(b"subdir"));
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir")),
             VisitChildrenSet::Set(set)
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir")),
             VisitChildrenSet::Recursive
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/foo")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"folder")),
             VisitChildrenSet::Empty
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/z")),
             VisitChildrenSet::This
         );
         assert_eq!(
             matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
             VisitChildrenSet::This
         );
     }
+
+    mod invariants {
+        pub mod visit_children_set {
+
+            use crate::{
+                matchers::{tests::Tree, Matcher, VisitChildrenSet},
+                utils::hg_path::HgPath,
+            };
+
+            #[allow(dead_code)]
+            #[derive(Debug)]
+            struct Error<'a, M> {
+                matcher: &'a M,
+                path: &'a HgPath,
+                matching: &'a Tree,
+                visit_children_set: &'a VisitChildrenSet,
+            }
+
+            fn holds(
+                matching: &Tree,
+                not_matching: &Tree,
+                vcs: &VisitChildrenSet,
+            ) -> bool {
+                match vcs {
+                    VisitChildrenSet::Empty => matching.is_empty(),
+                    VisitChildrenSet::This => {
+                        // `This` does not come with any obligations.
+                        true
+                    }
+                    VisitChildrenSet::Recursive => {
+                        // `Recursive` requires that *everything* in the
+                        // subtree matches. This
+                        // requirement is relied on for example in
+                        // DifferenceMatcher implementation.
+                        not_matching.is_empty()
+                    }
+                    VisitChildrenSet::Set(allowed_children) => {
+                        // `allowed_children` does not distinguish between
+                        // files and directories: if it's not included, it
+                        // must not be matched.
+                        for k in matching.dirs.keys() {
+                            if !(allowed_children.contains(k)) {
+                                return false;
+                            }
+                        }
+                        for k in matching.files.iter() {
+                            if !(allowed_children.contains(k)) {
+                                return false;
+                            }
+                        }
+                        true
+                    }
+                }
+            }
+
+            pub fn check<M: Matcher + std::fmt::Debug>(
+                matcher: &M,
+                path: &HgPath,
+                matching: &Tree,
+                not_matching: &Tree,
+                visit_children_set: &VisitChildrenSet,
+            ) {
+                if !holds(matching, not_matching, visit_children_set) {
+                    panic!(
+                        "{:#?}",
+                        Error {
+                            matcher,
+                            path,
+                            visit_children_set,
+                            matching
+                        }
+                    )
+                }
+            }
+        }
+    }
+
+    #[derive(Debug, Clone)]
+    pub struct Tree {
+        files: BTreeSet<HgPathBuf>,
+        dirs: BTreeMap<HgPathBuf, Tree>,
+    }
+
+    impl Tree {
+        fn len(&self) -> usize {
+            let mut n = 0;
+            n += self.files.len();
+            for d in self.dirs.values() {
+                n += d.len();
+            }
+            n
+        }
+
+        fn is_empty(&self) -> bool {
+            self.files.is_empty() && self.dirs.is_empty()
+        }
+
+        fn make(
+            files: BTreeSet<HgPathBuf>,
+            dirs: BTreeMap<HgPathBuf, Tree>,
+        ) -> Self {
+            Self {
+                files,
+                dirs: dirs
+                    .into_iter()
+                    .filter(|(_k, v)| (!(v.is_empty())))
+                    .collect(),
+            }
+        }
+
+        fn filter_and_check<M: Matcher + Debug>(
+            &self,
+            m: &M,
+            path: &HgPath,
+        ) -> (Self, Self) {
+            let (files1, files2): (BTreeSet<HgPathBuf>, BTreeSet<HgPathBuf>) =
+                self.files
+                    .iter()
+                    .map(|v| v.to_owned())
+                    .partition(|v| m.matches(&path.join(v)));
+            let (dirs1, dirs2): (
+                BTreeMap<HgPathBuf, Tree>,
+                BTreeMap<HgPathBuf, Tree>,
+            ) = self
+                .dirs
+                .iter()
+                .map(|(k, v)| {
+                    let path = path.join(k);
+                    let (t1, t2) = v.filter_and_check(m, &path);
+                    ((k.clone(), t1), (k.clone(), t2))
+                })
+                .unzip();
+            let matching = Self::make(files1, dirs1);
+            let not_matching = Self::make(files2, dirs2);
+            let vcs = m.visit_children_set(path);
+            invariants::visit_children_set::check(
+                m,
+                path,
+                &matching,
+                &not_matching,
+                &vcs,
+            );
+            (matching, not_matching)
+        }
+
+        fn check_matcher<M: Matcher + Debug>(
+            &self,
+            m: &M,
+            expect_count: usize,
+        ) {
+            let res = self.filter_and_check(m, &HgPathBuf::new());
+            if expect_count != res.0.len() {
+                eprintln!(
+                    "warning: expected {} matches, got {} for {:#?}",
+                    expect_count,
+                    res.0.len(),
+                    m
+                );
+            }
+        }
+    }
+
+    fn mkdir(children: &[(&[u8], &Tree)]) -> Tree {
+        let p = HgPathBuf::from_bytes;
+        let names = [
+            p(b"a"),
+            p(b"b.txt"),
+            p(b"file.txt"),
+            p(b"c.c"),
+            p(b"c.h"),
+            p(b"dir1"),
+            p(b"dir2"),
+            p(b"subdir"),
+        ];
+        let files: BTreeSet<HgPathBuf> = BTreeSet::from(names);
+        let dirs = children
+            .iter()
+            .map(|(name, t)| (p(name), (*t).clone()))
+            .collect();
+        Tree { files, dirs }
+    }
+
+    fn make_example_tree() -> Tree {
+        let leaf = mkdir(&[]);
+        let abc = mkdir(&[(b"d", &leaf)]);
+        let ab = mkdir(&[(b"c", &abc)]);
+        let a = mkdir(&[(b"b", &ab)]);
+        let dir = mkdir(&[(b"subdir", &leaf), (b"subdir.c", &leaf)]);
+        mkdir(&[(b"dir", &dir), (b"dir1", &dir), (b"dir2", &dir), (b"a", &a)])
+    }
+
+    #[test]
+    fn test_pattern_matcher_visit_children_set() {
+        let tree = make_example_tree();
+        let pattern_dir1_glob_c =
+            PatternMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::Glob,
+                b"dir1/*.c",
+                Path::new(""),
+            )])
+            .unwrap();
+        let pattern_dir1 = || {
+            PatternMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::Path,
+                b"dir1",
+                Path::new(""),
+            )])
+            .unwrap()
+        };
+        let pattern_dir1_a = PatternMatcher::new(vec![IgnorePattern::new(
+            PatternSyntax::Glob,
+            b"dir1/a",
+            Path::new(""),
+        )])
+        .unwrap();
+        let pattern_relglob_c = || {
+            PatternMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::RelGlob,
+                b"*.c",
+                Path::new(""),
+            )])
+            .unwrap()
+        };
+        let files = vec![HgPathBuf::from_bytes(b"dir/subdir/b.txt")];
+        let file_dir_subdir_b = FileMatcher::new(files).unwrap();
+
+        let files = vec![
+            HgPathBuf::from_bytes(b"file.txt"),
+            HgPathBuf::from_bytes(b"a/file.txt"),
+            HgPathBuf::from_bytes(b"a/b/file.txt"),
+            // No file in a/b/c
+            HgPathBuf::from_bytes(b"a/b/c/d/file.txt"),
+        ];
+        let file_abcdfile = FileMatcher::new(files).unwrap();
+        let rootfilesin_dir = PatternMatcher::new(vec![IgnorePattern::new(
+            PatternSyntax::RootFilesIn,
+            b"dir",
+            Path::new(""),
+        )])
+        .unwrap();
+
+        let pattern_filepath_dir_subdir =
+            PatternMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::FilePath,
+                b"dir/subdir",
+                Path::new(""),
+            )])
+            .unwrap();
+
+        let include_dir_subdir =
+            IncludeMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::RelPath,
+                b"dir/subdir",
+                Path::new(""),
+            )])
+            .unwrap();
+
+        let more_includematchers = [
+            IncludeMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::Glob,
+                b"dir/s*",
+                Path::new(""),
+            )])
+            .unwrap(),
+            // Test multiple patterns
+            IncludeMatcher::new(vec![
+                IgnorePattern::new(
+                    PatternSyntax::RelPath,
+                    b"dir",
+                    Path::new(""),
+                ),
+                IgnorePattern::new(PatternSyntax::Glob, b"s*", Path::new("")),
+            ])
+            .unwrap(),
+            // Test multiple patterns
+            IncludeMatcher::new(vec![IgnorePattern::new(
+                PatternSyntax::Glob,
+                b"**/*.c",
+                Path::new(""),
+            )])
+            .unwrap(),
+        ];
+
+        tree.check_matcher(&pattern_dir1(), 25);
+        tree.check_matcher(&pattern_dir1_a, 1);
+        tree.check_matcher(&pattern_dir1_glob_c, 2);
+        tree.check_matcher(&pattern_relglob_c(), 14);
+        tree.check_matcher(&AlwaysMatcher, 112);
+        tree.check_matcher(&NeverMatcher, 0);
+        tree.check_matcher(
+            &IntersectionMatcher::new(
+                Box::new(pattern_relglob_c()),
+                Box::new(pattern_dir1()),
+            ),
+            3,
+        );
+        tree.check_matcher(
+            &UnionMatcher::new(vec![
+                Box::new(pattern_relglob_c()),
+                Box::new(pattern_dir1()),
+            ]),
+            36,
+        );
+        tree.check_matcher(
+            &DifferenceMatcher::new(
+                Box::new(pattern_relglob_c()),
+                Box::new(pattern_dir1()),
+            ),
+            11,
+        );
+        tree.check_matcher(&file_dir_subdir_b, 1);
+        tree.check_matcher(&file_abcdfile, 4);
+        tree.check_matcher(&rootfilesin_dir, 8);
+        tree.check_matcher(&pattern_filepath_dir_subdir, 1);
+        tree.check_matcher(&include_dir_subdir, 9);
+        tree.check_matcher(&more_includematchers[0], 17);
+        tree.check_matcher(&more_includematchers[1], 25);
+        tree.check_matcher(&more_includematchers[2], 35);
+    }
 }

rust/hg-core/src/utils/files.rs

@@ -1,436 +1,440 b''
 // files.rs
 //
 // Copyright 2019
 // Raphaël Gomès <[e-mail]>,
 // Yuya Nishihara <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Functions for fiddling with files.
 
 use crate::utils::{
     hg_path::{path_to_hg_path_buf, HgPath, HgPathBuf, HgPathError},
     path_auditor::PathAuditor,
     replace_slice,
 };
 use lazy_static::lazy_static;
 use same_file::is_same_file;
 use std::borrow::{Cow, ToOwned};
 use std::ffi::{OsStr, OsString};
 use std::iter::FusedIterator;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 
 pub fn get_os_str_from_bytes(bytes: &[u8]) -> &OsStr {
     let os_str;
     #[cfg(unix)]
     {
         use std::os::unix::ffi::OsStrExt;
         os_str = std::ffi::OsStr::from_bytes(bytes);
     }
     // TODO Handle other platforms
     // TODO: convert from WTF8 to Windows MBCS (ANSI encoding).
     // Perhaps, the return type would have to be Result<PathBuf>.
     os_str
 }
 
 pub fn get_path_from_bytes(bytes: &[u8]) -> &Path {
     Path::new(get_os_str_from_bytes(bytes))
 }
 
 // TODO: need to convert from WTF8 to MBCS bytes on Windows.
 // that's why Vec<u8> is returned.
 #[cfg(unix)]
 pub fn get_bytes_from_path(path: impl AsRef<Path>) -> Vec<u8> {
     get_bytes_from_os_str(path.as_ref())
 }
 
 #[cfg(unix)]
 pub fn get_bytes_from_os_str(str: impl AsRef<OsStr>) -> Vec<u8> {
     use std::os::unix::ffi::OsStrExt;
     str.as_ref().as_bytes().to_vec()
 }
 
 #[cfg(unix)]
 pub fn get_bytes_from_os_string(str: OsString) -> Vec<u8> {
     use std::os::unix::ffi::OsStringExt;
     str.into_vec()
 }
 
 /// An iterator over repository path yielding itself and its ancestors.
 #[derive(Copy, Clone, Debug)]
 pub struct Ancestors<'a> {
     next: Option<&'a HgPath>,
 }
 
 impl<'a> Iterator for Ancestors<'a> {
     type Item = &'a HgPath;
 
     fn next(&mut self) -> Option<Self::Item> {
         let next = self.next;
         self.next = match self.next {
             Some(s) if s.is_empty() => None,
             Some(s) => {
                 let p = s.bytes().rposition(|c| *c == b'/').unwrap_or(0);
                 Some(HgPath::new(&s.as_bytes()[..p]))
             }
             None => None,
         };
         next
     }
 }
 
 impl<'a> FusedIterator for Ancestors<'a> {}
 
 /// An iterator over repository path yielding itself and its ancestors.
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct AncestorsWithBase<'a> {
     next: Option<(&'a HgPath, &'a HgPath)>,
 }
 
 impl<'a> Iterator for AncestorsWithBase<'a> {
     type Item = (&'a HgPath, &'a HgPath);
 
     fn next(&mut self) -> Option<Self::Item> {
         let next = self.next;
         self.next = match self.next {
             Some((s, _)) if s.is_empty() => None,
             Some((s, _)) => Some(s.split_filename()),
             None => None,
         };
         next
     }
 }
 
 impl<'a> FusedIterator for AncestorsWithBase<'a> {}
 
 /// Returns an iterator yielding ancestor directories of the given repository
 /// path.
 ///
 /// The path is separated by '/', and must not start with '/'.
 ///
 /// The path itself isn't included unless it is b"" (meaning the root
 /// directory.)
 pub fn find_dirs(path: &HgPath) -> Ancestors {
     let mut dirs = Ancestors { next: Some(path) };
     if !path.is_empty() {
         dirs.next(); // skip itself
     }
     dirs
 }
 
+pub fn dir_ancestors(path: &HgPath) -> Ancestors {
+    Ancestors { next: Some(path) }
+}
+
 /// Returns an iterator yielding ancestor directories of the given repository
 /// path.
 ///
 /// The path is separated by '/', and must not start with '/'.
 ///
 /// The path itself isn't included unless it is b"" (meaning the root
 /// directory.)
 pub(crate) fn find_dirs_with_base(path: &HgPath) -> AncestorsWithBase {
     let mut dirs = AncestorsWithBase {
         next: Some((path, HgPath::new(b""))),
     };
     if !path.is_empty() {
         dirs.next(); // skip itself
     }
     dirs
 }
 
 /// TODO more than ASCII?
 pub fn normalize_case(path: &HgPath) -> HgPathBuf {
     #[cfg(windows)] // NTFS compares via upper()
     return path.to_ascii_uppercase();
     #[cfg(unix)]
     path.to_ascii_lowercase()
 }
 
 lazy_static! {
     static ref IGNORED_CHARS: Vec<Vec<u8>> = {
         [
             0x200c, 0x200d, 0x200e, 0x200f, 0x202a, 0x202b, 0x202c, 0x202d,
             0x202e, 0x206a, 0x206b, 0x206c, 0x206d, 0x206e, 0x206f, 0xfeff,
         ]
         .iter()
         .map(|code| {
             std::char::from_u32(*code)
                 .unwrap()
                 .encode_utf8(&mut [0; 3])
                 .bytes()
                 .collect()
         })
         .collect()
     };
 }
 
 fn hfs_ignore_clean(bytes: &[u8]) -> Vec<u8> {
     let mut buf = bytes.to_owned();
     let needs_escaping = bytes.iter().any(|b| *b == b'\xe2' || *b == b'\xef');
     if needs_escaping {
         for forbidden in IGNORED_CHARS.iter() {
             replace_slice(&mut buf, forbidden, &[])
         }
         buf
     } else {
         buf
     }
 }
 
 pub fn lower_clean(bytes: &[u8]) -> Vec<u8> {
     hfs_ignore_clean(&bytes.to_ascii_lowercase())
 }
 
 /// Returns the canonical path of `name`, given `cwd` and `root`
 pub fn canonical_path(
     root: impl AsRef<Path>,
     cwd: impl AsRef<Path>,
     name: impl AsRef<Path>,
 ) -> Result<PathBuf, HgPathError> {
     // TODO add missing normalization for other platforms
     let root = root.as_ref();
     let cwd = cwd.as_ref();
     let name = name.as_ref();
 
     let name = if !name.is_absolute() {
         root.join(cwd).join(name)
     } else {
         name.to_owned()
     };
     let auditor = PathAuditor::new(root);
     if name != root && name.starts_with(root) {
         let name = name.strip_prefix(root).unwrap();
         auditor.audit_path(path_to_hg_path_buf(name)?)?;
         Ok(name.to_owned())
     } else if name == root {
         Ok("".into())
     } else {
         // Determine whether `name' is in the hierarchy at or beneath `root',
         // by iterating name=name.parent() until it returns `None` (can't
         // check name == '/', because that doesn't work on windows).
         let mut name = name.deref();
         let original_name = name.to_owned();
         loop {
             let same = is_same_file(name, root).unwrap_or(false);
             if same {
                 if name == original_name {
                     // `name` was actually the same as root (maybe a symlink)
                     return Ok("".into());
                 }
                 // `name` is a symlink to root, so `original_name` is under
                 // root
                 let rel_path = original_name.strip_prefix(name).unwrap();
                 auditor.audit_path(path_to_hg_path_buf(rel_path)?)?;
                 return Ok(rel_path.to_owned());
             }
             name = match name.parent() {
                 None => break,
                 Some(p) => p,
             };
         }
         // TODO hint to the user about using --cwd
         // Bubble up the responsibility to Python for now
         Err(HgPathError::NotUnderRoot {
             path: original_name,
             root: root.to_owned(),
         })
     }
 }
 
 /// Returns the representation of the path relative to the current working
 /// directory for display purposes.
 ///
 /// `cwd` is a `HgPath`, so it is considered relative to the root directory
 /// of the repository.
 ///
 /// # Examples
 ///
 /// ```
 /// use hg::utils::hg_path::HgPath;
 /// use hg::utils::files::relativize_path;
 /// use std::borrow::Cow;
 ///
 /// let file = HgPath::new(b"nested/file");
 /// let cwd = HgPath::new(b"");
 /// assert_eq!(relativize_path(file, cwd), Cow::Borrowed(b"nested/file"));
 ///
 /// let cwd = HgPath::new(b"nested");
 /// assert_eq!(relativize_path(file, cwd), Cow::Borrowed(b"file"));
 ///
 /// let cwd = HgPath::new(b"other");
 /// assert_eq!(relativize_path(file, cwd), Cow::Borrowed(b"../nested/file"));
 /// ```
 pub fn relativize_path(path: &HgPath, cwd: impl AsRef<HgPath>) -> Cow<[u8]> {
     if cwd.as_ref().is_empty() {
         Cow::Borrowed(path.as_bytes())
     } else {
         // This is not all accurate as to how large `res` will actually be, but
         // profiling `rhg files` on a large-ish repo shows it’s better than
         // starting from a zero-capacity `Vec` and letting `extend` reallocate
         // repeatedly.
         let guesstimate = path.as_bytes().len();
 
         let mut res: Vec<u8> = Vec::with_capacity(guesstimate);
         let mut path_iter = path.as_bytes().split(|b| *b == b'/').peekable();
         let mut cwd_iter =
             cwd.as_ref().as_bytes().split(|b| *b == b'/').peekable();
         loop {
             match (path_iter.peek(), cwd_iter.peek()) {
                 (Some(a), Some(b)) if a == b => (),
                 _ => break,
             }
             path_iter.next();
             cwd_iter.next();
         }
         let mut need_sep = false;
         for _ in cwd_iter {
             if need_sep {
                 res.extend(b"/")
             } else {
                 need_sep = true
             };
             res.extend(b"..");
         }
         for c in path_iter {
             if need_sep {
                 res.extend(b"/")
             } else {
                 need_sep = true
             };
             res.extend(c);
         }
         Cow::Owned(res)
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
 
     #[test]
     fn find_dirs_some() {
         let mut dirs = super::find_dirs(HgPath::new(b"foo/bar/baz"));
         assert_eq!(dirs.next(), Some(HgPath::new(b"foo/bar")));
         assert_eq!(dirs.next(), Some(HgPath::new(b"foo")));
         assert_eq!(dirs.next(), Some(HgPath::new(b"")));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn find_dirs_empty() {
         // looks weird, but mercurial.pathutil.finddirs(b"") yields b""
         let mut dirs = super::find_dirs(HgPath::new(b""));
         assert_eq!(dirs.next(), Some(HgPath::new(b"")));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn test_find_dirs_with_base_some() {
         let mut dirs = super::find_dirs_with_base(HgPath::new(b"foo/bar/baz"));
         assert_eq!(
             dirs.next(),
             Some((HgPath::new(b"foo/bar"), HgPath::new(b"baz")))
         );
         assert_eq!(
             dirs.next(),
             Some((HgPath::new(b"foo"), HgPath::new(b"bar")))
         );
         assert_eq!(dirs.next(), Some((HgPath::new(b""), HgPath::new(b"foo"))));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn test_find_dirs_with_base_empty() {
         let mut dirs = super::find_dirs_with_base(HgPath::new(b""));
         assert_eq!(dirs.next(), Some((HgPath::new(b""), HgPath::new(b""))));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn test_canonical_path() {
         let root = Path::new("/repo");
         let cwd = Path::new("/dir");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Err(HgPathError::NotUnderRoot {
                 path: PathBuf::from("/dir/filename"),
                 root: root.to_path_buf()
             })
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Err(HgPathError::NotUnderRoot {
                 path: PathBuf::from("/filename"),
                 root: root.to_path_buf()
             })
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/");
         let name = Path::new("repo/filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Ok(PathBuf::from("filename"))
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/repo");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Ok(PathBuf::from("filename"))
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/repo/subdir");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Ok(PathBuf::from("subdir/filename"))
         );
     }
 
     #[test]
     fn test_canonical_path_not_rooted() {
         use std::fs::create_dir;
         use tempfile::tempdir;
 
         let base_dir = tempdir().unwrap();
         let base_dir_path = base_dir.path();
         let beneath_repo = base_dir_path.join("a");
         let root = base_dir_path.join("a/b");
         let out_of_repo = base_dir_path.join("c");
         let under_repo_symlink = out_of_repo.join("d");
 
         create_dir(&beneath_repo).unwrap();
         create_dir(&root).unwrap();
 
         // TODO make portable
         std::os::unix::fs::symlink(&root, &out_of_repo).unwrap();
 
         assert_eq!(
             canonical_path(&root, Path::new(""), out_of_repo),
             Ok(PathBuf::from(""))
         );
         assert_eq!(
             canonical_path(&root, Path::new(""), &beneath_repo),
             Err(HgPathError::NotUnderRoot {
                 path: beneath_repo,
                 root: root.to_owned()
             })
         );
         assert_eq!(
             canonical_path(&root, Path::new(""), under_repo_symlink),
             Ok(PathBuf::from("d"))
         );
     }
 }

setup.py

@@ -1,1835 +1,1839 b''
 #
 # This is the mercurial setup script.
 #
 # 'python setup.py install', or
 # 'python setup.py --help' for more options
 import os
 
 # Mercurial can't work on 3.6.0 or 3.6.1 due to a bug in % formatting
 # in bytestrings.
 supportedpy = ','.join(
     [
         '>=3.6.2',
     ]
 )
 
 import sys, platform
 import sysconfig
 
 
 def sysstr(s):
     return s.decode('latin-1')
 
 
 def eprint(*args, **kwargs):
     kwargs['file'] = sys.stderr
     print(*args, **kwargs)
 
 
 import ssl
 
 # ssl.HAS_TLSv1* are preferred to check support but they were added in Python
 # 3.7. Prior to CPython commit 6e8cda91d92da72800d891b2fc2073ecbc134d98
 # (backported to the 3.7 branch), ssl.PROTOCOL_TLSv1_1 / ssl.PROTOCOL_TLSv1_2
 # were defined only if compiled against a OpenSSL version with TLS 1.1 / 1.2
 # support. At the mentioned commit, they were unconditionally defined.
 _notset = object()
 has_tlsv1_1 = getattr(ssl, 'HAS_TLSv1_1', _notset)
 if has_tlsv1_1 is _notset:
     has_tlsv1_1 = getattr(ssl, 'PROTOCOL_TLSv1_1', _notset) is not _notset
 has_tlsv1_2 = getattr(ssl, 'HAS_TLSv1_2', _notset)
 if has_tlsv1_2 is _notset:
     has_tlsv1_2 = getattr(ssl, 'PROTOCOL_TLSv1_2', _notset) is not _notset
 if not (has_tlsv1_1 or has_tlsv1_2):
     error = """
 The `ssl` module does not advertise support for TLS 1.1 or TLS 1.2.
 Please make sure that your Python installation was compiled against an OpenSSL
 version enabling these features (likely this requires the OpenSSL version to
 be at least 1.0.1).
 """
     print(error, file=sys.stderr)
     sys.exit(1)
 
 DYLIB_SUFFIX = sysconfig.get_config_vars()['EXT_SUFFIX']
 
 # Solaris Python packaging brain damage
 try:
     import hashlib
 
     sha = hashlib.sha1()
 except ImportError:
     try:
         import sha
 
         sha.sha  # silence unused import warning
     except ImportError:
         raise SystemExit(
             "Couldn't import standard hashlib (incomplete Python install)."
         )
 
 try:
     import zlib
 
     zlib.compressobj  # silence unused import warning
 except ImportError:
     raise SystemExit(
         "Couldn't import standard zlib (incomplete Python install)."
     )
 
 # The base IronPython distribution (as of 2.7.1) doesn't support bz2
 isironpython = False
 try:
     isironpython = (
         platform.python_implementation().lower().find("ironpython") != -1
     )
 except AttributeError:
     pass
 
 if isironpython:
     sys.stderr.write("warning: IronPython detected (no bz2 support)\n")
 else:
     try:
         import bz2
 
         bz2.BZ2Compressor  # silence unused import warning
     except ImportError:
         raise SystemExit(
             "Couldn't import standard bz2 (incomplete Python install)."
         )
 
 ispypy = "PyPy" in sys.version
 
 import ctypes
 import stat, subprocess, time
 import re
 import shutil
 import tempfile
 
 # We have issues with setuptools on some platforms and builders. Until
 # those are resolved, setuptools is opt-in except for platforms where
 # we don't have issues.
 issetuptools = os.name == 'nt' or 'FORCE_SETUPTOOLS' in os.environ
 if issetuptools:
     from setuptools import setup
 else:
     try:
         from distutils.core import setup
     except ModuleNotFoundError:
         from setuptools import setup
 from distutils.ccompiler import new_compiler
 from distutils.core import Command, Extension
 from distutils.dist import Distribution
 from distutils.command.build import build
 from distutils.command.build_ext import build_ext
 from distutils.command.build_py import build_py
 from distutils.command.build_scripts import build_scripts
 from distutils.command.install import install
 from distutils.command.install_lib import install_lib
 from distutils.command.install_scripts import install_scripts
 from distutils import log
 from distutils.spawn import spawn, find_executable
 from distutils import file_util
 from distutils.errors import (
     CCompilerError,
     DistutilsError,
     DistutilsExecError,
 )
 from distutils.sysconfig import get_python_inc
 
 
 def write_if_changed(path, content):
     """Write content to a file iff the content hasn't changed."""
     if os.path.exists(path):
         with open(path, 'rb') as fh:
             current = fh.read()
     else:
         current = b''
 
     if current != content:
         with open(path, 'wb') as fh:
             fh.write(content)
 
 
 scripts = ['hg']
 if os.name == 'nt':
     # We remove hg.bat if we are able to build hg.exe.
     scripts.append('contrib/win32/hg.bat')
 
 
 def cancompile(cc, code):
     tmpdir = tempfile.mkdtemp(prefix='hg-install-')
     devnull = oldstderr = None
     try:
         fname = os.path.join(tmpdir, 'testcomp.c')
         f = open(fname, 'w')
         f.write(code)
         f.close()
         # Redirect stderr to /dev/null to hide any error messages
         # from the compiler.
         # This will have to be changed if we ever have to check
         # for a function on Windows.
         devnull = open('/dev/null', 'w')
         oldstderr = os.dup(sys.stderr.fileno())
         os.dup2(devnull.fileno(), sys.stderr.fileno())
         objects = cc.compile([fname], output_dir=tmpdir)
         cc.link_executable(objects, os.path.join(tmpdir, "a.out"))
         return True
     except Exception:
         return False
     finally:
         if oldstderr is not None:
             os.dup2(oldstderr, sys.stderr.fileno())
         if devnull is not None:
             devnull.close()
         shutil.rmtree(tmpdir)
 
 
 # simplified version of distutils.ccompiler.CCompiler.has_function
 # that actually removes its temporary files.
 def hasfunction(cc, funcname):
     code = 'int main(void) { %s(); }\n' % funcname
     return cancompile(cc, code)
 
 
 def hasheader(cc, headername):
     code = '#include <%s>\nint main(void) { return 0; }\n' % headername
     return cancompile(cc, code)
 
 
 # py2exe needs to be installed to work
 try:
     import py2exe
 
     py2exe.patch_distutils()
     py2exeloaded = True
     # import py2exe's patched Distribution class
     from distutils.core import Distribution
 except ImportError:
     py2exeloaded = False
 
 
 def runcmd(cmd, env, cwd=None):
     p = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd
     )
     out, err = p.communicate()
     return p.returncode, out, err
 
 
 class hgcommand:
     def __init__(self, cmd, env):
         self.cmd = cmd
         self.env = env
 
     def __repr__(self):
         return f"<hgcommand cmd={self.cmd} env={self.env}>"
 
     def run(self, args):
         cmd = self.cmd + args
         returncode, out, err = runcmd(cmd, self.env)
         err = filterhgerr(err)
         if err:
             print("stderr from '%s':" % (' '.join(cmd)), file=sys.stderr)
             print(err, file=sys.stderr)
         if returncode != 0:
+            print(
+                "non zero-return '%s': %d" % (' '.join(cmd), returncode),
+                file=sys.stderr,
+            )
             return b''
         return out
 
 
 def filterhgerr(err):
     # If root is executing setup.py, but the repository is owned by
     # another user (as in "sudo python setup.py install") we will get
     # trust warnings since the .hg/hgrc file is untrusted. That is
     # fine, we don't want to load it anyway.  Python may warn about
     # a missing __init__.py in mercurial/locale, we also ignore that.
     err = [
         e
         for e in err.splitlines()
         if (
             not e.startswith(b'not trusting file')
             and not e.startswith(b'warning: Not importing')
             and not e.startswith(b'obsolete feature not enabled')
             and not e.startswith(b'*** failed to import extension')
             and not e.startswith(b'devel-warn:')
             and not (
                 e.startswith(b'(third party extension')
                 and e.endswith(b'or newer of Mercurial; disabling)')
             )
         )
     ]
     return b'\n'.join(b'  ' + e for e in err)
 
 
 def findhg():
     """Try to figure out how we should invoke hg for examining the local
     repository contents.
 
     Returns an hgcommand object."""
     # By default, prefer the "hg" command in the user's path.  This was
     # presumably the hg command that the user used to create this repository.
     #
     # This repository may require extensions or other settings that would not
     # be enabled by running the hg script directly from this local repository.
     hgenv = os.environ.copy()
     # Use HGPLAIN to disable hgrc settings that would change output formatting,
     # and disable localization for the same reasons.
     hgenv['HGPLAIN'] = '1'
     hgenv['LANGUAGE'] = 'C'
     hgcmd = ['hg']
     # Run a simple "hg log" command just to see if using hg from the user's
     # path works and can successfully interact with this repository.  Windows
     # gives precedence to hg.exe in the current directory, so fall back to the
     # python invocation of local hg, where pythonXY.dll can always be found.
     check_cmd = ['log', '-r.', '-Ttest']
     attempts = []
 
     def attempt(cmd, env):
         try:
             retcode, out, err = runcmd(hgcmd + check_cmd, hgenv)
             res = (True, retcode, out, err)
             if retcode == 0 and not filterhgerr(err):
                 return True
         except EnvironmentError as e:
             res = (False, e)
         attempts.append((cmd, res))
         return False
 
     if os.name != 'nt' or not os.path.exists("hg.exe"):
         if attempt(hgcmd + check_cmd, hgenv):
             return hgcommand(hgcmd, hgenv)
 
     # Fall back to trying the local hg installation (pure python)
     repo_hg = os.path.join(os.path.dirname(__file__), 'hg')
     hgenv = localhgenv()
     hgcmd = [sys.executable, repo_hg]
     if attempt(hgcmd + check_cmd, hgenv):
         return hgcommand(hgcmd, hgenv)
     # Fall back to trying the local hg installation (whatever we can)
     hgenv = localhgenv(pure_python=False)
     hgcmd = [sys.executable, repo_hg]
     if attempt(hgcmd + check_cmd, hgenv):
         return hgcommand(hgcmd, hgenv)
 
     eprint("/!\\")
     eprint(r"/!\ Unable to find a working hg binary")
     eprint(r"/!\ Version cannot be extracted from the repository")
     eprint(r"/!\ Re-run the setup once a first version is built")
     eprint(r"/!\ Attempts:")
     for i, e in enumerate(attempts):
         eprint(r"/!\   attempt #%d:" % (i))
         eprint(r"/!\     cmd:        ", e[0])
         res = e[1]
         if res[0]:
             eprint(r"/!\     return code:", res[1])
             eprint("/!\\     std output:\n%s" % (res[2].decode()), end="")
             eprint("/!\\     std error:\n%s" % (res[3].decode()), end="")
         else:
             eprint(r"/!\     exception:  ", res[1])
     return None
 
 
 def localhgenv(pure_python=True):
     """Get an environment dictionary to use for invoking or importing
     mercurial from the local repository."""
     # Execute hg out of this directory with a custom environment which takes
     # care to not use any hgrc files and do no localization.
     env = {
         'HGRCPATH': '',
         'LANGUAGE': 'C',
         'PATH': '',
     }  # make pypi modules that use os.environ['PATH'] happy
     if pure_python:
         env['HGMODULEPOLICY'] = 'py'
     if 'LD_LIBRARY_PATH' in os.environ:
         env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
     if 'SystemRoot' in os.environ:
         # SystemRoot is required by Windows to load various DLLs.  See:
         # https://bugs.python.org/issue13524#msg148850
         env['SystemRoot'] = os.environ['SystemRoot']
     return env
 
 
 version = ''
 
 
 def _try_get_version():
     hg = findhg()
     if hg is None:
         return ''
     hgid = None
     numerictags = []
     cmd = ['log', '-r', '.', '--template', '{tags}\n']
     pieces = sysstr(hg.run(cmd)).split()
     numerictags = [t for t in pieces if t[0:1].isdigit()]
     hgid = sysstr(hg.run(['id', '-i'])).strip()
     if hgid.count('+') == 2:
         hgid = hgid.replace("+", ".", 1)
     if not hgid:
         eprint("/!\\")
         eprint(r"/!\ Unable to determine hg version from local repository")
         eprint(r"/!\ Failed to retrieve current revision tags")
         return ''
     if numerictags:  # tag(s) found
         version = numerictags[-1]
         if hgid.endswith('+'):  # propagate the dirty status to the tag
             version += '+'
     else:  # no tag found on the checked out revision
         ltagcmd = ['log', '--rev', 'wdir()', '--template', '{latesttag}']
         ltag = sysstr(hg.run(ltagcmd))
         if not ltag:
             eprint("/!\\")
             eprint(r"/!\ Unable to determine hg version from local repository")
             eprint(
                 r"/!\ Failed to retrieve current revision distance to lated tag"
             )
             return ''
         changessincecmd = [
             'log',
             '-T',
             'x\n',
             '-r',
             "only(parents(),'%s')" % ltag,
         ]
         changessince = len(hg.run(changessincecmd).splitlines())
         version = '%s+hg%s.%s' % (ltag, changessince, hgid)
     if version.endswith('+'):
         version = version[:-1] + 'local' + time.strftime('%Y%m%d')
     return version
 
 
 if os.path.isdir('.hg'):
     version = _try_get_version()
 elif os.path.exists('.hg_archival.txt'):
     kw = dict(
         [[t.strip() for t in l.split(':', 1)] for l in open('.hg_archival.txt')]
     )
     if 'tag' in kw:
         version = kw['tag']
     elif 'latesttag' in kw:
         if 'changessincelatesttag' in kw:
             version = (
                 '%(latesttag)s+hg%(changessincelatesttag)s.%(node).12s' % kw
             )
         else:
             version = '%(latesttag)s+hg%(latesttagdistance)s.%(node).12s' % kw
     else:
         version = '0+hg' + kw.get('node', '')[:12]
 elif os.path.exists('mercurial/__version__.py'):
     with open('mercurial/__version__.py') as f:
         data = f.read()
     version = re.search('version = b"(.*)"', data).group(1)
 if not version:
     if os.environ.get("MERCURIAL_SETUP_MAKE_LOCAL") == "1":
         version = "0.0+0"
         eprint("/!\\")
         eprint(r"/!\ Using '0.0+0' as the default version")
         eprint(r"/!\ Re-run make local once that first version is built")
         eprint("/!\\")
     else:
         eprint("/!\\")
         eprint(r"/!\ Could not determine the Mercurial version")
         eprint(r"/!\ You need to build a local version first")
         eprint(r"/!\ Run `make local` and try again")
         eprint("/!\\")
         msg = "Run `make local` first to get a working local version"
         raise SystemExit(msg)
 
 versionb = version
 if not isinstance(versionb, bytes):
     versionb = versionb.encode('ascii')
 
 write_if_changed(
     'mercurial/__version__.py',
     b''.join(
         [
             b'# this file is autogenerated by setup.py\n'
             b'version = b"%s"\n' % versionb,
         ]
     ),
 )
 
 
 class hgbuild(build):
     # Insert hgbuildmo first so that files in mercurial/locale/ are found
     # when build_py is run next.
     sub_commands = [('build_mo', None)] + build.sub_commands
 
 
 class hgbuildmo(build):
 
     description = "build translations (.mo files)"
 
     def run(self):
         if not find_executable('msgfmt'):
             self.warn(
                 "could not find msgfmt executable, no translations "
                 "will be built"
             )
             return
 
         podir = 'i18n'
         if not os.path.isdir(podir):
             self.warn("could not find %s/ directory" % podir)
             return
 
         join = os.path.join
         for po in os.listdir(podir):
             if not po.endswith('.po'):
                 continue
             pofile = join(podir, po)
             modir = join('locale', po[:-3], 'LC_MESSAGES')
             mofile = join(modir, 'hg.mo')
             mobuildfile = join('mercurial', mofile)
             cmd = ['msgfmt', '-v', '-o', mobuildfile, pofile]
             if sys.platform != 'sunos5':
                 # msgfmt on Solaris does not know about -c
                 cmd.append('-c')
             self.mkpath(join('mercurial', modir))
             self.make_file([pofile], mobuildfile, spawn, (cmd,))
 
 
 class hgdist(Distribution):
     pure = False
     rust = False
     no_rust = False
     cffi = ispypy
 
     global_options = Distribution.global_options + [
         ('pure', None, "use pure (slow) Python code instead of C extensions"),
         ('rust', None, "use Rust extensions additionally to C extensions"),
         (
             'no-rust',
             None,
             "do not use Rust extensions additionally to C extensions",
         ),
     ]
 
     negative_opt = Distribution.negative_opt.copy()
     boolean_options = ['pure', 'rust', 'no-rust']
     negative_opt['no-rust'] = 'rust'
 
     def _set_command_options(self, command_obj, option_dict=None):
         # Not all distutils versions in the wild have boolean_options.
         # This should be cleaned up when we're Python 3 only.
         command_obj.boolean_options = (
             getattr(command_obj, 'boolean_options', []) + self.boolean_options
         )
         return Distribution._set_command_options(
             self, command_obj, option_dict=option_dict
         )
 
     def parse_command_line(self):
         ret = Distribution.parse_command_line(self)
         if not (self.rust or self.no_rust):
             hgrustext = os.environ.get('HGWITHRUSTEXT')
             # TODO record it for proper rebuild upon changes
             # (see mercurial/__modulepolicy__.py)
             if hgrustext != 'cpython' and hgrustext is not None:
                 if hgrustext:
                     msg = 'unknown HGWITHRUSTEXT value: %s' % hgrustext
                     print(msg, file=sys.stderr)
                 hgrustext = None
             self.rust = hgrustext is not None
             self.no_rust = not self.rust
         return ret
 
     def has_ext_modules(self):
         # self.ext_modules is emptied in hgbuildpy.finalize_options which is
         # too late for some cases
         return not self.pure and Distribution.has_ext_modules(self)
 
 
 # This is ugly as a one-liner. So use a variable.
 buildextnegops = dict(getattr(build_ext, 'negative_options', {}))
 buildextnegops['no-zstd'] = 'zstd'
 buildextnegops['no-rust'] = 'rust'
 
 
 class hgbuildext(build_ext):
     user_options = build_ext.user_options + [
         ('zstd', None, 'compile zstd bindings [default]'),
         ('no-zstd', None, 'do not compile zstd bindings'),
         (
             'rust',
             None,
             'compile Rust extensions if they are in use '
             '(requires Cargo) [default]',
         ),
         ('no-rust', None, 'do not compile Rust extensions'),
     ]
 
     boolean_options = build_ext.boolean_options + ['zstd', 'rust']
     negative_opt = buildextnegops
 
     def initialize_options(self):
         self.zstd = True
         self.rust = True
 
         return build_ext.initialize_options(self)
 
     def finalize_options(self):
         # Unless overridden by the end user, build extensions in parallel.
         # Only influences behavior on Python 3.5+.
         if getattr(self, 'parallel', None) is None:
             self.parallel = True
 
         return build_ext.finalize_options(self)
 
     def build_extensions(self):
         ruststandalones = [
             e for e in self.extensions if isinstance(e, RustStandaloneExtension)
         ]
         self.extensions = [
             e for e in self.extensions if e not in ruststandalones
         ]
         # Filter out zstd if disabled via argument.
         if not self.zstd:
             self.extensions = [
                 e for e in self.extensions if e.name != 'mercurial.zstd'
             ]
 
         # Build Rust standalone extensions if it'll be used
         # and its build is not explicitly disabled (for external build
         # as Linux distributions would do)
         if self.distribution.rust and self.rust:
             if not sys.platform.startswith('linux'):
                 self.warn(
                     "rust extensions have only been tested on Linux "
                     "and may not behave correctly on other platforms"
                 )
 
             for rustext in ruststandalones:
                 rustext.build('' if self.inplace else self.build_lib)
 
         return build_ext.build_extensions(self)
 
     def build_extension(self, ext):
         if (
             self.distribution.rust
             and self.rust
             and isinstance(ext, RustExtension)
         ):
             ext.rustbuild()
         try:
             build_ext.build_extension(self, ext)
         except CCompilerError:
             if not getattr(ext, 'optional', False):
                 raise
             log.warn(
                 "Failed to build optional extension '%s' (skipping)", ext.name
             )
 
 
 class hgbuildscripts(build_scripts):
     def run(self):
         if os.name != 'nt' or self.distribution.pure:
             return build_scripts.run(self)
 
         exebuilt = False
         try:
             self.run_command('build_hgexe')
             exebuilt = True
         except (DistutilsError, CCompilerError):
             log.warn('failed to build optional hg.exe')
 
         if exebuilt:
             # Copying hg.exe to the scripts build directory ensures it is
             # installed by the install_scripts command.
             hgexecommand = self.get_finalized_command('build_hgexe')
             dest = os.path.join(self.build_dir, 'hg.exe')
             self.mkpath(self.build_dir)
             self.copy_file(hgexecommand.hgexepath, dest)
 
             # Remove hg.bat because it is redundant with hg.exe.
             self.scripts.remove('contrib/win32/hg.bat')
 
         return build_scripts.run(self)
 
 
 class hgbuildpy(build_py):
     def finalize_options(self):
         build_py.finalize_options(self)
 
         if self.distribution.pure:
             self.distribution.ext_modules = []
         elif self.distribution.cffi:
             from mercurial.cffi import (
                 bdiffbuild,
                 mpatchbuild,
             )
 
             exts = [
                 mpatchbuild.ffi.distutils_extension(),
                 bdiffbuild.ffi.distutils_extension(),
             ]
             # cffi modules go here
             if sys.platform == 'darwin':
                 from mercurial.cffi import osutilbuild
 
                 exts.append(osutilbuild.ffi.distutils_extension())
             self.distribution.ext_modules = exts
         else:
             h = os.path.join(get_python_inc(), 'Python.h')
             if not os.path.exists(h):
                 raise SystemExit(
                     'Python headers are required to build '
                     'Mercurial but weren\'t found in %s' % h
                 )
 
     def run(self):
         basepath = os.path.join(self.build_lib, 'mercurial')
         self.mkpath(basepath)
 
         rust = self.distribution.rust
         if self.distribution.pure:
             modulepolicy = 'py'
         elif self.build_lib == '.':
             # in-place build should run without rebuilding and Rust extensions
             modulepolicy = 'rust+c-allow' if rust else 'allow'
         else:
             modulepolicy = 'rust+c' if rust else 'c'
 
         content = b''.join(
             [
                 b'# this file is autogenerated by setup.py\n',
                 b'modulepolicy = b"%s"\n' % modulepolicy.encode('ascii'),
             ]
         )
         write_if_changed(os.path.join(basepath, '__modulepolicy__.py'), content)
 
         build_py.run(self)
 
 
 class buildhgextindex(Command):
     description = 'generate prebuilt index of hgext (for frozen package)'
     user_options = []
     _indexfilename = 'hgext/__index__.py'
 
     def initialize_options(self):
         pass
 
     def finalize_options(self):
         pass
 
     def run(self):
         if os.path.exists(self._indexfilename):
             with open(self._indexfilename, 'w') as f:
                 f.write('# empty\n')
 
         # here no extension enabled, disabled() lists up everything
         code = (
             'import pprint; from mercurial import extensions; '
             'ext = extensions.disabled();'
             'ext.pop("__index__", None);'
             'pprint.pprint(ext)'
         )
         returncode, out, err = runcmd(
             [sys.executable, '-c', code], localhgenv()
         )
         if err or returncode != 0:
             raise DistutilsExecError(err)
 
         with open(self._indexfilename, 'wb') as f:
             f.write(b'# this file is autogenerated by setup.py\n')
             f.write(b'docs = ')
             f.write(out)
 
 
 class buildhgexe(build_ext):
     description = 'compile hg.exe from mercurial/exewrapper.c'
 
     LONG_PATHS_MANIFEST = """\
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
   <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
     <security>
       <requestedPrivileges>
         <requestedExecutionLevel
           level="asInvoker"
           uiAccess="false"
         />
       </requestedPrivileges>
     </security>
   </trustInfo>
   <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
     <application>
       <!-- Windows Vista -->
       <supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/>
       <!-- Windows 7 -->
       <supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
       <!-- Windows 8 -->
       <supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
       <!-- Windows 8.1 -->
       <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
       <!-- Windows 10 and Windows 11 -->
       <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
     </application>
   </compatibility>
   <application xmlns="urn:schemas-microsoft-com:asm.v3">
     <windowsSettings
         xmlns:ws2="http://schemas.microsoft.com/SMI/2016/WindowsSettings">
       <ws2:longPathAware>true</ws2:longPathAware>
     </windowsSettings>
   </application>
   <dependency>
     <dependentAssembly>
       <assemblyIdentity type="win32"
                         name="Microsoft.Windows.Common-Controls"
                         version="6.0.0.0"
                         processorArchitecture="*"
                         publicKeyToken="6595b64144ccf1df"
                         language="*" />
     </dependentAssembly>
   </dependency>
 </assembly>
 """
 
     def initialize_options(self):
         build_ext.initialize_options(self)
 
     def build_extensions(self):
         if os.name != 'nt':
             return
         if isinstance(self.compiler, HackedMingw32CCompiler):
             self.compiler.compiler_so = self.compiler.compiler  # no -mdll
             self.compiler.dll_libraries = []  # no -lmsrvc90
 
         pythonlib = None
 
         dirname = os.path.dirname(self.get_ext_fullpath('dummy'))
         self.hgtarget = os.path.join(dirname, 'hg')
 
         if getattr(sys, 'dllhandle', None):
             # Different Python installs can have different Python library
             # names. e.g. the official CPython distribution uses pythonXY.dll
             # and MinGW uses libpythonX.Y.dll.
             _kernel32 = ctypes.windll.kernel32
             _kernel32.GetModuleFileNameA.argtypes = [
                 ctypes.c_void_p,
                 ctypes.c_void_p,
                 ctypes.c_ulong,
             ]
             _kernel32.GetModuleFileNameA.restype = ctypes.c_ulong
             size = 1000
             buf = ctypes.create_string_buffer(size + 1)
             filelen = _kernel32.GetModuleFileNameA(
                 sys.dllhandle, ctypes.byref(buf), size
             )
 
             if filelen > 0 and filelen != size:
                 dllbasename = os.path.basename(buf.value)
                 if not dllbasename.lower().endswith(b'.dll'):
                     raise SystemExit(
                         'Python DLL does not end with .dll: %s' % dllbasename
                     )
                 pythonlib = dllbasename[:-4]
 
                 # Copy the pythonXY.dll next to the binary so that it runs
                 # without tampering with PATH.
                 dest = os.path.join(
                     os.path.dirname(self.hgtarget),
                     os.fsdecode(dllbasename),
                 )
 
                 if not os.path.exists(dest):
                     shutil.copy(buf.value, dest)
 
                 # Also overwrite python3.dll so that hgext.git is usable.
                 # TODO: also handle the MSYS flavor
                 python_x = os.path.join(
                     os.path.dirname(os.fsdecode(buf.value)),
                     "python3.dll",
                 )
 
                 if os.path.exists(python_x):
                     dest = os.path.join(
                         os.path.dirname(self.hgtarget),
                         os.path.basename(python_x),
                     )
 
                     shutil.copy(python_x, dest)
 
         if not pythonlib:
             log.warn(
                 'could not determine Python DLL filename; assuming pythonXY'
             )
 
             hv = sys.hexversion
             pythonlib = b'python%d%d' % (hv >> 24, (hv >> 16) & 0xFF)
 
         log.info('using %s as Python library name' % pythonlib)
         with open('mercurial/hgpythonlib.h', 'wb') as f:
             f.write(b'/* this file is autogenerated by setup.py */\n')
             f.write(b'#define HGPYTHONLIB "%s"\n' % pythonlib)
 
         objects = self.compiler.compile(
             ['mercurial/exewrapper.c'],
             output_dir=self.build_temp,
             macros=[('_UNICODE', None), ('UNICODE', None)],
         )
         self.compiler.link_executable(
             objects, self.hgtarget, libraries=[], output_dir=self.build_temp
         )
 
         self.addlongpathsmanifest()
 
     def addlongpathsmanifest(self):
         """Add manifest pieces so that hg.exe understands long paths
 
         Why resource #1 should be used for .exe manifests? I don't know and
         wasn't able to find an explanation for mortals. But it seems to work.
         """
         exefname = self.compiler.executable_filename(self.hgtarget)
         fdauto, manfname = tempfile.mkstemp(suffix='.hg.exe.manifest')
         os.close(fdauto)
         with open(manfname, 'w', encoding="UTF-8") as f:
             f.write(self.LONG_PATHS_MANIFEST)
         log.info("long paths manifest is written to '%s'" % manfname)
         outputresource = '-outputresource:%s;#1' % exefname
         log.info("running mt.exe to update hg.exe's manifest in-place")
 
         self.spawn(
             [
                 self.compiler.mt,
                 '-nologo',
                 '-manifest',
                 manfname,
                 outputresource,
             ]
         )
         log.info("done updating hg.exe's manifest")
         os.remove(manfname)
 
     @property
     def hgexepath(self):
         dir = os.path.dirname(self.get_ext_fullpath('dummy'))
         return os.path.join(self.build_temp, dir, 'hg.exe')
 
 
 class hgbuilddoc(Command):
     description = 'build documentation'
     user_options = [
         ('man', None, 'generate man pages'),
         ('html', None, 'generate html pages'),
     ]
 
     def initialize_options(self):
         self.man = None
         self.html = None
 
     def finalize_options(self):
         # If --man or --html are set, only generate what we're told to.
         # Otherwise generate everything.
         have_subset = self.man is not None or self.html is not None
 
         if have_subset:
             self.man = True if self.man else False
             self.html = True if self.html else False
         else:
             self.man = True
             self.html = True
 
     def run(self):
         def normalizecrlf(p):
             with open(p, 'rb') as fh:
                 orig = fh.read()
 
             if b'\r\n' not in orig:
                 return
 
             log.info('normalizing %s to LF line endings' % p)
             with open(p, 'wb') as fh:
                 fh.write(orig.replace(b'\r\n', b'\n'))
 
         def gentxt(root):
             txt = 'doc/%s.txt' % root
             log.info('generating %s' % txt)
             res, out, err = runcmd(
                 [sys.executable, 'gendoc.py', root], os.environ, cwd='doc'
             )
             if res:
                 raise SystemExit(
                     'error running gendoc.py: %s'
                     % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             with open(txt, 'wb') as fh:
                 fh.write(out)
 
         def gengendoc(root):
             gendoc = 'doc/%s.gendoc.txt' % root
 
             log.info('generating %s' % gendoc)
             res, out, err = runcmd(
                 [sys.executable, 'gendoc.py', '%s.gendoc' % root],
                 os.environ,
                 cwd='doc',
             )
             if res:
                 raise SystemExit(
                     'error running gendoc: %s'
                     % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             with open(gendoc, 'wb') as fh:
                 fh.write(out)
 
         def genman(root):
             log.info('generating doc/%s' % root)
             res, out, err = runcmd(
                 [
                     sys.executable,
                     'runrst',
                     'hgmanpage',
                     '--halt',
                     'warning',
                     '--strip-elements-with-class',
                     'htmlonly',
                     '%s.txt' % root,
                     root,
                 ],
                 os.environ,
                 cwd='doc',
             )
             if res:
                 raise SystemExit(
                     'error running runrst: %s'
                     % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             normalizecrlf('doc/%s' % root)
 
         def genhtml(root):
             log.info('generating doc/%s.html' % root)
             res, out, err = runcmd(
                 [
                     sys.executable,
                     'runrst',
                     'html',
                     '--halt',
                     'warning',
                     '--link-stylesheet',
                     '--stylesheet-path',
                     'style.css',
                     '%s.txt' % root,
                     '%s.html' % root,
                 ],
                 os.environ,
                 cwd='doc',
             )
             if res:
                 raise SystemExit(
                     'error running runrst: %s'
                     % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             normalizecrlf('doc/%s.html' % root)
 
         # This logic is duplicated in doc/Makefile.
         sources = {
             f
             for f in os.listdir('mercurial/helptext')
             if re.search(r'[0-9]\.txt$', f)
         }
 
         # common.txt is a one-off.
         gentxt('common')
 
         for source in sorted(sources):
             assert source[-4:] == '.txt'
             root = source[:-4]
 
             gentxt(root)
             gengendoc(root)
 
             if self.man:
                 genman(root)
             if self.html:
                 genhtml(root)
 
 
 class hginstall(install):
 
     user_options = install.user_options + [
         (
             'old-and-unmanageable',
             None,
             'noop, present for eggless setuptools compat',
         ),
         (
             'single-version-externally-managed',
             None,
             'noop, present for eggless setuptools compat',
         ),
     ]
 
     sub_commands = install.sub_commands + [
         ('install_completion', lambda self: True)
     ]
 
     # Also helps setuptools not be sad while we refuse to create eggs.
     single_version_externally_managed = True
 
     def get_sub_commands(self):
         # Screen out egg related commands to prevent egg generation.  But allow
         # mercurial.egg-info generation, since that is part of modern
         # packaging.
         excl = {'bdist_egg'}
         return filter(lambda x: x not in excl, install.get_sub_commands(self))
 
 
 class hginstalllib(install_lib):
     """
     This is a specialization of install_lib that replaces the copy_file used
     there so that it supports setting the mode of files after copying them,
     instead of just preserving the mode that the files originally had.  If your
     system has a umask of something like 027, preserving the permissions when
     copying will lead to a broken install.
 
     Note that just passing keep_permissions=False to copy_file would be
     insufficient, as it might still be applying a umask.
     """
 
     def run(self):
         realcopyfile = file_util.copy_file
 
         def copyfileandsetmode(*args, **kwargs):
             src, dst = args[0], args[1]
             dst, copied = realcopyfile(*args, **kwargs)
             if copied:
                 st = os.stat(src)
                 # Persist executable bit (apply it to group and other if user
                 # has it)
                 if st[stat.ST_MODE] & stat.S_IXUSR:
                     setmode = int('0755', 8)
                 else:
                     setmode = int('0644', 8)
                 m = stat.S_IMODE(st[stat.ST_MODE])
                 m = (m & ~int('0777', 8)) | setmode
                 os.chmod(dst, m)
 
         file_util.copy_file = copyfileandsetmode
         try:
             install_lib.run(self)
         finally:
             file_util.copy_file = realcopyfile
 
 
 class hginstallscripts(install_scripts):
     """
     This is a specialization of install_scripts that replaces the @LIBDIR@ with
     the configured directory for modules. If possible, the path is made relative
     to the directory for scripts.
     """
 
     def initialize_options(self):
         install_scripts.initialize_options(self)
 
         self.install_lib = None
 
     def finalize_options(self):
         install_scripts.finalize_options(self)
         self.set_undefined_options('install', ('install_lib', 'install_lib'))
 
     def run(self):
         install_scripts.run(self)
 
         # It only makes sense to replace @LIBDIR@ with the install path if
         # the install path is known. For wheels, the logic below calculates
         # the libdir to be "../..". This is because the internal layout of a
         # wheel archive looks like:
         #
         #   mercurial-3.6.1.data/scripts/hg
         #   mercurial/__init__.py
         #
         # When installing wheels, the subdirectories of the "<pkg>.data"
         # directory are translated to system local paths and files therein
         # are copied in place. The mercurial/* files are installed into the
         # site-packages directory. However, the site-packages directory
         # isn't known until wheel install time. This means we have no clue
         # at wheel generation time what the installed site-packages directory
         # will be. And, wheels don't appear to provide the ability to register
         # custom code to run during wheel installation. This all means that
         # we can't reliably set the libdir in wheels: the default behavior
         # of looking in sys.path must do.
 
         if (
             os.path.splitdrive(self.install_dir)[0]
             != os.path.splitdrive(self.install_lib)[0]
         ):
             # can't make relative paths from one drive to another, so use an
             # absolute path instead
             libdir = self.install_lib
         else:
             libdir = os.path.relpath(self.install_lib, self.install_dir)
 
         for outfile in self.outfiles:
             with open(outfile, 'rb') as fp:
                 data = fp.read()
 
             # skip binary files
             if b'\0' in data:
                 continue
 
             # During local installs, the shebang will be rewritten to the final
             # install path. During wheel packaging, the shebang has a special
             # value.
             if data.startswith(b'#!python'):
                 log.info(
                     'not rewriting @LIBDIR@ in %s because install path '
                     'not known' % outfile
                 )
                 continue
 
             data = data.replace(b'@LIBDIR@', libdir.encode('unicode_escape'))
             with open(outfile, 'wb') as fp:
                 fp.write(data)
 
 
 class hginstallcompletion(Command):
     description = 'Install shell completion'
 
     def initialize_options(self):
         self.install_dir = None
         self.outputs = []
 
     def finalize_options(self):
         self.set_undefined_options(
             'install_data', ('install_dir', 'install_dir')
         )
 
     def get_outputs(self):
         return self.outputs
 
     def run(self):
         for src, dir_path, dest in (
             (
                 'bash_completion',
                 ('share', 'bash-completion', 'completions'),
                 'hg',
             ),
             ('zsh_completion', ('share', 'zsh', 'site-functions'), '_hg'),
         ):
             dir = os.path.join(self.install_dir, *dir_path)
             self.mkpath(dir)
 
             dest = os.path.join(dir, dest)
             self.outputs.append(dest)
             self.copy_file(os.path.join('contrib', src), dest)
 
 
 # virtualenv installs custom distutils/__init__.py and
 # distutils/distutils.cfg files which essentially proxy back to the
 # "real" distutils in the main Python install. The presence of this
 # directory causes py2exe to pick up the "hacked" distutils package
 # from the virtualenv and "import distutils" will fail from the py2exe
 # build because the "real" distutils files can't be located.
 #
 # We work around this by monkeypatching the py2exe code finding Python
 # modules to replace the found virtualenv distutils modules with the
 # original versions via filesystem scanning. This is a bit hacky. But
 # it allows us to use virtualenvs for py2exe packaging, which is more
 # deterministic and reproducible.
 #
 # It's worth noting that the common StackOverflow suggestions for this
 # problem involve copying the original distutils files into the
 # virtualenv or into the staging directory after setup() is invoked.
 # The former is very brittle and can easily break setup(). Our hacking
 # of the found modules routine has a similar result as copying the files
 # manually. But it makes fewer assumptions about how py2exe works and
 # is less brittle.
 
 # This only catches virtualenvs made with virtualenv (as opposed to
 # venv, which is likely what Python 3 uses).
 py2exehacked = py2exeloaded and getattr(sys, 'real_prefix', None) is not None
 
 if py2exehacked:
     from distutils.command.py2exe import py2exe as buildpy2exe
     from py2exe.mf import Module as py2exemodule
 
     class hgbuildpy2exe(buildpy2exe):
         def find_needed_modules(self, mf, files, modules):
             res = buildpy2exe.find_needed_modules(self, mf, files, modules)
 
             # Replace virtualenv's distutils modules with the real ones.
             modules = {}
             for k, v in res.modules.items():
                 if k != 'distutils' and not k.startswith('distutils.'):
                     modules[k] = v
 
             res.modules = modules
 
             import opcode
 
             distutilsreal = os.path.join(
                 os.path.dirname(opcode.__file__), 'distutils'
             )
 
             for root, dirs, files in os.walk(distutilsreal):
                 for f in sorted(files):
                     if not f.endswith('.py'):
                         continue
 
                     full = os.path.join(root, f)
 
                     parents = ['distutils']
 
                     if root != distutilsreal:
                         rel = os.path.relpath(root, distutilsreal)
                         parents.extend(p for p in rel.split(os.sep))
 
                     modname = '%s.%s' % ('.'.join(parents), f[:-3])
 
                     if modname.startswith('distutils.tests.'):
                         continue
 
                     if modname.endswith('.__init__'):
                         modname = modname[: -len('.__init__')]
                         path = os.path.dirname(full)
                     else:
                         path = None
 
                     res.modules[modname] = py2exemodule(
                         modname, full, path=path
                     )
 
             if 'distutils' not in res.modules:
                 raise SystemExit('could not find distutils modules')
 
             return res
 
 
 cmdclass = {
     'build': hgbuild,
     'build_doc': hgbuilddoc,
     'build_mo': hgbuildmo,
     'build_ext': hgbuildext,
     'build_py': hgbuildpy,
     'build_scripts': hgbuildscripts,
     'build_hgextindex': buildhgextindex,
     'install': hginstall,
     'install_completion': hginstallcompletion,
     'install_lib': hginstalllib,
     'install_scripts': hginstallscripts,
     'build_hgexe': buildhgexe,
 }
 
 if py2exehacked:
     cmdclass['py2exe'] = hgbuildpy2exe
 
 packages = [
     'mercurial',
     'mercurial.admin',
     'mercurial.cext',
     'mercurial.cffi',
     'mercurial.defaultrc',
     'mercurial.dirstateutils',
     'mercurial.helptext',
     'mercurial.helptext.internals',
     'mercurial.hgweb',
     'mercurial.interfaces',
     'mercurial.pure',
     'mercurial.stabletailgraph',
     'mercurial.templates',
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
     'mercurial.thirdparty.tomli',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
     'mercurial.upgrade_utils',
     'mercurial.utils',
     'mercurial.revlogutils',
     'mercurial.testing',
     'hgext',
     'hgext.convert',
     'hgext.fsmonitor',
     'hgext.fastannotate',
     'hgext.fsmonitor.pywatchman',
     'hgext.git',
     'hgext.highlight',
     'hgext.hooklib',
     'hgext.largefiles',
     'hgext.lfs',
     'hgext.narrow',
     'hgext.remotefilelog',
     'hgext.zeroconf',
     'hgext3rd',
     'hgdemandimport',
 ]
 
 for name in os.listdir(os.path.join('mercurial', 'templates')):
     if name != '__pycache__' and os.path.isdir(
         os.path.join('mercurial', 'templates', name)
     ):
         packages.append('mercurial.templates.%s' % name)
 
 if 'HG_PY2EXE_EXTRA_INSTALL_PACKAGES' in os.environ:
     # py2exe can't cope with namespace packages very well, so we have to
     # install any hgext3rd.* extensions that we want in the final py2exe
     # image here. This is gross, but you gotta do what you gotta do.
     packages.extend(os.environ['HG_PY2EXE_EXTRA_INSTALL_PACKAGES'].split(' '))
 
 common_depends = [
     'mercurial/bitmanipulation.h',
     'mercurial/compat.h',
     'mercurial/cext/util.h',
 ]
 common_include_dirs = ['mercurial']
 
 common_cflags = []
 
 # MSVC 2008 still needs declarations at the top of the scope, but Python 3.9
 # makes declarations not at the top of a scope in the headers.
 if os.name != 'nt' and sys.version_info[1] < 9:
     common_cflags = ['-Werror=declaration-after-statement']
 
 osutil_cflags = []
 osutil_ldflags = []
 
 # platform specific macros
 for plat, func in [('bsd', 'setproctitle')]:
     if re.search(plat, sys.platform) and hasfunction(new_compiler(), func):
         osutil_cflags.append('-DHAVE_%s' % func.upper())
 
 for plat, macro, code in [
     (
         'bsd|darwin',
         'BSD_STATFS',
         '''
      #include <sys/param.h>
      #include <sys/mount.h>
      int main() { struct statfs s; return sizeof(s.f_fstypename); }
      ''',
     ),
     (
         'linux',
         'LINUX_STATFS',
         '''
      #include <linux/magic.h>
      #include <sys/vfs.h>
      int main() { struct statfs s; return sizeof(s.f_type); }
      ''',
     ),
 ]:
     if re.search(plat, sys.platform) and cancompile(new_compiler(), code):
         osutil_cflags.append('-DHAVE_%s' % macro)
 
 if sys.platform == 'darwin':
     osutil_ldflags += ['-framework', 'ApplicationServices']
 
 if sys.platform == 'sunos5':
     osutil_ldflags += ['-lsocket']
 
 xdiff_srcs = [
     'mercurial/thirdparty/xdiff/xdiffi.c',
     'mercurial/thirdparty/xdiff/xprepare.c',
     'mercurial/thirdparty/xdiff/xutils.c',
 ]
 
 xdiff_headers = [
     'mercurial/thirdparty/xdiff/xdiff.h',
     'mercurial/thirdparty/xdiff/xdiffi.h',
     'mercurial/thirdparty/xdiff/xinclude.h',
     'mercurial/thirdparty/xdiff/xmacros.h',
     'mercurial/thirdparty/xdiff/xprepare.h',
     'mercurial/thirdparty/xdiff/xtypes.h',
     'mercurial/thirdparty/xdiff/xutils.h',
 ]
 
 
 class RustCompilationError(CCompilerError):
     """Exception class for Rust compilation errors."""
 
 
 class RustExtension(Extension):
     """Base classes for concrete Rust Extension classes."""
 
     rusttargetdir = os.path.join('rust', 'target', 'release')
 
     def __init__(self, mpath, sources, rustlibname, subcrate, **kw):
         Extension.__init__(self, mpath, sources, **kw)
         srcdir = self.rustsrcdir = os.path.join('rust', subcrate)
 
         # adding Rust source and control files to depends so that the extension
         # gets rebuilt if they've changed
         self.depends.append(os.path.join(srcdir, 'Cargo.toml'))
         cargo_lock = os.path.join(srcdir, 'Cargo.lock')
         if os.path.exists(cargo_lock):
             self.depends.append(cargo_lock)
         for dirpath, subdir, fnames in os.walk(os.path.join(srcdir, 'src')):
             self.depends.extend(
                 os.path.join(dirpath, fname)
                 for fname in fnames
                 if os.path.splitext(fname)[1] == '.rs'
             )
 
     @staticmethod
     def rustdylibsuffix():
         """Return the suffix for shared libraries produced by rustc.
 
         See also: https://doc.rust-lang.org/reference/linkage.html
         """
         if sys.platform == 'darwin':
             return '.dylib'
         elif os.name == 'nt':
             return '.dll'
         else:
             return '.so'
 
     def rustbuild(self):
         env = os.environ.copy()
         if 'HGTEST_RESTOREENV' in env:
             # Mercurial tests change HOME to a temporary directory,
             # but, if installed with rustup, the Rust toolchain needs
             # HOME to be correct (otherwise the 'no default toolchain'
             # error message is issued and the build fails).
             # This happens currently with test-hghave.t, which does
             # invoke this build.
 
             # Unix only fix (os.path.expanduser not really reliable if
             # HOME is shadowed like this)
             import pwd
 
             env['HOME'] = pwd.getpwuid(os.getuid()).pw_dir
 
         cargocmd = ['cargo', 'rustc', '--release']
 
         rust_features = env.get("HG_RUST_FEATURES")
         if rust_features:
             cargocmd.extend(('--features', rust_features))
 
         cargocmd.append('--')
         if sys.platform == 'darwin':
             cargocmd.extend(
                 ("-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup")
             )
         try:
             subprocess.check_call(cargocmd, env=env, cwd=self.rustsrcdir)
         except FileNotFoundError:
             raise RustCompilationError("Cargo not found")
         except PermissionError:
             raise RustCompilationError(
                 "Cargo found, but permission to execute it is denied"
             )
         except subprocess.CalledProcessError:
             raise RustCompilationError(
                 "Cargo failed. Working directory: %r, "
                 "command: %r, environment: %r"
                 % (self.rustsrcdir, cargocmd, env)
             )
 
 
 class RustStandaloneExtension(RustExtension):
     def __init__(self, pydottedname, rustcrate, dylibname, **kw):
         RustExtension.__init__(
             self, pydottedname, [], dylibname, rustcrate, **kw
         )
         self.dylibname = dylibname
 
     def build(self, target_dir):
         self.rustbuild()
         target = [target_dir]
         target.extend(self.name.split('.'))
         target[-1] += DYLIB_SUFFIX
         target = os.path.join(*target)
         os.makedirs(os.path.dirname(target), exist_ok=True)
         shutil.copy2(
             os.path.join(
                 self.rusttargetdir, self.dylibname + self.rustdylibsuffix()
             ),
             target,
         )
 
 
 extmodules = [
     Extension(
         'mercurial.cext.base85',
         ['mercurial/cext/base85.c'],
         include_dirs=common_include_dirs,
         extra_compile_args=common_cflags,
         depends=common_depends,
     ),
     Extension(
         'mercurial.cext.bdiff',
         ['mercurial/bdiff.c', 'mercurial/cext/bdiff.c'] + xdiff_srcs,
         include_dirs=common_include_dirs,
         extra_compile_args=common_cflags,
         depends=common_depends + ['mercurial/bdiff.h'] + xdiff_headers,
     ),
     Extension(
         'mercurial.cext.mpatch',
         ['mercurial/mpatch.c', 'mercurial/cext/mpatch.c'],
         include_dirs=common_include_dirs,
         extra_compile_args=common_cflags,
         depends=common_depends,
     ),
     Extension(
         'mercurial.cext.parsers',
         [
             'mercurial/cext/charencode.c',
             'mercurial/cext/dirs.c',
             'mercurial/cext/manifest.c',
             'mercurial/cext/parsers.c',
             'mercurial/cext/pathencode.c',
             'mercurial/cext/revlog.c',
         ],
         include_dirs=common_include_dirs,
         extra_compile_args=common_cflags,
         depends=common_depends
         + [
             'mercurial/cext/charencode.h',
             'mercurial/cext/revlog.h',
         ],
     ),
     Extension(
         'mercurial.cext.osutil',
         ['mercurial/cext/osutil.c'],
         include_dirs=common_include_dirs,
         extra_compile_args=common_cflags + osutil_cflags,
         extra_link_args=osutil_ldflags,
         depends=common_depends,
     ),
     Extension(
         'mercurial.thirdparty.zope.interface._zope_interface_coptimizations',
         [
             'mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c',
         ],
         extra_compile_args=common_cflags,
     ),
     Extension(
         'mercurial.thirdparty.sha1dc',
         [
             'mercurial/thirdparty/sha1dc/cext.c',
             'mercurial/thirdparty/sha1dc/lib/sha1.c',
             'mercurial/thirdparty/sha1dc/lib/ubc_check.c',
         ],
         extra_compile_args=common_cflags,
     ),
     Extension(
         'hgext.fsmonitor.pywatchman.bser',
         ['hgext/fsmonitor/pywatchman/bser.c'],
         extra_compile_args=common_cflags,
     ),
     RustStandaloneExtension(
         'mercurial.rustext',
         'hg-cpython',
         'librusthg',
     ),
 ]
 
 
 sys.path.insert(0, 'contrib/python-zstandard')
 import setup_zstd
 
 zstd = setup_zstd.get_c_extension(
     name='mercurial.zstd', root=os.path.abspath(os.path.dirname(__file__))
 )
 zstd.extra_compile_args += common_cflags
 extmodules.append(zstd)
 
 try:
     from distutils import cygwinccompiler
 
     # the -mno-cygwin option has been deprecated for years
     mingw32compilerclass = cygwinccompiler.Mingw32CCompiler
 
     class HackedMingw32CCompiler(cygwinccompiler.Mingw32CCompiler):
         def __init__(self, *args, **kwargs):
             mingw32compilerclass.__init__(self, *args, **kwargs)
             for i in 'compiler compiler_so linker_exe linker_so'.split():
                 try:
                     getattr(self, i).remove('-mno-cygwin')
                 except ValueError:
                     pass
 
     cygwinccompiler.Mingw32CCompiler = HackedMingw32CCompiler
 except ImportError:
     # the cygwinccompiler package is not available on some Python
     # distributions like the ones from the optware project for Synology
     # DiskStation boxes
     class HackedMingw32CCompiler:
         pass
 
 
 if os.name == 'nt':
     # Allow compiler/linker flags to be added to Visual Studio builds.  Passing
     # extra_link_args to distutils.extensions.Extension() doesn't have any
     # effect.
     from distutils import msvccompiler
 
     msvccompilerclass = msvccompiler.MSVCCompiler
 
     class HackedMSVCCompiler(msvccompiler.MSVCCompiler):
         def initialize(self):
             msvccompilerclass.initialize(self)
             # "warning LNK4197: export 'func' specified multiple times"
             self.ldflags_shared.append('/ignore:4197')
             self.ldflags_shared_debug.append('/ignore:4197')
 
     msvccompiler.MSVCCompiler = HackedMSVCCompiler
 
 packagedata = {
     'mercurial': [
         'configitems.toml',
         'locale/*/LC_MESSAGES/hg.mo',
         'dummycert.pem',
     ],
     'mercurial.defaultrc': [
         '*.rc',
     ],
     'mercurial.helptext': [
         '*.txt',
     ],
     'mercurial.helptext.internals': [
         '*.txt',
     ],
     'mercurial.thirdparty.attr': [
         '*.pyi',
         'py.typed',
     ],
 }
 
 
 def ordinarypath(p):
     return p and p[0] != '.' and p[-1] != '~'
 
 
 for root in ('templates',):
     for curdir, dirs, files in os.walk(os.path.join('mercurial', root)):
         packagename = curdir.replace(os.sep, '.')
         packagedata[packagename] = list(filter(ordinarypath, files))
 
 datafiles = []
 
 # distutils expects version to be str/unicode. Converting it to
 # unicode on Python 2 still works because it won't contain any
 # non-ascii bytes and will be implicitly converted back to bytes
 # when operated on.
 assert isinstance(version, str)
 setupversion = version
 
 extra = {}
 
 py2exepackages = [
     'hgdemandimport',
     'hgext3rd',
     'hgext',
     'email',
     # implicitly imported per module policy
     # (cffi wouldn't be used as a frozen exe)
     'mercurial.cext',
     #'mercurial.cffi',
     'mercurial.pure',
 ]
 
 py2exe_includes = []
 
 py2exeexcludes = []
 py2exedllexcludes = ['crypt32.dll']
 
 if issetuptools:
     extra['python_requires'] = supportedpy
 
 if py2exeloaded:
     extra['console'] = [
         {
             'script': 'hg',
             'copyright': 'Copyright (C) 2005-2023 Olivia Mackall and others',
             'product_version': version,
         }
     ]
     # Sub command of 'build' because 'py2exe' does not handle sub_commands.
     # Need to override hgbuild because it has a private copy of
     # build.sub_commands.
     hgbuild.sub_commands.insert(0, ('build_hgextindex', None))
     # put dlls in sub directory so that they won't pollute PATH
     extra['zipfile'] = 'lib/library.zip'
 
     # We allow some configuration to be supplemented via environment
     # variables. This is better than setup.cfg files because it allows
     # supplementing configs instead of replacing them.
     extrapackages = os.environ.get('HG_PY2EXE_EXTRA_PACKAGES')
     if extrapackages:
         py2exepackages.extend(extrapackages.split(' '))
 
     extra_includes = os.environ.get('HG_PY2EXE_EXTRA_INCLUDES')
     if extra_includes:
         py2exe_includes.extend(extra_includes.split(' '))
 
     excludes = os.environ.get('HG_PY2EXE_EXTRA_EXCLUDES')
     if excludes:
         py2exeexcludes.extend(excludes.split(' '))
 
     dllexcludes = os.environ.get('HG_PY2EXE_EXTRA_DLL_EXCLUDES')
     if dllexcludes:
         py2exedllexcludes.extend(dllexcludes.split(' '))
 
 if os.environ.get('PYOXIDIZER'):
     hgbuild.sub_commands.insert(0, ('build_hgextindex', None))
 
 if os.name == 'nt':
     # Windows binary file versions for exe/dll files must have the
     # form W.X.Y.Z, where W,X,Y,Z are numbers in the range 0..65535
     setupversion = setupversion.split(r'+', 1)[0]
 
 setup(
     name='mercurial',
     version=setupversion,
     author='Olivia Mackall and many others',
     author_email='[e-mail]',
     url='https://mercurial-scm.org/',
     download_url='https://mercurial-scm.org/release/',
     description=(
         'Fast scalable distributed SCM (revision control, version '
         'control) system'
     ),
     long_description=(
         'Mercurial is a distributed SCM tool written in Python.'
         ' It is used by a number of large projects that require'
         ' fast, reliable distributed revision control, such as '
         'Mozilla.'
     ),
     license='GNU GPLv2 or any later version',
     classifiers=[
         'Development Status :: 6 - Mature',
         'Environment :: Console',
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: GNU General Public License (GPL)',
         'Natural Language :: Danish',
         'Natural Language :: English',
         'Natural Language :: German',
         'Natural Language :: Italian',
         'Natural Language :: Japanese',
         'Natural Language :: Portuguese (Brazilian)',
         'Operating System :: Microsoft :: Windows',
         'Operating System :: OS Independent',
         'Operating System :: POSIX',
         'Programming Language :: C',
         'Programming Language :: Python',
         'Topic :: Software Development :: Version Control',
     ],
     scripts=scripts,
     packages=packages,
     ext_modules=extmodules,
     data_files=datafiles,
     package_data=packagedata,
     cmdclass=cmdclass,
     distclass=hgdist,
     options={
         'py2exe': {
             'bundle_files': 3,
             'dll_excludes': py2exedllexcludes,
             'includes': py2exe_includes,
             'excludes': py2exeexcludes,
             'packages': py2exepackages,
         },
         'bdist_mpkg': {
             'zipdist': False,
             'license': 'COPYING',
             'readme': 'contrib/packaging/macosx/Readme.html',
             'welcome': 'contrib/packaging/macosx/Welcome.html',
         },
     },
     **extra,
 )

tests/sshprotoext.py

@@ -1,103 +1,102 b''
 # sshprotoext.py - Extension to test behavior of SSH protocol
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This extension replaces the SSH server started via `hg serve --stdio`.
 # The server behaves differently depending on environment variables.
 
 
 from mercurial import (
     error,
     extensions,
     registrar,
     sshpeer,
     wireprotoserver,
     wireprotov1server,
 )
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(b'sshpeer', b'mode', default=None)
 configitem(b'sshpeer', b'handshake-mode', default=None)
 
 
 class bannerserver(wireprotoserver.sshserver):
     """Server that sends a banner to stdout."""
 
     def serve_forever(self):
         for i in range(10):
-            self._fout.write(b'banner: line %d\n' % i)
+            self._ui.fout.write(b'banner: line %d\n' % i)
 
         super(bannerserver, self).serve_forever()
 
 
 class prehelloserver(wireprotoserver.sshserver):
     """Tests behavior when connecting to <0.9.1 servers.
 
     The ``hello`` wire protocol command was introduced in Mercurial
     0.9.1. Modern clients send the ``hello`` command when connecting
     to SSH servers. This mock server tests behavior of the handshake
     when ``hello`` is not supported.
     """
 
     def serve_forever(self):
-        l = self._fin.readline()
+        ui = self._ui
+        l = ui.fin.readline()
         assert l == b'hello\n'
         # Respond to unknown commands with an empty reply.
-        wireprotoserver._sshv1respondbytes(self._fout, b'')
-        l = self._fin.readline()
+        wireprotoserver._sshv1respondbytes(ui.fout, b'')
+        l = ui.fin.readline()
         assert l == b'between\n'
-        proto = wireprotoserver.sshv1protocolhandler(
-            self._ui, self._fin, self._fout
-        )
+        proto = wireprotoserver.sshv1protocolhandler(ui, ui.fin, ui.fout)
         rsp = wireprotov1server.dispatch(self._repo, proto, b'between')
-        wireprotoserver._sshv1respondbytes(self._fout, rsp.data)
+        wireprotoserver._sshv1respondbytes(ui.fout, rsp.data)
 
         super(prehelloserver, self).serve_forever()
 
 
 def performhandshake(orig, ui, stdin, stdout, stderr):
     """Wrapped version of sshpeer._performhandshake to send extra commands."""
     mode = ui.config(b'sshpeer', b'handshake-mode')
     if mode == b'pre-no-args':
         ui.debug(b'sending no-args command\n')
         stdin.write(b'no-args\n')
         stdin.flush()
         return orig(ui, stdin, stdout, stderr)
     elif mode == b'pre-multiple-no-args':
         ui.debug(b'sending unknown1 command\n')
         stdin.write(b'unknown1\n')
         ui.debug(b'sending unknown2 command\n')
         stdin.write(b'unknown2\n')
         ui.debug(b'sending unknown3 command\n')
         stdin.write(b'unknown3\n')
         stdin.flush()
         return orig(ui, stdin, stdout, stderr)
     else:
         raise error.ProgrammingError(b'unknown HANDSHAKECOMMANDMODE: %s' % mode)
 
 
 def extsetup(ui):
     # It's easier for tests to define the server behavior via environment
     # variables than config options. This is because `hg serve --stdio`
     # has to be invoked with a certain form for security reasons and
     # `dummyssh` can't just add `--config` flags to the command line.
     servermode = ui.environ.get(b'SSHSERVERMODE')
 
     if servermode == b'banner':
         wireprotoserver.sshserver = bannerserver
     elif servermode == b'no-hello':
         wireprotoserver.sshserver = prehelloserver
     elif servermode:
         raise error.ProgrammingError(b'unknown server mode: %s' % servermode)
 
     peermode = ui.config(b'sshpeer', b'mode')
 
     if peermode == b'extra-handshake-commands':
         extensions.wrapfunction(sshpeer, '_performhandshake', performhandshake)
     elif peermode:
         raise error.ProgrammingError(b'unknown peer mode: %s' % peermode)

tests/test-bundle-type.t

@@ -1,622 +1,712 b''
 bundle w/o type option
 
   $ hg init t1
   $ hg init t2
   $ cd t1
   $ echo blablablablabla > file.txt
   $ hg ci -A -m commit_root
   adding file.txt
   $ echo kapoue > file.txt
   $ hg ci -m commit_1
   $ echo scrabageul > file.txt
   $ hg ci -m commit_2
   $ hg up 'desc("commit_root")'
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo flagabalagla > file.txt
   $ hg ci -m commit_3
   created new head
   $ echo aliofia > file.txt
   $ hg ci -m commit_4
   $ echo alklqo > file.txt
   $ hg ci -m commit_5
   $ echo peakfeo > file.txt
   $ hg ci -m commit_6 --secret
   $ hg phase --public --rev 'desc(commit_3)'
   $ hg log -GT '[{phase}] {desc|firstline}\n'
   @  [secret] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [public] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [public] commit_root
   
 
 XXX the bundle generation is defined by a discovery round here. So the secret
 changeset should be excluded.
 
   $ hg bundle ../b1.hg ../t2
   searching for changes
   7 changesets found (known-bad-output !)
   6 changesets found (missing-correct-output !)
   $ cd ..
 
   $ hg -R t2 unbundle ./b1.hg
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads) (known-bad-output !)
   added 6 changesets with 6 changes to 1 files (+1 heads) (missing-correct-output !)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   $ hg -R t2 up
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   updated to "b9f5f740a8cd: commit_6"
   1 other heads for branch "default"
   $ hg -R t2 log -GT '[{phase}] {desc|firstline}\n'
   @  [draft] commit_6 (known-bad-output !)
   | (known-bad-output !)
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
 
 Unknown compression type is rejected
 
   $ hg init t3
   $ hg -R t3 -q unbundle ./b1.hg
   $ hg -R t3 bundle -a -t unknown out.hg
   abort: unknown is not a recognized bundle specification
   (see 'hg help bundlespec' for supported values for --type)
   [10]
 
   $ hg -R t3 bundle -a -t unknown-v2 out.hg
   abort: unknown compression is not supported
   (see 'hg help bundlespec' for supported values for --type)
   [10]
 
 test bundle types
 =================
 
 since we use --all, it is okay to include the secret changeset here. It is
 unfortunate that the phase information for the secret one is lost.
 
   $ testbundle() {
   >   echo % test bundle type $1
   >   echo '==================================='
   >   hg -R t1 bundle --all --type $1 ./b-$1.hg
   >   f -q -B6 -D ./b-$1.hg; echo
   >   hg debugbundle ./b-$1.hg
   >   hg debugbundle --spec ./b-$1.hg
   >   echo
   >   hg init repo-from-type-$1
   >   hg unbundle -R repo-from-type-$1 ./b-$1.hg
   >   hg -R repo-from-type-$1 log -GT '[{phase}] {desc|firstline}\n'
   >   echo
   > }
 
   $ for t in "None" "bzip2" "gzip" "none-v2" "v2" "v1" "gzip-v1" "v3"; do
   >   testbundle $t
   > done
   % test bundle type None
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   none-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type bzip2
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   bzip2-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type gzip
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {Compression: GZ}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   gzip-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type none-v2
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   none-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type v2
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   bzip2-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type v1
   ===================================
   7 changesets found
   HG10BZ
   ac39af4a9f7d2aaa7d244720e57838be9bf63b03
   901e97fadc587978ec52f2fa76af4aefc2d191e8
   a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
   66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
   624e609639853fe22c88d42a8fd1f53a0e9b7ebe
   2ea90778052ba7558fab36e3fd5d149512ff986b
   b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   bzip2-v1
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type gzip-v1
   ===================================
   7 changesets found
   HG10GZ
   ac39af4a9f7d2aaa7d244720e57838be9bf63b03
   901e97fadc587978ec52f2fa76af4aefc2d191e8
   a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
   66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
   624e609639853fe22c88d42a8fd1f53a0e9b7ebe
   2ea90778052ba7558fab36e3fd5d149512ff986b
   b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   gzip-v1
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type v3
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 7, targetphase: 2, version: 03} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   phase-heads -- {} (mandatory: True)
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d public
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55 draft
       2ea90778052ba7558fab36e3fd5d149512ff986b draft
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5 secret
   bzip2-v2;cg.version=03
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (4 drafts, 1 secrets)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [secret] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [public] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [public] commit_root
   
   
 
 Compression level can be adjusted for bundle2 bundles
 
   $ hg init test-complevel
   $ cd test-complevel
 
   $ cat > file0 << EOF
   > this is a file
   > with some text
   > and some more text
   > and other content
   > EOF
   $ cat > file1 << EOF
   > this is another file
   > with some other content
   > and repeated, repeated, repeated, repeated content
   > EOF
   $ hg -q commit -A -m initial
 
   $ hg bundle -a -t gzip-v2 gzip-v2.hg
   1 changesets found
   $ f --size gzip-v2.hg
   gzip-v2.hg: size=468
 
   $ hg --config experimental.bundlecomplevel=1 bundle -a -t gzip-v2 gzip-v2-level1.hg
   1 changesets found
   $ f --size gzip-v2-level1.hg
   gzip-v2-level1.hg: size=475
 
   $ hg --config experimental.bundlecomplevel.gzip=1 --config experimental.bundlelevel=9 bundle -a -t gzip-v2 gzip-v2-level1.hg
   1 changesets found
   $ f --size gzip-v2-level1.hg
   gzip-v2-level1.hg: size=475
 
   $ cd ..
 
 #if zstd
 
   $ for t in "zstd" "zstd-v2"; do
   >   testbundle $t
   > done
   % test bundle type zstd
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {Compression: ZS}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   zstd-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
   % test bundle type zstd-v2
   ===================================
   7 changesets found
   HG20\x00\x00 (esc)
   Stream params: {Compression: ZS}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   cache:rev-branch-cache -- {} (mandatory: False)
   zstd-v2
   
   adding changesets
   adding manifests
   adding file changes
   added 7 changesets with 7 changes to 1 files (+1 heads)
   new changesets ac39af4a9f7d:b9f5f740a8cd (7 drafts)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   o  [draft] commit_6
   |
   o  [draft] commit_5
   |
   o  [draft] commit_4
   |
   o  [draft] commit_3
   |
   | o  [draft] commit_2
   | |
   | o  [draft] commit_1
   |/
   o  [draft] commit_root
   
   
 
 Explicit request for zstd on non-generaldelta repos
 
   $ hg --config format.usegeneraldelta=false init nogd
   $ hg -q -R nogd pull t1
   $ hg -R nogd bundle -a -t zstd nogd-zstd
   6 changesets found
 
 zstd-v1 always fails
 
   $ hg -R t1 bundle -a -t zstd-v1 zstd-v1
   abort: compression engine zstd is not supported on v1 bundles
   (see 'hg help bundlespec' for supported values for --type)
   [10]
 
 zstd supports threading
 
   $ hg init test-compthreads
   $ cd test-compthreads
   $ hg debugbuilddag +3
   $ hg --config experimental.bundlecompthreads=1 bundle -a -t zstd-v2 zstd-v2-threaded.hg
   3 changesets found
   $ cd ..
 
 #else
 
 zstd is a valid engine but isn't available
 
   $ hg -R t1 bundle -a -t zstd irrelevant.hg
   abort: compression engine zstd could not be loaded
   [255]
 
 #endif
 
 test garbage file
 
   $ echo garbage > bgarbage
   $ hg init tgarbage
   $ cd tgarbage
   $ hg pull ../bgarbage
   pulling from ../bgarbage
   abort: ../bgarbage: not a Mercurial bundle
   [255]
   $ cd ..
 
 test invalid bundle type
 
   $ cd t1
   $ hg bundle -a -t garbage ../bgarbage
   abort: garbage is not a recognized bundle specification
   (see 'hg help bundlespec' for supported values for --type)
   [10]
   $ cd ..
 
 Test controlling the changegroup version
 
   $ hg -R t1 bundle --config experimental.changegroup3=yes -a -t v2 ./v2-cg-default.hg
   7 changesets found
   $ hg debugbundle ./v2-cg-default.hg --part-type changegroup
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   $ hg debugbundle ./v2-cg-default.hg --spec
   bzip2-v2
   $ hg -R t1 bundle --config experimental.changegroup3=yes -a -t 'v2;cg.version=02' ./v2-cg-02.hg
   7 changesets found
   $ hg debugbundle ./v2-cg-02.hg --part-type changegroup
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   $ hg debugbundle ./v2-cg-02.hg --spec
   bzip2-v2
   $ hg -R t1 bundle --config experimental.changegroup3=yes -a -t 'v2;cg.version=03' ./v2-cg-03.hg
   7 changesets found
   $ hg debugbundle ./v2-cg-03.hg --part-type changegroup
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 7, version: 03} (mandatory: True)
       ac39af4a9f7d2aaa7d244720e57838be9bf63b03
       901e97fadc587978ec52f2fa76af4aefc2d191e8
       a8c3a1ed30eb71f03f476c5fa7ead831ef991a55
       66e2c4b43e0cf8f0bdff0733a0b97ce57874e35d
       624e609639853fe22c88d42a8fd1f53a0e9b7ebe
       2ea90778052ba7558fab36e3fd5d149512ff986b
       b9f5f740a8cd76700020e3903ee55ecff78bd3e5
   $ hg debugbundle ./v2-cg-03.hg --spec
   bzip2-v2;cg.version=03
+
+tests controlling bundle contents
+=================================
+
+  $ hg debugupdatecache -R t1
+
+default content
+---------------
+
+  $ hg -R t1 bundle --all --quiet --type 'v2' ./v2.hg
+  $ hg debugbundle ./v2.hg --spec
+  bzip2-v2
+  $ hg debugbundle ./v2.hg --quiet
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
+  hgtagsfnodes -- {} (mandatory: False)
+  cache:rev-branch-cache -- {} (mandatory: False)
+
+  $ hg -R t1 bundle --all --quiet --type 'v3' ./v3.hg
+  $ hg debugbundle ./v3.hg --spec
+  bzip2-v2;cg.version=03
+  $ hg debugbundle ./v3.hg --quiet
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 7, targetphase: 2, version: 03} (mandatory: True)
+  hgtagsfnodes -- {} (mandatory: False)
+  cache:rev-branch-cache -- {} (mandatory: False)
+  phase-heads -- {} (mandatory: True)
+
+adding extra parts
+------------------
+
+We should have a "phase-heads" part here that we did not had in the default content
+
+  $ hg -R t1 bundle --all --quiet --type 'v2;phases=1' ./v2-phases.hg
+  $ hg debugbundle ./v2-phases.hg --spec
+  bzip2-v2
+  $ hg debugbundle ./v2-phases.hg --quiet
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 7, targetphase: 2, version: 02} (mandatory: True)
+  hgtagsfnodes -- {} (mandatory: False)
+  cache:rev-branch-cache -- {} (mandatory: False)
+  phase-heads -- {} (mandatory: True)
+
+skipping default inclusion
+--------------------------
+
+  $ hg -R t1 bundle --all --quiet --type 'v2;tagsfnodescache=false' ./v2-no-tfc.hg
+  $ hg debugbundle ./v2-no-tfc.hg --spec
+  bzip2-v2
+  $ hg debugbundle ./v2-no-tfc.hg --quiet
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 7, version: 02} (mandatory: True)
+  cache:rev-branch-cache -- {} (mandatory: False)
+
+  $ hg -R t1 bundle --all --quiet --type 'v3;phases=0' ./v3-no-phases.hg
+  $ hg debugbundle ./v3-no-phases.hg --spec
+  bzip2-v2;cg.version=03
+  $ hg debugbundle ./v3-no-phases.hg --quiet
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 7, version: 03} (mandatory: True)
+  hgtagsfnodes -- {} (mandatory: False)
+  cache:rev-branch-cache -- {} (mandatory: False)
+
+  $ hg -R t1 bundle --all --quiet --type 'v3;phases=no;tagsfnodescache=0' ./v3-multi-no.hg
+  $ hg debugbundle ./v3-multi-no.hg --spec
+  bzip2-v2;cg.version=03
+  $ hg debugbundle ./v3-multi-no.hg --quiet
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 7, version: 03} (mandatory: True)
+  cache:rev-branch-cache -- {} (mandatory: False)
+
+skipping changegroup
+--------------------
+
+  $ hg -R t1 bundle --all --quiet --type 'v2;changegroup=no' ./v2-no-cg.hg
+  $ hg debugbundle ./v2-no-cg.hg --spec
+  bzip2-v2;changegroup=no
+  $ hg debugbundle ./v2-no-cg.hg --quiet
+  Stream params: {Compression: BZ}
+  hgtagsfnodes -- {} (mandatory: False)
+  cache:rev-branch-cache -- {} (mandatory: False)
+
+  $ hg -R t1 bundle --all --quiet --type 'v3;changegroup=0' ./v3-no-cg.hg
+  $ hg debugbundle ./v3-no-cg.hg --spec
+  bzip2-v2;changegroup=no
+  $ hg debugbundle ./v3-no-cg.hg --quiet
+  Stream params: {Compression: BZ}
+  hgtagsfnodes -- {} (mandatory: False)
+  cache:rev-branch-cache -- {} (mandatory: False)
+  phase-heads -- {} (mandatory: True)

tests/test-match.py

@@ -1,1007 +1,1009 b''
 import unittest
 
 import silenttestrunner
 
 from mercurial import (
     match as matchmod,
     util,
 )
 
 
 noop_auditor = lambda name: None
 
 
 class BaseMatcherTests(unittest.TestCase):
     def testVisitdir(self):
         m = matchmod.basematcher()
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
 
     def testVisitchildrenset(self):
         m = matchmod.basematcher()
         self.assertEqual(m.visitchildrenset(b''), b'this')
         self.assertEqual(m.visitchildrenset(b'dir'), b'this')
 
 
 class AlwaysMatcherTests(unittest.TestCase):
     def testVisitdir(self):
         m = matchmod.alwaysmatcher()
         self.assertEqual(m.visitdir(b''), b'all')
         self.assertEqual(m.visitdir(b'dir'), b'all')
 
     def testVisitchildrenset(self):
         m = matchmod.alwaysmatcher()
         self.assertEqual(m.visitchildrenset(b''), b'all')
         self.assertEqual(m.visitchildrenset(b'dir'), b'all')
 
 
 class NeverMatcherTests(unittest.TestCase):
     def testVisitdir(self):
         m = matchmod.nevermatcher()
         self.assertFalse(m.visitdir(b''))
         self.assertFalse(m.visitdir(b'dir'))
 
     def testVisitchildrenset(self):
         m = matchmod.nevermatcher()
         self.assertEqual(m.visitchildrenset(b''), set())
         self.assertEqual(m.visitchildrenset(b'dir'), set())
 
 
 class PredicateMatcherTests(unittest.TestCase):
     # predicatematcher does not currently define either of these methods, so
     # this is equivalent to BaseMatcherTests.
 
     def testVisitdir(self):
         m = matchmod.predicatematcher(lambda *a: False)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
 
     def testVisitchildrenset(self):
         m = matchmod.predicatematcher(lambda *a: False)
         self.assertEqual(m.visitchildrenset(b''), b'this')
         self.assertEqual(m.visitchildrenset(b'dir'), b'this')
 
 
 class PatternMatcherTests(unittest.TestCase):
     def testVisitdirPrefix(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'path:dir/subdir']
         )
         assert isinstance(m, matchmod.patternmatcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertEqual(m.visitdir(b'dir/subdir'), b'all')
         # OPT: This should probably be 'all' if its parent is?
         self.assertTrue(m.visitdir(b'dir/subdir/x'))
         self.assertFalse(m.visitdir(b'folder'))
 
     def testVisitchildrensetPrefix(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'path:dir/subdir']
         )
         assert isinstance(m, matchmod.patternmatcher)
         self.assertEqual(m.visitchildrenset(b''), b'this')
         self.assertEqual(m.visitchildrenset(b'dir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'all')
         # OPT: This should probably be 'all' if its parent is?
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
         self.assertEqual(m.visitchildrenset(b'folder'), set())
 
     def testVisitdirRootfilesin(self):
         m = matchmod.match(
             util.localpath(b'/repo'),
             b'',
             patterns=[b'rootfilesin:dir/subdir'],
         )
         assert isinstance(m, matchmod.patternmatcher)
-        self.assertFalse(m.visitdir(b'dir/subdir/x'))
+        # OPT: we shouldn't visit [x] as a directory,
+        # but we should still visit it as a file.
+        # Unfortunately, `visitdir` is used for both.
+        self.assertTrue(m.visitdir(b'dir/subdir/x'))
         self.assertFalse(m.visitdir(b'folder'))
-        # FIXME: These should probably be True.
-        self.assertFalse(m.visitdir(b''))
-        self.assertFalse(m.visitdir(b'dir'))
-        self.assertFalse(m.visitdir(b'dir/subdir'))
+        self.assertTrue(m.visitdir(b''))
+        self.assertTrue(m.visitdir(b'dir'))
+        self.assertTrue(m.visitdir(b'dir/subdir'))
 
     def testVisitchildrensetRootfilesin(self):
         m = matchmod.match(
             util.localpath(b'/repo'),
             b'',
             patterns=[b'rootfilesin:dir/subdir'],
         )
         assert isinstance(m, matchmod.patternmatcher)
-        self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
+        self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
         self.assertEqual(m.visitchildrenset(b'folder'), set())
-        # FIXME: These should probably be {'dir'}, {'subdir'} and 'this',
-        # respectively, or at least 'this' for all three.
-        self.assertEqual(m.visitchildrenset(b''), set())
-        self.assertEqual(m.visitchildrenset(b'dir'), set())
-        self.assertEqual(m.visitchildrenset(b'dir/subdir'), set())
+        # OPT: These should probably be {'dir'}, {'subdir'} and 'this',
+        # respectively
+        self.assertEqual(m.visitchildrenset(b''), b'this')
+        self.assertEqual(m.visitchildrenset(b'dir'), b'this')
+        self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this')
 
     def testVisitdirGlob(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'glob:dir/z*']
         )
         assert isinstance(m, matchmod.patternmatcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertFalse(m.visitdir(b'folder'))
         # OPT: these should probably be False.
         self.assertTrue(m.visitdir(b'dir/subdir'))
         self.assertTrue(m.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetGlob(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'glob:dir/z*']
         )
         assert isinstance(m, matchmod.patternmatcher)
         self.assertEqual(m.visitchildrenset(b''), b'this')
         self.assertEqual(m.visitchildrenset(b'folder'), set())
         self.assertEqual(m.visitchildrenset(b'dir'), b'this')
         # OPT: these should probably be set().
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
 
     def testVisitdirFilepath(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'filepath:dir/z']
         )
         assert isinstance(m, matchmod.patternmatcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertFalse(m.visitdir(b'folder'))
         self.assertFalse(m.visitdir(b'dir/subdir'))
         self.assertFalse(m.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetFilepath(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'filepath:dir/z']
         )
         assert isinstance(m, matchmod.patternmatcher)
         self.assertEqual(m.visitchildrenset(b''), b'this')
         self.assertEqual(m.visitchildrenset(b'folder'), set())
         self.assertEqual(m.visitchildrenset(b'dir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
 
 
 class IncludeMatcherTests(unittest.TestCase):
     def testVisitdirPrefix(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertEqual(m.visitdir(b'dir/subdir'), b'all')
         # OPT: This should probably be 'all' if its parent is?
         self.assertTrue(m.visitdir(b'dir/subdir/x'))
         self.assertFalse(m.visitdir(b'folder'))
 
     def testVisitchildrensetPrefix(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertEqual(m.visitchildrenset(b''), {b'dir'})
         self.assertEqual(m.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'all')
         # OPT: This should probably be 'all' if its parent is?
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
         self.assertEqual(m.visitchildrenset(b'folder'), set())
 
     def testVisitdirRootfilesin(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir/subdir']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertTrue(m.visitdir(b'dir/subdir'))
         self.assertFalse(m.visitdir(b'dir/subdir/x'))
         self.assertFalse(m.visitdir(b'folder'))
 
     def testVisitchildrensetRootfilesin(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir/subdir']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertEqual(m.visitchildrenset(b''), {b'dir'})
         self.assertEqual(m.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
         self.assertEqual(m.visitchildrenset(b'folder'), set())
 
     def testVisitdirGlob(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'glob:dir/z*']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertFalse(m.visitdir(b'folder'))
         # OPT: these should probably be False.
         self.assertTrue(m.visitdir(b'dir/subdir'))
         self.assertTrue(m.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetGlob(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'glob:dir/z*']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertEqual(m.visitchildrenset(b''), {b'dir'})
         self.assertEqual(m.visitchildrenset(b'folder'), set())
         self.assertEqual(m.visitchildrenset(b'dir'), b'this')
         # OPT: these should probably be set().
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
 
     def testVisitdirFilepath(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'filepath:dir/z']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertFalse(m.visitdir(b'folder'))
         self.assertFalse(m.visitdir(b'dir/subdir'))
         self.assertFalse(m.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetFilepath(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'filepath:dir/z']
         )
         assert isinstance(m, matchmod.includematcher)
         self.assertEqual(m.visitchildrenset(b''), {b'dir'})
         self.assertEqual(m.visitchildrenset(b'folder'), set())
         self.assertEqual(m.visitchildrenset(b'dir'), {b'z'})
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
 
 
 class ExactMatcherTests(unittest.TestCase):
     def testVisitdir(self):
         m = matchmod.exact(files=[b'dir/subdir/foo.txt'])
         assert isinstance(m, matchmod.exactmatcher)
         self.assertTrue(m.visitdir(b''))
         self.assertTrue(m.visitdir(b'dir'))
         self.assertTrue(m.visitdir(b'dir/subdir'))
         self.assertFalse(m.visitdir(b'dir/subdir/foo.txt'))
         self.assertFalse(m.visitdir(b'dir/foo'))
         self.assertFalse(m.visitdir(b'dir/subdir/x'))
         self.assertFalse(m.visitdir(b'folder'))
 
     def testVisitchildrenset(self):
         m = matchmod.exact(files=[b'dir/subdir/foo.txt'])
         assert isinstance(m, matchmod.exactmatcher)
         self.assertEqual(m.visitchildrenset(b''), {b'dir'})
         self.assertEqual(m.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), {b'foo.txt'})
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
         self.assertEqual(m.visitchildrenset(b'dir/subdir/foo.txt'), set())
         self.assertEqual(m.visitchildrenset(b'folder'), set())
 
     def testVisitchildrensetFilesAndDirs(self):
         m = matchmod.exact(
             files=[
                 b'rootfile.txt',
                 b'a/file1.txt',
                 b'a/b/file2.txt',
                 # no file in a/b/c
                 b'a/b/c/d/file4.txt',
             ]
         )
         assert isinstance(m, matchmod.exactmatcher)
         self.assertEqual(m.visitchildrenset(b''), {b'a', b'rootfile.txt'})
         self.assertEqual(m.visitchildrenset(b'a'), {b'b', b'file1.txt'})
         self.assertEqual(m.visitchildrenset(b'a/b'), {b'c', b'file2.txt'})
         self.assertEqual(m.visitchildrenset(b'a/b/c'), {b'd'})
         self.assertEqual(m.visitchildrenset(b'a/b/c/d'), {b'file4.txt'})
         self.assertEqual(m.visitchildrenset(b'a/b/c/d/e'), set())
         self.assertEqual(m.visitchildrenset(b'folder'), set())
 
 
 class DifferenceMatcherTests(unittest.TestCase):
     def testVisitdirM2always(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.alwaysmatcher()
         dm = matchmod.differencematcher(m1, m2)
         # dm should be equivalent to a nevermatcher.
         self.assertFalse(dm.visitdir(b''))
         self.assertFalse(dm.visitdir(b'dir'))
         self.assertFalse(dm.visitdir(b'dir/subdir'))
         self.assertFalse(dm.visitdir(b'dir/subdir/z'))
         self.assertFalse(dm.visitdir(b'dir/foo'))
         self.assertFalse(dm.visitdir(b'dir/subdir/x'))
         self.assertFalse(dm.visitdir(b'folder'))
 
     def testVisitchildrensetM2always(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.alwaysmatcher()
         dm = matchmod.differencematcher(m1, m2)
         # dm should be equivalent to a nevermatcher.
         self.assertEqual(dm.visitchildrenset(b''), set())
         self.assertEqual(dm.visitchildrenset(b'dir'), set())
         self.assertEqual(dm.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/z'), set())
         self.assertEqual(dm.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/x'), set())
         self.assertEqual(dm.visitchildrenset(b'folder'), set())
 
     def testVisitdirM2never(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.nevermatcher()
         dm = matchmod.differencematcher(m1, m2)
         # dm should be equivalent to a alwaysmatcher.
         #
         # We're testing Equal-to-True instead of just 'assertTrue' since
         # assertTrue does NOT verify that it's a bool, just that it's truthy.
         # While we may want to eventually make these return 'all', they should
         # not currently do so.
         self.assertEqual(dm.visitdir(b''), b'all')
         self.assertEqual(dm.visitdir(b'dir'), b'all')
         self.assertEqual(dm.visitdir(b'dir/subdir'), b'all')
         self.assertEqual(dm.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(dm.visitdir(b'dir/foo'), b'all')
         self.assertEqual(dm.visitdir(b'dir/subdir/x'), b'all')
         self.assertEqual(dm.visitdir(b'folder'), b'all')
 
     def testVisitchildrensetM2never(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.nevermatcher()
         dm = matchmod.differencematcher(m1, m2)
         # dm should be equivalent to a alwaysmatcher.
         self.assertEqual(dm.visitchildrenset(b''), b'all')
         self.assertEqual(dm.visitchildrenset(b'dir'), b'all')
         self.assertEqual(dm.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(dm.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/x'), b'all')
         self.assertEqual(dm.visitchildrenset(b'folder'), b'all')
 
     def testVisitdirM2SubdirPrefix(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'path:dir/subdir']
         )
         dm = matchmod.differencematcher(m1, m2)
         self.assertEqual(dm.visitdir(b''), True)
         self.assertEqual(dm.visitdir(b'dir'), True)
         self.assertFalse(dm.visitdir(b'dir/subdir'))
         # OPT: We should probably return False for these; we don't because
         # patternmatcher.visitdir() (our m2) doesn't return 'all' for subdirs of
         # an 'all' pattern, just True.
         self.assertEqual(dm.visitdir(b'dir/subdir/z'), True)
         self.assertEqual(dm.visitdir(b'dir/subdir/x'), True)
         self.assertEqual(dm.visitdir(b'dir/foo'), b'all')
         self.assertEqual(dm.visitdir(b'folder'), b'all')
 
     def testVisitchildrensetM2SubdirPrefix(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'path:dir/subdir']
         )
         dm = matchmod.differencematcher(m1, m2)
         self.assertEqual(dm.visitchildrenset(b''), b'this')
         self.assertEqual(dm.visitchildrenset(b'dir'), b'this')
         self.assertEqual(dm.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(dm.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(dm.visitchildrenset(b'folder'), b'all')
         # OPT: We should probably return set() for these; we don't because
         # patternmatcher.visitdir() (our m2) doesn't return 'all' for subdirs of
         # an 'all' pattern, just 'this'.
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/z'), b'this')
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/x'), b'this')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir']
         )
         dm = matchmod.differencematcher(m1, m2)
         self.assertEqual(dm.visitdir(b''), True)
         self.assertEqual(dm.visitdir(b'dir'), True)
         self.assertEqual(dm.visitdir(b'dir/subdir'), b'all')
         self.assertFalse(dm.visitdir(b'dir/foo'))
         self.assertFalse(dm.visitdir(b'folder'))
         # OPT: We should probably return False for these; we don't because
         # patternmatcher.visitdir() (our m2) doesn't return 'all' for subdirs of
         # an 'all' pattern, just True.
         self.assertEqual(dm.visitdir(b'dir/subdir/z'), True)
         self.assertEqual(dm.visitdir(b'dir/subdir/x'), True)
 
     def testVisitchildrensetIncludeInclude(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir']
         )
         dm = matchmod.differencematcher(m1, m2)
         self.assertEqual(dm.visitchildrenset(b''), {b'dir'})
         self.assertEqual(dm.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(dm.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(dm.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(dm.visitchildrenset(b'folder'), set())
         # OPT: We should probably return set() for these; we don't because
         # patternmatcher.visitdir() (our m2) doesn't return 'all' for subdirs of
         # an 'all' pattern, just 'this'.
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/z'), b'this')
         self.assertEqual(dm.visitchildrenset(b'dir/subdir/x'), b'this')
 
 
 class IntersectionMatcherTests(unittest.TestCase):
     def testVisitdirM2always(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.alwaysmatcher()
         im = matchmod.intersectmatchers(m1, m2)
         # im should be equivalent to a alwaysmatcher.
         self.assertEqual(im.visitdir(b''), b'all')
         self.assertEqual(im.visitdir(b'dir'), b'all')
         self.assertEqual(im.visitdir(b'dir/subdir'), b'all')
         self.assertEqual(im.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(im.visitdir(b'dir/foo'), b'all')
         self.assertEqual(im.visitdir(b'dir/subdir/x'), b'all')
         self.assertEqual(im.visitdir(b'folder'), b'all')
 
     def testVisitchildrensetM2always(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.alwaysmatcher()
         im = matchmod.intersectmatchers(m1, m2)
         # im should be equivalent to a alwaysmatcher.
         self.assertEqual(im.visitchildrenset(b''), b'all')
         self.assertEqual(im.visitchildrenset(b'dir'), b'all')
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(im.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), b'all')
         self.assertEqual(im.visitchildrenset(b'folder'), b'all')
 
     def testVisitdirM2never(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.nevermatcher()
         im = matchmod.intersectmatchers(m1, m2)
         # im should be equivalent to a nevermatcher.
         self.assertFalse(im.visitdir(b''))
         self.assertFalse(im.visitdir(b'dir'))
         self.assertFalse(im.visitdir(b'dir/subdir'))
         self.assertFalse(im.visitdir(b'dir/subdir/z'))
         self.assertFalse(im.visitdir(b'dir/foo'))
         self.assertFalse(im.visitdir(b'dir/subdir/x'))
         self.assertFalse(im.visitdir(b'folder'))
 
     def testVisitchildrensetM2never(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.nevermatcher()
         im = matchmod.intersectmatchers(m1, m2)
         # im should be equivalent to a nevermqtcher.
         self.assertEqual(im.visitchildrenset(b''), set())
         self.assertEqual(im.visitchildrenset(b'dir'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), set())
         self.assertEqual(im.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), set())
         self.assertEqual(im.visitchildrenset(b'folder'), set())
 
     def testVisitdirM2SubdirPrefix(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'path:dir/subdir']
         )
         im = matchmod.intersectmatchers(m1, m2)
         self.assertEqual(im.visitdir(b''), True)
         self.assertEqual(im.visitdir(b'dir'), True)
         self.assertEqual(im.visitdir(b'dir/subdir'), b'all')
         self.assertFalse(im.visitdir(b'dir/foo'))
         self.assertFalse(im.visitdir(b'folder'))
         # OPT: We should probably return 'all' for these; we don't because
         # patternmatcher.visitdir() (our m2) doesn't return 'all' for subdirs of
         # an 'all' pattern, just True.
         self.assertEqual(im.visitdir(b'dir/subdir/z'), True)
         self.assertEqual(im.visitdir(b'dir/subdir/x'), True)
 
     def testVisitchildrensetM2SubdirPrefix(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         im = matchmod.intersectmatchers(m1, m2)
         self.assertEqual(im.visitchildrenset(b''), {b'dir'})
         self.assertEqual(im.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(im.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(im.visitchildrenset(b'folder'), set())
         # OPT: We should probably return 'all' for these
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), b'this')
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), b'this')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir']
         )
         im = matchmod.intersectmatchers(m1, m2)
         self.assertEqual(im.visitdir(b''), True)
         self.assertEqual(im.visitdir(b'dir'), True)
         self.assertFalse(im.visitdir(b'dir/subdir'))
         self.assertFalse(im.visitdir(b'dir/foo'))
         self.assertFalse(im.visitdir(b'folder'))
         self.assertFalse(im.visitdir(b'dir/subdir/z'))
         self.assertFalse(im.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetIncludeInclude(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir']
         )
         im = matchmod.intersectmatchers(m1, m2)
         self.assertEqual(im.visitchildrenset(b''), {b'dir'})
         self.assertEqual(im.visitchildrenset(b'dir'), b'this')
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(im.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(im.visitchildrenset(b'folder'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), set())
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude2(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:folder']
         )
         im = matchmod.intersectmatchers(m1, m2)
         # FIXME: is True correct here?
         self.assertEqual(im.visitdir(b''), True)
         self.assertFalse(im.visitdir(b'dir'))
         self.assertFalse(im.visitdir(b'dir/subdir'))
         self.assertFalse(im.visitdir(b'dir/foo'))
         self.assertFalse(im.visitdir(b'folder'))
         self.assertFalse(im.visitdir(b'dir/subdir/z'))
         self.assertFalse(im.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetIncludeInclude2(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:folder']
         )
         im = matchmod.intersectmatchers(m1, m2)
         # FIXME: is set() correct here?
         self.assertEqual(im.visitchildrenset(b''), set())
         self.assertEqual(im.visitchildrenset(b'dir'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(im.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(im.visitchildrenset(b'folder'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), set())
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude3(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         im = matchmod.intersectmatchers(m1, m2)
         self.assertEqual(im.visitdir(b''), True)
         self.assertEqual(im.visitdir(b'dir'), True)
         self.assertEqual(im.visitdir(b'dir/subdir'), True)
         self.assertFalse(im.visitdir(b'dir/foo'))
         self.assertFalse(im.visitdir(b'folder'))
         self.assertFalse(im.visitdir(b'dir/subdir/z'))
         # OPT: this should probably be 'all' not True.
         self.assertEqual(im.visitdir(b'dir/subdir/x'), True)
 
     def testVisitchildrensetIncludeInclude3(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         im = matchmod.intersectmatchers(m1, m2)
         self.assertEqual(im.visitchildrenset(b''), {b'dir'})
         self.assertEqual(im.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), {b'x'})
         self.assertEqual(im.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(im.visitchildrenset(b'folder'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), set())
         # OPT: this should probably be 'all' not 'this'.
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), b'this')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude4(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/z']
         )
         im = matchmod.intersectmatchers(m1, m2)
         # OPT: these next three could probably be False as well.
         self.assertEqual(im.visitdir(b''), True)
         self.assertEqual(im.visitdir(b'dir'), True)
         self.assertEqual(im.visitdir(b'dir/subdir'), True)
         self.assertFalse(im.visitdir(b'dir/foo'))
         self.assertFalse(im.visitdir(b'folder'))
         self.assertFalse(im.visitdir(b'dir/subdir/z'))
         self.assertFalse(im.visitdir(b'dir/subdir/x'))
 
     def testVisitchildrensetIncludeInclude4(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/z']
         )
         im = matchmod.intersectmatchers(m1, m2)
         # OPT: these next two could probably be set() as well.
         self.assertEqual(im.visitchildrenset(b''), {b'dir'})
         self.assertEqual(im.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(im.visitchildrenset(b'dir/subdir'), set())
         self.assertEqual(im.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(im.visitchildrenset(b'folder'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/z'), set())
         self.assertEqual(im.visitchildrenset(b'dir/subdir/x'), set())
 
 
 class UnionMatcherTests(unittest.TestCase):
     def testVisitdirM2always(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.alwaysmatcher()
         um = matchmod.unionmatcher([m1, m2])
         # um should be equivalent to a alwaysmatcher.
         self.assertEqual(um.visitdir(b''), b'all')
         self.assertEqual(um.visitdir(b'dir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitdir(b'dir/foo'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/x'), b'all')
         self.assertEqual(um.visitdir(b'folder'), b'all')
 
     def testVisitchildrensetM2always(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.alwaysmatcher()
         um = matchmod.unionmatcher([m1, m2])
         # um should be equivalent to a alwaysmatcher.
         self.assertEqual(um.visitchildrenset(b''), b'all')
         self.assertEqual(um.visitchildrenset(b'dir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'all')
         self.assertEqual(um.visitchildrenset(b'folder'), b'all')
 
     def testVisitdirM1never(self):
         m1 = matchmod.nevermatcher()
         m2 = matchmod.alwaysmatcher()
         um = matchmod.unionmatcher([m1, m2])
         # um should be equivalent to a alwaysmatcher.
         self.assertEqual(um.visitdir(b''), b'all')
         self.assertEqual(um.visitdir(b'dir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitdir(b'dir/foo'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/x'), b'all')
         self.assertEqual(um.visitdir(b'folder'), b'all')
 
     def testVisitchildrensetM1never(self):
         m1 = matchmod.nevermatcher()
         m2 = matchmod.alwaysmatcher()
         um = matchmod.unionmatcher([m1, m2])
         # um should be equivalent to a alwaysmatcher.
         self.assertEqual(um.visitchildrenset(b''), b'all')
         self.assertEqual(um.visitchildrenset(b'dir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'all')
         self.assertEqual(um.visitchildrenset(b'folder'), b'all')
 
     def testVisitdirM2never(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.nevermatcher()
         um = matchmod.unionmatcher([m1, m2])
         # um should be equivalent to a alwaysmatcher.
         self.assertEqual(um.visitdir(b''), b'all')
         self.assertEqual(um.visitdir(b'dir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitdir(b'dir/foo'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/x'), b'all')
         self.assertEqual(um.visitdir(b'folder'), b'all')
 
     def testVisitchildrensetM2never(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.nevermatcher()
         um = matchmod.unionmatcher([m1, m2])
         # um should be equivalent to a alwaysmatcher.
         self.assertEqual(um.visitchildrenset(b''), b'all')
         self.assertEqual(um.visitchildrenset(b'dir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'all')
         self.assertEqual(um.visitchildrenset(b'folder'), b'all')
 
     def testVisitdirM2SubdirPrefix(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', patterns=[b'path:dir/subdir']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitdir(b''), b'all')
         self.assertEqual(um.visitdir(b'dir'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertEqual(um.visitdir(b'dir/foo'), b'all')
         self.assertEqual(um.visitdir(b'folder'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/x'), b'all')
 
     def testVisitchildrensetM2SubdirPrefix(self):
         m1 = matchmod.alwaysmatcher()
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitchildrenset(b''), b'all')
         self.assertEqual(um.visitchildrenset(b'dir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), b'all')
         self.assertEqual(um.visitchildrenset(b'folder'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'all')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitdir(b''), True)
         self.assertEqual(um.visitdir(b'dir'), True)
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertFalse(um.visitdir(b'dir/foo'))
         self.assertFalse(um.visitdir(b'folder'))
         # OPT: These two should probably be 'all' not True.
         self.assertEqual(um.visitdir(b'dir/subdir/z'), True)
         self.assertEqual(um.visitdir(b'dir/subdir/x'), True)
 
     def testVisitchildrensetIncludeInclude(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'rootfilesin:dir']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitchildrenset(b''), {b'dir'})
         self.assertEqual(um.visitchildrenset(b'dir'), b'this')
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(um.visitchildrenset(b'folder'), set())
         # OPT: These next two could be 'all' instead of 'this'.
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'this')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'this')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude2(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:folder']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitdir(b''), True)
         self.assertEqual(um.visitdir(b'dir'), True)
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertFalse(um.visitdir(b'dir/foo'))
         self.assertEqual(um.visitdir(b'folder'), b'all')
         # OPT: These should probably be 'all' not True.
         self.assertEqual(um.visitdir(b'dir/subdir/z'), True)
         self.assertEqual(um.visitdir(b'dir/subdir/x'), True)
 
     def testVisitchildrensetIncludeInclude2(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:folder']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitchildrenset(b''), {b'folder', b'dir'})
         self.assertEqual(um.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(um.visitchildrenset(b'folder'), b'all')
         # OPT: These next two could be 'all' instead of 'this'.
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'this')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'this')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude3(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitdir(b''), True)
         self.assertEqual(um.visitdir(b'dir'), True)
         self.assertEqual(um.visitdir(b'dir/subdir'), b'all')
         self.assertFalse(um.visitdir(b'dir/foo'))
         self.assertFalse(um.visitdir(b'folder'))
         self.assertEqual(um.visitdir(b'dir/subdir/x'), b'all')
         # OPT: this should probably be 'all' not True.
         self.assertEqual(um.visitdir(b'dir/subdir/z'), True)
 
     def testVisitchildrensetIncludeInclude3(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitchildrenset(b''), {b'dir'})
         self.assertEqual(um.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(um.visitchildrenset(b'folder'), set())
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'all')
         # OPT: this should probably be 'all' not 'this'.
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'this')
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
     def testVisitdirIncludeInclude4(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/z']
         )
         um = matchmod.unionmatcher([m1, m2])
         # OPT: these next three could probably be False as well.
         self.assertEqual(um.visitdir(b''), True)
         self.assertEqual(um.visitdir(b'dir'), True)
         self.assertEqual(um.visitdir(b'dir/subdir'), True)
         self.assertFalse(um.visitdir(b'dir/foo'))
         self.assertFalse(um.visitdir(b'folder'))
         self.assertEqual(um.visitdir(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitdir(b'dir/subdir/x'), b'all')
 
     def testVisitchildrensetIncludeInclude4(self):
         m1 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/x']
         )
         m2 = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir/z']
         )
         um = matchmod.unionmatcher([m1, m2])
         self.assertEqual(um.visitchildrenset(b''), {b'dir'})
         self.assertEqual(um.visitchildrenset(b'dir'), {b'subdir'})
         self.assertEqual(um.visitchildrenset(b'dir/subdir'), {b'x', b'z'})
         self.assertEqual(um.visitchildrenset(b'dir/foo'), set())
         self.assertEqual(um.visitchildrenset(b'folder'), set())
         self.assertEqual(um.visitchildrenset(b'dir/subdir/z'), b'all')
         self.assertEqual(um.visitchildrenset(b'dir/subdir/x'), b'all')
 
 
 class SubdirMatcherTests(unittest.TestCase):
     def testVisitdir(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         sm = matchmod.subdirmatcher(b'dir', m)
 
         self.assertEqual(sm.visitdir(b''), True)
         self.assertEqual(sm.visitdir(b'subdir'), b'all')
         # OPT: These next two should probably be 'all' not True.
         self.assertEqual(sm.visitdir(b'subdir/x'), True)
         self.assertEqual(sm.visitdir(b'subdir/z'), True)
         self.assertFalse(sm.visitdir(b'foo'))
 
     def testVisitchildrenset(self):
         m = matchmod.match(
             util.localpath(b'/repo'), b'', include=[b'path:dir/subdir']
         )
         sm = matchmod.subdirmatcher(b'dir', m)
 
         self.assertEqual(sm.visitchildrenset(b''), {b'subdir'})
         self.assertEqual(sm.visitchildrenset(b'subdir'), b'all')
         # OPT: These next two should probably be 'all' not 'this'.
         self.assertEqual(sm.visitchildrenset(b'subdir/x'), b'this')
         self.assertEqual(sm.visitchildrenset(b'subdir/z'), b'this')
         self.assertEqual(sm.visitchildrenset(b'foo'), set())
 
 
 class PrefixdirMatcherTests(unittest.TestCase):
     def testVisitdir(self):
         m = matchmod.match(
             util.localpath(b'/root/d'),
             b'e/f',
             [b'../a.txt', b'b.txt'],
             auditor=noop_auditor,
         )
         pm = matchmod.prefixdirmatcher(b'd', m)
 
         # `m` elides 'd' because it's part of the root, and the rest of the
         # patterns are relative.
         self.assertEqual(bool(m(b'a.txt')), False)
         self.assertEqual(bool(m(b'b.txt')), False)
         self.assertEqual(bool(m(b'e/a.txt')), True)
         self.assertEqual(bool(m(b'e/b.txt')), False)
         self.assertEqual(bool(m(b'e/f/b.txt')), True)
 
         # The prefix matcher re-adds 'd' to the paths, so they need to be
         # specified when using the prefixdirmatcher.
         self.assertEqual(bool(pm(b'a.txt')), False)
         self.assertEqual(bool(pm(b'b.txt')), False)
         self.assertEqual(bool(pm(b'd/e/a.txt')), True)
         self.assertEqual(bool(pm(b'd/e/b.txt')), False)
         self.assertEqual(bool(pm(b'd/e/f/b.txt')), True)
 
         self.assertEqual(m.visitdir(b''), True)
         self.assertEqual(m.visitdir(b'e'), True)
         self.assertEqual(m.visitdir(b'e/f'), True)
         self.assertEqual(m.visitdir(b'e/f/g'), False)
 
         self.assertEqual(pm.visitdir(b''), True)
         self.assertEqual(pm.visitdir(b'd'), True)
         self.assertEqual(pm.visitdir(b'd/e'), True)
         self.assertEqual(pm.visitdir(b'd/e/f'), True)
         self.assertEqual(pm.visitdir(b'd/e/f/g'), False)
 
     def testVisitchildrenset(self):
         m = matchmod.match(
             util.localpath(b'/root/d'),
             b'e/f',
             [b'../a.txt', b'b.txt'],
             auditor=noop_auditor,
         )
         pm = matchmod.prefixdirmatcher(b'd', m)
 
         # OPT: visitchildrenset could possibly return {'e'} and {'f'} for these
         # next two, respectively; patternmatcher does not have this
         # optimization.
         self.assertEqual(m.visitchildrenset(b''), b'this')
         self.assertEqual(m.visitchildrenset(b'e'), b'this')
         self.assertEqual(m.visitchildrenset(b'e/f'), b'this')
         self.assertEqual(m.visitchildrenset(b'e/f/g'), set())
 
         # OPT: visitchildrenset could possibly return {'d'}, {'e'}, and {'f'}
         # for these next three, respectively; patternmatcher does not have this
         # optimization.
         self.assertEqual(pm.visitchildrenset(b''), b'this')
         self.assertEqual(pm.visitchildrenset(b'd'), b'this')
         self.assertEqual(pm.visitchildrenset(b'd/e'), b'this')
         self.assertEqual(pm.visitchildrenset(b'd/e/f'), b'this')
         self.assertEqual(pm.visitchildrenset(b'd/e/f/g'), set())
 
 
 if __name__ == '__main__':
     silenttestrunner.main(__name__)

tests/test-sshserver.py

@@ -1,62 +1,64 b''
 import io
 import unittest
 
 import silenttestrunner
 
 from mercurial import (
     wireprotoserver,
     wireprotov1server,
 )
 
 from mercurial.utils import procutil
 
 
 class SSHServerGetArgsTests(unittest.TestCase):
     def testparseknown(self):
         tests = [
             (b'* 0\nnodes 0\n', [b'', {}]),
             (
                 b'* 0\nnodes 40\n1111111111111111111111111111111111111111\n',
                 [b'1111111111111111111111111111111111111111', {}],
             ),
         ]
         for input, expected in tests:
             self.assertparse(b'known', input, expected)
 
     def assertparse(self, cmd, input, expected):
         server = mockserver(input)
-        proto = wireprotoserver.sshv1protocolhandler(
-            server._ui, server._fin, server._fout
-        )
+        ui = server._ui
+        proto = wireprotoserver.sshv1protocolhandler(ui, ui.fin, ui.fout)
         _func, spec = wireprotov1server.commands[cmd]
         self.assertEqual(proto.getargs(spec), expected)
 
 
 def mockserver(inbytes):
     ui = mockui(inbytes)
     repo = mockrepo(ui)
+    # note: this test unfortunately doesn't really test anything about
+    # `sshserver` class anymore: the entirety of logic of that class lives
+    # in `serveuntil`, and that function is not even called by this test.
     return wireprotoserver.sshserver(ui, repo)
 
 
 class mockrepo:
     def __init__(self, ui):
         self.ui = ui
 
 
 class mockui:
     def __init__(self, inbytes):
         self.fin = io.BytesIO(inbytes)
         self.fout = io.BytesIO()
         self.ferr = io.BytesIO()
 
     def protectfinout(self):
         return self.fin, self.fout
 
     def restorefinout(self, fin, fout):
         pass
 
 
 if __name__ == '__main__':
     # Don't call into msvcrt to set BytesIO to binary mode
     procutil.setbinary = lambda fp: True
     silenttestrunner.main(__name__)

tests/test-status.t

@@ -1,1031 +1,1051 b''
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
   $ cat >> $HGRCPATH << EOF
   > [format]
   > use-dirstate-v2=1
   > [storage]
   > dirstate-v2.slow-path=allow
   > EOF
 #endif
 
   $ hg init repo1
   $ cd repo1
   $ mkdir a b a/1 b/1 b/2
   $ touch in_root a/in_a b/in_b a/1/in_a_1 b/1/in_b_1 b/2/in_b_2
 
 hg status in repo root:
 
   $ hg status
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
 
 hg status . in repo root:
 
   $ hg status .
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
 
   $ hg status --cwd a
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd a .
   ? 1/in_a_1
   ? in_a
   $ hg status --cwd a ..
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
 
   $ hg status --cwd b
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd b .
   ? 1/in_b_1
   ? 2/in_b_2
   ? in_b
   $ hg status --cwd b ..
   ? ../a/1/in_a_1
   ? ../a/in_a
   ? 1/in_b_1
   ? 2/in_b_2
   ? in_b
   ? ../in_root
 
   $ hg status --cwd a/1
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd a/1 .
   ? in_a_1
   $ hg status --cwd a/1 ..
   ? in_a_1
   ? ../in_a
 
   $ hg status --cwd b/1
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd b/1 .
   ? in_b_1
   $ hg status --cwd b/1 ..
   ? in_b_1
   ? ../2/in_b_2
   ? ../in_b
 
   $ hg status --cwd b/2
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd b/2 .
   ? in_b_2
   $ hg status --cwd b/2 ..
   ? ../1/in_b_1
   ? in_b_2
   ? ../in_b
 
 combining patterns with root and patterns without a root works
 
   $ hg st a/in_a re:.*b$
   ? a/in_a
   ? b/in_b
 
 tweaking defaults works
   $ hg status --cwd a --config ui.tweakdefaults=yes
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
   $ HGPLAIN=1 hg status --cwd a --config ui.tweakdefaults=yes
   ? a/1/in_a_1 (glob)
   ? a/in_a (glob)
   ? b/1/in_b_1 (glob)
   ? b/2/in_b_2 (glob)
   ? b/in_b (glob)
   ? in_root
   $ HGPLAINEXCEPT=tweakdefaults hg status --cwd a --config ui.tweakdefaults=yes
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root (glob)
 
 relative paths can be requested
 
   $ hg status --cwd a --config ui.relative-paths=yes
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
 
   $ hg status --cwd a . --config ui.relative-paths=legacy
   ? 1/in_a_1
   ? in_a
   $ hg status --cwd a . --config ui.relative-paths=no
   ? a/1/in_a_1
   ? a/in_a
 
 commands.status.relative overrides ui.relative-paths
 
   $ cat >> $HGRCPATH <<EOF
   > [ui]
   > relative-paths = False
   > [commands]
   > status.relative = True
   > EOF
   $ hg status --cwd a
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
   $ HGPLAIN=1 hg status --cwd a
   ? a/1/in_a_1 (glob)
   ? a/in_a (glob)
   ? b/1/in_b_1 (glob)
   ? b/2/in_b_2 (glob)
   ? b/in_b (glob)
   ? in_root
 
 if relative paths are explicitly off, tweakdefaults doesn't change it
   $ cat >> $HGRCPATH <<EOF
   > [commands]
   > status.relative = False
   > EOF
   $ hg status --cwd a --config ui.tweakdefaults=yes
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
 
   $ cd ..
 
   $ hg init repo2
   $ cd repo2
   $ touch modified removed deleted ignored
   $ echo "^ignored$" > .hgignore
   $ hg ci -A -m 'initial checkin'
   adding .hgignore
   adding deleted
   adding modified
   adding removed
   $ touch modified added unknown ignored
   $ hg add added
   $ hg remove removed
   $ rm deleted
 
 hg status:
 
   $ hg status
   A added
   R removed
   ! deleted
   ? unknown
 
 hg status -n:
   $ env RHG_ON_UNSUPPORTED=abort hg status -n
   added
   removed
   deleted
   unknown
 
 hg status modified added removed deleted unknown never-existed ignored:
 
   $ hg status modified added removed deleted unknown never-existed ignored
   never-existed: * (glob)
   A added
   R removed
   ! deleted
   ? unknown
 
   $ hg copy modified copied
 
 hg status -C:
 
   $ hg status -C
   A added
   A copied
     modified
   R removed
   ! deleted
   ? unknown
 
 hg status -0:
 
   $ hg status -0 --config rhg.on-unsupported=abort
   A added\x00A copied\x00R removed\x00! deleted\x00? unknown\x00 (no-eol) (esc)
 
 hg status -A:
 
   $ hg status -A
   A added
   A copied
     modified
   R removed
   ! deleted
   ? unknown
   I ignored
   C .hgignore
   C modified
 
   $ hg status -A -T '{status} {path} {node|shortest}\n'
   A added ffff
   A copied ffff
   R removed ffff
   ! deleted ffff
   ? unknown ffff
   I ignored ffff
   C .hgignore ffff
   C modified ffff
 
   $ hg status -A -Tjson
   [
    {
     "itemtype": "file",
     "path": "added",
     "status": "A"
    },
    {
     "itemtype": "file",
     "path": "copied",
     "source": "modified",
     "status": "A"
    },
    {
     "itemtype": "file",
     "path": "removed",
     "status": "R"
    },
    {
     "itemtype": "file",
     "path": "deleted",
     "status": "!"
    },
    {
     "itemtype": "file",
     "path": "unknown",
     "status": "?"
    },
    {
     "itemtype": "file",
     "path": "ignored",
     "status": "I"
    },
    {
     "itemtype": "file",
     "path": ".hgignore",
     "status": "C"
    },
    {
     "itemtype": "file",
     "path": "modified",
     "status": "C"
    }
   ]
 
   $ hg status -A -Tpickle > pickle
   >>> import pickle
   >>> from mercurial import util
   >>> data = sorted((x[b'status'].decode(), x[b'path'].decode()) for x in pickle.load(open("pickle", r"rb")))
   >>> for s, p in data: print("%s %s" % (s, p))
   ! deleted
   ? pickle
   ? unknown
   A added
   A copied
   C .hgignore
   C modified
   I ignored
   R removed
   $ rm pickle
 
   $ echo "^ignoreddir$" > .hgignore
   $ mkdir ignoreddir
   $ touch ignoreddir/file
 
 Test templater support:
 
   $ hg status -AT "[{status}]\t{if(source, '{source} -> ')}{path}\n"
   [M]	.hgignore
   [A]	added
   [A]	modified -> copied
   [R]	removed
   [!]	deleted
   [?]	ignored
   [?]	unknown
   [I]	ignoreddir/file
   [C]	modified
   $ hg status -AT default
   M .hgignore
   A added
   A copied
     modified
   R removed
   ! deleted
   ? ignored
   ? unknown
   I ignoreddir/file
   C modified
   $ hg status -T compact
   abort: "status" not in template map
   [255]
 
 hg status ignoreddir/file:
 
   $ hg status ignoreddir/file
 
 hg status -i ignoreddir/file:
 
   $ hg status -i ignoreddir/file
   I ignoreddir/file
   $ cd ..
 
 Check 'status -q' and some combinations
 
   $ hg init repo3
   $ cd repo3
   $ touch modified removed deleted ignored
   $ echo "^ignored$" > .hgignore
   $ hg commit -A -m 'initial checkin'
   adding .hgignore
   adding deleted
   adding modified
   adding removed
   $ touch added unknown ignored
   $ hg add added
   $ echo "test" >> modified
   $ hg remove removed
   $ rm deleted
   $ hg copy modified copied
 
 Specify working directory revision explicitly, that should be the same as
 "hg status"
 
   $ hg status --change "wdir()"
   M modified
   A added
   A copied
   R removed
   ! deleted
   ? unknown
 
 Run status with 2 different flags.
 Check if result is the same or different.
 If result is not as expected, raise error
 
   $ assert() {
   >     hg status $1 > ../a
   >     hg status $2 > ../b
   >     if diff ../a ../b > /dev/null; then
   >         out=0
   >     else
   >         out=1
   >     fi
   >     if [ $3 -eq 0 ]; then
   >         df="same"
   >     else
   >         df="different"
   >     fi
   >     if [ $out -ne $3 ]; then
   >         echo "Error on $1 and $2, should be $df."
   >     fi
   > }
 
 Assert flag1 flag2 [0-same | 1-different]
 
   $ assert "-q" "-mard"      0
   $ assert "-A" "-marduicC"  0
   $ assert "-qA" "-mardcC"   0
   $ assert "-qAui" "-A"      0
   $ assert "-qAu" "-marducC" 0
   $ assert "-qAi" "-mardicC" 0
   $ assert "-qu" "-u"        0
   $ assert "-q" "-u"         1
   $ assert "-m" "-a"         1
   $ assert "-r" "-d"         1
   $ cd ..
 
   $ hg init repo4
   $ cd repo4
   $ touch modified removed deleted
   $ hg ci -q -A -m 'initial checkin'
   $ touch added unknown
   $ hg add added
   $ hg remove removed
   $ rm deleted
   $ echo x > modified
   $ hg copy modified copied
   $ hg ci -m 'test checkin' -d "1000001 0"
   $ rm *
   $ touch unrelated
   $ hg ci -q -A -m 'unrelated checkin' -d "1000002 0"
 
 hg status --change 1:
 
   $ hg status --change 1
   M modified
   A added
   A copied
   R removed
 
 hg status --change 1 unrelated:
 
   $ hg status --change 1 unrelated
 
 hg status -C --change 1 added modified copied removed deleted:
 
   $ hg status -C --change 1 added modified copied removed deleted
   M modified
   A added
   A copied
     modified
   R removed
 
 hg status -A --change 1 and revset:
 
   $ hg status -A --change '1|1'
   M modified
   A added
   A copied
     modified
   R removed
   C deleted
 
   $ cd ..
 
 hg status with --rev and reverted changes:
 
   $ hg init reverted-changes-repo
   $ cd reverted-changes-repo
   $ echo a > file
   $ hg add file
   $ hg ci -m a
   $ echo b > file
   $ hg ci -m b
 
 reverted file should appear clean
 
   $ hg revert -r 0 .
   reverting file
   $ hg status -A --rev 0
   C file
 
 #if execbit
 reverted file with changed flag should appear modified
 
   $ chmod +x file
   $ hg status -A --rev 0
   M file
 
   $ hg revert -r 0 .
   reverting file
 
 reverted and committed file with changed flag should appear modified
 
   $ hg co -C .
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ chmod +x file
   $ hg ci -m 'change flag'
   $ hg status -A --rev 1 --rev 2
   M file
   $ hg diff -r 1 -r 2
 
 #endif
 
   $ cd ..
 
 hg status of binary file starting with '\1\n', a separator for metadata:
 
   $ hg init repo5
   $ cd repo5
   >>> open("010a", r"wb").write(b"\1\nfoo") and None
   $ hg ci -q -A -m 'initial checkin'
   $ hg status -A
   C 010a
 
   >>> open("010a", r"wb").write(b"\1\nbar") and None
   $ hg status -A
   M 010a
   $ hg ci -q -m 'modify 010a'
   $ hg status -A --rev 0:1
   M 010a
 
   $ touch empty
   $ hg ci -q -A -m 'add another file'
   $ hg status -A --rev 1:2 010a
   C 010a
 
   $ cd ..
 
 test "hg status" with "directory pattern" which matches against files
 only known on target revision.
 
   $ hg init repo6
   $ cd repo6
 
   $ echo a > a.txt
   $ hg add a.txt
   $ hg commit -m '#0'
   $ mkdir -p 1/2/3/4/5
   $ echo b > 1/2/3/4/5/b.txt
   $ hg add 1/2/3/4/5/b.txt
   $ hg commit -m '#1'
 
   $ hg update -C 0 > /dev/null
   $ hg status -A
   C a.txt
 
 the directory matching against specified pattern should be removed,
 because directory existence prevents 'dirstate.walk()' from showing
 warning message about such pattern.
 
   $ test ! -d 1
   $ hg status -A --rev 1 1/2/3/4/5/b.txt
   R 1/2/3/4/5/b.txt
   $ hg status -A --rev 1 1/2/3/4/5
   R 1/2/3/4/5/b.txt
   $ hg status -A --rev 1 1/2/3
   R 1/2/3/4/5/b.txt
   $ hg status -A --rev 1 1
   R 1/2/3/4/5/b.txt
 
   $ hg status --config ui.formatdebug=True --rev 1 1
   status = [
       {
           'itemtype': 'file',
           'path': '1/2/3/4/5/b.txt',
           'status': 'R'
       },
   ]
 
 #if windows
   $ hg --config ui.slash=false status -A --rev 1 1
   R 1\2\3\4\5\b.txt
 #endif
 
   $ cd ..
 
 Status after move overwriting a file (issue4458)
 =================================================
 
 
   $ hg init issue4458
   $ cd issue4458
   $ echo a > a
   $ echo b > b
   $ hg commit -Am base
   adding a
   adding b
 
 
 with --force
 
   $ hg mv b --force a
   $ hg st --copies
   M a
     b
   R b
   $ hg revert --all
   reverting a
   undeleting b
   $ rm *.orig
 
 without force
 
   $ hg rm a
   $ hg st --copies
   R a
   $ hg mv b a
   $ hg st --copies
   M a
     b
   R b
 
 using ui.statuscopies setting
   $ hg st --config ui.statuscopies=true
   M a
     b
   R b
   $ hg st --config ui.statuscopies=true --no-copies
   M a
   R b
   $ hg st --config ui.statuscopies=false
   M a
   R b
   $ hg st --config ui.statuscopies=false --copies
   M a
     b
   R b
   $ hg st --config ui.tweakdefaults=yes
   M a
     b
   R b
 
 using log status template (issue5155)
   $ hg log -Tstatus -r 'wdir()' -C
   changeset:   2147483647:ffffffffffff
   parent:      0:8c55c58b4c0e
   user:        test
   date:        * (glob)
   files:
   M a
     b
   R b
   
   $ hg log -GTstatus -r 'wdir()' -C
   o  changeset:   2147483647:ffffffffffff
   |  parent:      0:8c55c58b4c0e
   ~  user:        test
      date:        * (glob)
      files:
      M a
        b
      R b
   
 
 Other "bug" highlight, the revision status does not report the copy information.
 This is buggy behavior.
 
   $ hg commit -m 'blah'
   $ hg st --copies --change .
   M a
   R b
 
 using log status template, the copy information is displayed correctly.
   $ hg log -Tstatus -r. -C
   changeset:   1:6685fde43d21
   tag:         tip
   user:        test
   date:        * (glob)
   summary:     blah
   files:
   M a
     b
   R b
   
 
   $ cd ..
 
 Make sure .hg doesn't show up even as a symlink
 
   $ hg init repo0
   $ mkdir symlink-repo0
   $ cd symlink-repo0
   $ ln -s ../repo0/.hg
   $ hg status
 
 If the size hasn’t changed but mtime has, status needs to read the contents
 of the file to check whether it has changed
 
   $ echo 1 > a
   $ echo 1 > b
   $ touch -t 200102030000 a b
   $ hg commit -Aqm '#0'
   $ echo 2 > a
   $ touch -t 200102040000 a b
   $ hg status
   M a
 
 Asking specifically for the status of a deleted/removed file
 
   $ rm a
   $ rm b
   $ hg status a
   ! a
   $ hg rm a
   $ hg rm b
   $ hg status a
   R a
   $ hg commit -qm '#1'
   $ hg status a
   a: $ENOENT$
 
 Check using include flag with pattern when status does not need to traverse
 the working directory (issue6483)
 
   $ cd ..
   $ hg init issue6483
   $ cd issue6483
   $ touch a.py b.rs
   $ hg add a.py b.rs
   $ hg st -aI "*.py"
   A a.py
 
 Also check exclude pattern
 
   $ hg st -aX "*.rs"
   A a.py
 
 issue6335
 When a directory containing a tracked file gets symlinked, as of 5.8
 `hg st` only gives the correct answer about clean (or deleted) files
 if also listing unknowns.
 The tree-based dirstate and status algorithm fix this:
 
 #if symlink no-dirstate-v1 rust
 
   $ cd ..
   $ hg init issue6335
   $ cd issue6335
   $ mkdir foo
   $ touch foo/a
   $ hg ci -Ama
   adding foo/a
   $ mv foo bar
   $ ln -s bar foo
   $ hg status
   ! foo/a
   ? bar/a
   ? foo
 
   $ hg status -c  # incorrect output without the Rust implementation
   $ hg status -cu
   ? bar/a
   ? foo
   $ hg status -d  # incorrect output without the Rust implementation
   ! foo/a
   $ hg status -du
   ! foo/a
   ? bar/a
   ? foo
 
 #endif
 
 
 Create a repo with files in each possible status
 
   $ cd ..
   $ hg init repo7
   $ cd repo7
   $ mkdir subdir
   $ touch clean modified deleted removed
   $ touch subdir/clean subdir/modified subdir/deleted subdir/removed
   $ echo ignored > .hgignore
   $ hg ci -Aqm '#0'
   $ echo 1 > modified
   $ echo 1 > subdir/modified
   $ rm deleted
   $ rm subdir/deleted
   $ hg rm removed
   $ hg rm subdir/removed
   $ touch unknown ignored
   $ touch subdir/unknown subdir/ignored
 
 Check the output
 
   $ hg status
   M modified
   M subdir/modified
   R removed
   R subdir/removed
   ! deleted
   ! subdir/deleted
   ? subdir/unknown
   ? unknown
 
   $ hg status -mard
   M modified
   M subdir/modified
   R removed
   R subdir/removed
   ! deleted
   ! subdir/deleted
 
   $ hg status -A
   M modified
   M subdir/modified
   R removed
   R subdir/removed
   ! deleted
   ! subdir/deleted
   ? subdir/unknown
   ? unknown
   I ignored
   I subdir/ignored
   C .hgignore
   C clean
   C subdir/clean
 
+Test various matchers interatction with dirstate code:
+
+  $ hg status path:subdir
+  M subdir/modified
+  R subdir/removed
+  ! subdir/deleted
+  ? subdir/unknown
+
+  $ hg status 'glob:subdir/*'
+  M subdir/modified
+  R subdir/removed
+  ! subdir/deleted
+  ? subdir/unknown
+
+  $ hg status rootfilesin:subdir
+  M subdir/modified
+  R subdir/removed
+  ! subdir/deleted
+  ? subdir/unknown
+
 Note: `hg status some-name` creates a patternmatcher which is not supported
 yet by the Rust implementation of status, but includematcher is supported.
 --include is used below for that reason
 
 #if unix-permissions
 
 Not having permission to read a directory that contains tracked files makes
 status emit a warning then behave as if the directory was empty or removed
 entirely:
 
   $ chmod 0 subdir
   $ hg status --include subdir
   subdir: $EACCES$
   R subdir/removed
   ! subdir/clean
   ! subdir/deleted
   ! subdir/modified
   $ chmod 755 subdir
 
 #endif
 
 Remove a directory that contains tracked files
 
   $ rm -r subdir
   $ hg status --include subdir
   R subdir/removed
   ! subdir/clean
   ! subdir/deleted
   ! subdir/modified
 
 … and replace it by a file
 
   $ touch subdir
   $ hg status --include subdir
   R subdir/removed
   ! subdir/clean
   ! subdir/deleted
   ! subdir/modified
   ? subdir
 
 Replaced a deleted or removed file with a directory
 
   $ mkdir deleted removed
   $ touch deleted/1 removed/1
   $ hg status --include deleted --include removed
   R removed
   ! deleted
   ? deleted/1
   ? removed/1
   $ hg add removed/1
   $ hg status --include deleted --include removed
   A removed/1
   R removed
   ! deleted
   ? deleted/1
 
 Deeply nested files in an ignored directory are still listed on request
 
   $ echo ignored-dir >> .hgignore
   $ mkdir ignored-dir
   $ mkdir ignored-dir/subdir
   $ touch ignored-dir/subdir/1
   $ hg status --ignored
   I ignored
   I ignored-dir/subdir/1
 
 Check using include flag while listing ignored composes correctly (issue6514)
 
   $ cd ..
   $ hg init issue6514
   $ cd issue6514
   $ mkdir ignored-folder
   $ touch A.hs B.hs C.hs ignored-folder/other.txt ignored-folder/ctest.hs
   $ cat >.hgignore <<EOF
   > A.hs
   > B.hs
   > ignored-folder/
   > EOF
   $ hg st -i -I 're:.*\.hs$'
   I A.hs
   I B.hs
   I ignored-folder/ctest.hs
 
 #if rust dirstate-v2
 
 Check read_dir caching
 
   $ cd ..
   $ hg init repo8
   $ cd repo8
   $ mkdir subdir
   $ touch subdir/a subdir/b
   $ hg ci -Aqm '#0'
 
 The cached mtime is initially unset
 
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 unset               subdir
 
 It is still not set when there are unknown files
 
   $ touch subdir/unknown
   $ hg status
   ? subdir/unknown
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 unset               subdir
 
 Now the directory is eligible for caching, so its mtime is saved in the dirstate
 
   $ rm subdir/unknown
   $ sleep 0.1 # ensure the kernel’s internal clock for mtimes has ticked
   $ hg status
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 set                 subdir
 
 This time the command should be ever so slightly faster since it does not need `read_dir("subdir")`
 
   $ hg status
 
 Creating a new file changes the directory’s mtime, invalidating the cache
 
   $ touch subdir/unknown
   $ hg status
   ? subdir/unknown
 
   $ rm subdir/unknown
   $ hg status
 
 Removing a node from the dirstate resets the cache for its parent directory
 
   $ hg forget subdir/a
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 set                 subdir
   $ hg ci -qm '#1'
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 unset               subdir
   $ hg status
   ? subdir/a
 
 Changing the hgignore rules makes us recompute the status (and rewrite the dirstate).
 
   $ rm subdir/a
   $ mkdir another-subdir
   $ touch another-subdir/something-else
 
   $ cat > "$TESTTMP"/extra-hgignore <<EOF
   > something-else
   > EOF
 
   $ hg status --config ui.ignore.global="$TESTTMP"/extra-hgignore
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 set                 subdir
 
   $ hg status
   ? another-subdir/something-else
 
 One invocation of status is enough to populate the cache even if it's invalidated
 in the same run.
 
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 set                 subdir
 
 #endif
 
 
 Test copy source formatting.
   $ cd ..
   $ hg init copy-source-repo
   $ cd copy-source-repo
   $ mkdir -p foo/bar
   $ cd foo/bar
   $ touch file
   $ hg addremove
   adding foo/bar/file
   $ hg commit -m 'add file'
   $ hg mv file copy
 
 Copy source respects relative path setting.
   $ hg st --config ui.statuscopies=true --config commands.status.relative=true
   A copy
     file
   R file
 
 Copy source is not shown when --no-status is passed.
   $ hg st --config ui.statuscopies=true --no-status
   foo/bar/copy
   foo/bar/file