rust-status: use bare hg status fastpath from Python

This change also adds a test case for subrepos.
Repeating the benchmark information from the `hg-core` commit:

On the Netbeans repository:
C: 840ms
Rust+C: 556ms

Mozilla Central with the one pattern that causes a fallback removed:
C: 2.315s
Rust+C: 1.700s

Differential Revision: https://phab.mercurial-scm.org/D7931

mercurial/dirstate.py

@@ -1,1866 +1,1901 b''
 # dirstate.py - working directory tracking for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import errno
 import os
 import stat
 
 from .i18n import _
 from .node import nullid
 from .pycompat import delattr
 
 from hgdemandimport import tracing
 
 from . import (
     encoding,
     error,
     match as matchmod,
     pathutil,
     policy,
     pycompat,
     scmutil,
+    sparse,
     txnutil,
     util,
 )
 
 from .interfaces import (
     dirstate as intdirstate,
     util as interfaceutil,
 )
 
 parsers = policy.importmod('parsers')
 rustmod = policy.importrust('dirstate')
 
 propertycache = util.propertycache
 filecache = scmutil.filecache
 _rangemask = 0x7FFFFFFF
 
 dirstatetuple = parsers.dirstatetuple
 
 
 class repocache(filecache):
     """filecache for files in .hg/"""
 
     def join(self, obj, fname):
         return obj._opener.join(fname)
 
 
 class rootcache(filecache):
     """filecache for files in the repository root"""
 
     def join(self, obj, fname):
         return obj._join(fname)
 
 
 def _getfsnow(vfs):
     '''Get "now" timestamp on filesystem'''
     tmpfd, tmpname = vfs.mkstemp()
     try:
         return os.fstat(tmpfd)[stat.ST_MTIME]
     finally:
         os.close(tmpfd)
         vfs.unlink(tmpname)
 
 
 @interfaceutil.implementer(intdirstate.idirstate)
 class dirstate(object):
     def __init__(self, opener, ui, root, validate, sparsematchfn):
         '''Create a new dirstate object.
 
         opener is an open()-like callable that can be used to open the
         dirstate file; root is the root of the directory tracked by
         the dirstate.
         '''
         self._opener = opener
         self._validate = validate
         self._root = root
         self._sparsematchfn = sparsematchfn
         # ntpath.join(root, '') of Python 2.7.9 does not add sep if root is
         # UNC path pointing to root share (issue4557)
         self._rootdir = pathutil.normasprefix(root)
         self._dirty = False
         self._lastnormaltime = 0
         self._ui = ui
         self._filecache = {}
         self._parentwriters = 0
         self._filename = b'dirstate'
         self._pendingfilename = b'%s.pending' % self._filename
         self._plchangecallbacks = {}
         self._origpl = None
         self._updatedfiles = set()
         self._mapcls = dirstatemap
         # Access and cache cwd early, so we don't access it for the first time
         # after a working-copy update caused it to not exist (accessing it then
         # raises an exception).
         self._cwd
 
     @contextlib.contextmanager
     def parentchange(self):
         '''Context manager for handling dirstate parents.
 
         If an exception occurs in the scope of the context manager,
         the incoherent dirstate won't be written when wlock is
         released.
         '''
         self._parentwriters += 1
         yield
         # Typically we want the "undo" step of a context manager in a
         # finally block so it happens even when an exception
         # occurs. In this case, however, we only want to decrement
         # parentwriters if the code in the with statement exits
         # normally, so we don't have a try/finally here on purpose.
         self._parentwriters -= 1
 
     def pendingparentchange(self):
         '''Returns true if the dirstate is in the middle of a set of changes
         that modify the dirstate parent.
         '''
         return self._parentwriters > 0
 
     @propertycache
     def _map(self):
         """Return the dirstate contents (see documentation for dirstatemap)."""
         self._map = self._mapcls(self._ui, self._opener, self._root)
         return self._map
 
     @property
     def _sparsematcher(self):
         """The matcher for the sparse checkout.
 
         The working directory may not include every file from a manifest. The
         matcher obtained by this property will match a path if it is to be
         included in the working directory.
         """
         # TODO there is potential to cache this property. For now, the matcher
         # is resolved on every access. (But the called function does use a
         # cache to keep the lookup fast.)
         return self._sparsematchfn()
 
     @repocache(b'branch')
     def _branch(self):
         try:
             return self._opener.read(b"branch").strip() or b"default"
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             return b"default"
 
     @property
     def _pl(self):
         return self._map.parents()
 
     def hasdir(self, d):
         return self._map.hastrackeddir(d)
 
     @rootcache(b'.hgignore')
     def _ignore(self):
         files = self._ignorefiles()
         if not files:
             return matchmod.never()
 
         pats = [b'include:%s' % f for f in files]
         return matchmod.match(self._root, b'', [], pats, warn=self._ui.warn)
 
     @propertycache
     def _slash(self):
         return self._ui.configbool(b'ui', b'slash') and pycompat.ossep != b'/'
 
     @propertycache
     def _checklink(self):
         return util.checklink(self._root)
 
     @propertycache
     def _checkexec(self):
         return util.checkexec(self._root)
 
     @propertycache
     def _checkcase(self):
         return not util.fscasesensitive(self._join(b'.hg'))
 
     def _join(self, f):
         # much faster than os.path.join()
         # it's safe because f is always a relative path
         return self._rootdir + f
 
     def flagfunc(self, buildfallback):
         if self._checklink and self._checkexec:
 
             def f(x):
                 try:
                     st = os.lstat(self._join(x))
                     if util.statislink(st):
                         return b'l'
                     if util.statisexec(st):
                         return b'x'
                 except OSError:
                     pass
                 return b''
 
             return f
 
         fallback = buildfallback()
         if self._checklink:
 
             def f(x):
                 if os.path.islink(self._join(x)):
                     return b'l'
                 if b'x' in fallback(x):
                     return b'x'
                 return b''
 
             return f
         if self._checkexec:
 
             def f(x):
                 if b'l' in fallback(x):
                     return b'l'
                 if util.isexec(self._join(x)):
                     return b'x'
                 return b''
 
             return f
         else:
             return fallback
 
     @propertycache
     def _cwd(self):
         # internal config: ui.forcecwd
         forcecwd = self._ui.config(b'ui', b'forcecwd')
         if forcecwd:
             return forcecwd
         return encoding.getcwd()
 
     def getcwd(self):
         '''Return the path from which a canonical path is calculated.
 
         This path should be used to resolve file patterns or to convert
         canonical paths back to file paths for display. It shouldn't be
         used to get real file paths. Use vfs functions instead.
         '''
         cwd = self._cwd
         if cwd == self._root:
             return b''
         # self._root ends with a path separator if self._root is '/' or 'C:\'
         rootsep = self._root
         if not util.endswithsep(rootsep):
             rootsep += pycompat.ossep
         if cwd.startswith(rootsep):
             return cwd[len(rootsep) :]
         else:
             # we're outside the repo. return an absolute path.
             return cwd
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         path = util.pathto(self._root, cwd, f)
         if self._slash:
             return util.pconvert(path)
         return path
 
     def __getitem__(self, key):
         '''Return the current state of key (a filename) in the dirstate.
 
         States are:
           n  normal
           m  needs merging
           r  marked for removal
           a  marked for addition
           ?  not tracked
         '''
         return self._map.get(key, (b"?",))[0]
 
     def __contains__(self, key):
         return key in self._map
 
     def __iter__(self):
         return iter(sorted(self._map))
 
     def items(self):
         return pycompat.iteritems(self._map)
 
     iteritems = items
 
     def parents(self):
         return [self._validate(p) for p in self._pl]
 
     def p1(self):
         return self._validate(self._pl[0])
 
     def p2(self):
         return self._validate(self._pl[1])
 
     def branch(self):
         return encoding.tolocal(self._branch)
 
     def setparents(self, p1, p2=nullid):
         """Set dirstate parents to p1 and p2.
 
         When moving from two parents to one, 'm' merged entries a
         adjusted to normal and previous copy records discarded and
         returned by the call.
 
         See localrepo.setparents()
         """
         if self._parentwriters == 0:
             raise ValueError(
                 b"cannot set dirstate parent outside of "
                 b"dirstate.parentchange context manager"
             )
 
         self._dirty = True
         oldp2 = self._pl[1]
         if self._origpl is None:
             self._origpl = self._pl
         self._map.setparents(p1, p2)
         copies = {}
         if oldp2 != nullid and p2 == nullid:
             candidatefiles = self._map.nonnormalset.union(
                 self._map.otherparentset
             )
             for f in candidatefiles:
                 s = self._map.get(f)
                 if s is None:
                     continue
 
                 # Discard 'm' markers when moving away from a merge state
                 if s[0] == b'm':
                     source = self._map.copymap.get(f)
                     if source:
                         copies[f] = source
                     self.normallookup(f)
                 # Also fix up otherparent markers
                 elif s[0] == b'n' and s[2] == -2:
                     source = self._map.copymap.get(f)
                     if source:
                         copies[f] = source
                     self.add(f)
         return copies
 
     def setbranch(self, branch):
         self.__class__._branch.set(self, encoding.fromlocal(branch))
         f = self._opener(b'branch', b'w', atomictemp=True, checkambig=True)
         try:
             f.write(self._branch + b'\n')
             f.close()
 
             # make sure filecache has the correct stat info for _branch after
             # replacing the underlying file
             ce = self._filecache[b'_branch']
             if ce:
                 ce.refresh()
         except:  # re-raises
             f.discard()
             raise
 
     def invalidate(self):
         '''Causes the next access to reread the dirstate.
 
         This is different from localrepo.invalidatedirstate() because it always
         rereads the dirstate. Use localrepo.invalidatedirstate() if you want to
         check whether the dirstate has changed before rereading it.'''
 
         for a in ("_map", "_branch", "_ignore"):
             if a in self.__dict__:
                 delattr(self, a)
         self._lastnormaltime = 0
         self._dirty = False
         self._updatedfiles.clear()
         self._parentwriters = 0
         self._origpl = None
 
     def copy(self, source, dest):
         """Mark dest as a copy of source. Unmark dest if source is None."""
         if source == dest:
             return
         self._dirty = True
         if source is not None:
             self._map.copymap[dest] = source
             self._updatedfiles.add(source)
             self._updatedfiles.add(dest)
         elif self._map.copymap.pop(dest, None):
             self._updatedfiles.add(dest)
 
     def copied(self, file):
         return self._map.copymap.get(file, None)
 
     def copies(self):
         return self._map.copymap
 
     def _addpath(self, f, state, mode, size, mtime):
         oldstate = self[f]
         if state == b'a' or oldstate == b'r':
             scmutil.checkfilename(f)
             if self._map.hastrackeddir(f):
                 raise error.Abort(
                     _(b'directory %r already in dirstate') % pycompat.bytestr(f)
                 )
             # shadows
             for d in pathutil.finddirs(f):
                 if self._map.hastrackeddir(d):
                     break
                 entry = self._map.get(d)
                 if entry is not None and entry[0] != b'r':
                     raise error.Abort(
                         _(b'file %r in dirstate clashes with %r')
                         % (pycompat.bytestr(d), pycompat.bytestr(f))
                     )
         self._dirty = True
         self._updatedfiles.add(f)
         self._map.addfile(f, oldstate, state, mode, size, mtime)
 
     def normal(self, f, parentfiledata=None):
         '''Mark a file normal and clean.
 
         parentfiledata: (mode, size, mtime) of the clean file
 
         parentfiledata should be computed from memory (for mode,
         size), as or close as possible from the point where we
         determined the file was clean, to limit the risk of the
         file having been changed by an external process between the
         moment where the file was determined to be clean and now.'''
         if parentfiledata:
             (mode, size, mtime) = parentfiledata
         else:
             s = os.lstat(self._join(f))
             mode = s.st_mode
             size = s.st_size
             mtime = s[stat.ST_MTIME]
         self._addpath(f, b'n', mode, size & _rangemask, mtime & _rangemask)
         self._map.copymap.pop(f, None)
         if f in self._map.nonnormalset:
             self._map.nonnormalset.remove(f)
         if mtime > self._lastnormaltime:
             # Remember the most recent modification timeslot for status(),
             # to make sure we won't miss future size-preserving file content
             # modifications that happen within the same timeslot.
             self._lastnormaltime = mtime
 
     def normallookup(self, f):
         '''Mark a file normal, but possibly dirty.'''
         if self._pl[1] != nullid:
             # if there is a merge going on and the file was either
             # in state 'm' (-1) or coming from other parent (-2) before
             # being removed, restore that state.
             entry = self._map.get(f)
             if entry is not None:
                 if entry[0] == b'r' and entry[2] in (-1, -2):
                     source = self._map.copymap.get(f)
                     if entry[2] == -1:
                         self.merge(f)
                     elif entry[2] == -2:
                         self.otherparent(f)
                     if source:
                         self.copy(source, f)
                     return
                 if entry[0] == b'm' or entry[0] == b'n' and entry[2] == -2:
                     return
         self._addpath(f, b'n', 0, -1, -1)
         self._map.copymap.pop(f, None)
 
     def otherparent(self, f):
         '''Mark as coming from the other parent, always dirty.'''
         if self._pl[1] == nullid:
             raise error.Abort(
                 _(b"setting %r to other parent only allowed in merges") % f
             )
         if f in self and self[f] == b'n':
             # merge-like
             self._addpath(f, b'm', 0, -2, -1)
         else:
             # add-like
             self._addpath(f, b'n', 0, -2, -1)
         self._map.copymap.pop(f, None)
 
     def add(self, f):
         '''Mark a file added.'''
         self._addpath(f, b'a', 0, -1, -1)
         self._map.copymap.pop(f, None)
 
     def remove(self, f):
         '''Mark a file removed.'''
         self._dirty = True
         oldstate = self[f]
         size = 0
         if self._pl[1] != nullid:
             entry = self._map.get(f)
             if entry is not None:
                 # backup the previous state
                 if entry[0] == b'm':  # merge
                     size = -1
                 elif entry[0] == b'n' and entry[2] == -2:  # other parent
                     size = -2
                     self._map.otherparentset.add(f)
         self._updatedfiles.add(f)
         self._map.removefile(f, oldstate, size)
         if size == 0:
             self._map.copymap.pop(f, None)
 
     def merge(self, f):
         '''Mark a file merged.'''
         if self._pl[1] == nullid:
             return self.normallookup(f)
         return self.otherparent(f)
 
     def drop(self, f):
         '''Drop a file from the dirstate'''
         oldstate = self[f]
         if self._map.dropfile(f, oldstate):
             self._dirty = True
             self._updatedfiles.add(f)
             self._map.copymap.pop(f, None)
 
     def _discoverpath(self, path, normed, ignoremissing, exists, storemap):
         if exists is None:
             exists = os.path.lexists(os.path.join(self._root, path))
         if not exists:
             # Maybe a path component exists
             if not ignoremissing and b'/' in path:
                 d, f = path.rsplit(b'/', 1)
                 d = self._normalize(d, False, ignoremissing, None)
                 folded = d + b"/" + f
             else:
                 # No path components, preserve original case
                 folded = path
         else:
             # recursively normalize leading directory components
             # against dirstate
             if b'/' in normed:
                 d, f = normed.rsplit(b'/', 1)
                 d = self._normalize(d, False, ignoremissing, True)
                 r = self._root + b"/" + d
                 folded = d + b"/" + util.fspath(f, r)
             else:
                 folded = util.fspath(normed, self._root)
             storemap[normed] = folded
 
         return folded
 
     def _normalizefile(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._map.filefoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 folded = self._discoverpath(
                     path, normed, ignoremissing, exists, self._map.filefoldmap
                 )
         return folded
 
     def _normalize(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._map.filefoldmap.get(normed, None)
         if folded is None:
             folded = self._map.dirfoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 # store discovered result in dirfoldmap so that future
                 # normalizefile calls don't start matching directories
                 folded = self._discoverpath(
                     path, normed, ignoremissing, exists, self._map.dirfoldmap
                 )
         return folded
 
     def normalize(self, path, isknown=False, ignoremissing=False):
         '''
         normalize the case of a pathname when on a casefolding filesystem
 
         isknown specifies whether the filename came from walking the
         disk, to avoid extra filesystem access.
 
         If ignoremissing is True, missing path are returned
         unchanged. Otherwise, we try harder to normalize possibly
         existing path components.
 
         The normalized case is determined based on the following precedence:
 
         - version of name already stored in the dirstate
         - version of name stored on disk
         - version provided via command arguments
         '''
 
         if self._checkcase:
             return self._normalize(path, isknown, ignoremissing)
         return path
 
     def clear(self):
         self._map.clear()
         self._lastnormaltime = 0
         self._updatedfiles.clear()
         self._dirty = True
 
     def rebuild(self, parent, allfiles, changedfiles=None):
         if changedfiles is None:
             # Rebuild entire dirstate
             to_lookup = allfiles
             to_drop = []
             lastnormaltime = self._lastnormaltime
             self.clear()
             self._lastnormaltime = lastnormaltime
         elif len(changedfiles) < 10:
             # Avoid turning allfiles into a set, which can be expensive if it's
             # large.
             to_lookup = []
             to_drop = []
             for f in changedfiles:
                 if f in allfiles:
                     to_lookup.append(f)
                 else:
                     to_drop.append(f)
         else:
             changedfilesset = set(changedfiles)
             to_lookup = changedfilesset & set(allfiles)
             to_drop = changedfilesset - to_lookup
 
         if self._origpl is None:
             self._origpl = self._pl
         self._map.setparents(parent, nullid)
 
         for f in to_lookup:
             self.normallookup(f)
         for f in to_drop:
             self.drop(f)
 
         self._dirty = True
 
     def identity(self):
         '''Return identity of dirstate itself to detect changing in storage
 
         If identity of previous dirstate is equal to this, writing
         changes based on the former dirstate out can keep consistency.
         '''
         return self._map.identity
 
     def write(self, tr):
         if not self._dirty:
             return
 
         filename = self._filename
         if tr:
             # 'dirstate.write()' is not only for writing in-memory
             # changes out, but also for dropping ambiguous timestamp.
             # delayed writing re-raise "ambiguous timestamp issue".
             # See also the wiki page below for detail:
             # https://www.mercurial-scm.org/wiki/DirstateTransactionPlan
 
             # emulate dropping timestamp in 'parsers.pack_dirstate'
             now = _getfsnow(self._opener)
             self._map.clearambiguoustimes(self._updatedfiles, now)
 
             # emulate that all 'dirstate.normal' results are written out
             self._lastnormaltime = 0
             self._updatedfiles.clear()
 
             # delay writing in-memory changes out
             tr.addfilegenerator(
                 b'dirstate',
                 (self._filename,),
                 self._writedirstate,
                 location=b'plain',
             )
             return
 
         st = self._opener(filename, b"w", atomictemp=True, checkambig=True)
         self._writedirstate(st)
 
     def addparentchangecallback(self, category, callback):
         """add a callback to be called when the wd parents are changed
 
         Callback will be called with the following arguments:
             dirstate, (oldp1, oldp2), (newp1, newp2)
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._plchangecallbacks[category] = callback
 
     def _writedirstate(self, st):
         # notify callbacks about parents change
         if self._origpl is not None and self._origpl != self._pl:
             for c, callback in sorted(
                 pycompat.iteritems(self._plchangecallbacks)
             ):
                 callback(self, self._origpl, self._pl)
             self._origpl = None
         # use the modification time of the newly created temporary file as the
         # filesystem's notion of 'now'
         now = util.fstat(st)[stat.ST_MTIME] & _rangemask
 
         # enough 'delaywrite' prevents 'pack_dirstate' from dropping
         # timestamp of each entries in dirstate, because of 'now > mtime'
         delaywrite = self._ui.configint(b'debug', b'dirstate.delaywrite')
         if delaywrite > 0:
             # do we have any files to delay for?
             for f, e in pycompat.iteritems(self._map):
                 if e[0] == b'n' and e[3] == now:
                     import time  # to avoid useless import
 
                     # rather than sleep n seconds, sleep until the next
                     # multiple of n seconds
                     clock = time.time()
                     start = int(clock) - (int(clock) % delaywrite)
                     end = start + delaywrite
                     time.sleep(end - clock)
                     now = end  # trust our estimate that the end is near now
                     break
 
         self._map.write(st, now)
         self._lastnormaltime = 0
         self._dirty = False
 
     def _dirignore(self, f):
         if self._ignore(f):
             return True
         for p in pathutil.finddirs(f):
             if self._ignore(p):
                 return True
         return False
 
     def _ignorefiles(self):
         files = []
         if os.path.exists(self._join(b'.hgignore')):
             files.append(self._join(b'.hgignore'))
         for name, path in self._ui.configitems(b"ui"):
             if name == b'ignore' or name.startswith(b'ignore.'):
                 # we need to use os.path.join here rather than self._join
                 # because path is arbitrary and user-specified
                 files.append(os.path.join(self._rootdir, util.expandpath(path)))
         return files
 
     def _ignorefileandline(self, f):
         files = collections.deque(self._ignorefiles())
         visited = set()
         while files:
             i = files.popleft()
             patterns = matchmod.readpatternfile(
                 i, self._ui.warn, sourceinfo=True
             )
             for pattern, lineno, line in patterns:
                 kind, p = matchmod._patsplit(pattern, b'glob')
                 if kind == b"subinclude":
                     if p not in visited:
                         files.append(p)
                     continue
                 m = matchmod.match(
                     self._root, b'', [], [pattern], warn=self._ui.warn
                 )
                 if m(f):
                     return (i, lineno, line)
             visited.add(i)
         return (None, -1, b"")
 
     def _walkexplicit(self, match, subrepos):
         '''Get stat data about the files explicitly specified by match.
 
         Return a triple (results, dirsfound, dirsnotfound).
         - results is a mapping from filename to stat result. It also contains
           listings mapping subrepos and .hg to None.
         - dirsfound is a list of files found to be directories.
         - dirsnotfound is a list of files that the dirstate thinks are
           directories and that were not found.'''
 
         def badtype(mode):
             kind = _(b'unknown')
             if stat.S_ISCHR(mode):
                 kind = _(b'character device')
             elif stat.S_ISBLK(mode):
                 kind = _(b'block device')
             elif stat.S_ISFIFO(mode):
                 kind = _(b'fifo')
             elif stat.S_ISSOCK(mode):
                 kind = _(b'socket')
             elif stat.S_ISDIR(mode):
                 kind = _(b'directory')
             return _(b'unsupported file type (type is %s)') % kind
 
         badfn = match.bad
         dmap = self._map
         lstat = os.lstat
         getkind = stat.S_IFMT
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
         dirsfound = []
         foundadd = dirsfound.append
         dirsnotfound = []
         notfoundadd = dirsnotfound.append
 
         if not match.isexact() and self._checkcase:
             normalize = self._normalize
         else:
             normalize = None
 
         files = sorted(match.files())
         subrepos.sort()
         i, j = 0, 0
         while i < len(files) and j < len(subrepos):
             subpath = subrepos[j] + b"/"
             if files[i] < subpath:
                 i += 1
                 continue
             while i < len(files) and files[i].startswith(subpath):
                 del files[i]
             j += 1
 
         if not files or b'' in files:
             files = [b'']
             # constructing the foldmap is expensive, so don't do it for the
             # common case where files is ['']
             normalize = None
         results = dict.fromkeys(subrepos)
         results[b'.hg'] = None
 
         for ff in files:
             if normalize:
                 nf = normalize(ff, False, True)
             else:
                 nf = ff
             if nf in results:
                 continue
 
             try:
                 st = lstat(join(nf))
                 kind = getkind(st.st_mode)
                 if kind == dirkind:
                     if nf in dmap:
                         # file replaced by dir on disk but still in dirstate
                         results[nf] = None
                     foundadd((nf, ff))
                 elif kind == regkind or kind == lnkkind:
                     results[nf] = st
                 else:
                     badfn(ff, badtype(kind))
                     if nf in dmap:
                         results[nf] = None
             except OSError as inst:  # nf not found on disk - it is dirstate only
                 if nf in dmap:  # does it exactly match a missing file?
                     results[nf] = None
                 else:  # does it match a missing directory?
                     if self._map.hasdir(nf):
                         notfoundadd(nf)
                     else:
                         badfn(ff, encoding.strtolocal(inst.strerror))
 
         # match.files() may contain explicitly-specified paths that shouldn't
         # be taken; drop them from the list of files found. dirsfound/notfound
         # aren't filtered here because they will be tested later.
         if match.anypats():
             for f in list(results):
                 if f == b'.hg' or f in subrepos:
                     # keep sentinel to disable further out-of-repo walks
                     continue
                 if not match(f):
                     del results[f]
 
         # Case insensitive filesystems cannot rely on lstat() failing to detect
         # a case-only rename.  Prune the stat object for any file that does not
         # match the case in the filesystem, if there are multiple files that
         # normalize to the same path.
         if match.isexact() and self._checkcase:
             normed = {}
 
             for f, st in pycompat.iteritems(results):
                 if st is None:
                     continue
 
                 nc = util.normcase(f)
                 paths = normed.get(nc)
 
                 if paths is None:
                     paths = set()
                     normed[nc] = paths
 
                 paths.add(f)
 
             for norm, paths in pycompat.iteritems(normed):
                 if len(paths) > 1:
                     for path in paths:
                         folded = self._discoverpath(
                             path, norm, True, None, self._map.dirfoldmap
                         )
                         if path != folded:
                             results[path] = None
 
         return results, dirsfound, dirsnotfound
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         '''
         Walk recursively through the directory tree, finding all files
         matched by match.
 
         If full is False, maybe skip some known-clean files.
 
         Return a dict mapping filename to stat-like object (either
         mercurial.osutil.stat instance or return value of os.stat()).
 
         '''
         # full is a flag that extensions that hook into walk can use -- this
         # implementation doesn't use it at all. This satisfies the contract
         # because we only guarantee a "maybe".
 
         if ignored:
             ignore = util.never
             dirignore = util.never
         elif unknown:
             ignore = self._ignore
             dirignore = self._dirignore
         else:
             # if not unknown and not ignored, drop dir recursion and step 2
             ignore = util.always
             dirignore = util.always
 
         matchfn = match.matchfn
         matchalways = match.always()
         matchtdir = match.traversedir
         dmap = self._map
         listdir = util.listdir
         lstat = os.lstat
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
 
         exact = skipstep3 = False
         if match.isexact():  # match.exact
             exact = True
             dirignore = util.always  # skip step 2
         elif match.prefix():  # match.match, no patterns
             skipstep3 = True
 
         if not exact and self._checkcase:
             normalize = self._normalize
             normalizefile = self._normalizefile
             skipstep3 = False
         else:
             normalize = self._normalize
             normalizefile = None
 
         # step 1: find all explicit files
         results, work, dirsnotfound = self._walkexplicit(match, subrepos)
         if matchtdir:
             for d in work:
                 matchtdir(d[0])
             for d in dirsnotfound:
                 matchtdir(d)
 
         skipstep3 = skipstep3 and not (work or dirsnotfound)
         work = [d for d in work if not dirignore(d[0])]
 
         # step 2: visit subdirectories
         def traverse(work, alreadynormed):
             wadd = work.append
             while work:
                 tracing.counter('dirstate.walk work', len(work))
                 nd = work.pop()
                 visitentries = match.visitchildrenset(nd)
                 if not visitentries:
                     continue
                 if visitentries == b'this' or visitentries == b'all':
                     visitentries = None
                 skip = None
                 if nd != b'':
                     skip = b'.hg'
                 try:
                     with tracing.log('dirstate.walk.traverse listdir %s', nd):
                         entries = listdir(join(nd), stat=True, skip=skip)
                 except OSError as inst:
                     if inst.errno in (errno.EACCES, errno.ENOENT):
                         match.bad(
                             self.pathto(nd), encoding.strtolocal(inst.strerror)
                         )
                         continue
                     raise
                 for f, kind, st in entries:
                     # Some matchers may return files in the visitentries set,
                     # instead of 'this', if the matcher explicitly mentions them
                     # and is not an exactmatcher. This is acceptable; we do not
                     # make any hard assumptions about file-or-directory below
                     # based on the presence of `f` in visitentries. If
                     # visitchildrenset returned a set, we can always skip the
                     # entries *not* in the set it provided regardless of whether
                     # they're actually a file or a directory.
                     if visitentries and f not in visitentries:
                         continue
                     if normalizefile:
                         # even though f might be a directory, we're only
                         # interested in comparing it to files currently in the
                         # dmap -- therefore normalizefile is enough
                         nf = normalizefile(
                             nd and (nd + b"/" + f) or f, True, True
                         )
                     else:
                         nf = nd and (nd + b"/" + f) or f
                     if nf not in results:
                         if kind == dirkind:
                             if not ignore(nf):
                                 if matchtdir:
                                     matchtdir(nf)
                                 wadd(nf)
                             if nf in dmap and (matchalways or matchfn(nf)):
                                 results[nf] = None
                         elif kind == regkind or kind == lnkkind:
                             if nf in dmap:
                                 if matchalways or matchfn(nf):
                                     results[nf] = st
                             elif (matchalways or matchfn(nf)) and not ignore(
                                 nf
                             ):
                                 # unknown file -- normalize if necessary
                                 if not alreadynormed:
                                     nf = normalize(nf, False, True)
                                 results[nf] = st
                         elif nf in dmap and (matchalways or matchfn(nf)):
                             results[nf] = None
 
         for nd, d in work:
             # alreadynormed means that processwork doesn't have to do any
             # expensive directory normalization
             alreadynormed = not normalize or nd == d
             traverse([d], alreadynormed)
 
         for s in subrepos:
             del results[s]
         del results[b'.hg']
 
         # step 3: visit remaining files from dmap
         if not skipstep3 and not exact:
             # If a dmap file is not in results yet, it was either
             # a) not matching matchfn b) ignored, c) missing, or d) under a
             # symlink directory.
             if not results and matchalways:
                 visit = [f for f in dmap]
             else:
                 visit = [f for f in dmap if f not in results and matchfn(f)]
             visit.sort()
 
             if unknown:
                 # unknown == True means we walked all dirs under the roots
                 # that wasn't ignored, and everything that matched was stat'ed
                 # and is already in results.
                 # The rest must thus be ignored or under a symlink.
                 audit_path = pathutil.pathauditor(self._root, cached=True)
 
                 for nf in iter(visit):
                     # If a stat for the same file was already added with a
                     # different case, don't add one for this, since that would
                     # make it appear as if the file exists under both names
                     # on disk.
                     if (
                         normalizefile
                         and normalizefile(nf, True, True) in results
                     ):
                         results[nf] = None
                     # Report ignored items in the dmap as long as they are not
                     # under a symlink directory.
                     elif audit_path.check(nf):
                         try:
                             results[nf] = lstat(join(nf))
                             # file was just ignored, no links, and exists
                         except OSError:
                             # file doesn't exist
                             results[nf] = None
                     else:
                         # It's either missing or under a symlink directory
                         # which we in this case report as missing
                         results[nf] = None
             else:
                 # We may not have walked the full directory tree above,
                 # so stat and check everything we missed.
                 iv = iter(visit)
                 for st in util.statfiles([join(i) for i in visit]):
                     results[next(iv)] = st
         return results
 
-    def _rust_status(self, matcher, list_clean):
+    def _rust_status(self, matcher, list_clean, list_ignored, list_unknown):
         # Force Rayon (Rust parallelism library) to respect the number of
         # workers. This is a temporary workaround until Rust code knows
         # how to read the config file.
         numcpus = self._ui.configint(b"worker", b"numcpus")
         if numcpus is not None:
             encoding.environ.setdefault(b'RAYON_NUM_THREADS', b'%d' % numcpus)
 
         workers_enabled = self._ui.configbool(b"worker", b"enabled", True)
         if not workers_enabled:
             encoding.environ[b"RAYON_NUM_THREADS"] = b"1"
 
         (
             lookup,
             modified,
             added,
             removed,
             deleted,
+            clean,
+            ignored,
             unknown,
-            clean,
+            warnings,
+            bad,
         ) = rustmod.status(
             self._map._rustmap,
             matcher,
             self._rootdir,
-            bool(list_clean),
+            self._ignorefiles(),
+            self._checkexec,
             self._lastnormaltime,
-            self._checkexec,
+            bool(list_clean),
+            bool(list_ignored),
+            bool(list_unknown),
         )
+        if self._ui.warn:
+            for item in warnings:
+                if isinstance(item, tuple):
+                    file_path, syntax = item
+                    msg = _(b"%s: ignoring invalid syntax '%s'\n") % (
+                        file_path,
+                        syntax,
+                    )
+                    self._ui.warn(msg)
+                else:
+                    msg = _(b"skipping unreadable pattern file '%s': %s\n")
+                    self._ui.warn(
+                        msg
+                        % (
+                            pathutil.canonpath(
+                                self._rootdir, self._rootdir, item
+                            ),
+                            b"No such file or directory",
+                        )
+                    )
+
+        for (fn, message) in bad:
+            matcher.bad(fn, encoding.strtolocal(message))
 
         status = scmutil.status(
             modified=modified,
             added=added,
             removed=removed,
             deleted=deleted,
             unknown=unknown,
-            ignored=[],
+            ignored=ignored,
             clean=clean,
         )
         return (lookup, status)
 
     def status(self, match, subrepos, ignored, clean, unknown):
         '''Determine the status of the working copy relative to the
         dirstate and return a pair of (unsure, status), where status is of type
         scmutil.status and:
 
           unsure:
             files that might have been modified since the dirstate was
             written, but need to be read to be sure (size is the same
             but mtime differs)
           status.modified:
             files that have definitely been modified since the dirstate
             was written (different size or mode)
           status.clean:
             files that have definitely not been modified since the
             dirstate was written
         '''
         listignored, listclean, listunknown = ignored, clean, unknown
         lookup, modified, added, unknown, ignored = [], [], [], [], []
         removed, deleted, clean = [], [], []
 
         dmap = self._map
         dmap.preload()
 
         use_rust = True
 
-        allowed_matchers = (matchmod.alwaysmatcher, matchmod.exactmatcher)
+        allowed_matchers = (
+            matchmod.alwaysmatcher,
+            matchmod.exactmatcher,
+            matchmod.includematcher,
+        )
 
         if rustmod is None:
             use_rust = False
+        elif self._checkcase:
+            # Case-insensitive filesystems are not handled yet
+            use_rust = False
         elif subrepos:
             use_rust = False
-        elif bool(listunknown):
-            # Pathauditor does not exist yet in Rust, unknown files
-            # can't be trusted.
+        elif sparse.enabled:
             use_rust = False
-        elif self._ignorefiles() and listignored:
-            # Rust has no ignore mechanism yet, so don't use Rust for
-            # commands that need ignore.
+        elif match.traversedir is not None:
             use_rust = False
         elif not isinstance(match, allowed_matchers):
             # Matchers have yet to be implemented
             use_rust = False
 
         if use_rust:
-            return self._rust_status(match, listclean)
+            try:
+                return self._rust_status(
+                    match, listclean, listignored, listunknown
+                )
+            except rustmod.FallbackError:
+                pass
 
         def noop(f):
             pass
 
         dcontains = dmap.__contains__
         dget = dmap.__getitem__
         ladd = lookup.append  # aka "unsure"
         madd = modified.append
         aadd = added.append
         uadd = unknown.append if listunknown else noop
         iadd = ignored.append if listignored else noop
         radd = removed.append
         dadd = deleted.append
         cadd = clean.append if listclean else noop
         mexact = match.exact
         dirignore = self._dirignore
         checkexec = self._checkexec
         copymap = self._map.copymap
         lastnormaltime = self._lastnormaltime
 
         # We need to do full walks when either
         # - we're listing all clean files, or
         # - match.traversedir does something, because match.traversedir should
         #   be called for every dir in the working dir
         full = listclean or match.traversedir is not None
         for fn, st in pycompat.iteritems(
             self.walk(match, subrepos, listunknown, listignored, full=full)
         ):
             if not dcontains(fn):
                 if (listignored or mexact(fn)) and dirignore(fn):
                     if listignored:
                         iadd(fn)
                 else:
                     uadd(fn)
                 continue
 
             # This is equivalent to 'state, mode, size, time = dmap[fn]' but not
             # written like that for performance reasons. dmap[fn] is not a
             # Python tuple in compiled builds. The CPython UNPACK_SEQUENCE
             # opcode has fast paths when the value to be unpacked is a tuple or
             # a list, but falls back to creating a full-fledged iterator in
             # general. That is much slower than simply accessing and storing the
             # tuple members one by one.
             t = dget(fn)
             state = t[0]
             mode = t[1]
             size = t[2]
             time = t[3]
 
             if not st and state in b"nma":
                 dadd(fn)
             elif state == b'n':
                 if (
                     size >= 0
                     and (
                         (size != st.st_size and size != st.st_size & _rangemask)
                         or ((mode ^ st.st_mode) & 0o100 and checkexec)
                     )
                     or size == -2  # other parent
                     or fn in copymap
                 ):
                     madd(fn)
                 elif (
                     time != st[stat.ST_MTIME]
                     and time != st[stat.ST_MTIME] & _rangemask
                 ):
                     ladd(fn)
                 elif st[stat.ST_MTIME] == lastnormaltime:
                     # fn may have just been marked as normal and it may have
                     # changed in the same second without changing its size.
                     # This can happen if we quickly do multiple commits.
                     # Force lookup, so we don't miss such a racy file change.
                     ladd(fn)
                 elif listclean:
                     cadd(fn)
             elif state == b'm':
                 madd(fn)
             elif state == b'a':
                 aadd(fn)
             elif state == b'r':
                 radd(fn)
-
-        return (
-            lookup,
-            scmutil.status(
-                modified, added, removed, deleted, unknown, ignored, clean
-            ),
+        status = scmutil.status(
+            modified, added, removed, deleted, unknown, ignored, clean
         )
+        return (lookup, status)
 
     def matches(self, match):
         '''
         return files in the dirstate (in whatever state) filtered by match
         '''
         dmap = self._map
         if rustmod is not None:
             dmap = self._map._rustmap
 
         if match.always():
             return dmap.keys()
         files = match.files()
         if match.isexact():
             # fast path -- filter the other way around, since typically files is
             # much smaller than dmap
             return [f for f in files if f in dmap]
         if match.prefix() and all(fn in dmap for fn in files):
             # fast path -- all the values are known to be files, so just return
             # that
             return list(files)
         return [f for f in dmap if match(f)]
 
     def _actualfilename(self, tr):
         if tr:
             return self._pendingfilename
         else:
             return self._filename
 
     def savebackup(self, tr, backupname):
         '''Save current dirstate into backup file'''
         filename = self._actualfilename(tr)
         assert backupname != filename
 
         # use '_writedirstate' instead of 'write' to write changes certainly,
         # because the latter omits writing out if transaction is running.
         # output file will be used to create backup of dirstate at this point.
         if self._dirty or not self._opener.exists(filename):
             self._writedirstate(
                 self._opener(filename, b"w", atomictemp=True, checkambig=True)
             )
 
         if tr:
             # ensure that subsequent tr.writepending returns True for
             # changes written out above, even if dirstate is never
             # changed after this
             tr.addfilegenerator(
                 b'dirstate',
                 (self._filename,),
                 self._writedirstate,
                 location=b'plain',
             )
 
             # ensure that pending file written above is unlinked at
             # failure, even if tr.writepending isn't invoked until the
             # end of this transaction
             tr.registertmp(filename, location=b'plain')
 
         self._opener.tryunlink(backupname)
         # hardlink backup is okay because _writedirstate is always called
         # with an "atomictemp=True" file.
         util.copyfile(
             self._opener.join(filename),
             self._opener.join(backupname),
             hardlink=True,
         )
 
     def restorebackup(self, tr, backupname):
         '''Restore dirstate by backup file'''
         # this "invalidate()" prevents "wlock.release()" from writing
         # changes of dirstate out after restoring from backup file
         self.invalidate()
         filename = self._actualfilename(tr)
         o = self._opener
         if util.samefile(o.join(backupname), o.join(filename)):
             o.unlink(backupname)
         else:
             o.rename(backupname, filename, checkambig=True)
 
     def clearbackup(self, tr, backupname):
         '''Clear backup file'''
         self._opener.unlink(backupname)
 
 
 class dirstatemap(object):
     """Map encapsulating the dirstate's contents.
 
     The dirstate contains the following state:
 
     - `identity` is the identity of the dirstate file, which can be used to
       detect when changes have occurred to the dirstate file.
 
     - `parents` is a pair containing the parents of the working copy. The
       parents are updated by calling `setparents`.
 
     - the state map maps filenames to tuples of (state, mode, size, mtime),
       where state is a single character representing 'normal', 'added',
       'removed', or 'merged'. It is read by treating the dirstate as a
       dict.  File state is updated by calling the `addfile`, `removefile` and
       `dropfile` methods.
 
     - `copymap` maps destination filenames to their source filename.
 
     The dirstate also provides the following views onto the state:
 
     - `nonnormalset` is a set of the filenames that have state other
       than 'normal', or are normal but have an mtime of -1 ('normallookup').
 
     - `otherparentset` is a set of the filenames that are marked as coming
       from the second parent when the dirstate is currently being merged.
 
     - `filefoldmap` is a dict mapping normalized filenames to the denormalized
       form that they appear as in the dirstate.
 
     - `dirfoldmap` is a dict mapping normalized directory names to the
       denormalized form that they appear as in the dirstate.
     """
 
     def __init__(self, ui, opener, root):
         self._ui = ui
         self._opener = opener
         self._root = root
         self._filename = b'dirstate'
 
         self._parents = None
         self._dirtyparents = False
 
         # for consistent view between _pl() and _read() invocations
         self._pendingmode = None
 
     @propertycache
     def _map(self):
         self._map = {}
         self.read()
         return self._map
 
     @propertycache
     def copymap(self):
         self.copymap = {}
         self._map
         return self.copymap
 
     def clear(self):
         self._map.clear()
         self.copymap.clear()
         self.setparents(nullid, nullid)
         util.clearcachedproperty(self, b"_dirs")
         util.clearcachedproperty(self, b"_alldirs")
         util.clearcachedproperty(self, b"filefoldmap")
         util.clearcachedproperty(self, b"dirfoldmap")
         util.clearcachedproperty(self, b"nonnormalset")
         util.clearcachedproperty(self, b"otherparentset")
 
     def items(self):
         return pycompat.iteritems(self._map)
 
     # forward for python2,3 compat
     iteritems = items
 
     def __len__(self):
         return len(self._map)
 
     def __iter__(self):
         return iter(self._map)
 
     def get(self, key, default=None):
         return self._map.get(key, default)
 
     def __contains__(self, key):
         return key in self._map
 
     def __getitem__(self, key):
         return self._map[key]
 
     def keys(self):
         return self._map.keys()
 
     def preload(self):
         """Loads the underlying data, if it's not already loaded"""
         self._map
 
     def addfile(self, f, oldstate, state, mode, size, mtime):
         """Add a tracked file to the dirstate."""
         if oldstate in b"?r" and "_dirs" in self.__dict__:
             self._dirs.addpath(f)
         if oldstate == b"?" and "_alldirs" in self.__dict__:
             self._alldirs.addpath(f)
         self._map[f] = dirstatetuple(state, mode, size, mtime)
         if state != b'n' or mtime == -1:
             self.nonnormalset.add(f)
         if size == -2:
             self.otherparentset.add(f)
 
     def removefile(self, f, oldstate, size):
         """
         Mark a file as removed in the dirstate.
 
         The `size` parameter is used to store sentinel values that indicate
         the file's previous state.  In the future, we should refactor this
         to be more explicit about what that state is.
         """
         if oldstate not in b"?r" and "_dirs" in self.__dict__:
             self._dirs.delpath(f)
         if oldstate == b"?" and "_alldirs" in self.__dict__:
             self._alldirs.addpath(f)
         if "filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             self.filefoldmap.pop(normed, None)
         self._map[f] = dirstatetuple(b'r', 0, size, 0)
         self.nonnormalset.add(f)
 
     def dropfile(self, f, oldstate):
         """
         Remove a file from the dirstate.  Returns True if the file was
         previously recorded.
         """
         exists = self._map.pop(f, None) is not None
         if exists:
             if oldstate != b"r" and "_dirs" in self.__dict__:
                 self._dirs.delpath(f)
             if "_alldirs" in self.__dict__:
                 self._alldirs.delpath(f)
         if "filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             self.filefoldmap.pop(normed, None)
         self.nonnormalset.discard(f)
         return exists
 
     def clearambiguoustimes(self, files, now):
         for f in files:
             e = self.get(f)
             if e is not None and e[0] == b'n' and e[3] == now:
                 self._map[f] = dirstatetuple(e[0], e[1], e[2], -1)
                 self.nonnormalset.add(f)
 
     def nonnormalentries(self):
         '''Compute the nonnormal dirstate entries from the dmap'''
         try:
             return parsers.nonnormalotherparententries(self._map)
         except AttributeError:
             nonnorm = set()
             otherparent = set()
             for fname, e in pycompat.iteritems(self._map):
                 if e[0] != b'n' or e[3] == -1:
                     nonnorm.add(fname)
                 if e[0] == b'n' and e[2] == -2:
                     otherparent.add(fname)
             return nonnorm, otherparent
 
     @propertycache
     def filefoldmap(self):
         """Returns a dictionary mapping normalized case paths to their
         non-normalized versions.
         """
         try:
             makefilefoldmap = parsers.make_file_foldmap
         except AttributeError:
             pass
         else:
             return makefilefoldmap(
                 self._map, util.normcasespec, util.normcasefallback
             )
 
         f = {}
         normcase = util.normcase
         for name, s in pycompat.iteritems(self._map):
             if s[0] != b'r':
                 f[normcase(name)] = name
         f[b'.'] = b'.'  # prevents useless util.fspath() invocation
         return f
 
     def hastrackeddir(self, d):
         """
         Returns True if the dirstate contains a tracked (not removed) file
         in this directory.
         """
         return d in self._dirs
 
     def hasdir(self, d):
         """
         Returns True if the dirstate contains a file (tracked or removed)
         in this directory.
         """
         return d in self._alldirs
 
     @propertycache
     def _dirs(self):
         return pathutil.dirs(self._map, b'r')
 
     @propertycache
     def _alldirs(self):
         return pathutil.dirs(self._map)
 
     def _opendirstatefile(self):
         fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
         if self._pendingmode is not None and self._pendingmode != mode:
             fp.close()
             raise error.Abort(
                 _(b'working directory state may be changed parallelly')
             )
         self._pendingmode = mode
         return fp
 
     def parents(self):
         if not self._parents:
             try:
                 fp = self._opendirstatefile()
                 st = fp.read(40)
                 fp.close()
             except IOError as err:
                 if err.errno != errno.ENOENT:
                     raise
                 # File doesn't exist, so the current state is empty
                 st = b''
 
             l = len(st)
             if l == 40:
                 self._parents = (st[:20], st[20:40])
             elif l == 0:
                 self._parents = (nullid, nullid)
             else:
                 raise error.Abort(
                     _(b'working directory state appears damaged!')
                 )
 
         return self._parents
 
     def setparents(self, p1, p2):
         self._parents = (p1, p2)
         self._dirtyparents = True
 
     def read(self):
         # ignore HG_PENDING because identity is used only for writing
         self.identity = util.filestat.frompath(
             self._opener.join(self._filename)
         )
 
         try:
             fp = self._opendirstatefile()
             try:
                 st = fp.read()
             finally:
                 fp.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         if not st:
             return
 
         if util.safehasattr(parsers, b'dict_new_presized'):
             # Make an estimate of the number of files in the dirstate based on
             # its size. From a linear regression on a set of real-world repos,
             # all over 10,000 files, the size of a dirstate entry is 85
             # bytes. The cost of resizing is significantly higher than the cost
             # of filling in a larger presized dict, so subtract 20% from the
             # size.
             #
             # This heuristic is imperfect in many ways, so in a future dirstate
             # format update it makes sense to just record the number of entries
             # on write.
             self._map = parsers.dict_new_presized(len(st) // 71)
 
         # Python's garbage collector triggers a GC each time a certain number
         # of container objects (the number being defined by
         # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
         # for each file in the dirstate. The C version then immediately marks
         # them as not to be tracked by the collector. However, this has no
         # effect on when GCs are triggered, only on what objects the GC looks
         # into. This means that O(number of files) GCs are unavoidable.
         # Depending on when in the process's lifetime the dirstate is parsed,
         # this can get very expensive. As a workaround, disable GC while
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
         parse_dirstate = util.nogc(parsers.parse_dirstate)
         p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
 
         # Avoid excess attribute lookups by fast pathing certain checks
         self.__contains__ = self._map.__contains__
         self.__getitem__ = self._map.__getitem__
         self.get = self._map.get
 
     def write(self, st, now):
         st.write(
             parsers.pack_dirstate(self._map, self.copymap, self.parents(), now)
         )
         st.close()
         self._dirtyparents = False
         self.nonnormalset, self.otherparentset = self.nonnormalentries()
 
     @propertycache
     def nonnormalset(self):
         nonnorm, otherparents = self.nonnormalentries()
         self.otherparentset = otherparents
         return nonnorm
 
     @propertycache
     def otherparentset(self):
         nonnorm, otherparents = self.nonnormalentries()
         self.nonnormalset = nonnorm
         return otherparents
 
     @propertycache
     def identity(self):
         self._map
         return self.identity
 
     @propertycache
     def dirfoldmap(self):
         f = {}
         normcase = util.normcase
         for name in self._dirs:
             f[normcase(name)] = name
         return f
 
 
 if rustmod is not None:
 
     class dirstatemap(object):
         def __init__(self, ui, opener, root):
             self._ui = ui
             self._opener = opener
             self._root = root
             self._filename = b'dirstate'
             self._parents = None
             self._dirtyparents = False
 
             # for consistent view between _pl() and _read() invocations
             self._pendingmode = None
 
         def addfile(self, *args, **kwargs):
             return self._rustmap.addfile(*args, **kwargs)
 
         def removefile(self, *args, **kwargs):
             return self._rustmap.removefile(*args, **kwargs)
 
         def dropfile(self, *args, **kwargs):
             return self._rustmap.dropfile(*args, **kwargs)
 
         def clearambiguoustimes(self, *args, **kwargs):
             return self._rustmap.clearambiguoustimes(*args, **kwargs)
 
         def nonnormalentries(self):
             return self._rustmap.nonnormalentries()
 
         def get(self, *args, **kwargs):
             return self._rustmap.get(*args, **kwargs)
 
         @propertycache
         def _rustmap(self):
             self._rustmap = rustmod.DirstateMap(self._root)
             self.read()
             return self._rustmap
 
         @property
         def copymap(self):
             return self._rustmap.copymap()
 
         def preload(self):
             self._rustmap
 
         def clear(self):
             self._rustmap.clear()
             self.setparents(nullid, nullid)
             util.clearcachedproperty(self, b"_dirs")
             util.clearcachedproperty(self, b"_alldirs")
             util.clearcachedproperty(self, b"dirfoldmap")
 
         def items(self):
             return self._rustmap.items()
 
         def keys(self):
             return iter(self._rustmap)
 
         def __contains__(self, key):
             return key in self._rustmap
 
         def __getitem__(self, item):
             return self._rustmap[item]
 
         def __len__(self):
             return len(self._rustmap)
 
         def __iter__(self):
             return iter(self._rustmap)
 
         # forward for python2,3 compat
         iteritems = items
 
         def _opendirstatefile(self):
             fp, mode = txnutil.trypending(
                 self._root, self._opener, self._filename
             )
             if self._pendingmode is not None and self._pendingmode != mode:
                 fp.close()
                 raise error.Abort(
                     _(b'working directory state may be changed parallelly')
                 )
             self._pendingmode = mode
             return fp
 
         def setparents(self, p1, p2):
             self._rustmap.setparents(p1, p2)
             self._parents = (p1, p2)
             self._dirtyparents = True
 
         def parents(self):
             if not self._parents:
                 try:
                     fp = self._opendirstatefile()
                     st = fp.read(40)
                     fp.close()
                 except IOError as err:
                     if err.errno != errno.ENOENT:
                         raise
                     # File doesn't exist, so the current state is empty
                     st = b''
 
                 try:
                     self._parents = self._rustmap.parents(st)
                 except ValueError:
                     raise error.Abort(
                         _(b'working directory state appears damaged!')
                     )
 
             return self._parents
 
         def read(self):
             # ignore HG_PENDING because identity is used only for writing
             self.identity = util.filestat.frompath(
                 self._opener.join(self._filename)
             )
 
             try:
                 fp = self._opendirstatefile()
                 try:
                     st = fp.read()
                 finally:
                     fp.close()
             except IOError as err:
                 if err.errno != errno.ENOENT:
                     raise
                 return
             if not st:
                 return
 
             parse_dirstate = util.nogc(self._rustmap.read)
             parents = parse_dirstate(st)
             if parents and not self._dirtyparents:
                 self.setparents(*parents)
 
             self.__contains__ = self._rustmap.__contains__
             self.__getitem__ = self._rustmap.__getitem__
             self.get = self._rustmap.get
 
         def write(self, st, now):
             parents = self.parents()
             st.write(self._rustmap.write(parents[0], parents[1], now))
             st.close()
             self._dirtyparents = False
 
         @propertycache
         def filefoldmap(self):
             """Returns a dictionary mapping normalized case paths to their
             non-normalized versions.
             """
             return self._rustmap.filefoldmapasdict()
 
         def hastrackeddir(self, d):
             self._dirs  # Trigger Python's propertycache
             return self._rustmap.hastrackeddir(d)
 
         def hasdir(self, d):
             self._dirs  # Trigger Python's propertycache
             return self._rustmap.hasdir(d)
 
         @propertycache
         def _dirs(self):
             return self._rustmap.getdirs()
 
         @propertycache
         def _alldirs(self):
             return self._rustmap.getalldirs()
 
         @propertycache
         def identity(self):
             self._rustmap
             return self.identity
 
         @property
         def nonnormalset(self):
             nonnorm = self._rustmap.non_normal_entries()
             return nonnorm
 
         @propertycache
         def otherparentset(self):
             otherparents = self._rustmap.other_parent_entries()
             return otherparents
 
         @propertycache
         def dirfoldmap(self):
             f = {}
             normcase = util.normcase
             for name in self._dirs:
                 f[normcase(name)] = name
             return f

mercurial/match.py

@@ -1,1601 +1,1604 b''
 # match.py - filename matching
 #
 #  Copyright 2008, 2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import copy
 import itertools
 import os
 import re
 
 from .i18n import _
 from .pycompat import open
 from . import (
     encoding,
     error,
     pathutil,
     policy,
     pycompat,
     util,
 )
 from .utils import stringutil
 
 rustmod = policy.importrust('dirstate')
 
 allpatternkinds = (
     b're',
     b'glob',
     b'path',
     b'relglob',
     b'relpath',
     b'relre',
     b'rootglob',
     b'listfile',
     b'listfile0',
     b'set',
     b'include',
     b'subinclude',
     b'rootfilesin',
 )
 cwdrelativepatternkinds = (b'relpath', b'glob')
 
 propertycache = util.propertycache
 
 
 def _rematcher(regex):
     '''compile the regexp with the best available regexp engine and return a
     matcher function'''
     m = util.re.compile(regex)
     try:
         # slightly faster, provided by facebook's re2 bindings
         return m.test_match
     except AttributeError:
         return m.match
 
 
 def _expandsets(cwd, kindpats, ctx=None, listsubrepos=False, badfn=None):
     '''Returns the kindpats list with the 'set' patterns expanded to matchers'''
     matchers = []
     other = []
 
     for kind, pat, source in kindpats:
         if kind == b'set':
             if ctx is None:
                 raise error.ProgrammingError(
                     b"fileset expression with no context"
                 )
             matchers.append(ctx.matchfileset(cwd, pat, badfn=badfn))
 
             if listsubrepos:
                 for subpath in ctx.substate:
                     sm = ctx.sub(subpath).matchfileset(cwd, pat, badfn=badfn)
                     pm = prefixdirmatcher(subpath, sm, badfn=badfn)
                     matchers.append(pm)
 
             continue
         other.append((kind, pat, source))
     return matchers, other
 
 
 def _expandsubinclude(kindpats, root):
     '''Returns the list of subinclude matcher args and the kindpats without the
     subincludes in it.'''
     relmatchers = []
     other = []
 
     for kind, pat, source in kindpats:
         if kind == b'subinclude':
             sourceroot = pathutil.dirname(util.normpath(source))
             pat = util.pconvert(pat)
             path = pathutil.join(sourceroot, pat)
 
             newroot = pathutil.dirname(path)
             matcherargs = (newroot, b'', [], [b'include:%s' % path])
 
             prefix = pathutil.canonpath(root, root, newroot)
             if prefix:
                 prefix += b'/'
             relmatchers.append((prefix, matcherargs))
         else:
             other.append((kind, pat, source))
 
     return relmatchers, other
 
 
 def _kindpatsalwaysmatch(kindpats):
     """"Checks whether the kindspats match everything, as e.g.
     'relpath:.' does.
     """
     for kind, pat, source in kindpats:
         if pat != b'' or kind not in [b'relpath', b'glob']:
             return False
     return True
 
 
 def _buildkindpatsmatcher(
     matchercls, root, cwd, kindpats, ctx=None, listsubrepos=False, badfn=None,
 ):
     matchers = []
     fms, kindpats = _expandsets(
         cwd, kindpats, ctx=ctx, listsubrepos=listsubrepos, badfn=badfn,
     )
     if kindpats:
         m = matchercls(root, kindpats, badfn=badfn)
         matchers.append(m)
     if fms:
         matchers.extend(fms)
     if not matchers:
         return nevermatcher(badfn=badfn)
     if len(matchers) == 1:
         return matchers[0]
     return unionmatcher(matchers)
 
 
 def match(
     root,
     cwd,
     patterns=None,
     include=None,
     exclude=None,
     default=b'glob',
     auditor=None,
     ctx=None,
     listsubrepos=False,
     warn=None,
     badfn=None,
     icasefs=False,
 ):
     r"""build an object to match a set of file patterns
 
     arguments:
     root - the canonical root of the tree you're matching against
     cwd - the current working directory, if relevant
     patterns - patterns to find
     include - patterns to include (unless they are excluded)
     exclude - patterns to exclude (even if they are included)
     default - if a pattern in patterns has no explicit type, assume this one
     auditor - optional path auditor
     ctx - optional changecontext
     listsubrepos - if True, recurse into subrepositories
     warn - optional function used for printing warnings
     badfn - optional bad() callback for this matcher instead of the default
     icasefs - make a matcher for wdir on case insensitive filesystems, which
         normalizes the given patterns to the case in the filesystem
 
     a pattern is one of:
     'glob:<glob>' - a glob relative to cwd
     're:<regexp>' - a regular expression
     'path:<path>' - a path relative to repository root, which is matched
                     recursively
     'rootfilesin:<path>' - a path relative to repository root, which is
                     matched non-recursively (will not match subdirectories)
     'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs)
     'relpath:<path>' - a path relative to cwd
     'relre:<regexp>' - a regexp that needn't match the start of a name
     'set:<fileset>' - a fileset expression
     'include:<path>' - a file of patterns to read and include
     'subinclude:<path>' - a file of patterns to match against files under
                           the same directory
     '<something>' - a pattern of the specified default type
 
     >>> def _match(root, *args, **kwargs):
     ...     return match(util.localpath(root), *args, **kwargs)
 
     Usually a patternmatcher is returned:
     >>> _match(b'/foo', b'.', [b're:.*\.c$', b'path:foo/a', b'*.py'])
     <patternmatcher patterns='.*\\.c$|foo/a(?:/|$)|[^/]*\\.py$'>
 
     Combining 'patterns' with 'include' (resp. 'exclude') gives an
     intersectionmatcher (resp. a differencematcher):
     >>> type(_match(b'/foo', b'.', [b're:.*\.c$'], include=[b'path:lib']))
     <class 'mercurial.match.intersectionmatcher'>
     >>> type(_match(b'/foo', b'.', [b're:.*\.c$'], exclude=[b'path:build']))
     <class 'mercurial.match.differencematcher'>
 
     Notice that, if 'patterns' is empty, an alwaysmatcher is returned:
     >>> _match(b'/foo', b'.', [])
     <alwaysmatcher>
 
     The 'default' argument determines which kind of pattern is assumed if a
     pattern has no prefix:
     >>> _match(b'/foo', b'.', [b'.*\.c$'], default=b're')
     <patternmatcher patterns='.*\\.c$'>
     >>> _match(b'/foo', b'.', [b'main.py'], default=b'relpath')
     <patternmatcher patterns='main\\.py(?:/|$)'>
     >>> _match(b'/foo', b'.', [b'main.py'], default=b're')
     <patternmatcher patterns='main.py'>
 
     The primary use of matchers is to check whether a value (usually a file
     name) matches againset one of the patterns given at initialization. There
     are two ways of doing this check.
 
     >>> m = _match(b'/foo', b'', [b're:.*\.c$', b'relpath:a'])
 
     1. Calling the matcher with a file name returns True if any pattern
     matches that file name:
     >>> m(b'a')
     True
     >>> m(b'main.c')
     True
     >>> m(b'test.py')
     False
 
     2. Using the exact() method only returns True if the file name matches one
     of the exact patterns (i.e. not re: or glob: patterns):
     >>> m.exact(b'a')
     True
     >>> m.exact(b'main.c')
     False
     """
     assert os.path.isabs(root)
     cwd = os.path.join(root, util.localpath(cwd))
     normalize = _donormalize
     if icasefs:
         dirstate = ctx.repo().dirstate
         dsnormalize = dirstate.normalize
 
         def normalize(patterns, default, root, cwd, auditor, warn):
             kp = _donormalize(patterns, default, root, cwd, auditor, warn)
             kindpats = []
             for kind, pats, source in kp:
                 if kind not in (b're', b'relre'):  # regex can't be normalized
                     p = pats
                     pats = dsnormalize(pats)
 
                     # Preserve the original to handle a case only rename.
                     if p != pats and p in dirstate:
                         kindpats.append((kind, p, source))
 
                 kindpats.append((kind, pats, source))
             return kindpats
 
     if patterns:
         kindpats = normalize(patterns, default, root, cwd, auditor, warn)
         if _kindpatsalwaysmatch(kindpats):
             m = alwaysmatcher(badfn)
         else:
             m = _buildkindpatsmatcher(
                 patternmatcher,
                 root,
                 cwd,
                 kindpats,
                 ctx=ctx,
                 listsubrepos=listsubrepos,
                 badfn=badfn,
             )
     else:
         # It's a little strange that no patterns means to match everything.
         # Consider changing this to match nothing (probably using nevermatcher).
         m = alwaysmatcher(badfn)
 
     if include:
         kindpats = normalize(include, b'glob', root, cwd, auditor, warn)
         im = _buildkindpatsmatcher(
             includematcher,
             root,
             cwd,
             kindpats,
             ctx=ctx,
             listsubrepos=listsubrepos,
             badfn=None,
         )
         m = intersectmatchers(m, im)
     if exclude:
         kindpats = normalize(exclude, b'glob', root, cwd, auditor, warn)
         em = _buildkindpatsmatcher(
             includematcher,
             root,
             cwd,
             kindpats,
             ctx=ctx,
             listsubrepos=listsubrepos,
             badfn=None,
         )
         m = differencematcher(m, em)
     return m
 
 
 def exact(files, badfn=None):
     return exactmatcher(files, badfn=badfn)
 
 
 def always(badfn=None):
     return alwaysmatcher(badfn)
 
 
 def never(badfn=None):
     return nevermatcher(badfn)
 
 
 def badmatch(match, badfn):
     """Make a copy of the given matcher, replacing its bad method with the given
     one.
     """
     m = copy.copy(match)
     m.bad = badfn
     return m
 
 
 def _donormalize(patterns, default, root, cwd, auditor=None, warn=None):
     '''Convert 'kind:pat' from the patterns list to tuples with kind and
     normalized and rooted patterns and with listfiles expanded.'''
     kindpats = []
     for kind, pat in [_patsplit(p, default) for p in patterns]:
         if kind in cwdrelativepatternkinds:
             pat = pathutil.canonpath(root, cwd, pat, auditor=auditor)
         elif kind in (b'relglob', b'path', b'rootfilesin', b'rootglob'):
             pat = util.normpath(pat)
         elif kind in (b'listfile', b'listfile0'):
             try:
                 files = util.readfile(pat)
                 if kind == b'listfile0':
                     files = files.split(b'\0')
                 else:
                     files = files.splitlines()
                 files = [f for f in files if f]
             except EnvironmentError:
                 raise error.Abort(_(b"unable to read file list (%s)") % pat)
             for k, p, source in _donormalize(
                 files, default, root, cwd, auditor, warn
             ):
                 kindpats.append((k, p, pat))
             continue
         elif kind == b'include':
             try:
                 fullpath = os.path.join(root, util.localpath(pat))
                 includepats = readpatternfile(fullpath, warn)
                 for k, p, source in _donormalize(
                     includepats, default, root, cwd, auditor, warn
                 ):
                     kindpats.append((k, p, source or pat))
             except error.Abort as inst:
                 raise error.Abort(
                     b'%s: %s'
                     % (pat, inst[0])  # pytype: disable=unsupported-operands
                 )
             except IOError as inst:
                 if warn:
                     warn(
                         _(b"skipping unreadable pattern file '%s': %s\n")
                         % (pat, stringutil.forcebytestr(inst.strerror))
                     )
             continue
         # else: re or relre - which cannot be normalized
         kindpats.append((kind, pat, b''))
     return kindpats
 
 
 class basematcher(object):
     def __init__(self, badfn=None):
         if badfn is not None:
             self.bad = badfn
 
     def __call__(self, fn):
         return self.matchfn(fn)
 
     # Callbacks related to how the matcher is used by dirstate.walk.
     # Subscribers to these events must monkeypatch the matcher object.
     def bad(self, f, msg):
         '''Callback from dirstate.walk for each explicit file that can't be
         found/accessed, with an error message.'''
 
     # If an traversedir is set, it will be called when a directory discovered
     # by recursive traversal is visited.
     traversedir = None
 
     @propertycache
     def _files(self):
         return []
 
     def files(self):
         '''Explicitly listed files or patterns or roots:
         if no patterns or .always(): empty list,
         if exact: list exact files,
         if not .anypats(): list all files and dirs,
         else: optimal roots'''
         return self._files
 
     @propertycache
     def _fileset(self):
         return set(self._files)
 
     def exact(self, f):
         '''Returns True if f is in .files().'''
         return f in self._fileset
 
     def matchfn(self, f):
         return False
 
     def visitdir(self, dir):
         '''Decides whether a directory should be visited based on whether it
         has potential matches in it or one of its subdirectories. This is
         based on the match's primary, included, and excluded patterns.
 
         Returns the string 'all' if the given directory and all subdirectories
         should be visited. Otherwise returns True or False indicating whether
         the given directory should be visited.
         '''
         return True
 
     def visitchildrenset(self, dir):
         '''Decides whether a directory should be visited based on whether it
         has potential matches in it or one of its subdirectories, and
         potentially lists which subdirectories of that directory should be
         visited. This is based on the match's primary, included, and excluded
         patterns.
 
         This function is very similar to 'visitdir', and the following mapping
         can be applied:
 
              visitdir | visitchildrenlist
             ----------+-------------------
              False    | set()
              'all'    | 'all'
              True     | 'this' OR non-empty set of subdirs -or files- to visit
 
         Example:
           Assume matchers ['path:foo/bar', 'rootfilesin:qux'], we would return
           the following values (assuming the implementation of visitchildrenset
           is capable of recognizing this; some implementations are not).
 
           '' -> {'foo', 'qux'}
           'baz' -> set()
           'foo' -> {'bar'}
           # Ideally this would be 'all', but since the prefix nature of matchers
           # is applied to the entire matcher, we have to downgrade this to
           # 'this' due to the non-prefix 'rootfilesin'-kind matcher being mixed
           # in.
           'foo/bar' -> 'this'
           'qux' -> 'this'
 
         Important:
           Most matchers do not know if they're representing files or
           directories. They see ['path:dir/f'] and don't know whether 'f' is a
           file or a directory, so visitchildrenset('dir') for most matchers will
           return {'f'}, but if the matcher knows it's a file (like exactmatcher
           does), it may return 'this'. Do not rely on the return being a set
           indicating that there are no files in this dir to investigate (or
           equivalently that if there are files to investigate in 'dir' that it
           will always return 'this').
         '''
         return b'this'
 
     def always(self):
         '''Matcher will match everything and .files() will be empty --
         optimization might be possible.'''
         return False
 
     def isexact(self):
         '''Matcher will match exactly the list of files in .files() --
         optimization might be possible.'''
         return False
 
     def prefix(self):
         '''Matcher will match the paths in .files() recursively --
         optimization might be possible.'''
         return False
 
     def anypats(self):
         '''None of .always(), .isexact(), and .prefix() is true --
         optimizations will be difficult.'''
         return not self.always() and not self.isexact() and not self.prefix()
 
 
 class alwaysmatcher(basematcher):
     '''Matches everything.'''
 
     def __init__(self, badfn=None):
         super(alwaysmatcher, self).__init__(badfn)
 
     def always(self):
         return True
 
     def matchfn(self, f):
         return True
 
     def visitdir(self, dir):
         return b'all'
 
     def visitchildrenset(self, dir):
         return b'all'
 
     def __repr__(self):
         return r'<alwaysmatcher>'
 
 
 class nevermatcher(basematcher):
     '''Matches nothing.'''
 
     def __init__(self, badfn=None):
         super(nevermatcher, self).__init__(badfn)
 
     # It's a little weird to say that the nevermatcher is an exact matcher
     # or a prefix matcher, but it seems to make sense to let callers take
     # fast paths based on either. There will be no exact matches, nor any
     # prefixes (files() returns []), so fast paths iterating over them should
     # be efficient (and correct).
     def isexact(self):
         return True
 
     def prefix(self):
         return True
 
     def visitdir(self, dir):
         return False
 
     def visitchildrenset(self, dir):
         return set()
 
     def __repr__(self):
         return r'<nevermatcher>'
 
 
 class predicatematcher(basematcher):
     """A matcher adapter for a simple boolean function"""
 
     def __init__(self, predfn, predrepr=None, badfn=None):
         super(predicatematcher, self).__init__(badfn)
         self.matchfn = predfn
         self._predrepr = predrepr
 
     @encoding.strmethod
     def __repr__(self):
         s = stringutil.buildrepr(self._predrepr) or pycompat.byterepr(
             self.matchfn
         )
         return b'<predicatenmatcher pred=%s>' % s
 
 
 class patternmatcher(basematcher):
     r"""Matches a set of (kind, pat, source) against a 'root' directory.
 
     >>> kindpats = [
     ...     (b're', br'.*\.c$', b''),
     ...     (b'path', b'foo/a', b''),
     ...     (b'relpath', b'b', b''),
     ...     (b'glob', b'*.h', b''),
     ... ]
     >>> m = patternmatcher(b'foo', kindpats)
     >>> m(b'main.c')  # matches re:.*\.c$
     True
     >>> m(b'b.txt')
     False
     >>> m(b'foo/a')  # matches path:foo/a
     True
     >>> m(b'a')  # does not match path:b, since 'root' is 'foo'
     False
     >>> m(b'b')  # matches relpath:b, since 'root' is 'foo'
     True
     >>> m(b'lib.h')  # matches glob:*.h
     True
 
     >>> m.files()
     ['', 'foo/a', 'b', '']
     >>> m.exact(b'foo/a')
     True
     >>> m.exact(b'b')
     True
     >>> m.exact(b'lib.h')  # exact matches are for (rel)path kinds
     False
     """
 
     def __init__(self, root, kindpats, badfn=None):
         super(patternmatcher, self).__init__(badfn)
 
         self._files = _explicitfiles(kindpats)
         self._prefix = _prefix(kindpats)
         self._pats, self.matchfn = _buildmatch(kindpats, b'$', root)
 
     @propertycache
     def _dirs(self):
         return set(pathutil.dirs(self._fileset))
 
     def visitdir(self, dir):
         if self._prefix and dir in self._fileset:
             return b'all'
         return (
             dir in self._fileset
             or dir in self._dirs
             or any(
                 parentdir in self._fileset
                 for parentdir in pathutil.finddirs(dir)
             )
         )
 
     def visitchildrenset(self, dir):
         ret = self.visitdir(dir)
         if ret is True:
             return b'this'
         elif not ret:
             return set()
         assert ret == b'all'
         return b'all'
 
     def prefix(self):
         return self._prefix
 
     @encoding.strmethod
     def __repr__(self):
         return b'<patternmatcher patterns=%r>' % pycompat.bytestr(self._pats)
 
 
 # This is basically a reimplementation of pathutil.dirs that stores the
 # children instead of just a count of them, plus a small optional optimization
 # to avoid some directories we don't need.
 class _dirchildren(object):
     def __init__(self, paths, onlyinclude=None):
         self._dirs = {}
         self._onlyinclude = onlyinclude or []
         addpath = self.addpath
         for f in paths:
             addpath(f)
 
     def addpath(self, path):
         if path == b'':
             return
         dirs = self._dirs
         findsplitdirs = _dirchildren._findsplitdirs
         for d, b in findsplitdirs(path):
             if d not in self._onlyinclude:
                 continue
             dirs.setdefault(d, set()).add(b)
 
     @staticmethod
     def _findsplitdirs(path):
         # yields (dirname, basename) tuples, walking back to the root.  This is
         # very similar to pathutil.finddirs, except:
         #  - produces a (dirname, basename) tuple, not just 'dirname'
         # Unlike manifest._splittopdir, this does not suffix `dirname` with a
         # slash.
         oldpos = len(path)
         pos = path.rfind(b'/')
         while pos != -1:
             yield path[:pos], path[pos + 1 : oldpos]
             oldpos = pos
             pos = path.rfind(b'/', 0, pos)
         yield b'', path[:oldpos]
 
     def get(self, path):
         return self._dirs.get(path, set())
 
 
 class includematcher(basematcher):
     def __init__(self, root, kindpats, badfn=None):
         super(includematcher, self).__init__(badfn)
-
+        if rustmod is not None:
+            # We need to pass the patterns to Rust because they can contain
+            # patterns from the user interface
+            self._kindpats = kindpats
         self._pats, self.matchfn = _buildmatch(kindpats, b'(?:/|$)', root)
         self._prefix = _prefix(kindpats)
         roots, dirs, parents = _rootsdirsandparents(kindpats)
         # roots are directories which are recursively included.
         self._roots = set(roots)
         # dirs are directories which are non-recursively included.
         self._dirs = set(dirs)
         # parents are directories which are non-recursively included because
         # they are needed to get to items in _dirs or _roots.
         self._parents = parents
 
     def visitdir(self, dir):
         if self._prefix and dir in self._roots:
             return b'all'
         return (
             dir in self._roots
             or dir in self._dirs
             or dir in self._parents
             or any(
                 parentdir in self._roots for parentdir in pathutil.finddirs(dir)
             )
         )
 
     @propertycache
     def _allparentschildren(self):
         # It may seem odd that we add dirs, roots, and parents, and then
         # restrict to only parents. This is to catch the case of:
         #   dirs = ['foo/bar']
         #   parents = ['foo']
         # if we asked for the children of 'foo', but had only added
         # self._parents, we wouldn't be able to respond ['bar'].
         return _dirchildren(
             itertools.chain(self._dirs, self._roots, self._parents),
             onlyinclude=self._parents,
         )
 
     def visitchildrenset(self, dir):
         if self._prefix and dir in self._roots:
             return b'all'
         # Note: this does *not* include the 'dir in self._parents' case from
         # visitdir, that's handled below.
         if (
             b'' in self._roots
             or dir in self._roots
             or dir in self._dirs
             or any(
                 parentdir in self._roots for parentdir in pathutil.finddirs(dir)
             )
         ):
             return b'this'
 
         if dir in self._parents:
             return self._allparentschildren.get(dir) or set()
         return set()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<includematcher includes=%r>' % pycompat.bytestr(self._pats)
 
 
 class exactmatcher(basematcher):
     r'''Matches the input files exactly. They are interpreted as paths, not
     patterns (so no kind-prefixes).
 
     >>> m = exactmatcher([b'a.txt', br're:.*\.c$'])
     >>> m(b'a.txt')
     True
     >>> m(b'b.txt')
     False
 
     Input files that would be matched are exactly those returned by .files()
     >>> m.files()
     ['a.txt', 're:.*\\.c$']
 
     So pattern 're:.*\.c$' is not considered as a regex, but as a file name
     >>> m(b'main.c')
     False
     >>> m(br're:.*\.c$')
     True
     '''
 
     def __init__(self, files, badfn=None):
         super(exactmatcher, self).__init__(badfn)
 
         if isinstance(files, list):
             self._files = files
         else:
             self._files = list(files)
 
     matchfn = basematcher.exact
 
     @propertycache
     def _dirs(self):
         return set(pathutil.dirs(self._fileset))
 
     def visitdir(self, dir):
         return dir in self._dirs
 
     def visitchildrenset(self, dir):
         if not self._fileset or dir not in self._dirs:
             return set()
 
         candidates = self._fileset | self._dirs - {b''}
         if dir != b'':
             d = dir + b'/'
             candidates = {c[len(d) :] for c in candidates if c.startswith(d)}
         # self._dirs includes all of the directories, recursively, so if
         # we're attempting to match foo/bar/baz.txt, it'll have '', 'foo',
         # 'foo/bar' in it. Thus we can safely ignore a candidate that has a
         # '/' in it, indicating a it's for a subdir-of-a-subdir; the
         # immediate subdir will be in there without a slash.
         ret = {c for c in candidates if b'/' not in c}
         # We really do not expect ret to be empty, since that would imply that
         # there's something in _dirs that didn't have a file in _fileset.
         assert ret
         return ret
 
     def isexact(self):
         return True
 
     @encoding.strmethod
     def __repr__(self):
         return b'<exactmatcher files=%r>' % self._files
 
 
 class differencematcher(basematcher):
     '''Composes two matchers by matching if the first matches and the second
     does not.
 
     The second matcher's non-matching-attributes (bad, traversedir) are ignored.
     '''
 
     def __init__(self, m1, m2):
         super(differencematcher, self).__init__()
         self._m1 = m1
         self._m2 = m2
         self.bad = m1.bad
         self.traversedir = m1.traversedir
 
     def matchfn(self, f):
         return self._m1(f) and not self._m2(f)
 
     @propertycache
     def _files(self):
         if self.isexact():
             return [f for f in self._m1.files() if self(f)]
         # If m1 is not an exact matcher, we can't easily figure out the set of
         # files, because its files() are not always files. For example, if
         # m1 is "path:dir" and m2 is "rootfileins:.", we don't
         # want to remove "dir" from the set even though it would match m2,
         # because the "dir" in m1 may not be a file.
         return self._m1.files()
 
     def visitdir(self, dir):
         if self._m2.visitdir(dir) == b'all':
             return False
         elif not self._m2.visitdir(dir):
             # m2 does not match dir, we can return 'all' here if possible
             return self._m1.visitdir(dir)
         return bool(self._m1.visitdir(dir))
 
     def visitchildrenset(self, dir):
         m2_set = self._m2.visitchildrenset(dir)
         if m2_set == b'all':
             return set()
         m1_set = self._m1.visitchildrenset(dir)
         # Possible values for m1: 'all', 'this', set(...), set()
         # Possible values for m2:        'this', set(...), set()
         # If m2 has nothing under here that we care about, return m1, even if
         # it's 'all'. This is a change in behavior from visitdir, which would
         # return True, not 'all', for some reason.
         if not m2_set:
             return m1_set
         if m1_set in [b'all', b'this']:
             # Never return 'all' here if m2_set is any kind of non-empty (either
             # 'this' or set(foo)), since m2 might return set() for a
             # subdirectory.
             return b'this'
         # Possible values for m1:         set(...), set()
         # Possible values for m2: 'this', set(...)
         # We ignore m2's set results. They're possibly incorrect:
         #  m1 = path:dir/subdir, m2=rootfilesin:dir, visitchildrenset(''):
         #    m1 returns {'dir'}, m2 returns {'dir'}, if we subtracted we'd
         #    return set(), which is *not* correct, we still need to visit 'dir'!
         return m1_set
 
     def isexact(self):
         return self._m1.isexact()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<differencematcher m1=%r, m2=%r>' % (self._m1, self._m2)
 
 
 def intersectmatchers(m1, m2):
     '''Composes two matchers by matching if both of them match.
 
     The second matcher's non-matching-attributes (bad, traversedir) are ignored.
     '''
     if m1 is None or m2 is None:
         return m1 or m2
     if m1.always():
         m = copy.copy(m2)
         # TODO: Consider encapsulating these things in a class so there's only
         # one thing to copy from m1.
         m.bad = m1.bad
         m.traversedir = m1.traversedir
         return m
     if m2.always():
         m = copy.copy(m1)
         return m
     return intersectionmatcher(m1, m2)
 
 
 class intersectionmatcher(basematcher):
     def __init__(self, m1, m2):
         super(intersectionmatcher, self).__init__()
         self._m1 = m1
         self._m2 = m2
         self.bad = m1.bad
         self.traversedir = m1.traversedir
 
     @propertycache
     def _files(self):
         if self.isexact():
             m1, m2 = self._m1, self._m2
             if not m1.isexact():
                 m1, m2 = m2, m1
             return [f for f in m1.files() if m2(f)]
         # It neither m1 nor m2 is an exact matcher, we can't easily intersect
         # the set of files, because their files() are not always files. For
         # example, if intersecting a matcher "-I glob:foo.txt" with matcher of
         # "path:dir2", we don't want to remove "dir2" from the set.
         return self._m1.files() + self._m2.files()
 
     def matchfn(self, f):
         return self._m1(f) and self._m2(f)
 
     def visitdir(self, dir):
         visit1 = self._m1.visitdir(dir)
         if visit1 == b'all':
             return self._m2.visitdir(dir)
         # bool() because visit1=True + visit2='all' should not be 'all'
         return bool(visit1 and self._m2.visitdir(dir))
 
     def visitchildrenset(self, dir):
         m1_set = self._m1.visitchildrenset(dir)
         if not m1_set:
             return set()
         m2_set = self._m2.visitchildrenset(dir)
         if not m2_set:
             return set()
 
         if m1_set == b'all':
             return m2_set
         elif m2_set == b'all':
             return m1_set
 
         if m1_set == b'this' or m2_set == b'this':
             return b'this'
 
         assert isinstance(m1_set, set) and isinstance(m2_set, set)
         return m1_set.intersection(m2_set)
 
     def always(self):
         return self._m1.always() and self._m2.always()
 
     def isexact(self):
         return self._m1.isexact() or self._m2.isexact()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<intersectionmatcher m1=%r, m2=%r>' % (self._m1, self._m2)
 
 
 class subdirmatcher(basematcher):
     """Adapt a matcher to work on a subdirectory only.
 
     The paths are remapped to remove/insert the path as needed:
 
     >>> from . import pycompat
     >>> m1 = match(util.localpath(b'/root'), b'', [b'a.txt', b'sub/b.txt'], auditor=lambda name: None)
     >>> m2 = subdirmatcher(b'sub', m1)
     >>> m2(b'a.txt')
     False
     >>> m2(b'b.txt')
     True
     >>> m2.matchfn(b'a.txt')
     False
     >>> m2.matchfn(b'b.txt')
     True
     >>> m2.files()
     ['b.txt']
     >>> m2.exact(b'b.txt')
     True
     >>> def bad(f, msg):
     ...     print(pycompat.sysstr(b"%s: %s" % (f, msg)))
     >>> m1.bad = bad
     >>> m2.bad(b'x.txt', b'No such file')
     sub/x.txt: No such file
     """
 
     def __init__(self, path, matcher):
         super(subdirmatcher, self).__init__()
         self._path = path
         self._matcher = matcher
         self._always = matcher.always()
 
         self._files = [
             f[len(path) + 1 :]
             for f in matcher._files
             if f.startswith(path + b"/")
         ]
 
         # If the parent repo had a path to this subrepo and the matcher is
         # a prefix matcher, this submatcher always matches.
         if matcher.prefix():
             self._always = any(f == path for f in matcher._files)
 
     def bad(self, f, msg):
         self._matcher.bad(self._path + b"/" + f, msg)
 
     def matchfn(self, f):
         # Some information is lost in the superclass's constructor, so we
         # can not accurately create the matching function for the subdirectory
         # from the inputs. Instead, we override matchfn() and visitdir() to
         # call the original matcher with the subdirectory path prepended.
         return self._matcher.matchfn(self._path + b"/" + f)
 
     def visitdir(self, dir):
         if dir == b'':
             dir = self._path
         else:
             dir = self._path + b"/" + dir
         return self._matcher.visitdir(dir)
 
     def visitchildrenset(self, dir):
         if dir == b'':
             dir = self._path
         else:
             dir = self._path + b"/" + dir
         return self._matcher.visitchildrenset(dir)
 
     def always(self):
         return self._always
 
     def prefix(self):
         return self._matcher.prefix() and not self._always
 
     @encoding.strmethod
     def __repr__(self):
         return b'<subdirmatcher path=%r, matcher=%r>' % (
             self._path,
             self._matcher,
         )
 
 
 class prefixdirmatcher(basematcher):
     """Adapt a matcher to work on a parent directory.
 
     The matcher's non-matching-attributes (bad, traversedir) are ignored.
 
     The prefix path should usually be the relative path from the root of
     this matcher to the root of the wrapped matcher.
 
     >>> m1 = match(util.localpath(b'/root/d/e'), b'f', [b'../a.txt', b'b.txt'], auditor=lambda name: None)
     >>> m2 = prefixdirmatcher(b'd/e', m1)
     >>> m2(b'a.txt')
     False
     >>> m2(b'd/e/a.txt')
     True
     >>> m2(b'd/e/b.txt')
     False
     >>> m2.files()
     ['d/e/a.txt', 'd/e/f/b.txt']
     >>> m2.exact(b'd/e/a.txt')
     True
     >>> m2.visitdir(b'd')
     True
     >>> m2.visitdir(b'd/e')
     True
     >>> m2.visitdir(b'd/e/f')
     True
     >>> m2.visitdir(b'd/e/g')
     False
     >>> m2.visitdir(b'd/ef')
     False
     """
 
     def __init__(self, path, matcher, badfn=None):
         super(prefixdirmatcher, self).__init__(badfn)
         if not path:
             raise error.ProgrammingError(b'prefix path must not be empty')
         self._path = path
         self._pathprefix = path + b'/'
         self._matcher = matcher
 
     @propertycache
     def _files(self):
         return [self._pathprefix + f for f in self._matcher._files]
 
     def matchfn(self, f):
         if not f.startswith(self._pathprefix):
             return False
         return self._matcher.matchfn(f[len(self._pathprefix) :])
 
     @propertycache
     def _pathdirs(self):
         return set(pathutil.finddirs(self._path))
 
     def visitdir(self, dir):
         if dir == self._path:
             return self._matcher.visitdir(b'')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitdir(dir[len(self._pathprefix) :])
         return dir in self._pathdirs
 
     def visitchildrenset(self, dir):
         if dir == self._path:
             return self._matcher.visitchildrenset(b'')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitchildrenset(dir[len(self._pathprefix) :])
         if dir in self._pathdirs:
             return b'this'
         return set()
 
     def isexact(self):
         return self._matcher.isexact()
 
     def prefix(self):
         return self._matcher.prefix()
 
     @encoding.strmethod
     def __repr__(self):
         return b'<prefixdirmatcher path=%r, matcher=%r>' % (
             pycompat.bytestr(self._path),
             self._matcher,
         )
 
 
 class unionmatcher(basematcher):
     """A matcher that is the union of several matchers.
 
     The non-matching-attributes (bad, traversedir) are taken from the first
     matcher.
     """
 
     def __init__(self, matchers):
         m1 = matchers[0]
         super(unionmatcher, self).__init__()
         self.traversedir = m1.traversedir
         self._matchers = matchers
 
     def matchfn(self, f):
         for match in self._matchers:
             if match(f):
                 return True
         return False
 
     def visitdir(self, dir):
         r = False
         for m in self._matchers:
             v = m.visitdir(dir)
             if v == b'all':
                 return v
             r |= v
         return r
 
     def visitchildrenset(self, dir):
         r = set()
         this = False
         for m in self._matchers:
             v = m.visitchildrenset(dir)
             if not v:
                 continue
             if v == b'all':
                 return v
             if this or v == b'this':
                 this = True
                 # don't break, we might have an 'all' in here.
                 continue
             assert isinstance(v, set)
             r = r.union(v)
         if this:
             return b'this'
         return r
 
     @encoding.strmethod
     def __repr__(self):
         return b'<unionmatcher matchers=%r>' % self._matchers
 
 
 def patkind(pattern, default=None):
     r'''If pattern is 'kind:pat' with a known kind, return kind.
 
     >>> patkind(br're:.*\.c$')
     're'
     >>> patkind(b'glob:*.c')
     'glob'
     >>> patkind(b'relpath:test.py')
     'relpath'
     >>> patkind(b'main.py')
     >>> patkind(b'main.py', default=b're')
     're'
     '''
     return _patsplit(pattern, default)[0]
 
 
 def _patsplit(pattern, default):
     """Split a string into the optional pattern kind prefix and the actual
     pattern."""
     if b':' in pattern:
         kind, pat = pattern.split(b':', 1)
         if kind in allpatternkinds:
             return kind, pat
     return default, pattern
 
 
 def _globre(pat):
     r'''Convert an extended glob string to a regexp string.
 
     >>> from . import pycompat
     >>> def bprint(s):
     ...     print(pycompat.sysstr(s))
     >>> bprint(_globre(br'?'))
     .
     >>> bprint(_globre(br'*'))
     [^/]*
     >>> bprint(_globre(br'**'))
     .*
     >>> bprint(_globre(br'**/a'))
     (?:.*/)?a
     >>> bprint(_globre(br'a/**/b'))
     a/(?:.*/)?b
     >>> bprint(_globre(br'[a*?!^][^b][!c]'))
     [a*?!^][\^b][^c]
     >>> bprint(_globre(br'{a,b}'))
     (?:a|b)
     >>> bprint(_globre(br'.\*\?'))
     \.\*\?
     '''
     i, n = 0, len(pat)
     res = b''
     group = 0
     escape = util.stringutil.regexbytesescapemap.get
 
     def peek():
         return i < n and pat[i : i + 1]
 
     while i < n:
         c = pat[i : i + 1]
         i += 1
         if c not in b'*?[{},\\':
             res += escape(c, c)
         elif c == b'*':
             if peek() == b'*':
                 i += 1
                 if peek() == b'/':
                     i += 1
                     res += b'(?:.*/)?'
                 else:
                     res += b'.*'
             else:
                 res += b'[^/]*'
         elif c == b'?':
             res += b'.'
         elif c == b'[':
             j = i
             if j < n and pat[j : j + 1] in b'!]':
                 j += 1
             while j < n and pat[j : j + 1] != b']':
                 j += 1
             if j >= n:
                 res += b'\\['
             else:
                 stuff = pat[i:j].replace(b'\\', b'\\\\')
                 i = j + 1
                 if stuff[0:1] == b'!':
                     stuff = b'^' + stuff[1:]
                 elif stuff[0:1] == b'^':
                     stuff = b'\\' + stuff
                 res = b'%s[%s]' % (res, stuff)
         elif c == b'{':
             group += 1
             res += b'(?:'
         elif c == b'}' and group:
             res += b')'
             group -= 1
         elif c == b',' and group:
             res += b'|'
         elif c == b'\\':
             p = peek()
             if p:
                 i += 1
                 res += escape(p, p)
             else:
                 res += escape(c, c)
         else:
             res += escape(c, c)
     return res
 
 
 def _regex(kind, pat, globsuffix):
     '''Convert a (normalized) pattern of any kind into a
     regular expression.
     globsuffix is appended to the regexp of globs.'''
     if not pat and kind in (b'glob', b'relpath'):
         return b''
     if kind == b're':
         return pat
     if kind in (b'path', b'relpath'):
         if pat == b'.':
             return b''
         return util.stringutil.reescape(pat) + b'(?:/|$)'
     if kind == b'rootfilesin':
         if pat == b'.':
             escaped = b''
         else:
             # Pattern is a directory name.
             escaped = util.stringutil.reescape(pat) + b'/'
         # Anything after the pattern must be a non-directory.
         return escaped + b'[^/]+$'
     if kind == b'relglob':
         globre = _globre(pat)
         if globre.startswith(b'[^/]*'):
             # When pat has the form *XYZ (common), make the returned regex more
             # legible by returning the regex for **XYZ instead of **/*XYZ.
             return b'.*' + globre[len(b'[^/]*') :] + globsuffix
         return b'(?:|.*/)' + globre + globsuffix
     if kind == b'relre':
         if pat.startswith(b'^'):
             return pat
         return b'.*' + pat
     if kind in (b'glob', b'rootglob'):
         return _globre(pat) + globsuffix
     raise error.ProgrammingError(b'not a regex pattern: %s:%s' % (kind, pat))
 
 
 def _buildmatch(kindpats, globsuffix, root):
     '''Return regexp string and a matcher function for kindpats.
     globsuffix is appended to the regexp of globs.'''
     matchfuncs = []
 
     subincludes, kindpats = _expandsubinclude(kindpats, root)
     if subincludes:
         submatchers = {}
 
         def matchsubinclude(f):
             for prefix, matcherargs in subincludes:
                 if f.startswith(prefix):
                     mf = submatchers.get(prefix)
                     if mf is None:
                         mf = match(*matcherargs)
                         submatchers[prefix] = mf
 
                     if mf(f[len(prefix) :]):
                         return True
             return False
 
         matchfuncs.append(matchsubinclude)
 
     regex = b''
     if kindpats:
         if all(k == b'rootfilesin' for k, p, s in kindpats):
             dirs = {p for k, p, s in kindpats}
 
             def mf(f):
                 i = f.rfind(b'/')
                 if i >= 0:
                     dir = f[:i]
                 else:
                     dir = b'.'
                 return dir in dirs
 
             regex = b'rootfilesin: %s' % stringutil.pprint(list(sorted(dirs)))
             matchfuncs.append(mf)
         else:
             regex, mf = _buildregexmatch(kindpats, globsuffix)
             matchfuncs.append(mf)
 
     if len(matchfuncs) == 1:
         return regex, matchfuncs[0]
     else:
         return regex, lambda f: any(mf(f) for mf in matchfuncs)
 
 
 MAX_RE_SIZE = 20000
 
 
 def _joinregexes(regexps):
     """gather multiple regular expressions into a single one"""
     return b'|'.join(regexps)
 
 
 def _buildregexmatch(kindpats, globsuffix):
     """Build a match function from a list of kinds and kindpats,
     return regexp string and a matcher function.
 
     Test too large input
     >>> _buildregexmatch([
     ...     (b'relglob', b'?' * MAX_RE_SIZE, b'')
     ... ], b'$')
     Traceback (most recent call last):
     ...
     Abort: matcher pattern is too long (20009 bytes)
     """
     try:
         allgroups = []
         regexps = [_regex(k, p, globsuffix) for (k, p, s) in kindpats]
         fullregexp = _joinregexes(regexps)
 
         startidx = 0
         groupsize = 0
         for idx, r in enumerate(regexps):
             piecesize = len(r)
             if piecesize > MAX_RE_SIZE:
                 msg = _(b"matcher pattern is too long (%d bytes)") % piecesize
                 raise error.Abort(msg)
             elif (groupsize + piecesize) > MAX_RE_SIZE:
                 group = regexps[startidx:idx]
                 allgroups.append(_joinregexes(group))
                 startidx = idx
                 groupsize = 0
             groupsize += piecesize + 1
 
         if startidx == 0:
             matcher = _rematcher(fullregexp)
             func = lambda s: bool(matcher(s))
         else:
             group = regexps[startidx:]
             allgroups.append(_joinregexes(group))
             allmatchers = [_rematcher(g) for g in allgroups]
             func = lambda s: any(m(s) for m in allmatchers)
         return fullregexp, func
     except re.error:
         for k, p, s in kindpats:
             try:
                 _rematcher(_regex(k, p, globsuffix))
             except re.error:
                 if s:
                     raise error.Abort(
                         _(b"%s: invalid pattern (%s): %s") % (s, k, p)
                     )
                 else:
                     raise error.Abort(_(b"invalid pattern (%s): %s") % (k, p))
         raise error.Abort(_(b"invalid pattern"))
 
 
 def _patternrootsanddirs(kindpats):
     '''Returns roots and directories corresponding to each pattern.
 
     This calculates the roots and directories exactly matching the patterns and
     returns a tuple of (roots, dirs) for each. It does not return other
     directories which may also need to be considered, like the parent
     directories.
     '''
     r = []
     d = []
     for kind, pat, source in kindpats:
         if kind in (b'glob', b'rootglob'):  # find the non-glob prefix
             root = []
             for p in pat.split(b'/'):
                 if b'[' in p or b'{' in p or b'*' in p or b'?' in p:
                     break
                 root.append(p)
             r.append(b'/'.join(root))
         elif kind in (b'relpath', b'path'):
             if pat == b'.':
                 pat = b''
             r.append(pat)
         elif kind in (b'rootfilesin',):
             if pat == b'.':
                 pat = b''
             d.append(pat)
         else:  # relglob, re, relre
             r.append(b'')
     return r, d
 
 
 def _roots(kindpats):
     '''Returns root directories to match recursively from the given patterns.'''
     roots, dirs = _patternrootsanddirs(kindpats)
     return roots
 
 
 def _rootsdirsandparents(kindpats):
     '''Returns roots and exact directories from patterns.
 
     `roots` are directories to match recursively, `dirs` should
     be matched non-recursively, and `parents` are the implicitly required
     directories to walk to items in either roots or dirs.
 
     Returns a tuple of (roots, dirs, parents).
 
     >>> r = _rootsdirsandparents(
     ...     [(b'glob', b'g/h/*', b''), (b'glob', b'g/h', b''),
     ...      (b'glob', b'g*', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['g/h', 'g/h', ''], []) ['', 'g']
     >>> r = _rootsdirsandparents(
     ...     [(b'rootfilesin', b'g/h', b''), (b'rootfilesin', b'', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     ([], ['g/h', '']) ['', 'g']
     >>> r = _rootsdirsandparents(
     ...     [(b'relpath', b'r', b''), (b'path', b'p/p', b''),
     ...      (b'path', b'', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['r', 'p/p', ''], []) ['', 'p']
     >>> r = _rootsdirsandparents(
     ...     [(b'relglob', b'rg*', b''), (b're', b're/', b''),
     ...      (b'relre', b'rr', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['', '', ''], []) ['']
     '''
     r, d = _patternrootsanddirs(kindpats)
 
     p = set()
     # Add the parents as non-recursive/exact directories, since they must be
     # scanned to get to either the roots or the other exact directories.
     p.update(pathutil.dirs(d))
     p.update(pathutil.dirs(r))
 
     # FIXME: all uses of this function convert these to sets, do so before
     # returning.
     # FIXME: all uses of this function do not need anything in 'roots' and
     # 'dirs' to also be in 'parents', consider removing them before returning.
     return r, d, p
 
 
 def _explicitfiles(kindpats):
     '''Returns the potential explicit filenames from the patterns.
 
     >>> _explicitfiles([(b'path', b'foo/bar', b'')])
     ['foo/bar']
     >>> _explicitfiles([(b'rootfilesin', b'foo/bar', b'')])
     []
     '''
     # Keep only the pattern kinds where one can specify filenames (vs only
     # directory names).
     filable = [kp for kp in kindpats if kp[0] not in (b'rootfilesin',)]
     return _roots(filable)
 
 
 def _prefix(kindpats):
     '''Whether all the patterns match a prefix (i.e. recursively)'''
     for kind, pat, source in kindpats:
         if kind not in (b'path', b'relpath'):
             return False
     return True
 
 
 _commentre = None
 
 
 def readpatternfile(filepath, warn, sourceinfo=False):
     '''parse a pattern file, returning a list of
     patterns. These patterns should be given to compile()
     to be validated and converted into a match function.
 
     trailing white space is dropped.
     the escape character is backslash.
     comments start with #.
     empty lines are skipped.
 
     lines can be of the following formats:
 
     syntax: regexp # defaults following lines to non-rooted regexps
     syntax: glob   # defaults following lines to non-rooted globs
     re:pattern     # non-rooted regular expression
     glob:pattern   # non-rooted glob
     rootglob:pat   # rooted glob (same root as ^ in regexps)
     pattern        # pattern of the current default type
 
     if sourceinfo is set, returns a list of tuples:
     (pattern, lineno, originalline).
     This is useful to debug ignore patterns.
     '''
 
     syntaxes = {
         b're': b'relre:',
         b'regexp': b'relre:',
         b'glob': b'relglob:',
         b'rootglob': b'rootglob:',
         b'include': b'include',
         b'subinclude': b'subinclude',
     }
     syntax = b'relre:'
     patterns = []
 
     fp = open(filepath, b'rb')
     for lineno, line in enumerate(util.iterfile(fp), start=1):
         if b"#" in line:
             global _commentre
             if not _commentre:
                 _commentre = util.re.compile(br'((?:^|[^\\])(?:\\\\)*)#.*')
             # remove comments prefixed by an even number of escapes
             m = _commentre.search(line)
             if m:
                 line = line[: m.end(1)]
             # fixup properly escaped comments that survived the above
             line = line.replace(b"\\#", b"#")
         line = line.rstrip()
         if not line:
             continue
 
         if line.startswith(b'syntax:'):
             s = line[7:].strip()
             try:
                 syntax = syntaxes[s]
             except KeyError:
                 if warn:
                     warn(
                         _(b"%s: ignoring invalid syntax '%s'\n") % (filepath, s)
                     )
             continue
 
         linesyntax = syntax
         for s, rels in pycompat.iteritems(syntaxes):
             if line.startswith(rels):
                 linesyntax = rels
                 line = line[len(rels) :]
                 break
             elif line.startswith(s + b':'):
                 linesyntax = rels
                 line = line[len(s) + 1 :]
                 break
         if sourceinfo:
             patterns.append((linesyntax + line, lineno, line))
         else:
             patterns.append(linesyntax + line)
     fp.close()
     return patterns

tests/test-subrepo-deep-nested-change.t

@@ -1,1177 +1,1182 b''
   $ cat >> $HGRCPATH <<EOF
   > [extdiff]
   > # for portability:
   > pdiff = sh "$RUNTESTDIR/pdiff"
   > [progress]
   > disable=False
   > assume-tty = 1
   > delay = 0
   > # set changedelay really large so we don't see nested topics
   > changedelay = 30000
   > format = topic bar number
   > refresh = 0
   > width = 60
   > EOF
 
 Preparing the subrepository 'sub2'
 
   $ hg init sub2
   $ echo sub2 > sub2/sub2
   $ hg add -R sub2
   adding sub2/sub2
   $ hg commit -R sub2 -m "sub2 import"
 
 Preparing the 'sub1' repo which depends on the subrepo 'sub2'
 
   $ hg init sub1
   $ echo sub1 > sub1/sub1
   $ echo "sub2 = ../sub2" > sub1/.hgsub
   $ hg clone sub2 sub1/sub2
   \r (no-eol) (esc)
   linking [ <=>                                           ] 1\r (no-eol) (esc)
   linking [  <=>                                          ] 2\r (no-eol) (esc)
   linking [   <=>                                         ] 3\r (no-eol) (esc)
   linking [    <=>                                        ] 4\r (no-eol) (esc)
   linking [     <=>                                       ] 5\r (no-eol) (esc)
   linking [      <=>                                      ] 6\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   updating [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg add -R sub1
   adding sub1/.hgsub
   adding sub1/sub1
   $ hg commit -R sub1 -m "sub1 import"
 
 Preparing the 'main' repo which depends on the subrepo 'sub1'
 
   $ hg init main
   $ echo main > main/main
   $ echo "sub1 = ../sub1" > main/.hgsub
   $ hg clone sub1 main/sub1
   \r (no-eol) (esc)
   linking [ <=>                                           ] 1\r (no-eol) (esc)
   linking [  <=>                                          ] 2\r (no-eol) (esc)
   linking [   <=>                                         ] 3\r (no-eol) (esc)
   linking [    <=>                                        ] 4\r (no-eol) (esc)
   linking [     <=>                                       ] 5\r (no-eol) (esc)
   linking [      <=>                                      ] 6\r (no-eol) (esc)
   linking [       <=>                                     ] 7\r (no-eol) (esc)
   linking [        <=>                                    ] 8\r (no-eol) (esc)
   linking [         <=>                                   ] 9\r (no-eol) (esc) (reposimplestore !)
   linking [          <=>                                 ] 10\r (no-eol) (esc) (reposimplestore !)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   updating [===========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   linking [ <=>                                           ] 1\r (no-eol) (esc)
   linking [  <=>                                          ] 2\r (no-eol) (esc)
   linking [   <=>                                         ] 3\r (no-eol) (esc)
   linking [    <=>                                        ] 4\r (no-eol) (esc)
   linking [     <=>                                       ] 5\r (no-eol) (esc)
   linking [      <=>                                      ] 6\r (no-eol) (esc)
   updating [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   updating to branch default
   cloning subrepo sub2 from $TESTTMP/sub2
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg add -R main
   adding main/.hgsub
   adding main/main
   $ hg commit -R main -m "main import"
 
 #if serve
 
 Unfortunately, subrepos not at their nominal location cannot be cloned.  But
 they are still served from their location within the local repository.  The only
 reason why 'main' can be cloned via the filesystem is because 'sub1' and 'sub2'
 are also available as siblings of 'main'.
 
   $ hg serve -R main --debug -S -p $HGPORT -d --pid-file=hg1.pid -E error.log -A access.log
   adding  = $TESTTMP/main
   adding sub1 = $TESTTMP/main/sub1
   adding sub1/sub2 = $TESTTMP/main/sub1/sub2
   listening at http://*:$HGPORT/ (bound to *:$HGPORT) (glob) (?)
   adding  = $TESTTMP/main (?)
   adding sub1 = $TESTTMP/main/sub1 (?)
   adding sub1/sub2 = $TESTTMP/main/sub1/sub2 (?)
   $ cat hg1.pid >> $DAEMON_PIDS
 
   $ hg clone http://localhost:$HGPORT httpclone --config progress.disable=True
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 3 changes to 3 files
   new changesets 7f491f53a367
   updating to branch default
   cloning subrepo sub1 from http://localhost:$HGPORT/../sub1
   abort: HTTP Error 404: Not Found
   [255]
 
   $ cat access.log
   * "GET /?cmd=capabilities HTTP/1.1" 200 - (glob)
   * "GET /?cmd=batch HTTP/1.1" 200 - * (glob)
   * "GET /?cmd=getbundle HTTP/1.1" 200 - * (glob)
   * "GET /../sub1?cmd=capabilities HTTP/1.1" 404 - (glob)
   $ cat error.log
 
   $ killdaemons.py
   $ rm hg1.pid error.log access.log
 #endif
 
 Cleaning both repositories, just as a clone -U
 
   $ hg up -C -R sub2 null
   \r (no-eol) (esc)
   updating [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ hg up -C -R sub1 null
   \r (no-eol) (esc)
   updating [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   updating [===========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   0 files updated, 0 files merged, 3 files removed, 0 files unresolved
   $ hg up -C -R main null
   \r (no-eol) (esc)
   updating [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   updating [===========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   updating [===========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   0 files updated, 0 files merged, 3 files removed, 0 files unresolved
   $ rm -rf main/sub1
   $ rm -rf sub1/sub2
 
 Clone main
 
   $ hg --config extensions.largefiles= clone main cloned
   \r (no-eol) (esc)
   linking [ <=>                                           ] 1\r (no-eol) (esc)
   linking [  <=>                                          ] 2\r (no-eol) (esc)
   linking [   <=>                                         ] 3\r (no-eol) (esc)
   linking [    <=>                                        ] 4\r (no-eol) (esc)
   linking [     <=>                                       ] 5\r (no-eol) (esc)
   linking [      <=>                                      ] 6\r (no-eol) (esc)
   linking [       <=>                                     ] 7\r (no-eol) (esc)
   linking [        <=>                                    ] 8\r (no-eol) (esc)
   linking [         <=>                                   ] 9\r (no-eol) (esc) (reposimplestore !)
   linking [          <=>                                 ] 10\r (no-eol) (esc) (reposimplestore !)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   updating [===========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   linking [ <=>                                           ] 1\r (no-eol) (esc)
   linking [  <=>                                          ] 2\r (no-eol) (esc)
   linking [   <=>                                         ] 3\r (no-eol) (esc)
   linking [    <=>                                        ] 4\r (no-eol) (esc)
   linking [     <=>                                       ] 5\r (no-eol) (esc)
   linking [      <=>                                      ] 6\r (no-eol) (esc)
   linking [       <=>                                     ] 7\r (no-eol) (esc)
   linking [        <=>                                    ] 8\r (no-eol) (esc)
   linking [         <=>                                   ] 9\r (no-eol) (esc) (reposimplestore !)
   linking [          <=>                                 ] 10\r (no-eol) (esc) (reposimplestore !)
   updating [===========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   linking [         <=>                                   ] 1\r (no-eol) (esc) (reporevlogstore !)
   linking [          <=>                                  ] 2\r (no-eol) (esc) (reporevlogstore !)
   linking [           <=>                                 ] 3\r (no-eol) (esc) (reporevlogstore !)
   linking [            <=>                                ] 4\r (no-eol) (esc) (reporevlogstore !)
   linking [             <=>                               ] 5\r (no-eol) (esc) (reporevlogstore !)
   linking [              <=>                              ] 6\r (no-eol) (esc) (reporevlogstore !)
   linking [           <=>                                 ] 1\r (no-eol) (esc) (reposimplestore !)
   linking [            <=>                                ] 2\r (no-eol) (esc) (reposimplestore !)
   linking [             <=>                               ] 3\r (no-eol) (esc) (reposimplestore !)
   linking [              <=>                              ] 4\r (no-eol) (esc) (reposimplestore !)
   linking [               <=>                             ] 5\r (no-eol) (esc) (reposimplestore !)
   linking [                <=>                            ] 6\r (no-eol) (esc) (reposimplestore !)
   updating [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   updating to branch default
   cloning subrepo sub1 from $TESTTMP/sub1
   cloning subrepo sub1/sub2 from $TESTTMP/sub2
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 Largefiles is NOT enabled in the clone if the source repo doesn't require it
   $ grep largefiles cloned/.hg/hgrc
   [1]
 
 Checking cloned repo ids
 
   $ printf "cloned " ; hg id -R cloned
   cloned 7f491f53a367 tip
   $ printf "cloned/sub1 " ; hg id -R cloned/sub1
   cloned/sub1 fc3b4ce2696f tip
   $ printf "cloned/sub1/sub2 " ; hg id -R cloned/sub1/sub2
   cloned/sub1/sub2 c57a0840e3ba tip
 
 debugsub output for main and sub1
 
   $ hg debugsub -R cloned
   path sub1
    source   ../sub1
    revision fc3b4ce2696f7741438c79207583768f2ce6b0dd
   $ hg debugsub -R cloned/sub1
   path sub2
    source   ../sub2
    revision c57a0840e3badd667ef3c3ef65471609acb2ba3c
 
 Modifying deeply nested 'sub2'
 
   $ echo modified > cloned/sub1/sub2/sub2
   $ hg commit --subrepos -m "deep nested modif should trigger a commit" -R cloned
   committing subrepository sub1
   committing subrepository sub1/sub2
 
 Checking modified node ids
 
   $ printf "cloned " ; hg id -R cloned
   cloned ffe6649062fe tip
   $ printf "cloned/sub1 " ; hg id -R cloned/sub1
   cloned/sub1 2ecb03bf44a9 tip
   $ printf "cloned/sub1/sub2 " ; hg id -R cloned/sub1/sub2
   cloned/sub1/sub2 53dd3430bcaf tip
 
 debugsub output for main and sub1
 
   $ hg debugsub -R cloned
   path sub1
    source   ../sub1
    revision 2ecb03bf44a94e749e8669481dd9069526ce7cb9
   $ hg debugsub -R cloned/sub1
   path sub2
    source   ../sub2
    revision 53dd3430bcaf5ab4a7c48262bcad6d441f510487
 
 Check that deep archiving works
 
   $ cd cloned
   $ echo 'test' > sub1/sub2/test.txt
   $ hg --config extensions.largefiles=! add sub1/sub2/test.txt
   $ mkdir sub1/sub2/folder
   $ echo 'subfolder' > sub1/sub2/folder/test.txt
   $ hg ci -ASm "add test.txt"
   adding sub1/sub2/folder/test.txt
   committing subrepository sub1
   committing subrepository sub1/sub2
 
   $ rm -r main
   $ hg archive -S -qr 'wdir()' ../wdir
   $ cat ../wdir/.hg_archival.txt
   repo: 7f491f53a367861f47ee64a80eb997d1f341b77a
   node: 9bb10eebee29dc0f1201dcf5977b811a540255fd+
   branch: default
   latesttag: null
   latesttagdistance: 4
   changessincelatesttag: 4
   $ hg update -Cq .
 
 A deleted subrepo file is flagged as dirty, like the top level repo
 
   $ rm -r ../wdir sub1/sub2/folder/test.txt
   $ hg archive -S -qr 'wdir()' ../wdir
   $ cat ../wdir/.hg_archival.txt
   repo: 7f491f53a367861f47ee64a80eb997d1f341b77a
   node: 9bb10eebee29dc0f1201dcf5977b811a540255fd+
   branch: default
   latesttag: null
   latesttagdistance: 4
   changessincelatesttag: 4
   $ hg update -Cq .
   $ rm -r ../wdir
 
   $ hg archive -S -qr 'wdir()' ../wdir \
   >            --config 'experimental.archivemetatemplate=archived {node|short}\n'
   $ cat ../wdir/.hg_archival.txt
   archived ffffffffffff
   $ rm -r ../wdir
 
 .. but first take a detour through some deep removal testing
 
   $ hg remove -S -I 're:.*.txt' .
   \r (no-eol) (esc)
   searching [==========================================>] 1/1\r (no-eol) (esc)
   searching [==========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [=====================>                      ] 1/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   removing sub1/sub2/folder/test.txt
   removing sub1/sub2/test.txt
   $ hg status -S
   R sub1/sub2/folder/test.txt
   R sub1/sub2/test.txt
   $ hg update -Cq
   $ hg remove -I 're:.*.txt' sub1
   \r (no-eol) (esc)
   searching [==========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ hg status -S
   $ hg remove sub1/sub2/folder/test.txt
   \r (no-eol) (esc)
   searching [==========================================>] 1/1\r (no-eol) (esc)
   searching [==========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ hg remove sub1/.hgsubstate
   \r (no-eol) (esc)
   searching [==========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ mv sub1/.hgsub sub1/x.hgsub
   $ hg status -S
   warning: subrepo spec file 'sub1/.hgsub' not found
   R sub1/.hgsubstate
   R sub1/sub2/folder/test.txt
   ! sub1/.hgsub
   ? sub1/x.hgsub
+  $ hg status -R sub1
+  warning: subrepo spec file 'sub1/.hgsub' not found
+  R .hgsubstate
+  ! .hgsub
+  ? x.hgsub
   $ mv sub1/x.hgsub sub1/.hgsub
   $ hg update -Cq
   $ touch sub1/foo
   $ hg forget sub1/sub2/folder/test.txt
   $ rm sub1/sub2/test.txt
 
 Test relative path printing + subrepos
   $ mkdir -p foo/bar
   $ cd foo
   $ touch bar/abc
   $ hg addremove -S ..
   \r (no-eol) (esc)
   searching for exact renames [========================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   adding ../sub1/sub2/folder/test.txt
   removing ../sub1/sub2/test.txt
   adding ../sub1/foo
   adding bar/abc
   $ cd ..
   $ hg status -S
   A foo/bar/abc
   A sub1/foo
   R sub1/sub2/test.txt
 
 Archive wdir() with subrepos
   $ hg rm main
   \r (no-eol) (esc)
   deleting [===========================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ hg archive -S -r 'wdir()' ../wdir
   \r (no-eol) (esc)
   archiving [                                           ] 0/3\r (no-eol) (esc)
   archiving [=============>                             ] 1/3\r (no-eol) (esc)
   archiving [===========================>               ] 2/3\r (no-eol) (esc)
   archiving [==========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/4\r (no-eol) (esc)
   archiving (sub1) [========>                           ] 1/4\r (no-eol) (esc)
   archiving (sub1) [=================>                  ] 2/4\r (no-eol) (esc)
   archiving (sub1) [==========================>         ] 3/4\r (no-eol) (esc)
   archiving (sub1) [===================================>] 4/4\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/2\r (no-eol) (esc)
   archiving (sub1/sub2) [==============>                ] 1/2\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ diff -r . ../wdir | egrep -v '\.hg$|^Common subdirectories:'
   Only in ../wdir: .hg_archival.txt
 
   $ find ../wdir -type f | sort
   ../wdir/.hg_archival.txt
   ../wdir/.hgsub
   ../wdir/.hgsubstate
   ../wdir/foo/bar/abc
   ../wdir/sub1/.hgsub
   ../wdir/sub1/.hgsubstate
   ../wdir/sub1/foo
   ../wdir/sub1/sub1
   ../wdir/sub1/sub2/folder/test.txt
   ../wdir/sub1/sub2/sub2
 
   $ cat ../wdir/.hg_archival.txt
   repo: 7f491f53a367861f47ee64a80eb997d1f341b77a
   node: 9bb10eebee29dc0f1201dcf5977b811a540255fd+
   branch: default
   latesttag: null
   latesttagdistance: 4
   changessincelatesttag: 4
 
 Attempting to archive 'wdir()' with a missing file is handled gracefully
   $ rm sub1/sub1
   $ rm -r ../wdir
   $ hg archive -v -S -r 'wdir()' ../wdir
   \r (no-eol) (esc)
   archiving [                                           ] 0/3\r (no-eol) (esc)
   archiving [=============>                             ] 1/3\r (no-eol) (esc)
   archiving [===========================>               ] 2/3\r (no-eol) (esc)
   archiving [==========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/3\r (no-eol) (esc)
   archiving (sub1) [===========>                        ] 1/3\r (no-eol) (esc)
   archiving (sub1) [=======================>            ] 2/3\r (no-eol) (esc)
   archiving (sub1) [===================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/2\r (no-eol) (esc)
   archiving (sub1/sub2) [==============>                ] 1/2\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ find ../wdir -type f | sort
   ../wdir/.hg_archival.txt
   ../wdir/.hgsub
   ../wdir/.hgsubstate
   ../wdir/foo/bar/abc
   ../wdir/sub1/.hgsub
   ../wdir/sub1/.hgsubstate
   ../wdir/sub1/foo
   ../wdir/sub1/sub2/folder/test.txt
   ../wdir/sub1/sub2/sub2
 
 Continue relative path printing + subrepos
   $ hg update -Cq
   $ rm -r ../wdir
   $ hg archive -S -r 'wdir()' ../wdir
   \r (no-eol) (esc)
   archiving [                                           ] 0/3\r (no-eol) (esc)
   archiving [=============>                             ] 1/3\r (no-eol) (esc)
   archiving [===========================>               ] 2/3\r (no-eol) (esc)
   archiving [==========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/3\r (no-eol) (esc)
   archiving (sub1) [===========>                        ] 1/3\r (no-eol) (esc)
   archiving (sub1) [=======================>            ] 2/3\r (no-eol) (esc)
   archiving (sub1) [===================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/3\r (no-eol) (esc)
   archiving (sub1/sub2) [=========>                     ] 1/3\r (no-eol) (esc)
   archiving (sub1/sub2) [===================>           ] 2/3\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ cat ../wdir/.hg_archival.txt
   repo: 7f491f53a367861f47ee64a80eb997d1f341b77a
   node: 9bb10eebee29dc0f1201dcf5977b811a540255fd
   branch: default
   latesttag: null
   latesttagdistance: 4
   changessincelatesttag: 4
 
   $ touch sub1/sub2/folder/bar
   $ hg addremove sub1/sub2
   adding sub1/sub2/folder/bar
   $ hg status -S
   A sub1/sub2/folder/bar
   ? foo/bar/abc
   ? sub1/foo
   $ hg update -Cq
   $ hg addremove sub1
   adding sub1/sub2/folder/bar
   adding sub1/foo
   $ hg update -Cq
   $ rm sub1/sub2/folder/test.txt
   $ rm sub1/sub2/test.txt
   $ hg ci -ASm "remove test.txt"
   adding sub1/sub2/folder/bar
   removing sub1/sub2/folder/test.txt
   removing sub1/sub2/test.txt
   adding sub1/foo
   adding foo/bar/abc
   committing subrepository sub1
   committing subrepository sub1/sub2
 
   $ hg forget sub1/sub2/sub2
   $ echo x > sub1/sub2/x.txt
   $ hg add sub1/sub2/x.txt
 
 Files sees uncommitted adds and removes in subrepos
   $ hg files -S
   .hgsub
   .hgsubstate
   foo/bar/abc
   main
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/foo
   sub1/sub1
   sub1/sub2/folder/bar
   sub1/sub2/x.txt
 
   $ hg files -S "set:eol('dos') or eol('unix') or size('<= 0')"
   .hgsub
   .hgsubstate
   foo/bar/abc
   main
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/foo
   sub1/sub1
   sub1/sub2/folder/bar
   sub1/sub2/x.txt
 
   $ hg files -r '.^' -S "set:eol('dos') or eol('unix')"
   .hgsub
   .hgsubstate
   main
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/sub1
   sub1/sub2/folder/test.txt
   sub1/sub2/sub2
   sub1/sub2/test.txt
 
   $ hg files sub1
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/foo
   sub1/sub1
   sub1/sub2/folder/bar
   sub1/sub2/x.txt
 
   $ hg files sub1/sub2
   sub1/sub2/folder/bar
   sub1/sub2/x.txt
 
   $ hg files
   .hgsub
   .hgsubstate
   foo/bar/abc
   main
 
   $ hg files -S -r '.^' sub1/sub2/folder
   sub1/sub2/folder/test.txt
 
   $ hg files -S -r '.^' sub1/sub2/missing
   sub1/sub2/missing: no such file in rev 78026e779ea6
   [1]
 
   $ hg files -r '.^' sub1/
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/sub1
   sub1/sub2/folder/test.txt
   sub1/sub2/sub2
   sub1/sub2/test.txt
 
   $ hg files -r '.^' sub1/sub2
   sub1/sub2/folder/test.txt
   sub1/sub2/sub2
   sub1/sub2/test.txt
 
   $ hg rollback -q
   $ hg up -Cq
 
   $ hg --config extensions.largefiles=! archive -S ../archive_all
   \r (no-eol) (esc)
   archiving [                                           ] 0/3\r (no-eol) (esc)
   archiving [=============>                             ] 1/3\r (no-eol) (esc)
   archiving [===========================>               ] 2/3\r (no-eol) (esc)
   archiving [==========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/3\r (no-eol) (esc)
   archiving (sub1) [===========>                        ] 1/3\r (no-eol) (esc)
   archiving (sub1) [=======================>            ] 2/3\r (no-eol) (esc)
   archiving (sub1) [===================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/3\r (no-eol) (esc)
   archiving (sub1/sub2) [=========>                     ] 1/3\r (no-eol) (esc)
   archiving (sub1/sub2) [===================>           ] 2/3\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ find ../archive_all | sort
   ../archive_all
   ../archive_all/.hg_archival.txt
   ../archive_all/.hgsub
   ../archive_all/.hgsubstate
   ../archive_all/main
   ../archive_all/sub1
   ../archive_all/sub1/.hgsub
   ../archive_all/sub1/.hgsubstate
   ../archive_all/sub1/sub1
   ../archive_all/sub1/sub2
   ../archive_all/sub1/sub2/folder
   ../archive_all/sub1/sub2/folder/test.txt
   ../archive_all/sub1/sub2/sub2
   ../archive_all/sub1/sub2/test.txt
 
 Check that archive -X works in deep subrepos
 
   $ hg --config extensions.largefiles=! archive -S -X '**test*' ../archive_exclude
   \r (no-eol) (esc)
   archiving [                                           ] 0/3\r (no-eol) (esc)
   archiving [=============>                             ] 1/3\r (no-eol) (esc)
   archiving [===========================>               ] 2/3\r (no-eol) (esc)
   archiving [==========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/3\r (no-eol) (esc)
   archiving (sub1) [===========>                        ] 1/3\r (no-eol) (esc)
   archiving (sub1) [=======================>            ] 2/3\r (no-eol) (esc)
   archiving (sub1) [===================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/1\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ find ../archive_exclude | sort
   ../archive_exclude
   ../archive_exclude/.hg_archival.txt
   ../archive_exclude/.hgsub
   ../archive_exclude/.hgsubstate
   ../archive_exclude/main
   ../archive_exclude/sub1
   ../archive_exclude/sub1/.hgsub
   ../archive_exclude/sub1/.hgsubstate
   ../archive_exclude/sub1/sub1
   ../archive_exclude/sub1/sub2
   ../archive_exclude/sub1/sub2/sub2
 
   $ hg --config extensions.largefiles=! archive -S -I '**test*' ../archive_include
   \r (no-eol) (esc)
   archiving (sub1) [ <=>                                  ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/2\r (no-eol) (esc)
   archiving (sub1/sub2) [==============>                ] 1/2\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ find ../archive_include | sort
   ../archive_include
   ../archive_include/sub1
   ../archive_include/sub1/sub2
   ../archive_include/sub1/sub2/folder
   ../archive_include/sub1/sub2/folder/test.txt
   ../archive_include/sub1/sub2/test.txt
 
 Check that deep archive works with largefiles (which overrides hgsubrepo impl)
 This also tests the repo.ui regression in 43fb170a23bd, and that lf subrepo
 subrepos are archived properly.
 Note that add --large through a subrepo currently adds the file as a normal file
 
   $ echo "large" > sub1/sub2/large.bin
   $ hg --config extensions.largefiles= add --large -R sub1/sub2 sub1/sub2/large.bin
   $ echo "large" > large.bin
   $ hg --config extensions.largefiles= add --large large.bin
   $ hg --config extensions.largefiles= ci -S -m "add large files"
   committing subrepository sub1
   committing subrepository sub1/sub2
 
   $ hg --config extensions.largefiles= archive -S ../archive_lf
   $ find ../archive_lf | sort
   ../archive_lf
   ../archive_lf/.hg_archival.txt
   ../archive_lf/.hgsub
   ../archive_lf/.hgsubstate
   ../archive_lf/large.bin
   ../archive_lf/main
   ../archive_lf/sub1
   ../archive_lf/sub1/.hgsub
   ../archive_lf/sub1/.hgsubstate
   ../archive_lf/sub1/sub1
   ../archive_lf/sub1/sub2
   ../archive_lf/sub1/sub2/folder
   ../archive_lf/sub1/sub2/folder/test.txt
   ../archive_lf/sub1/sub2/large.bin
   ../archive_lf/sub1/sub2/sub2
   ../archive_lf/sub1/sub2/test.txt
   $ rm -rf ../archive_lf
 
 Exclude large files from main and sub-sub repo
 
   $ hg --config extensions.largefiles= archive -S -X '**.bin' ../archive_lf
   $ find ../archive_lf | sort
   ../archive_lf
   ../archive_lf/.hg_archival.txt
   ../archive_lf/.hgsub
   ../archive_lf/.hgsubstate
   ../archive_lf/main
   ../archive_lf/sub1
   ../archive_lf/sub1/.hgsub
   ../archive_lf/sub1/.hgsubstate
   ../archive_lf/sub1/sub1
   ../archive_lf/sub1/sub2
   ../archive_lf/sub1/sub2/folder
   ../archive_lf/sub1/sub2/folder/test.txt
   ../archive_lf/sub1/sub2/sub2
   ../archive_lf/sub1/sub2/test.txt
   $ rm -rf ../archive_lf
 
 Exclude normal files from main and sub-sub repo
 
   $ hg --config extensions.largefiles= archive -S -X '**.txt' -p '.' ../archive_lf.tgz
   $ tar -tzf ../archive_lf.tgz | sort
   .hgsub
   .hgsubstate
   large.bin
   main
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/sub1
   sub1/sub2/large.bin
   sub1/sub2/sub2
 
 Include normal files from within a largefiles subrepo
 
   $ hg --config extensions.largefiles= archive -S -I '**.txt' ../archive_lf
   $ find ../archive_lf | sort
   ../archive_lf
   ../archive_lf/.hg_archival.txt
   ../archive_lf/sub1
   ../archive_lf/sub1/sub2
   ../archive_lf/sub1/sub2/folder
   ../archive_lf/sub1/sub2/folder/test.txt
   ../archive_lf/sub1/sub2/test.txt
   $ rm -rf ../archive_lf
 
 Include large files from within a largefiles subrepo
 
   $ hg --config extensions.largefiles= archive -S -I '**.bin' ../archive_lf
   $ find ../archive_lf | sort
   ../archive_lf
   ../archive_lf/large.bin
   ../archive_lf/sub1
   ../archive_lf/sub1/sub2
   ../archive_lf/sub1/sub2/large.bin
   $ rm -rf ../archive_lf
 
 Find an exact largefile match in a largefiles subrepo
 
   $ hg --config extensions.largefiles= archive -S -I 'sub1/sub2/large.bin' ../archive_lf
   $ find ../archive_lf | sort
   ../archive_lf
   ../archive_lf/sub1
   ../archive_lf/sub1/sub2
   ../archive_lf/sub1/sub2/large.bin
   $ rm -rf ../archive_lf
 
 The local repo enables largefiles if a largefiles repo is cloned
 
   $ hg showconfig extensions
   extensions.largefiles=
 
   $ hg --config extensions.largefiles= clone -qU . ../lfclone
   $ grep largefiles ../lfclone/.hg/requires
   largefiles
 
 Find an exact match to a standin (should archive nothing)
   $ hg --config extensions.largefiles= archive -S -I 'sub/sub2/.hglf/large.bin' ../archive_lf
   $ find ../archive_lf 2> /dev/null | sort
 
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > largefiles=
   > [largefiles]
   > patterns=glob:**.dat
   > EOF
 
 Test forget through a deep subrepo with the largefiles extension, both a
 largefile and a normal file.  Then a largefile that hasn't been committed yet.
   $ touch sub1/sub2/untracked.txt
   $ touch sub1/sub2/large.dat
   $ hg forget sub1/sub2/large.bin sub1/sub2/test.txt sub1/sub2/untracked.txt
   not removing sub1/sub2/untracked.txt: file is already untracked
   [1]
   $ hg add --large --dry-run -v sub1/sub2/untracked.txt
   adding sub1/sub2/untracked.txt as a largefile
   $ hg add --large -v sub1/sub2/untracked.txt
   adding sub1/sub2/untracked.txt as a largefile
   $ hg add --normal -v sub1/sub2/large.dat
   adding sub1/sub2/large.dat
   $ hg forget -v sub1/sub2/untracked.txt
   removing sub1/sub2/untracked.txt
   $ hg status -S
   A sub1/sub2/large.dat
   R sub1/sub2/large.bin
   R sub1/sub2/test.txt
   ? foo/bar/abc
   ? sub1/sub2/untracked.txt
   ? sub1/sub2/x.txt
   $ hg add sub1/sub2
 
   $ hg archive -S -r 'wdir()' ../wdir2
   $ diff -r . ../wdir2 | egrep -v '\.hg$|^Common subdirectories:'
   Only in ../wdir2: .hg_archival.txt
   Only in .: .hglf
   Only in .: foo
   Only in ./sub1/sub2: large.bin
   Only in ./sub1/sub2: test.txt
   Only in ./sub1/sub2: untracked.txt
   Only in ./sub1/sub2: x.txt
   $ find ../wdir2 -type f | sort
   ../wdir2/.hg_archival.txt
   ../wdir2/.hgsub
   ../wdir2/.hgsubstate
   ../wdir2/large.bin
   ../wdir2/main
   ../wdir2/sub1/.hgsub
   ../wdir2/sub1/.hgsubstate
   ../wdir2/sub1/sub1
   ../wdir2/sub1/sub2/folder/test.txt
   ../wdir2/sub1/sub2/large.dat
   ../wdir2/sub1/sub2/sub2
   $ hg status -S -mac -n | sort
   .hgsub
   .hgsubstate
   large.bin
   main
   sub1/.hgsub
   sub1/.hgsubstate
   sub1/sub1
   sub1/sub2/folder/test.txt
   sub1/sub2/large.dat
   sub1/sub2/sub2
 
   $ hg ci -Sqm 'forget testing'
 
 Test 'wdir()' modified file archiving with largefiles
   $ echo 'mod' > main
   $ echo 'mod' > large.bin
   $ echo 'mod' > sub1/sub2/large.dat
   $ hg archive -S -r 'wdir()' ../wdir3
   $ diff -r . ../wdir3 | egrep -v '\.hg$|^Common subdirectories'
   Only in ../wdir3: .hg_archival.txt
   Only in .: .hglf
   Only in .: foo
   Only in ./sub1/sub2: large.bin
   Only in ./sub1/sub2: test.txt
   Only in ./sub1/sub2: untracked.txt
   Only in ./sub1/sub2: x.txt
   $ find ../wdir3 -type f | sort
   ../wdir3/.hg_archival.txt
   ../wdir3/.hgsub
   ../wdir3/.hgsubstate
   ../wdir3/large.bin
   ../wdir3/main
   ../wdir3/sub1/.hgsub
   ../wdir3/sub1/.hgsubstate
   ../wdir3/sub1/sub1
   ../wdir3/sub1/sub2/folder/test.txt
   ../wdir3/sub1/sub2/large.dat
   ../wdir3/sub1/sub2/sub2
   $ hg up -Cq
 
 Test issue4330: commit a directory where only normal files have changed
   $ touch foo/bar/large.dat
   $ hg add --large foo/bar/large.dat
   $ hg ci -m 'add foo/bar/large.dat'
   $ touch a.txt
   $ touch a.dat
   $ hg add -v foo/bar/abc a.txt a.dat
   adding a.dat as a largefile
   adding a.txt
   adding foo/bar/abc
   $ hg ci -m 'dir commit with only normal file deltas' foo/bar
   $ hg status
   A a.dat
   A a.txt
 
 Test a directory commit with a changed largefile and a changed normal file
   $ echo changed > foo/bar/large.dat
   $ echo changed > foo/bar/abc
   $ hg ci -m 'dir commit with normal and lf file deltas' foo
   $ hg status
   A a.dat
   A a.txt
 
   $ hg ci -m "add a.*"
   $ hg mv a.dat b.dat
   $ hg mv foo/bar/abc foo/bar/def
   $ hg status -C
   A b.dat
     a.dat
   A foo/bar/def
     foo/bar/abc
   R a.dat
   R foo/bar/abc
 
   $ hg ci -m "move large and normal"
   $ hg status -C --rev '.^' --rev .
   A b.dat
     a.dat
   A foo/bar/def
     foo/bar/abc
   R a.dat
   R foo/bar/abc
 
 
   $ echo foo > main
   $ hg ci -m "mod parent only"
   $ hg init sub3
   $ echo "sub3 = sub3" >> .hgsub
   $ echo xyz > sub3/a.txt
   $ hg add sub3/a.txt
   $ hg ci -Sm "add sub3"
   committing subrepository sub3
   $ cat .hgsub | grep -v sub3 > .hgsub1
   $ mv .hgsub1 .hgsub
   $ hg ci -m "remove sub3"
 
   $ hg log -r "subrepo()" --style compact
   0   7f491f53a367   1970-01-01 00:00 +0000   test
     main import
   
   1   ffe6649062fe   1970-01-01 00:00 +0000   test
     deep nested modif should trigger a commit
   
   2   9bb10eebee29   1970-01-01 00:00 +0000   test
     add test.txt
   
   3   7c64f035294f   1970-01-01 00:00 +0000   test
     add large files
   
   4   f734a59e2e35   1970-01-01 00:00 +0000   test
     forget testing
   
   11   9685a22af5db   1970-01-01 00:00 +0000   test
     add sub3
   
   12[tip]   2e0485b475b9   1970-01-01 00:00 +0000   test
     remove sub3
   
   $ hg log -r "subrepo('sub3')" --style compact
   11   9685a22af5db   1970-01-01 00:00 +0000   test
     add sub3
   
   12[tip]   2e0485b475b9   1970-01-01 00:00 +0000   test
     remove sub3
   
   $ hg log -r "subrepo('bogus')" --style compact
 
 
 Test .hgsubstate in the R state
 
   $ hg rm .hgsub .hgsubstate
   \r (no-eol) (esc)
   deleting [=====================>                      ] 1/2\r (no-eol) (esc)
   deleting [===========================================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   $ hg ci -m 'trash subrepo tracking'
 
   $ hg log -r "subrepo('re:sub\d+')" --style compact
   0   7f491f53a367   1970-01-01 00:00 +0000   test
     main import
   
   1   ffe6649062fe   1970-01-01 00:00 +0000   test
     deep nested modif should trigger a commit
   
   2   9bb10eebee29   1970-01-01 00:00 +0000   test
     add test.txt
   
   3   7c64f035294f   1970-01-01 00:00 +0000   test
     add large files
   
   4   f734a59e2e35   1970-01-01 00:00 +0000   test
     forget testing
   
   11   9685a22af5db   1970-01-01 00:00 +0000   test
     add sub3
   
   12   2e0485b475b9   1970-01-01 00:00 +0000   test
     remove sub3
   
   13[tip]   a68b2c361653   1970-01-01 00:00 +0000   test
     trash subrepo tracking
   
 
 Restore the trashed subrepo tracking
 
   $ hg rollback -q
   $ hg update -Cq .
 
 Interaction with extdiff, largefiles and subrepos
 
   $ hg --config extensions.extdiff= pdiff -S
 
   $ hg --config extensions.extdiff= pdiff -r '.^' -S
   \r (no-eol) (esc)
   archiving [                                           ] 0/2\r (no-eol) (esc)
   archiving [====================>                      ] 1/2\r (no-eol) (esc)
   archiving [==========================================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [ <=>                                  ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [ <=>                             ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub3) [ <=>                                  ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving [                                           ] 0/2\r (no-eol) (esc)
   archiving [====================>                      ] 1/2\r (no-eol) (esc)
   archiving [==========================================>] 2/2\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [ <=>                                  ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [ <=>                             ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   diff -Nru cloned.*/.hgsub cloned/.hgsub (glob)
   --- cloned.*/.hgsub	* (glob)
   +++ cloned/.hgsub	* (glob)
   @@ -1,2 +1* @@ (glob)
    sub1 = ../sub1
   -sub3 = sub3
   diff -Nru cloned.*/.hgsubstate cloned/.hgsubstate (glob)
   --- cloned.*/.hgsubstate	* (glob)
   +++ cloned/.hgsubstate	* (glob)
   @@ -1,2 +1* @@ (glob)
    7a36fa02b66e61f27f3d4a822809f159479b8ab2 sub1
   -b1a26de6f2a045a9f079323693614ee322f1ff7e sub3
   [1]
 
   $ hg --config extensions.extdiff= pdiff -r 0 -r '.^' -S
   \r (no-eol) (esc)
   archiving [                                           ] 0/3\r (no-eol) (esc)
   archiving [=============>                             ] 1/3\r (no-eol) (esc)
   archiving [===========================>               ] 2/3\r (no-eol) (esc)
   archiving [==========================================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/1\r (no-eol) (esc)
   archiving (sub1) [===================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/1\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving [                                           ] 0/8\r (no-eol) (esc)
   archiving [====>                                      ] 1/8\r (no-eol) (esc)
   archiving [=========>                                 ] 2/8\r (no-eol) (esc)
   archiving [===============>                           ] 3/8\r (no-eol) (esc)
   archiving [====================>                      ] 4/8\r (no-eol) (esc)
   archiving [=========================>                 ] 5/8\r (no-eol) (esc)
   archiving [===============================>           ] 6/8\r (no-eol) (esc)
   archiving [====================================>      ] 7/8\r (no-eol) (esc)
   archiving [==========================================>] 8/8\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1) [                                    ] 0/1\r (no-eol) (esc)
   archiving (sub1) [===================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/3\r (no-eol) (esc)
   archiving (sub1/sub2) [=========>                     ] 1/3\r (no-eol) (esc)
   archiving (sub1/sub2) [===================>           ] 2/3\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 3/3\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub3) [                                    ] 0/1\r (no-eol) (esc)
   archiving (sub3) [===================================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   diff -Nru cloned.*/.hglf/b.dat cloned.*/.hglf/b.dat (glob)
   --- cloned.*/.hglf/b.dat	* (glob)
   +++ cloned.*/.hglf/b.dat	* (glob)
   @@ -*,0 +1* @@ (glob)
   +da39a3ee5e6b4b0d3255bfef95601890afd80709
   diff -Nru cloned.*/.hglf/foo/bar/large.dat cloned.*/.hglf/foo/bar/large.dat (glob)
   --- cloned.*/.hglf/foo/bar/large.dat	* (glob)
   +++ cloned.*/.hglf/foo/bar/large.dat	* (glob)
   @@ -*,0 +1* @@ (glob)
   +2f6933b5ee0f5fdd823d9717d8729f3c2523811b
   diff -Nru cloned.*/.hglf/large.bin cloned.*/.hglf/large.bin (glob)
   --- cloned.*/.hglf/large.bin	* (glob)
   +++ cloned.*/.hglf/large.bin	* (glob)
   @@ -*,0 +1* @@ (glob)
   +7f7097b041ccf68cc5561e9600da4655d21c6d18
   diff -Nru cloned.*/.hgsub cloned.*/.hgsub (glob)
   --- cloned.*/.hgsub	* (glob)
   +++ cloned.*/.hgsub	* (glob)
   @@ -1* +1,2 @@ (glob)
    sub1 = ../sub1
   +sub3 = sub3
   diff -Nru cloned.*/.hgsubstate cloned.*/.hgsubstate (glob)
   --- cloned.*/.hgsubstate	* (glob)
   +++ cloned.*/.hgsubstate	* (glob)
   @@ -1* +1,2 @@ (glob)
   -fc3b4ce2696f7741438c79207583768f2ce6b0dd sub1
   +7a36fa02b66e61f27f3d4a822809f159479b8ab2 sub1
   +b1a26de6f2a045a9f079323693614ee322f1ff7e sub3
   diff -Nru cloned.*/foo/bar/def cloned.*/foo/bar/def (glob)
   --- cloned.*/foo/bar/def	* (glob)
   +++ cloned.*/foo/bar/def	* (glob)
   @@ -*,0 +1* @@ (glob)
   +changed
   diff -Nru cloned.*/main cloned.*/main (glob)
   --- cloned.*/main	* (glob)
   +++ cloned.*/main	* (glob)
   @@ -1* +1* @@ (glob)
   -main
   +foo
   diff -Nru cloned.*/sub1/.hgsubstate cloned.*/sub1/.hgsubstate (glob)
   --- cloned.*/sub1/.hgsubstate	* (glob)
   +++ cloned.*/sub1/.hgsubstate	* (glob)
   @@ -1* +1* @@ (glob)
   -c57a0840e3badd667ef3c3ef65471609acb2ba3c sub2
   +c77908c81ccea3794a896c79e98b0e004aee2e9e sub2
   diff -Nru cloned.*/sub1/sub2/folder/test.txt cloned.*/sub1/sub2/folder/test.txt (glob)
   --- cloned.*/sub1/sub2/folder/test.txt	* (glob)
   +++ cloned.*/sub1/sub2/folder/test.txt	* (glob)
   @@ -*,0 +1* @@ (glob)
   +subfolder
   diff -Nru cloned.*/sub1/sub2/sub2 cloned.*/sub1/sub2/sub2 (glob)
   --- cloned.*/sub1/sub2/sub2	* (glob)
   +++ cloned.*/sub1/sub2/sub2	* (glob)
   @@ -1* +1* @@ (glob)
   -sub2
   +modified
   diff -Nru cloned.*/sub3/a.txt cloned.*/sub3/a.txt (glob)
   --- cloned.*/sub3/a.txt	* (glob)
   +++ cloned.*/sub3/a.txt	* (glob)
   @@ -*,0 +1* @@ (glob)
   +xyz
   [1]
 
   $ echo mod > sub1/sub2/sub2
   $ hg --config extensions.extdiff= pdiff -S
   \r (no-eol) (esc)
   archiving (sub1) [ <=>                                  ] 0\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   \r (no-eol) (esc)
   archiving (sub1/sub2) [                               ] 0/1\r (no-eol) (esc)
   archiving (sub1/sub2) [==============================>] 1/1\r (no-eol) (esc)
                                                               \r (no-eol) (esc)
   --- */cloned.*/sub1/sub2/sub2	* (glob)
   +++ */cloned/sub1/sub2/sub2	* (glob)
   @@ -1* +1* @@ (glob)
   -modified
   +mod
   [1]
 
   $ cd ..