rust: using policy.importrust from Python callers

This commit converts all current Python callers of
mercurial.rustext to the new policy.importrust system.

After this point, going through policy.importrust
or policy.importmod (in some more distant future)
is mandatory for callers of Rust code outside of
Python tests.

We felt it to be appropriate to keep Rust-specific tests
run inconditionally if the Rust extensions are present.

mercurial/dirstate.py

@@ -1,1521 +1,1506
 # dirstate.py - working directory tracking for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import errno
 import os
 import stat
 
 from .i18n import _
 from .node import nullid
 from . import (
     encoding,
     error,
     match as matchmod,
     pathutil,
     policy,
     pycompat,
     scmutil,
     txnutil,
     util,
 )
 
-try:
-    from . import rustext
-    rustext.__name__  # force actual import (see hgdemandimport)
-except ImportError:
-    rustext = None
-
 parsers = policy.importmod(r'parsers')
+dirstatemod = policy.importrust(r'dirstate', default=parsers)
 
 propertycache = util.propertycache
 filecache = scmutil.filecache
 _rangemask = 0x7fffffff
 
 dirstatetuple = parsers.dirstatetuple
 
 class repocache(filecache):
     """filecache for files in .hg/"""
     def join(self, obj, fname):
         return obj._opener.join(fname)
 
 class rootcache(filecache):
     """filecache for files in the repository root"""
     def join(self, obj, fname):
         return obj._join(fname)
 
 def _getfsnow(vfs):
     '''Get "now" timestamp on filesystem'''
     tmpfd, tmpname = vfs.mkstemp()
     try:
         return os.fstat(tmpfd)[stat.ST_MTIME]
     finally:
         os.close(tmpfd)
         vfs.unlink(tmpname)
 
 class dirstate(object):
 
     def __init__(self, opener, ui, root, validate, sparsematchfn):
         '''Create a new dirstate object.
 
         opener is an open()-like callable that can be used to open the
         dirstate file; root is the root of the directory tracked by
         the dirstate.
         '''
         self._opener = opener
         self._validate = validate
         self._root = root
         self._sparsematchfn = sparsematchfn
         # ntpath.join(root, '') of Python 2.7.9 does not add sep if root is
         # UNC path pointing to root share (issue4557)
         self._rootdir = pathutil.normasprefix(root)
         self._dirty = False
         self._lastnormaltime = 0
         self._ui = ui
         self._filecache = {}
         self._parentwriters = 0
         self._filename = 'dirstate'
         self._pendingfilename = '%s.pending' % self._filename
         self._plchangecallbacks = {}
         self._origpl = None
         self._updatedfiles = set()
         self._mapcls = dirstatemap
         # Access and cache cwd early, so we don't access it for the first time
         # after a working-copy update caused it to not exist (accessing it then
         # raises an exception).
         self._cwd
 
     @contextlib.contextmanager
     def parentchange(self):
         '''Context manager for handling dirstate parents.
 
         If an exception occurs in the scope of the context manager,
         the incoherent dirstate won't be written when wlock is
         released.
         '''
         self._parentwriters += 1
         yield
         # Typically we want the "undo" step of a context manager in a
         # finally block so it happens even when an exception
         # occurs. In this case, however, we only want to decrement
         # parentwriters if the code in the with statement exits
         # normally, so we don't have a try/finally here on purpose.
         self._parentwriters -= 1
 
     def pendingparentchange(self):
         '''Returns true if the dirstate is in the middle of a set of changes
         that modify the dirstate parent.
         '''
         return self._parentwriters > 0
 
     @propertycache
     def _map(self):
         """Return the dirstate contents (see documentation for dirstatemap)."""
         self._map = self._mapcls(self._ui, self._opener, self._root)
         return self._map
 
     @property
     def _sparsematcher(self):
         """The matcher for the sparse checkout.
 
         The working directory may not include every file from a manifest. The
         matcher obtained by this property will match a path if it is to be
         included in the working directory.
         """
         # TODO there is potential to cache this property. For now, the matcher
         # is resolved on every access. (But the called function does use a
         # cache to keep the lookup fast.)
         return self._sparsematchfn()
 
     @repocache('branch')
     def _branch(self):
         try:
             return self._opener.read("branch").strip() or "default"
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             return "default"
 
     @property
     def _pl(self):
         return self._map.parents()
 
     def hasdir(self, d):
         return self._map.hastrackeddir(d)
 
     @rootcache('.hgignore')
     def _ignore(self):
         files = self._ignorefiles()
         if not files:
             return matchmod.never()
 
         pats = ['include:%s' % f for f in files]
         return matchmod.match(self._root, '', [], pats, warn=self._ui.warn)
 
     @propertycache
     def _slash(self):
         return self._ui.configbool('ui', 'slash') and pycompat.ossep != '/'
 
     @propertycache
     def _checklink(self):
         return util.checklink(self._root)
 
     @propertycache
     def _checkexec(self):
         return util.checkexec(self._root)
 
     @propertycache
     def _checkcase(self):
         return not util.fscasesensitive(self._join('.hg'))
 
     def _join(self, f):
         # much faster than os.path.join()
         # it's safe because f is always a relative path
         return self._rootdir + f
 
     def flagfunc(self, buildfallback):
         if self._checklink and self._checkexec:
             def f(x):
                 try:
                     st = os.lstat(self._join(x))
                     if util.statislink(st):
                         return 'l'
                     if util.statisexec(st):
                         return 'x'
                 except OSError:
                     pass
                 return ''
             return f
 
         fallback = buildfallback()
         if self._checklink:
             def f(x):
                 if os.path.islink(self._join(x)):
                     return 'l'
                 if 'x' in fallback(x):
                     return 'x'
                 return ''
             return f
         if self._checkexec:
             def f(x):
                 if 'l' in fallback(x):
                     return 'l'
                 if util.isexec(self._join(x)):
                     return 'x'
                 return ''
             return f
         else:
             return fallback
 
     @propertycache
     def _cwd(self):
         # internal config: ui.forcecwd
         forcecwd = self._ui.config('ui', 'forcecwd')
         if forcecwd:
             return forcecwd
         return encoding.getcwd()
 
     def getcwd(self):
         '''Return the path from which a canonical path is calculated.
 
         This path should be used to resolve file patterns or to convert
         canonical paths back to file paths for display. It shouldn't be
         used to get real file paths. Use vfs functions instead.
         '''
         cwd = self._cwd
         if cwd == self._root:
             return ''
         # self._root ends with a path separator if self._root is '/' or 'C:\'
         rootsep = self._root
         if not util.endswithsep(rootsep):
             rootsep += pycompat.ossep
         if cwd.startswith(rootsep):
             return cwd[len(rootsep):]
         else:
             # we're outside the repo. return an absolute path.
             return cwd
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         path = util.pathto(self._root, cwd, f)
         if self._slash:
             return util.pconvert(path)
         return path
 
     def __getitem__(self, key):
         '''Return the current state of key (a filename) in the dirstate.
 
         States are:
           n  normal
           m  needs merging
           r  marked for removal
           a  marked for addition
           ?  not tracked
         '''
         return self._map.get(key, ("?",))[0]
 
     def __contains__(self, key):
         return key in self._map
 
     def __iter__(self):
         return iter(sorted(self._map))
 
     def items(self):
         return self._map.iteritems()
 
     iteritems = items
 
     def parents(self):
         return [self._validate(p) for p in self._pl]
 
     def p1(self):
         return self._validate(self._pl[0])
 
     def p2(self):
         return self._validate(self._pl[1])
 
     def branch(self):
         return encoding.tolocal(self._branch)
 
     def setparents(self, p1, p2=nullid):
         """Set dirstate parents to p1 and p2.
 
         When moving from two parents to one, 'm' merged entries a
         adjusted to normal and previous copy records discarded and
         returned by the call.
 
         See localrepo.setparents()
         """
         if self._parentwriters == 0:
             raise ValueError("cannot set dirstate parent outside of "
                              "dirstate.parentchange context manager")
 
         self._dirty = True
         oldp2 = self._pl[1]
         if self._origpl is None:
             self._origpl = self._pl
         self._map.setparents(p1, p2)
         copies = {}
         if oldp2 != nullid and p2 == nullid:
             candidatefiles = self._map.nonnormalset.union(
                                 self._map.otherparentset)
             for f in candidatefiles:
                 s = self._map.get(f)
                 if s is None:
                     continue
 
                 # Discard 'm' markers when moving away from a merge state
                 if s[0] == 'm':
                     source = self._map.copymap.get(f)
                     if source:
                         copies[f] = source
                     self.normallookup(f)
                 # Also fix up otherparent markers
                 elif s[0] == 'n' and s[2] == -2:
                     source = self._map.copymap.get(f)
                     if source:
                         copies[f] = source
                     self.add(f)
         return copies
 
     def setbranch(self, branch):
         self.__class__._branch.set(self, encoding.fromlocal(branch))
         f = self._opener('branch', 'w', atomictemp=True, checkambig=True)
         try:
             f.write(self._branch + '\n')
             f.close()
 
             # make sure filecache has the correct stat info for _branch after
             # replacing the underlying file
             ce = self._filecache['_branch']
             if ce:
                 ce.refresh()
         except: # re-raises
             f.discard()
             raise
 
     def invalidate(self):
         '''Causes the next access to reread the dirstate.
 
         This is different from localrepo.invalidatedirstate() because it always
         rereads the dirstate. Use localrepo.invalidatedirstate() if you want to
         check whether the dirstate has changed before rereading it.'''
 
         for a in (r"_map", r"_branch", r"_ignore"):
             if a in self.__dict__:
                 delattr(self, a)
         self._lastnormaltime = 0
         self._dirty = False
         self._updatedfiles.clear()
         self._parentwriters = 0
         self._origpl = None
 
     def copy(self, source, dest):
         """Mark dest as a copy of source. Unmark dest if source is None."""
         if source == dest:
             return
         self._dirty = True
         if source is not None:
             self._map.copymap[dest] = source
             self._updatedfiles.add(source)
             self._updatedfiles.add(dest)
         elif self._map.copymap.pop(dest, None):
             self._updatedfiles.add(dest)
 
     def copied(self, file):
         return self._map.copymap.get(file, None)
 
     def copies(self):
         return self._map.copymap
 
     def _addpath(self, f, state, mode, size, mtime):
         oldstate = self[f]
         if state == 'a' or oldstate == 'r':
             scmutil.checkfilename(f)
             if self._map.hastrackeddir(f):
                 raise error.Abort(_('directory %r already in dirstate') %
                                   pycompat.bytestr(f))
             # shadows
             for d in util.finddirs(f):
                 if self._map.hastrackeddir(d):
                     break
                 entry = self._map.get(d)
                 if entry is not None and entry[0] != 'r':
                     raise error.Abort(
                         _('file %r in dirstate clashes with %r') %
                         (pycompat.bytestr(d), pycompat.bytestr(f)))
         self._dirty = True
         self._updatedfiles.add(f)
         self._map.addfile(f, oldstate, state, mode, size, mtime)
 
     def normal(self, f):
         '''Mark a file normal and clean.'''
         s = os.lstat(self._join(f))
         mtime = s[stat.ST_MTIME]
         self._addpath(f, 'n', s.st_mode,
                       s.st_size & _rangemask, mtime & _rangemask)
         self._map.copymap.pop(f, None)
         if f in self._map.nonnormalset:
             self._map.nonnormalset.remove(f)
         if mtime > self._lastnormaltime:
             # Remember the most recent modification timeslot for status(),
             # to make sure we won't miss future size-preserving file content
             # modifications that happen within the same timeslot.
             self._lastnormaltime = mtime
 
     def normallookup(self, f):
         '''Mark a file normal, but possibly dirty.'''
         if self._pl[1] != nullid:
             # if there is a merge going on and the file was either
             # in state 'm' (-1) or coming from other parent (-2) before
             # being removed, restore that state.
             entry = self._map.get(f)
             if entry is not None:
                 if entry[0] == 'r' and entry[2] in (-1, -2):
                     source = self._map.copymap.get(f)
                     if entry[2] == -1:
                         self.merge(f)
                     elif entry[2] == -2:
                         self.otherparent(f)
                     if source:
                         self.copy(source, f)
                     return
                 if entry[0] == 'm' or entry[0] == 'n' and entry[2] == -2:
                     return
         self._addpath(f, 'n', 0, -1, -1)
         self._map.copymap.pop(f, None)
 
     def otherparent(self, f):
         '''Mark as coming from the other parent, always dirty.'''
         if self._pl[1] == nullid:
             raise error.Abort(_("setting %r to other parent "
                                "only allowed in merges") % f)
         if f in self and self[f] == 'n':
             # merge-like
             self._addpath(f, 'm', 0, -2, -1)
         else:
             # add-like
             self._addpath(f, 'n', 0, -2, -1)
         self._map.copymap.pop(f, None)
 
     def add(self, f):
         '''Mark a file added.'''
         self._addpath(f, 'a', 0, -1, -1)
         self._map.copymap.pop(f, None)
 
     def remove(self, f):
         '''Mark a file removed.'''
         self._dirty = True
         oldstate = self[f]
         size = 0
         if self._pl[1] != nullid:
             entry = self._map.get(f)
             if entry is not None:
                 # backup the previous state
                 if entry[0] == 'm': # merge
                     size = -1
                 elif entry[0] == 'n' and entry[2] == -2: # other parent
                     size = -2
                     self._map.otherparentset.add(f)
         self._updatedfiles.add(f)
         self._map.removefile(f, oldstate, size)
         if size == 0:
             self._map.copymap.pop(f, None)
 
     def merge(self, f):
         '''Mark a file merged.'''
         if self._pl[1] == nullid:
             return self.normallookup(f)
         return self.otherparent(f)
 
     def drop(self, f):
         '''Drop a file from the dirstate'''
         oldstate = self[f]
         if self._map.dropfile(f, oldstate):
             self._dirty = True
             self._updatedfiles.add(f)
             self._map.copymap.pop(f, None)
 
     def _discoverpath(self, path, normed, ignoremissing, exists, storemap):
         if exists is None:
             exists = os.path.lexists(os.path.join(self._root, path))
         if not exists:
             # Maybe a path component exists
             if not ignoremissing and '/' in path:
                 d, f = path.rsplit('/', 1)
                 d = self._normalize(d, False, ignoremissing, None)
                 folded = d + "/" + f
             else:
                 # No path components, preserve original case
                 folded = path
         else:
             # recursively normalize leading directory components
             # against dirstate
             if '/' in normed:
                 d, f = normed.rsplit('/', 1)
                 d = self._normalize(d, False, ignoremissing, True)
                 r = self._root + "/" + d
                 folded = d + "/" + util.fspath(f, r)
             else:
                 folded = util.fspath(normed, self._root)
             storemap[normed] = folded
 
         return folded
 
     def _normalizefile(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._map.filefoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 folded = self._discoverpath(path, normed, ignoremissing, exists,
                                             self._map.filefoldmap)
         return folded
 
     def _normalize(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._map.filefoldmap.get(normed, None)
         if folded is None:
             folded = self._map.dirfoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 # store discovered result in dirfoldmap so that future
                 # normalizefile calls don't start matching directories
                 folded = self._discoverpath(path, normed, ignoremissing, exists,
                                             self._map.dirfoldmap)
         return folded
 
     def normalize(self, path, isknown=False, ignoremissing=False):
         '''
         normalize the case of a pathname when on a casefolding filesystem
 
         isknown specifies whether the filename came from walking the
         disk, to avoid extra filesystem access.
 
         If ignoremissing is True, missing path are returned
         unchanged. Otherwise, we try harder to normalize possibly
         existing path components.
 
         The normalized case is determined based on the following precedence:
 
         - version of name already stored in the dirstate
         - version of name stored on disk
         - version provided via command arguments
         '''
 
         if self._checkcase:
             return self._normalize(path, isknown, ignoremissing)
         return path
 
     def clear(self):
         self._map.clear()
         self._lastnormaltime = 0
         self._updatedfiles.clear()
         self._dirty = True
 
     def rebuild(self, parent, allfiles, changedfiles=None):
         if changedfiles is None:
             # Rebuild entire dirstate
             changedfiles = allfiles
             lastnormaltime = self._lastnormaltime
             self.clear()
             self._lastnormaltime = lastnormaltime
 
         if self._origpl is None:
             self._origpl = self._pl
         self._map.setparents(parent, nullid)
         for f in changedfiles:
             if f in allfiles:
                 self.normallookup(f)
             else:
                 self.drop(f)
 
         self._dirty = True
 
     def identity(self):
         '''Return identity of dirstate itself to detect changing in storage
 
         If identity of previous dirstate is equal to this, writing
         changes based on the former dirstate out can keep consistency.
         '''
         return self._map.identity
 
     def write(self, tr):
         if not self._dirty:
             return
 
         filename = self._filename
         if tr:
             # 'dirstate.write()' is not only for writing in-memory
             # changes out, but also for dropping ambiguous timestamp.
             # delayed writing re-raise "ambiguous timestamp issue".
             # See also the wiki page below for detail:
             # https://www.mercurial-scm.org/wiki/DirstateTransactionPlan
 
             # emulate dropping timestamp in 'parsers.pack_dirstate'
             now = _getfsnow(self._opener)
             self._map.clearambiguoustimes(self._updatedfiles, now)
 
             # emulate that all 'dirstate.normal' results are written out
             self._lastnormaltime = 0
             self._updatedfiles.clear()
 
             # delay writing in-memory changes out
             tr.addfilegenerator('dirstate', (self._filename,),
                                 self._writedirstate, location='plain')
             return
 
         st = self._opener(filename, "w", atomictemp=True, checkambig=True)
         self._writedirstate(st)
 
     def addparentchangecallback(self, category, callback):
         """add a callback to be called when the wd parents are changed
 
         Callback will be called with the following arguments:
             dirstate, (oldp1, oldp2), (newp1, newp2)
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._plchangecallbacks[category] = callback
 
     def _writedirstate(self, st):
         # notify callbacks about parents change
         if self._origpl is not None and self._origpl != self._pl:
             for c, callback in sorted(self._plchangecallbacks.iteritems()):
                 callback(self, self._origpl, self._pl)
             self._origpl = None
         # use the modification time of the newly created temporary file as the
         # filesystem's notion of 'now'
         now = util.fstat(st)[stat.ST_MTIME] & _rangemask
 
         # enough 'delaywrite' prevents 'pack_dirstate' from dropping
         # timestamp of each entries in dirstate, because of 'now > mtime'
         delaywrite = self._ui.configint('debug', 'dirstate.delaywrite')
         if delaywrite > 0:
             # do we have any files to delay for?
             for f, e in self._map.iteritems():
                 if e[0] == 'n' and e[3] == now:
                     import time # to avoid useless import
                     # rather than sleep n seconds, sleep until the next
                     # multiple of n seconds
                     clock = time.time()
                     start = int(clock) - (int(clock) % delaywrite)
                     end = start + delaywrite
                     time.sleep(end - clock)
                     now = end # trust our estimate that the end is near now
                     break
 
         self._map.write(st, now)
         self._lastnormaltime = 0
         self._dirty = False
 
     def _dirignore(self, f):
         if self._ignore(f):
             return True
         for p in util.finddirs(f):
             if self._ignore(p):
                 return True
         return False
 
     def _ignorefiles(self):
         files = []
         if os.path.exists(self._join('.hgignore')):
             files.append(self._join('.hgignore'))
         for name, path in self._ui.configitems("ui"):
             if name == 'ignore' or name.startswith('ignore.'):
                 # we need to use os.path.join here rather than self._join
                 # because path is arbitrary and user-specified
                 files.append(os.path.join(self._rootdir, util.expandpath(path)))
         return files
 
     def _ignorefileandline(self, f):
         files = collections.deque(self._ignorefiles())
         visited = set()
         while files:
             i = files.popleft()
             patterns = matchmod.readpatternfile(i, self._ui.warn,
                                                 sourceinfo=True)
             for pattern, lineno, line in patterns:
                 kind, p = matchmod._patsplit(pattern, 'glob')
                 if kind == "subinclude":
                     if p not in visited:
                         files.append(p)
                     continue
                 m = matchmod.match(self._root, '', [], [pattern],
                                    warn=self._ui.warn)
                 if m(f):
                     return (i, lineno, line)
             visited.add(i)
         return (None, -1, "")
 
     def _walkexplicit(self, match, subrepos):
         '''Get stat data about the files explicitly specified by match.
 
         Return a triple (results, dirsfound, dirsnotfound).
         - results is a mapping from filename to stat result. It also contains
           listings mapping subrepos and .hg to None.
         - dirsfound is a list of files found to be directories.
         - dirsnotfound is a list of files that the dirstate thinks are
           directories and that were not found.'''
 
         def badtype(mode):
             kind = _('unknown')
             if stat.S_ISCHR(mode):
                 kind = _('character device')
             elif stat.S_ISBLK(mode):
                 kind = _('block device')
             elif stat.S_ISFIFO(mode):
                 kind = _('fifo')
             elif stat.S_ISSOCK(mode):
                 kind = _('socket')
             elif stat.S_ISDIR(mode):
                 kind = _('directory')
             return _('unsupported file type (type is %s)') % kind
 
         matchedir = match.explicitdir
         badfn = match.bad
         dmap = self._map
         lstat = os.lstat
         getkind = stat.S_IFMT
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
         dirsfound = []
         foundadd = dirsfound.append
         dirsnotfound = []
         notfoundadd = dirsnotfound.append
 
         if not match.isexact() and self._checkcase:
             normalize = self._normalize
         else:
             normalize = None
 
         files = sorted(match.files())
         subrepos.sort()
         i, j = 0, 0
         while i < len(files) and j < len(subrepos):
             subpath = subrepos[j] + "/"
             if files[i] < subpath:
                 i += 1
                 continue
             while i < len(files) and files[i].startswith(subpath):
                 del files[i]
             j += 1
 
         if not files or '' in files:
             files = ['']
             # constructing the foldmap is expensive, so don't do it for the
             # common case where files is ['']
             normalize = None
         results = dict.fromkeys(subrepos)
         results['.hg'] = None
 
         for ff in files:
             if normalize:
                 nf = normalize(ff, False, True)
             else:
                 nf = ff
             if nf in results:
                 continue
 
             try:
                 st = lstat(join(nf))
                 kind = getkind(st.st_mode)
                 if kind == dirkind:
                     if nf in dmap:
                         # file replaced by dir on disk but still in dirstate
                         results[nf] = None
                     if matchedir:
                         matchedir(nf)
                     foundadd((nf, ff))
                 elif kind == regkind or kind == lnkkind:
                     results[nf] = st
                 else:
                     badfn(ff, badtype(kind))
                     if nf in dmap:
                         results[nf] = None
             except OSError as inst: # nf not found on disk - it is dirstate only
                 if nf in dmap: # does it exactly match a missing file?
                     results[nf] = None
                 else: # does it match a missing directory?
                     if self._map.hasdir(nf):
                         if matchedir:
                             matchedir(nf)
                         notfoundadd(nf)
                     else:
                         badfn(ff, encoding.strtolocal(inst.strerror))
 
         # match.files() may contain explicitly-specified paths that shouldn't
         # be taken; drop them from the list of files found. dirsfound/notfound
         # aren't filtered here because they will be tested later.
         if match.anypats():
             for f in list(results):
                 if f == '.hg' or f in subrepos:
                     # keep sentinel to disable further out-of-repo walks
                     continue
                 if not match(f):
                     del results[f]
 
         # Case insensitive filesystems cannot rely on lstat() failing to detect
         # a case-only rename.  Prune the stat object for any file that does not
         # match the case in the filesystem, if there are multiple files that
         # normalize to the same path.
         if match.isexact() and self._checkcase:
             normed = {}
 
             for f, st in results.iteritems():
                 if st is None:
                     continue
 
                 nc = util.normcase(f)
                 paths = normed.get(nc)
 
                 if paths is None:
                     paths = set()
                     normed[nc] = paths
 
                 paths.add(f)
 
             for norm, paths in normed.iteritems():
                 if len(paths) > 1:
                     for path in paths:
                         folded = self._discoverpath(path, norm, True, None,
                                                     self._map.dirfoldmap)
                         if path != folded:
                             results[path] = None
 
         return results, dirsfound, dirsnotfound
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         '''
         Walk recursively through the directory tree, finding all files
         matched by match.
 
         If full is False, maybe skip some known-clean files.
 
         Return a dict mapping filename to stat-like object (either
         mercurial.osutil.stat instance or return value of os.stat()).
 
         '''
         # full is a flag that extensions that hook into walk can use -- this
         # implementation doesn't use it at all. This satisfies the contract
         # because we only guarantee a "maybe".
 
         if ignored:
             ignore = util.never
             dirignore = util.never
         elif unknown:
             ignore = self._ignore
             dirignore = self._dirignore
         else:
             # if not unknown and not ignored, drop dir recursion and step 2
             ignore = util.always
             dirignore = util.always
 
         matchfn = match.matchfn
         matchalways = match.always()
         matchtdir = match.traversedir
         dmap = self._map
         listdir = util.listdir
         lstat = os.lstat
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
 
         exact = skipstep3 = False
         if match.isexact(): # match.exact
             exact = True
             dirignore = util.always # skip step 2
         elif match.prefix(): # match.match, no patterns
             skipstep3 = True
 
         if not exact and self._checkcase:
             normalize = self._normalize
             normalizefile = self._normalizefile
             skipstep3 = False
         else:
             normalize = self._normalize
             normalizefile = None
 
         # step 1: find all explicit files
         results, work, dirsnotfound = self._walkexplicit(match, subrepos)
 
         skipstep3 = skipstep3 and not (work or dirsnotfound)
         work = [d for d in work if not dirignore(d[0])]
 
         # step 2: visit subdirectories
         def traverse(work, alreadynormed):
             wadd = work.append
             while work:
                 nd = work.pop()
                 visitentries = match.visitchildrenset(nd)
                 if not visitentries:
                     continue
                 if visitentries == 'this' or visitentries == 'all':
                     visitentries = None
                 skip = None
                 if nd != '':
                     skip = '.hg'
                 try:
                     entries = listdir(join(nd), stat=True, skip=skip)
                 except OSError as inst:
                     if inst.errno in (errno.EACCES, errno.ENOENT):
                         match.bad(self.pathto(nd),
                                   encoding.strtolocal(inst.strerror))
                         continue
                     raise
                 for f, kind, st in entries:
                     # Some matchers may return files in the visitentries set,
                     # instead of 'this', if the matcher explicitly mentions them
                     # and is not an exactmatcher. This is acceptable; we do not
                     # make any hard assumptions about file-or-directory below
                     # based on the presence of `f` in visitentries. If
                     # visitchildrenset returned a set, we can always skip the
                     # entries *not* in the set it provided regardless of whether
                     # they're actually a file or a directory.
                     if visitentries and f not in visitentries:
                         continue
                     if normalizefile:
                         # even though f might be a directory, we're only
                         # interested in comparing it to files currently in the
                         # dmap -- therefore normalizefile is enough
                         nf = normalizefile(nd and (nd + "/" + f) or f, True,
                                            True)
                     else:
                         nf = nd and (nd + "/" + f) or f
                     if nf not in results:
                         if kind == dirkind:
                             if not ignore(nf):
                                 if matchtdir:
                                     matchtdir(nf)
                                 wadd(nf)
                             if nf in dmap and (matchalways or matchfn(nf)):
                                 results[nf] = None
                         elif kind == regkind or kind == lnkkind:
                             if nf in dmap:
                                 if matchalways or matchfn(nf):
                                     results[nf] = st
                             elif ((matchalways or matchfn(nf))
                                   and not ignore(nf)):
                                 # unknown file -- normalize if necessary
                                 if not alreadynormed:
                                     nf = normalize(nf, False, True)
                                 results[nf] = st
                         elif nf in dmap and (matchalways or matchfn(nf)):
                             results[nf] = None
 
         for nd, d in work:
             # alreadynormed means that processwork doesn't have to do any
             # expensive directory normalization
             alreadynormed = not normalize or nd == d
             traverse([d], alreadynormed)
 
         for s in subrepos:
             del results[s]
         del results['.hg']
 
         # step 3: visit remaining files from dmap
         if not skipstep3 and not exact:
             # If a dmap file is not in results yet, it was either
             # a) not matching matchfn b) ignored, c) missing, or d) under a
             # symlink directory.
             if not results and matchalways:
                 visit = [f for f in dmap]
             else:
                 visit = [f for f in dmap if f not in results and matchfn(f)]
             visit.sort()
 
             if unknown:
                 # unknown == True means we walked all dirs under the roots
                 # that wasn't ignored, and everything that matched was stat'ed
                 # and is already in results.
                 # The rest must thus be ignored or under a symlink.
                 audit_path = pathutil.pathauditor(self._root, cached=True)
 
                 for nf in iter(visit):
                     # If a stat for the same file was already added with a
                     # different case, don't add one for this, since that would
                     # make it appear as if the file exists under both names
                     # on disk.
                     if (normalizefile and
                         normalizefile(nf, True, True) in results):
                         results[nf] = None
                     # Report ignored items in the dmap as long as they are not
                     # under a symlink directory.
                     elif audit_path.check(nf):
                         try:
                             results[nf] = lstat(join(nf))
                             # file was just ignored, no links, and exists
                         except OSError:
                             # file doesn't exist
                             results[nf] = None
                     else:
                         # It's either missing or under a symlink directory
                         # which we in this case report as missing
                         results[nf] = None
             else:
                 # We may not have walked the full directory tree above,
                 # so stat and check everything we missed.
                 iv = iter(visit)
                 for st in util.statfiles([join(i) for i in visit]):
                     results[next(iv)] = st
         return results
 
     def status(self, match, subrepos, ignored, clean, unknown):
         '''Determine the status of the working copy relative to the
         dirstate and return a pair of (unsure, status), where status is of type
         scmutil.status and:
 
           unsure:
             files that might have been modified since the dirstate was
             written, but need to be read to be sure (size is the same
             but mtime differs)
           status.modified:
             files that have definitely been modified since the dirstate
             was written (different size or mode)
           status.clean:
             files that have definitely not been modified since the
             dirstate was written
         '''
         listignored, listclean, listunknown = ignored, clean, unknown
         lookup, modified, added, unknown, ignored = [], [], [], [], []
         removed, deleted, clean = [], [], []
 
         dmap = self._map
         dmap.preload()
         dcontains = dmap.__contains__
         dget = dmap.__getitem__
         ladd = lookup.append            # aka "unsure"
         madd = modified.append
         aadd = added.append
         uadd = unknown.append
         iadd = ignored.append
         radd = removed.append
         dadd = deleted.append
         cadd = clean.append
         mexact = match.exact
         dirignore = self._dirignore
         checkexec = self._checkexec
         copymap = self._map.copymap
         lastnormaltime = self._lastnormaltime
 
         # We need to do full walks when either
         # - we're listing all clean files, or
         # - match.traversedir does something, because match.traversedir should
         #   be called for every dir in the working dir
         full = listclean or match.traversedir is not None
         for fn, st in self.walk(match, subrepos, listunknown, listignored,
                                 full=full).iteritems():
             if not dcontains(fn):
                 if (listignored or mexact(fn)) and dirignore(fn):
                     if listignored:
                         iadd(fn)
                 else:
                     uadd(fn)
                 continue
 
             # This is equivalent to 'state, mode, size, time = dmap[fn]' but not
             # written like that for performance reasons. dmap[fn] is not a
             # Python tuple in compiled builds. The CPython UNPACK_SEQUENCE
             # opcode has fast paths when the value to be unpacked is a tuple or
             # a list, but falls back to creating a full-fledged iterator in
             # general. That is much slower than simply accessing and storing the
             # tuple members one by one.
             t = dget(fn)
             state = t[0]
             mode = t[1]
             size = t[2]
             time = t[3]
 
             if not st and state in "nma":
                 dadd(fn)
             elif state == 'n':
                 if (size >= 0 and
                     ((size != st.st_size and size != st.st_size & _rangemask)
                      or ((mode ^ st.st_mode) & 0o100 and checkexec))
                     or size == -2 # other parent
                     or fn in copymap):
                     madd(fn)
                 elif (time != st[stat.ST_MTIME]
                       and time != st[stat.ST_MTIME] & _rangemask):
                     ladd(fn)
                 elif st[stat.ST_MTIME] == lastnormaltime:
                     # fn may have just been marked as normal and it may have
                     # changed in the same second without changing its size.
                     # This can happen if we quickly do multiple commits.
                     # Force lookup, so we don't miss such a racy file change.
                     ladd(fn)
                 elif listclean:
                     cadd(fn)
             elif state == 'm':
                 madd(fn)
             elif state == 'a':
                 aadd(fn)
             elif state == 'r':
                 radd(fn)
 
         return (lookup, scmutil.status(modified, added, removed, deleted,
                                        unknown, ignored, clean))
 
     def matches(self, match):
         '''
         return files in the dirstate (in whatever state) filtered by match
         '''
         dmap = self._map
         if match.always():
             return dmap.keys()
         files = match.files()
         if match.isexact():
             # fast path -- filter the other way around, since typically files is
             # much smaller than dmap
             return [f for f in files if f in dmap]
         if match.prefix() and all(fn in dmap for fn in files):
             # fast path -- all the values are known to be files, so just return
             # that
             return list(files)
         return [f for f in dmap if match(f)]
 
     def _actualfilename(self, tr):
         if tr:
             return self._pendingfilename
         else:
             return self._filename
 
     def savebackup(self, tr, backupname):
         '''Save current dirstate into backup file'''
         filename = self._actualfilename(tr)
         assert backupname != filename
 
         # use '_writedirstate' instead of 'write' to write changes certainly,
         # because the latter omits writing out if transaction is running.
         # output file will be used to create backup of dirstate at this point.
         if self._dirty or not self._opener.exists(filename):
             self._writedirstate(self._opener(filename, "w", atomictemp=True,
                                              checkambig=True))
 
         if tr:
             # ensure that subsequent tr.writepending returns True for
             # changes written out above, even if dirstate is never
             # changed after this
             tr.addfilegenerator('dirstate', (self._filename,),
                                 self._writedirstate, location='plain')
 
             # ensure that pending file written above is unlinked at
             # failure, even if tr.writepending isn't invoked until the
             # end of this transaction
             tr.registertmp(filename, location='plain')
 
         self._opener.tryunlink(backupname)
         # hardlink backup is okay because _writedirstate is always called
         # with an "atomictemp=True" file.
         util.copyfile(self._opener.join(filename),
                       self._opener.join(backupname), hardlink=True)
 
     def restorebackup(self, tr, backupname):
         '''Restore dirstate by backup file'''
         # this "invalidate()" prevents "wlock.release()" from writing
         # changes of dirstate out after restoring from backup file
         self.invalidate()
         filename = self._actualfilename(tr)
         o = self._opener
         if util.samefile(o.join(backupname), o.join(filename)):
             o.unlink(backupname)
         else:
             o.rename(backupname, filename, checkambig=True)
 
     def clearbackup(self, tr, backupname):
         '''Clear backup file'''
         self._opener.unlink(backupname)
 
 class dirstatemap(object):
     """Map encapsulating the dirstate's contents.
 
     The dirstate contains the following state:
 
     - `identity` is the identity of the dirstate file, which can be used to
       detect when changes have occurred to the dirstate file.
 
     - `parents` is a pair containing the parents of the working copy. The
       parents are updated by calling `setparents`.
 
     - the state map maps filenames to tuples of (state, mode, size, mtime),
       where state is a single character representing 'normal', 'added',
       'removed', or 'merged'. It is read by treating the dirstate as a
       dict.  File state is updated by calling the `addfile`, `removefile` and
       `dropfile` methods.
 
     - `copymap` maps destination filenames to their source filename.
 
     The dirstate also provides the following views onto the state:
 
     - `nonnormalset` is a set of the filenames that have state other
       than 'normal', or are normal but have an mtime of -1 ('normallookup').
 
     - `otherparentset` is a set of the filenames that are marked as coming
       from the second parent when the dirstate is currently being merged.
 
     - `filefoldmap` is a dict mapping normalized filenames to the denormalized
       form that they appear as in the dirstate.
 
     - `dirfoldmap` is a dict mapping normalized directory names to the
       denormalized form that they appear as in the dirstate.
     """
 
     def __init__(self, ui, opener, root):
         self._ui = ui
         self._opener = opener
         self._root = root
         self._filename = 'dirstate'
 
         self._parents = None
         self._dirtyparents = False
 
         # for consistent view between _pl() and _read() invocations
         self._pendingmode = None
 
     @propertycache
     def _map(self):
         self._map = {}
         self.read()
         return self._map
 
     @propertycache
     def copymap(self):
         self.copymap = {}
         self._map
         return self.copymap
 
     def clear(self):
         self._map.clear()
         self.copymap.clear()
         self.setparents(nullid, nullid)
         util.clearcachedproperty(self, "_dirs")
         util.clearcachedproperty(self, "_alldirs")
         util.clearcachedproperty(self, "filefoldmap")
         util.clearcachedproperty(self, "dirfoldmap")
         util.clearcachedproperty(self, "nonnormalset")
         util.clearcachedproperty(self, "otherparentset")
 
     def items(self):
         return self._map.iteritems()
 
     # forward for python2,3 compat
     iteritems = items
 
     def __len__(self):
         return len(self._map)
 
     def __iter__(self):
         return iter(self._map)
 
     def get(self, key, default=None):
         return self._map.get(key, default)
 
     def __contains__(self, key):
         return key in self._map
 
     def __getitem__(self, key):
         return self._map[key]
 
     def keys(self):
         return self._map.keys()
 
     def preload(self):
         """Loads the underlying data, if it's not already loaded"""
         self._map
 
     def addfile(self, f, oldstate, state, mode, size, mtime):
         """Add a tracked file to the dirstate."""
         if oldstate in "?r" and r"_dirs" in self.__dict__:
             self._dirs.addpath(f)
         if oldstate == "?" and r"_alldirs" in self.__dict__:
             self._alldirs.addpath(f)
         self._map[f] = dirstatetuple(state, mode, size, mtime)
         if state != 'n' or mtime == -1:
             self.nonnormalset.add(f)
         if size == -2:
             self.otherparentset.add(f)
 
     def removefile(self, f, oldstate, size):
         """
         Mark a file as removed in the dirstate.
 
         The `size` parameter is used to store sentinel values that indicate
         the file's previous state.  In the future, we should refactor this
         to be more explicit about what that state is.
         """
         if oldstate not in "?r" and r"_dirs" in self.__dict__:
             self._dirs.delpath(f)
         if oldstate == "?" and r"_alldirs" in self.__dict__:
             self._alldirs.addpath(f)
         if r"filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             self.filefoldmap.pop(normed, None)
         self._map[f] = dirstatetuple('r', 0, size, 0)
         self.nonnormalset.add(f)
 
     def dropfile(self, f, oldstate):
         """
         Remove a file from the dirstate.  Returns True if the file was
         previously recorded.
         """
         exists = self._map.pop(f, None) is not None
         if exists:
             if oldstate != "r" and r"_dirs" in self.__dict__:
                 self._dirs.delpath(f)
             if r"_alldirs" in self.__dict__:
                 self._alldirs.delpath(f)
         if r"filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             self.filefoldmap.pop(normed, None)
         self.nonnormalset.discard(f)
         return exists
 
     def clearambiguoustimes(self, files, now):
         for f in files:
             e = self.get(f)
             if e is not None and e[0] == 'n' and e[3] == now:
                 self._map[f] = dirstatetuple(e[0], e[1], e[2], -1)
                 self.nonnormalset.add(f)
 
     def nonnormalentries(self):
         '''Compute the nonnormal dirstate entries from the dmap'''
         try:
             return parsers.nonnormalotherparententries(self._map)
         except AttributeError:
             nonnorm = set()
             otherparent = set()
             for fname, e in self._map.iteritems():
                 if e[0] != 'n' or e[3] == -1:
                     nonnorm.add(fname)
                 if e[0] == 'n' and e[2] == -2:
                     otherparent.add(fname)
             return nonnorm, otherparent
 
     @propertycache
     def filefoldmap(self):
         """Returns a dictionary mapping normalized case paths to their
         non-normalized versions.
         """
         try:
             makefilefoldmap = parsers.make_file_foldmap
         except AttributeError:
             pass
         else:
             return makefilefoldmap(self._map, util.normcasespec,
                                    util.normcasefallback)
 
         f = {}
         normcase = util.normcase
         for name, s in self._map.iteritems():
             if s[0] != 'r':
                 f[normcase(name)] = name
         f['.'] = '.' # prevents useless util.fspath() invocation
         return f
 
     def hastrackeddir(self, d):
         """
         Returns True if the dirstate contains a tracked (not removed) file
         in this directory.
         """
         return d in self._dirs
 
     def hasdir(self, d):
         """
         Returns True if the dirstate contains a file (tracked or removed)
         in this directory.
         """
         return d in self._alldirs
 
     @propertycache
     def _dirs(self):
         return util.dirs(self._map, 'r')
 
     @propertycache
     def _alldirs(self):
         return util.dirs(self._map)
 
     def _opendirstatefile(self):
         fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
         if self._pendingmode is not None and self._pendingmode != mode:
             fp.close()
             raise error.Abort(_('working directory state may be '
                                 'changed parallelly'))
         self._pendingmode = mode
         return fp
 
     def parents(self):
         if not self._parents:
             try:
                 fp = self._opendirstatefile()
                 st = fp.read(40)
                 fp.close()
             except IOError as err:
                 if err.errno != errno.ENOENT:
                     raise
                 # File doesn't exist, so the current state is empty
                 st = ''
 
             l = len(st)
             if l == 40:
                 self._parents = (st[:20], st[20:40])
             elif l == 0:
                 self._parents = (nullid, nullid)
             else:
                 raise error.Abort(_('working directory state appears '
                                     'damaged!'))
 
         return self._parents
 
     def setparents(self, p1, p2):
         self._parents = (p1, p2)
         self._dirtyparents = True
 
     def read(self):
         # ignore HG_PENDING because identity is used only for writing
         self.identity = util.filestat.frompath(
             self._opener.join(self._filename))
 
         try:
             fp = self._opendirstatefile()
             try:
                 st = fp.read()
             finally:
                 fp.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         if not st:
             return
 
         if util.safehasattr(parsers, 'dict_new_presized'):
             # Make an estimate of the number of files in the dirstate based on
             # its size. From a linear regression on a set of real-world repos,
             # all over 10,000 files, the size of a dirstate entry is 85
             # bytes. The cost of resizing is significantly higher than the cost
             # of filling in a larger presized dict, so subtract 20% from the
             # size.
             #
             # This heuristic is imperfect in many ways, so in a future dirstate
             # format update it makes sense to just record the number of entries
             # on write.
             self._map = parsers.dict_new_presized(len(st) // 71)
 
         # Python's garbage collector triggers a GC each time a certain number
         # of container objects (the number being defined by
         # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
         # for each file in the dirstate. The C version then immediately marks
         # them as not to be tracked by the collector. However, this has no
         # effect on when GCs are triggered, only on what objects the GC looks
         # into. This means that O(number of files) GCs are unavoidable.
         # Depending on when in the process's lifetime the dirstate is parsed,
         # this can get very expensive. As a workaround, disable GC while
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
-        if rustext is not None:
-            parse_dirstate = rustext.dirstate.parse_dirstate
-        else:
-            parse_dirstate = parsers.parse_dirstate
-
-        parse_dirstate = util.nogc(parse_dirstate)
+        parse_dirstate = util.nogc(dirstatemod.parse_dirstate)
         p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
 
         # Avoid excess attribute lookups by fast pathing certain checks
         self.__contains__ = self._map.__contains__
         self.__getitem__ = self._map.__getitem__
         self.get = self._map.get
 
     def write(self, st, now):
-        if rustext is not None:
-            pack_dirstate = rustext.dirstate.pack_dirstate
-        else:
-            pack_dirstate = parsers.pack_dirstate
-
-        st.write(pack_dirstate(self._map, self.copymap,
+        st.write(dirstatemod.pack_dirstate(self._map, self.copymap,
                                        self.parents(), now))
         st.close()
         self._dirtyparents = False
         self.nonnormalset, self.otherparentset = self.nonnormalentries()
 
     @propertycache
     def nonnormalset(self):
         nonnorm, otherparents = self.nonnormalentries()
         self.otherparentset = otherparents
         return nonnorm
 
     @propertycache
     def otherparentset(self):
         nonnorm, otherparents = self.nonnormalentries()
         self.nonnormalset = nonnorm
         return otherparents
 
     @propertycache
     def identity(self):
         self._map
         return self.identity
 
     @propertycache
     def dirfoldmap(self):
         f = {}
         normcase = util.normcase
         for name in self._dirs:
             f[normcase(name)] = name
         return f

mercurial/match.py

@@ -1,1529 +1,1526
 # match.py - filename matching
 #
 #  Copyright 2008, 2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import copy
 import itertools
 import os
 import re
 
 from .i18n import _
 from . import (
     encoding,
     error,
     pathutil,
+    policy,
     pycompat,
     util,
 )
 from .utils import (
     stringutil,
 )
 
-try:
-    from . import rustext
-    rustext.__name__  # force actual import (see hgdemandimport)
-except ImportError:
-    rustext = None
+rustmod = policy.importrust('filepatterns')
 
 allpatternkinds = ('re', 'glob', 'path', 'relglob', 'relpath', 'relre',
                    'rootglob',
                    'listfile', 'listfile0', 'set', 'include', 'subinclude',
                    'rootfilesin')
 cwdrelativepatternkinds = ('relpath', 'glob')
 
 propertycache = util.propertycache
 
 def _rematcher(regex):
     '''compile the regexp with the best available regexp engine and return a
     matcher function'''
     m = util.re.compile(regex)
     try:
         # slightly faster, provided by facebook's re2 bindings
         return m.test_match
     except AttributeError:
         return m.match
 
 def _expandsets(kindpats, ctx=None, listsubrepos=False, badfn=None):
     '''Returns the kindpats list with the 'set' patterns expanded to matchers'''
     matchers = []
     other = []
 
     for kind, pat, source in kindpats:
         if kind == 'set':
             if ctx is None:
                 raise error.ProgrammingError("fileset expression with no "
                                              "context")
             matchers.append(ctx.matchfileset(pat, badfn=badfn))
 
             if listsubrepos:
                 for subpath in ctx.substate:
                     sm = ctx.sub(subpath).matchfileset(pat, badfn=badfn)
                     pm = prefixdirmatcher(subpath, sm, badfn=badfn)
                     matchers.append(pm)
 
             continue
         other.append((kind, pat, source))
     return matchers, other
 
 def _expandsubinclude(kindpats, root):
     '''Returns the list of subinclude matcher args and the kindpats without the
     subincludes in it.'''
     relmatchers = []
     other = []
 
     for kind, pat, source in kindpats:
         if kind == 'subinclude':
             sourceroot = pathutil.dirname(util.normpath(source))
             pat = util.pconvert(pat)
             path = pathutil.join(sourceroot, pat)
 
             newroot = pathutil.dirname(path)
             matcherargs = (newroot, '', [], ['include:%s' % path])
 
             prefix = pathutil.canonpath(root, root, newroot)
             if prefix:
                 prefix += '/'
             relmatchers.append((prefix, matcherargs))
         else:
             other.append((kind, pat, source))
 
     return relmatchers, other
 
 def _kindpatsalwaysmatch(kindpats):
     """"Checks whether the kindspats match everything, as e.g.
     'relpath:.' does.
     """
     for kind, pat, source in kindpats:
         if pat != '' or kind not in ['relpath', 'glob']:
             return False
     return True
 
 def _buildkindpatsmatcher(matchercls, root, kindpats, ctx=None,
                           listsubrepos=False, badfn=None):
     matchers = []
     fms, kindpats = _expandsets(kindpats, ctx=ctx,
                                 listsubrepos=listsubrepos, badfn=badfn)
     if kindpats:
         m = matchercls(root, kindpats, badfn=badfn)
         matchers.append(m)
     if fms:
         matchers.extend(fms)
     if not matchers:
         return nevermatcher(badfn=badfn)
     if len(matchers) == 1:
         return matchers[0]
     return unionmatcher(matchers)
 
 def match(root, cwd, patterns=None, include=None, exclude=None, default='glob',
           auditor=None, ctx=None, listsubrepos=False, warn=None,
           badfn=None, icasefs=False):
     r"""build an object to match a set of file patterns
 
     arguments:
     root - the canonical root of the tree you're matching against
     cwd - the current working directory, if relevant
     patterns - patterns to find
     include - patterns to include (unless they are excluded)
     exclude - patterns to exclude (even if they are included)
     default - if a pattern in patterns has no explicit type, assume this one
     auditor - optional path auditor
     ctx - optional changecontext
     listsubrepos - if True, recurse into subrepositories
     warn - optional function used for printing warnings
     badfn - optional bad() callback for this matcher instead of the default
     icasefs - make a matcher for wdir on case insensitive filesystems, which
         normalizes the given patterns to the case in the filesystem
 
     a pattern is one of:
     'glob:<glob>' - a glob relative to cwd
     're:<regexp>' - a regular expression
     'path:<path>' - a path relative to repository root, which is matched
                     recursively
     'rootfilesin:<path>' - a path relative to repository root, which is
                     matched non-recursively (will not match subdirectories)
     'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs)
     'relpath:<path>' - a path relative to cwd
     'relre:<regexp>' - a regexp that needn't match the start of a name
     'set:<fileset>' - a fileset expression
     'include:<path>' - a file of patterns to read and include
     'subinclude:<path>' - a file of patterns to match against files under
                           the same directory
     '<something>' - a pattern of the specified default type
 
     Usually a patternmatcher is returned:
     >>> match(b'foo', b'.', [b're:.*\.c$', b'path:foo/a', b'*.py'])
     <patternmatcher patterns='.*\\.c$|foo/a(?:/|$)|[^/]*\\.py$'>
 
     Combining 'patterns' with 'include' (resp. 'exclude') gives an
     intersectionmatcher (resp. a differencematcher):
     >>> type(match(b'foo', b'.', [b're:.*\.c$'], include=[b'path:lib']))
     <class 'mercurial.match.intersectionmatcher'>
     >>> type(match(b'foo', b'.', [b're:.*\.c$'], exclude=[b'path:build']))
     <class 'mercurial.match.differencematcher'>
 
     Notice that, if 'patterns' is empty, an alwaysmatcher is returned:
     >>> match(b'foo', b'.', [])
     <alwaysmatcher>
 
     The 'default' argument determines which kind of pattern is assumed if a
     pattern has no prefix:
     >>> match(b'foo', b'.', [b'.*\.c$'], default=b're')
     <patternmatcher patterns='.*\\.c$'>
     >>> match(b'foo', b'.', [b'main.py'], default=b'relpath')
     <patternmatcher patterns='main\\.py(?:/|$)'>
     >>> match(b'foo', b'.', [b'main.py'], default=b're')
     <patternmatcher patterns='main.py'>
 
     The primary use of matchers is to check whether a value (usually a file
     name) matches againset one of the patterns given at initialization. There
     are two ways of doing this check.
 
     >>> m = match(b'foo', b'', [b're:.*\.c$', b'relpath:a'])
 
     1. Calling the matcher with a file name returns True if any pattern
     matches that file name:
     >>> m(b'a')
     True
     >>> m(b'main.c')
     True
     >>> m(b'test.py')
     False
 
     2. Using the exact() method only returns True if the file name matches one
     of the exact patterns (i.e. not re: or glob: patterns):
     >>> m.exact(b'a')
     True
     >>> m.exact(b'main.c')
     False
     """
     normalize = _donormalize
     if icasefs:
         dirstate = ctx.repo().dirstate
         dsnormalize = dirstate.normalize
 
         def normalize(patterns, default, root, cwd, auditor, warn):
             kp = _donormalize(patterns, default, root, cwd, auditor, warn)
             kindpats = []
             for kind, pats, source in kp:
                 if kind not in ('re', 'relre'):  # regex can't be normalized
                     p = pats
                     pats = dsnormalize(pats)
 
                     # Preserve the original to handle a case only rename.
                     if p != pats and p in dirstate:
                         kindpats.append((kind, p, source))
 
                 kindpats.append((kind, pats, source))
             return kindpats
 
     if patterns:
         kindpats = normalize(patterns, default, root, cwd, auditor, warn)
         if _kindpatsalwaysmatch(kindpats):
             m = alwaysmatcher(badfn)
         else:
             m = _buildkindpatsmatcher(patternmatcher, root, kindpats, ctx=ctx,
                                       listsubrepos=listsubrepos, badfn=badfn)
     else:
         # It's a little strange that no patterns means to match everything.
         # Consider changing this to match nothing (probably using nevermatcher).
         m = alwaysmatcher(badfn)
 
     if include:
         kindpats = normalize(include, 'glob', root, cwd, auditor, warn)
         im = _buildkindpatsmatcher(includematcher, root, kindpats, ctx=ctx,
                                    listsubrepos=listsubrepos, badfn=None)
         m = intersectmatchers(m, im)
     if exclude:
         kindpats = normalize(exclude, 'glob', root, cwd, auditor, warn)
         em = _buildkindpatsmatcher(includematcher, root, kindpats, ctx=ctx,
                                    listsubrepos=listsubrepos, badfn=None)
         m = differencematcher(m, em)
     return m
 
 def exact(files, badfn=None):
     return exactmatcher(files, badfn=badfn)
 
 def always(badfn=None):
     return alwaysmatcher(badfn)
 
 def never(badfn=None):
     return nevermatcher(badfn)
 
 def badmatch(match, badfn):
     """Make a copy of the given matcher, replacing its bad method with the given
     one.
     """
     m = copy.copy(match)
     m.bad = badfn
     return m
 
 def _donormalize(patterns, default, root, cwd, auditor=None, warn=None):
     '''Convert 'kind:pat' from the patterns list to tuples with kind and
     normalized and rooted patterns and with listfiles expanded.'''
     kindpats = []
     for kind, pat in [_patsplit(p, default) for p in patterns]:
         if kind in cwdrelativepatternkinds:
             pat = pathutil.canonpath(root, cwd, pat, auditor=auditor)
         elif kind in ('relglob', 'path', 'rootfilesin', 'rootglob'):
             pat = util.normpath(pat)
         elif kind in ('listfile', 'listfile0'):
             try:
                 files = util.readfile(pat)
                 if kind == 'listfile0':
                     files = files.split('\0')
                 else:
                     files = files.splitlines()
                 files = [f for f in files if f]
             except EnvironmentError:
                 raise error.Abort(_("unable to read file list (%s)") % pat)
             for k, p, source in _donormalize(files, default, root, cwd,
                                              auditor, warn):
                 kindpats.append((k, p, pat))
             continue
         elif kind == 'include':
             try:
                 fullpath = os.path.join(root, util.localpath(pat))
                 includepats = readpatternfile(fullpath, warn)
                 for k, p, source in _donormalize(includepats, default,
                                                  root, cwd, auditor, warn):
                     kindpats.append((k, p, source or pat))
             except error.Abort as inst:
                 raise error.Abort('%s: %s' % (pat, inst[0]))
             except IOError as inst:
                 if warn:
                     warn(_("skipping unreadable pattern file '%s': %s\n") %
                          (pat, stringutil.forcebytestr(inst.strerror)))
             continue
         # else: re or relre - which cannot be normalized
         kindpats.append((kind, pat, ''))
     return kindpats
 
 class basematcher(object):
 
     def __init__(self, badfn=None):
         if badfn is not None:
             self.bad = badfn
 
     def __call__(self, fn):
         return self.matchfn(fn)
     # Callbacks related to how the matcher is used by dirstate.walk.
     # Subscribers to these events must monkeypatch the matcher object.
     def bad(self, f, msg):
         '''Callback from dirstate.walk for each explicit file that can't be
         found/accessed, with an error message.'''
 
     # If an explicitdir is set, it will be called when an explicitly listed
     # directory is visited.
     explicitdir = None
 
     # If an traversedir is set, it will be called when a directory discovered
     # by recursive traversal is visited.
     traversedir = None
 
     @propertycache
     def _files(self):
         return []
 
     def files(self):
         '''Explicitly listed files or patterns or roots:
         if no patterns or .always(): empty list,
         if exact: list exact files,
         if not .anypats(): list all files and dirs,
         else: optimal roots'''
         return self._files
 
     @propertycache
     def _fileset(self):
         return set(self._files)
 
     def exact(self, f):
         '''Returns True if f is in .files().'''
         return f in self._fileset
 
     def matchfn(self, f):
         return False
 
     def visitdir(self, dir):
         '''Decides whether a directory should be visited based on whether it
         has potential matches in it or one of its subdirectories. This is
         based on the match's primary, included, and excluded patterns.
 
         Returns the string 'all' if the given directory and all subdirectories
         should be visited. Otherwise returns True or False indicating whether
         the given directory should be visited.
         '''
         return True
 
     def visitchildrenset(self, dir):
         '''Decides whether a directory should be visited based on whether it
         has potential matches in it or one of its subdirectories, and
         potentially lists which subdirectories of that directory should be
         visited. This is based on the match's primary, included, and excluded
         patterns.
 
         This function is very similar to 'visitdir', and the following mapping
         can be applied:
 
              visitdir | visitchildrenlist
             ----------+-------------------
              False    | set()
              'all'    | 'all'
              True     | 'this' OR non-empty set of subdirs -or files- to visit
 
         Example:
           Assume matchers ['path:foo/bar', 'rootfilesin:qux'], we would return
           the following values (assuming the implementation of visitchildrenset
           is capable of recognizing this; some implementations are not).
 
           '' -> {'foo', 'qux'}
           'baz' -> set()
           'foo' -> {'bar'}
           # Ideally this would be 'all', but since the prefix nature of matchers
           # is applied to the entire matcher, we have to downgrade this to
           # 'this' due to the non-prefix 'rootfilesin'-kind matcher being mixed
           # in.
           'foo/bar' -> 'this'
           'qux' -> 'this'
 
         Important:
           Most matchers do not know if they're representing files or
           directories. They see ['path:dir/f'] and don't know whether 'f' is a
           file or a directory, so visitchildrenset('dir') for most matchers will
           return {'f'}, but if the matcher knows it's a file (like exactmatcher
           does), it may return 'this'. Do not rely on the return being a set
           indicating that there are no files in this dir to investigate (or
           equivalently that if there are files to investigate in 'dir' that it
           will always return 'this').
         '''
         return 'this'
 
     def always(self):
         '''Matcher will match everything and .files() will be empty --
         optimization might be possible.'''
         return False
 
     def isexact(self):
         '''Matcher will match exactly the list of files in .files() --
         optimization might be possible.'''
         return False
 
     def prefix(self):
         '''Matcher will match the paths in .files() recursively --
         optimization might be possible.'''
         return False
 
     def anypats(self):
         '''None of .always(), .isexact(), and .prefix() is true --
         optimizations will be difficult.'''
         return not self.always() and not self.isexact() and not self.prefix()
 
 class alwaysmatcher(basematcher):
     '''Matches everything.'''
 
     def __init__(self, badfn=None):
         super(alwaysmatcher, self).__init__(badfn)
 
     def always(self):
         return True
 
     def matchfn(self, f):
         return True
 
     def visitdir(self, dir):
         return 'all'
 
     def visitchildrenset(self, dir):
         return 'all'
 
     def __repr__(self):
         return r'<alwaysmatcher>'
 
 class nevermatcher(basematcher):
     '''Matches nothing.'''
 
     def __init__(self, badfn=None):
         super(nevermatcher, self).__init__(badfn)
 
     # It's a little weird to say that the nevermatcher is an exact matcher
     # or a prefix matcher, but it seems to make sense to let callers take
     # fast paths based on either. There will be no exact matches, nor any
     # prefixes (files() returns []), so fast paths iterating over them should
     # be efficient (and correct).
     def isexact(self):
         return True
 
     def prefix(self):
         return True
 
     def visitdir(self, dir):
         return False
 
     def visitchildrenset(self, dir):
         return set()
 
     def __repr__(self):
         return r'<nevermatcher>'
 
 class predicatematcher(basematcher):
     """A matcher adapter for a simple boolean function"""
 
     def __init__(self, predfn, predrepr=None, badfn=None):
         super(predicatematcher, self).__init__(badfn)
         self.matchfn = predfn
         self._predrepr = predrepr
 
     @encoding.strmethod
     def __repr__(self):
         s = (stringutil.buildrepr(self._predrepr)
              or pycompat.byterepr(self.matchfn))
         return '<predicatenmatcher pred=%s>' % s
 
 def normalizerootdir(dir, funcname):
     if dir == '.':
         util.nouideprecwarn("match.%s() no longer accepts "
                             "'.', use '' instead." % funcname, '5.1')
         return ''
     return dir
 
 
 class patternmatcher(basematcher):
     """Matches a set of (kind, pat, source) against a 'root' directory.
 
     >>> kindpats = [
     ...     (b're', br'.*\.c$', b''),
     ...     (b'path', b'foo/a', b''),
     ...     (b'relpath', b'b', b''),
     ...     (b'glob', b'*.h', b''),
     ... ]
     >>> m = patternmatcher(b'foo', kindpats)
     >>> m(b'main.c')  # matches re:.*\.c$
     True
     >>> m(b'b.txt')
     False
     >>> m(b'foo/a')  # matches path:foo/a
     True
     >>> m(b'a')  # does not match path:b, since 'root' is 'foo'
     False
     >>> m(b'b')  # matches relpath:b, since 'root' is 'foo'
     True
     >>> m(b'lib.h')  # matches glob:*.h
     True
 
     >>> m.files()
     ['', 'foo/a', 'b', '']
     >>> m.exact(b'foo/a')
     True
     >>> m.exact(b'b')
     True
     >>> m.exact(b'lib.h')  # exact matches are for (rel)path kinds
     False
     """
 
     def __init__(self, root, kindpats, badfn=None):
         super(patternmatcher, self).__init__(badfn)
 
         self._files = _explicitfiles(kindpats)
         self._prefix = _prefix(kindpats)
         self._pats, self.matchfn = _buildmatch(kindpats, '$', root)
 
     @propertycache
     def _dirs(self):
         return set(util.dirs(self._fileset))
 
     def visitdir(self, dir):
         dir = normalizerootdir(dir, 'visitdir')
         if self._prefix and dir in self._fileset:
             return 'all'
         return (dir in self._fileset or
                 dir in self._dirs or
                 any(parentdir in self._fileset
                     for parentdir in util.finddirs(dir)))
 
     def visitchildrenset(self, dir):
         ret = self.visitdir(dir)
         if ret is True:
             return 'this'
         elif not ret:
             return set()
         assert ret == 'all'
         return 'all'
 
     def prefix(self):
         return self._prefix
 
     @encoding.strmethod
     def __repr__(self):
         return ('<patternmatcher patterns=%r>' % pycompat.bytestr(self._pats))
 
 # This is basically a reimplementation of util.dirs that stores the children
 # instead of just a count of them, plus a small optional optimization to avoid
 # some directories we don't need.
 class _dirchildren(object):
     def __init__(self, paths, onlyinclude=None):
         self._dirs = {}
         self._onlyinclude = onlyinclude or []
         addpath = self.addpath
         for f in paths:
             addpath(f)
 
     def addpath(self, path):
         if path == '':
             return
         dirs = self._dirs
         findsplitdirs = _dirchildren._findsplitdirs
         for d, b in findsplitdirs(path):
             if d not in self._onlyinclude:
                 continue
             dirs.setdefault(d, set()).add(b)
 
     @staticmethod
     def _findsplitdirs(path):
         # yields (dirname, basename) tuples, walking back to the root.  This is
         # very similar to util.finddirs, except:
         #  - produces a (dirname, basename) tuple, not just 'dirname'
         # Unlike manifest._splittopdir, this does not suffix `dirname` with a
         # slash.
         oldpos = len(path)
         pos = path.rfind('/')
         while pos != -1:
             yield path[:pos], path[pos + 1:oldpos]
             oldpos = pos
             pos = path.rfind('/', 0, pos)
         yield '', path[:oldpos]
 
     def get(self, path):
         return self._dirs.get(path, set())
 
 class includematcher(basematcher):
 
     def __init__(self, root, kindpats, badfn=None):
         super(includematcher, self).__init__(badfn)
 
         self._pats, self.matchfn = _buildmatch(kindpats, '(?:/|$)', root)
         self._prefix = _prefix(kindpats)
         roots, dirs, parents = _rootsdirsandparents(kindpats)
         # roots are directories which are recursively included.
         self._roots = set(roots)
         # dirs are directories which are non-recursively included.
         self._dirs = set(dirs)
         # parents are directories which are non-recursively included because
         # they are needed to get to items in _dirs or _roots.
         self._parents = parents
 
     def visitdir(self, dir):
         dir = normalizerootdir(dir, 'visitdir')
         if self._prefix and dir in self._roots:
             return 'all'
         return (dir in self._roots or
                 dir in self._dirs or
                 dir in self._parents or
                 any(parentdir in self._roots
                     for parentdir in util.finddirs(dir)))
 
     @propertycache
     def _allparentschildren(self):
         # It may seem odd that we add dirs, roots, and parents, and then
         # restrict to only parents. This is to catch the case of:
         #   dirs = ['foo/bar']
         #   parents = ['foo']
         # if we asked for the children of 'foo', but had only added
         # self._parents, we wouldn't be able to respond ['bar'].
         return _dirchildren(
                 itertools.chain(self._dirs, self._roots, self._parents),
                 onlyinclude=self._parents)
 
     def visitchildrenset(self, dir):
         if self._prefix and dir in self._roots:
             return 'all'
         # Note: this does *not* include the 'dir in self._parents' case from
         # visitdir, that's handled below.
         if ('' in self._roots or
             dir in self._roots or
             dir in self._dirs or
             any(parentdir in self._roots
                 for parentdir in util.finddirs(dir))):
             return 'this'
 
         if dir in self._parents:
             return self._allparentschildren.get(dir) or set()
         return set()
 
     @encoding.strmethod
     def __repr__(self):
         return ('<includematcher includes=%r>' % pycompat.bytestr(self._pats))
 
 class exactmatcher(basematcher):
     r'''Matches the input files exactly. They are interpreted as paths, not
     patterns (so no kind-prefixes).
 
     >>> m = exactmatcher([b'a.txt', br're:.*\.c$'])
     >>> m(b'a.txt')
     True
     >>> m(b'b.txt')
     False
 
     Input files that would be matched are exactly those returned by .files()
     >>> m.files()
     ['a.txt', 're:.*\\.c$']
 
     So pattern 're:.*\.c$' is not considered as a regex, but as a file name
     >>> m(b'main.c')
     False
     >>> m(br're:.*\.c$')
     True
     '''
 
     def __init__(self, files, badfn=None):
         super(exactmatcher, self).__init__(badfn)
 
         if isinstance(files, list):
             self._files = files
         else:
             self._files = list(files)
 
     matchfn = basematcher.exact
 
     @propertycache
     def _dirs(self):
         return set(util.dirs(self._fileset))
 
     def visitdir(self, dir):
         dir = normalizerootdir(dir, 'visitdir')
         return dir in self._dirs
 
     def visitchildrenset(self, dir):
         dir = normalizerootdir(dir, 'visitchildrenset')
 
         if not self._fileset or dir not in self._dirs:
             return set()
 
         candidates = self._fileset | self._dirs - {''}
         if dir != '':
             d = dir + '/'
             candidates = set(c[len(d):] for c in candidates if
                              c.startswith(d))
         # self._dirs includes all of the directories, recursively, so if
         # we're attempting to match foo/bar/baz.txt, it'll have '', 'foo',
         # 'foo/bar' in it. Thus we can safely ignore a candidate that has a
         # '/' in it, indicating a it's for a subdir-of-a-subdir; the
         # immediate subdir will be in there without a slash.
         ret = {c for c in candidates if '/' not in c}
         # We really do not expect ret to be empty, since that would imply that
         # there's something in _dirs that didn't have a file in _fileset.
         assert ret
         return ret
 
     def isexact(self):
         return True
 
     @encoding.strmethod
     def __repr__(self):
         return ('<exactmatcher files=%r>' % self._files)
 
 class differencematcher(basematcher):
     '''Composes two matchers by matching if the first matches and the second
     does not.
 
     The second matcher's non-matching-attributes (bad, explicitdir,
     traversedir) are ignored.
     '''
     def __init__(self, m1, m2):
         super(differencematcher, self).__init__()
         self._m1 = m1
         self._m2 = m2
         self.bad = m1.bad
         self.explicitdir = m1.explicitdir
         self.traversedir = m1.traversedir
 
     def matchfn(self, f):
         return self._m1(f) and not self._m2(f)
 
     @propertycache
     def _files(self):
         if self.isexact():
             return [f for f in self._m1.files() if self(f)]
         # If m1 is not an exact matcher, we can't easily figure out the set of
         # files, because its files() are not always files. For example, if
         # m1 is "path:dir" and m2 is "rootfileins:.", we don't
         # want to remove "dir" from the set even though it would match m2,
         # because the "dir" in m1 may not be a file.
         return self._m1.files()
 
     def visitdir(self, dir):
         if self._m2.visitdir(dir) == 'all':
             return False
         elif not self._m2.visitdir(dir):
             # m2 does not match dir, we can return 'all' here if possible
             return self._m1.visitdir(dir)
         return bool(self._m1.visitdir(dir))
 
     def visitchildrenset(self, dir):
         m2_set = self._m2.visitchildrenset(dir)
         if m2_set == 'all':
             return set()
         m1_set = self._m1.visitchildrenset(dir)
         # Possible values for m1: 'all', 'this', set(...), set()
         # Possible values for m2:        'this', set(...), set()
         # If m2 has nothing under here that we care about, return m1, even if
         # it's 'all'. This is a change in behavior from visitdir, which would
         # return True, not 'all', for some reason.
         if not m2_set:
             return m1_set
         if m1_set in ['all', 'this']:
             # Never return 'all' here if m2_set is any kind of non-empty (either
             # 'this' or set(foo)), since m2 might return set() for a
             # subdirectory.
             return 'this'
         # Possible values for m1:         set(...), set()
         # Possible values for m2: 'this', set(...)
         # We ignore m2's set results. They're possibly incorrect:
         #  m1 = path:dir/subdir, m2=rootfilesin:dir, visitchildrenset(''):
         #    m1 returns {'dir'}, m2 returns {'dir'}, if we subtracted we'd
         #    return set(), which is *not* correct, we still need to visit 'dir'!
         return m1_set
 
     def isexact(self):
         return self._m1.isexact()
 
     @encoding.strmethod
     def __repr__(self):
         return ('<differencematcher m1=%r, m2=%r>' % (self._m1, self._m2))
 
 def intersectmatchers(m1, m2):
     '''Composes two matchers by matching if both of them match.
 
     The second matcher's non-matching-attributes (bad, explicitdir,
     traversedir) are ignored.
     '''
     if m1 is None or m2 is None:
         return m1 or m2
     if m1.always():
         m = copy.copy(m2)
         # TODO: Consider encapsulating these things in a class so there's only
         # one thing to copy from m1.
         m.bad = m1.bad
         m.explicitdir = m1.explicitdir
         m.traversedir = m1.traversedir
         return m
     if m2.always():
         m = copy.copy(m1)
         return m
     return intersectionmatcher(m1, m2)
 
 class intersectionmatcher(basematcher):
     def __init__(self, m1, m2):
         super(intersectionmatcher, self).__init__()
         self._m1 = m1
         self._m2 = m2
         self.bad = m1.bad
         self.explicitdir = m1.explicitdir
         self.traversedir = m1.traversedir
 
     @propertycache
     def _files(self):
         if self.isexact():
             m1, m2 = self._m1, self._m2
             if not m1.isexact():
                 m1, m2 = m2, m1
             return [f for f in m1.files() if m2(f)]
         # It neither m1 nor m2 is an exact matcher, we can't easily intersect
         # the set of files, because their files() are not always files. For
         # example, if intersecting a matcher "-I glob:foo.txt" with matcher of
         # "path:dir2", we don't want to remove "dir2" from the set.
         return self._m1.files() + self._m2.files()
 
     def matchfn(self, f):
         return self._m1(f) and self._m2(f)
 
     def visitdir(self, dir):
         visit1 = self._m1.visitdir(dir)
         if visit1 == 'all':
             return self._m2.visitdir(dir)
         # bool() because visit1=True + visit2='all' should not be 'all'
         return bool(visit1 and self._m2.visitdir(dir))
 
     def visitchildrenset(self, dir):
         m1_set = self._m1.visitchildrenset(dir)
         if not m1_set:
             return set()
         m2_set = self._m2.visitchildrenset(dir)
         if not m2_set:
             return set()
 
         if m1_set == 'all':
             return m2_set
         elif m2_set == 'all':
             return m1_set
 
         if m1_set == 'this' or m2_set == 'this':
             return 'this'
 
         assert isinstance(m1_set, set) and isinstance(m2_set, set)
         return m1_set.intersection(m2_set)
 
     def always(self):
         return self._m1.always() and self._m2.always()
 
     def isexact(self):
         return self._m1.isexact() or self._m2.isexact()
 
     @encoding.strmethod
     def __repr__(self):
         return ('<intersectionmatcher m1=%r, m2=%r>' % (self._m1, self._m2))
 
 class subdirmatcher(basematcher):
     """Adapt a matcher to work on a subdirectory only.
 
     The paths are remapped to remove/insert the path as needed:
 
     >>> from . import pycompat
     >>> m1 = match(b'root', b'', [b'a.txt', b'sub/b.txt'])
     >>> m2 = subdirmatcher(b'sub', m1)
     >>> m2(b'a.txt')
     False
     >>> m2(b'b.txt')
     True
     >>> m2.matchfn(b'a.txt')
     False
     >>> m2.matchfn(b'b.txt')
     True
     >>> m2.files()
     ['b.txt']
     >>> m2.exact(b'b.txt')
     True
     >>> def bad(f, msg):
     ...     print(pycompat.sysstr(b"%s: %s" % (f, msg)))
     >>> m1.bad = bad
     >>> m2.bad(b'x.txt', b'No such file')
     sub/x.txt: No such file
     """
 
     def __init__(self, path, matcher):
         super(subdirmatcher, self).__init__()
         self._path = path
         self._matcher = matcher
         self._always = matcher.always()
 
         self._files = [f[len(path) + 1:] for f in matcher._files
                        if f.startswith(path + "/")]
 
         # If the parent repo had a path to this subrepo and the matcher is
         # a prefix matcher, this submatcher always matches.
         if matcher.prefix():
             self._always = any(f == path for f in matcher._files)
 
     def bad(self, f, msg):
         self._matcher.bad(self._path + "/" + f, msg)
 
     def matchfn(self, f):
         # Some information is lost in the superclass's constructor, so we
         # can not accurately create the matching function for the subdirectory
         # from the inputs. Instead, we override matchfn() and visitdir() to
         # call the original matcher with the subdirectory path prepended.
         return self._matcher.matchfn(self._path + "/" + f)
 
     def visitdir(self, dir):
         dir = normalizerootdir(dir, 'visitdir')
         if dir == '':
             dir = self._path
         else:
             dir = self._path + "/" + dir
         return self._matcher.visitdir(dir)
 
     def visitchildrenset(self, dir):
         dir = normalizerootdir(dir, 'visitchildrenset')
         if dir == '':
             dir = self._path
         else:
             dir = self._path + "/" + dir
         return self._matcher.visitchildrenset(dir)
 
     def always(self):
         return self._always
 
     def prefix(self):
         return self._matcher.prefix() and not self._always
 
     @encoding.strmethod
     def __repr__(self):
         return ('<subdirmatcher path=%r, matcher=%r>' %
                 (self._path, self._matcher))
 
 class prefixdirmatcher(basematcher):
     """Adapt a matcher to work on a parent directory.
 
     The matcher's non-matching-attributes (bad, explicitdir, traversedir) are
     ignored.
 
     The prefix path should usually be the relative path from the root of
     this matcher to the root of the wrapped matcher.
 
     >>> m1 = match(util.localpath(b'root/d/e'), b'f', [b'../a.txt', b'b.txt'])
     >>> m2 = prefixdirmatcher(b'd/e', m1)
     >>> m2(b'a.txt')
     False
     >>> m2(b'd/e/a.txt')
     True
     >>> m2(b'd/e/b.txt')
     False
     >>> m2.files()
     ['d/e/a.txt', 'd/e/f/b.txt']
     >>> m2.exact(b'd/e/a.txt')
     True
     >>> m2.visitdir(b'd')
     True
     >>> m2.visitdir(b'd/e')
     True
     >>> m2.visitdir(b'd/e/f')
     True
     >>> m2.visitdir(b'd/e/g')
     False
     >>> m2.visitdir(b'd/ef')
     False
     """
 
     def __init__(self, path, matcher, badfn=None):
         super(prefixdirmatcher, self).__init__(badfn)
         if not path:
             raise error.ProgrammingError('prefix path must not be empty')
         self._path = path
         self._pathprefix = path + '/'
         self._matcher = matcher
 
     @propertycache
     def _files(self):
         return [self._pathprefix + f for f in self._matcher._files]
 
     def matchfn(self, f):
         if not f.startswith(self._pathprefix):
             return False
         return self._matcher.matchfn(f[len(self._pathprefix):])
 
     @propertycache
     def _pathdirs(self):
         return set(util.finddirs(self._path))
 
     def visitdir(self, dir):
         if dir == self._path:
             return self._matcher.visitdir('')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitdir(dir[len(self._pathprefix):])
         return dir in self._pathdirs
 
     def visitchildrenset(self, dir):
         if dir == self._path:
             return self._matcher.visitchildrenset('')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitchildrenset(dir[len(self._pathprefix):])
         if dir in self._pathdirs:
             return 'this'
         return set()
 
     def isexact(self):
         return self._matcher.isexact()
 
     def prefix(self):
         return self._matcher.prefix()
 
     @encoding.strmethod
     def __repr__(self):
         return ('<prefixdirmatcher path=%r, matcher=%r>'
                 % (pycompat.bytestr(self._path), self._matcher))
 
 class unionmatcher(basematcher):
     """A matcher that is the union of several matchers.
 
     The non-matching-attributes (bad, explicitdir, traversedir) are taken from
     the first matcher.
     """
 
     def __init__(self, matchers):
         m1 = matchers[0]
         super(unionmatcher, self).__init__()
         self.explicitdir = m1.explicitdir
         self.traversedir = m1.traversedir
         self._matchers = matchers
 
     def matchfn(self, f):
         for match in self._matchers:
             if match(f):
                 return True
         return False
 
     def visitdir(self, dir):
         r = False
         for m in self._matchers:
             v = m.visitdir(dir)
             if v == 'all':
                 return v
             r |= v
         return r
 
     def visitchildrenset(self, dir):
         r = set()
         this = False
         for m in self._matchers:
             v = m.visitchildrenset(dir)
             if not v:
                 continue
             if v == 'all':
                 return v
             if this or v == 'this':
                 this = True
                 # don't break, we might have an 'all' in here.
                 continue
             assert isinstance(v, set)
             r = r.union(v)
         if this:
             return 'this'
         return r
 
     @encoding.strmethod
     def __repr__(self):
         return ('<unionmatcher matchers=%r>' % self._matchers)
 
 def patkind(pattern, default=None):
     '''If pattern is 'kind:pat' with a known kind, return kind.
 
     >>> patkind(br're:.*\.c$')
     're'
     >>> patkind(b'glob:*.c')
     'glob'
     >>> patkind(b'relpath:test.py')
     'relpath'
     >>> patkind(b'main.py')
     >>> patkind(b'main.py', default=b're')
     're'
     '''
     return _patsplit(pattern, default)[0]
 
 def _patsplit(pattern, default):
     """Split a string into the optional pattern kind prefix and the actual
     pattern."""
     if ':' in pattern:
         kind, pat = pattern.split(':', 1)
         if kind in allpatternkinds:
             return kind, pat
     return default, pattern
 
 def _globre(pat):
     r'''Convert an extended glob string to a regexp string.
 
     >>> from . import pycompat
     >>> def bprint(s):
     ...     print(pycompat.sysstr(s))
     >>> bprint(_globre(br'?'))
     .
     >>> bprint(_globre(br'*'))
     [^/]*
     >>> bprint(_globre(br'**'))
     .*
     >>> bprint(_globre(br'**/a'))
     (?:.*/)?a
     >>> bprint(_globre(br'a/**/b'))
     a/(?:.*/)?b
     >>> bprint(_globre(br'[a*?!^][^b][!c]'))
     [a*?!^][\^b][^c]
     >>> bprint(_globre(br'{a,b}'))
     (?:a|b)
     >>> bprint(_globre(br'.\*\?'))
     \.\*\?
     '''
     i, n = 0, len(pat)
     res = ''
     group = 0
     escape = util.stringutil.regexbytesescapemap.get
     def peek():
         return i < n and pat[i:i + 1]
     while i < n:
         c = pat[i:i + 1]
         i += 1
         if c not in '*?[{},\\':
             res += escape(c, c)
         elif c == '*':
             if peek() == '*':
                 i += 1
                 if peek() == '/':
                     i += 1
                     res += '(?:.*/)?'
                 else:
                     res += '.*'
             else:
                 res += '[^/]*'
         elif c == '?':
             res += '.'
         elif c == '[':
             j = i
             if j < n and pat[j:j + 1] in '!]':
                 j += 1
             while j < n and pat[j:j + 1] != ']':
                 j += 1
             if j >= n:
                 res += '\\['
             else:
                 stuff = pat[i:j].replace('\\','\\\\')
                 i = j + 1
                 if stuff[0:1] == '!':
                     stuff = '^' + stuff[1:]
                 elif stuff[0:1] == '^':
                     stuff = '\\' + stuff
                 res = '%s[%s]' % (res, stuff)
         elif c == '{':
             group += 1
             res += '(?:'
         elif c == '}' and group:
             res += ')'
             group -= 1
         elif c == ',' and group:
             res += '|'
         elif c == '\\':
             p = peek()
             if p:
                 i += 1
                 res += escape(p, p)
             else:
                 res += escape(c, c)
         else:
             res += escape(c, c)
     return res
 
 def _regex(kind, pat, globsuffix):
     '''Convert a (normalized) pattern of any kind into a
     regular expression.
     globsuffix is appended to the regexp of globs.'''
 
-    if rustext is not None:
+    if rustmod is not None:
         try:
-            return rustext.filepatterns.build_single_regex(
+            return rustmod.build_single_regex(
                 kind,
                 pat,
                 globsuffix
             )
-        except rustext.filepatterns.PatternError:
+        except rustmod.PatternError:
             raise error.ProgrammingError(
                 'not a regex pattern: %s:%s' % (kind, pat)
             )
 
     if not pat and kind in ('glob', 'relpath'):
         return ''
     if kind == 're':
         return pat
     if kind in ('path', 'relpath'):
         if pat == '.':
             return ''
         return util.stringutil.reescape(pat) + '(?:/|$)'
     if kind == 'rootfilesin':
         if pat == '.':
             escaped = ''
         else:
             # Pattern is a directory name.
             escaped = util.stringutil.reescape(pat) + '/'
         # Anything after the pattern must be a non-directory.
         return escaped + '[^/]+$'
     if kind == 'relglob':
         return '(?:|.*/)' + _globre(pat) + globsuffix
     if kind == 'relre':
         if pat.startswith('^'):
             return pat
         return '.*' + pat
     if kind in ('glob', 'rootglob'):
         return _globre(pat) + globsuffix
     raise error.ProgrammingError('not a regex pattern: %s:%s' % (kind, pat))
 
 def _buildmatch(kindpats, globsuffix, root):
     '''Return regexp string and a matcher function for kindpats.
     globsuffix is appended to the regexp of globs.'''
     matchfuncs = []
 
     subincludes, kindpats = _expandsubinclude(kindpats, root)
     if subincludes:
         submatchers = {}
         def matchsubinclude(f):
             for prefix, matcherargs in subincludes:
                 if f.startswith(prefix):
                     mf = submatchers.get(prefix)
                     if mf is None:
                         mf = match(*matcherargs)
                         submatchers[prefix] = mf
 
                     if mf(f[len(prefix):]):
                         return True
             return False
         matchfuncs.append(matchsubinclude)
 
     regex = ''
     if kindpats:
         if all(k == 'rootfilesin' for k, p, s in kindpats):
             dirs = {p for k, p, s in kindpats}
             def mf(f):
                 i = f.rfind('/')
                 if i >= 0:
                     dir = f[:i]
                 else:
                     dir = '.'
                 return dir in dirs
             regex = b'rootfilesin: %s' % stringutil.pprint(list(sorted(dirs)))
             matchfuncs.append(mf)
         else:
             regex, mf = _buildregexmatch(kindpats, globsuffix)
             matchfuncs.append(mf)
 
     if len(matchfuncs) == 1:
         return regex, matchfuncs[0]
     else:
         return regex, lambda f: any(mf(f) for mf in matchfuncs)
 
 MAX_RE_SIZE = 20000
 
 def _joinregexes(regexps):
     """gather multiple regular expressions into a single one"""
     return '|'.join(regexps)
 
 def _buildregexmatch(kindpats, globsuffix):
     """Build a match function from a list of kinds and kindpats,
     return regexp string and a matcher function.
 
     Test too large input
     >>> _buildregexmatch([
     ...     (b'relglob', b'?' * MAX_RE_SIZE, b'')
     ... ], b'$')
     Traceback (most recent call last):
     ...
     Abort: matcher pattern is too long (20009 bytes)
     """
     try:
         allgroups = []
         regexps = [_regex(k, p, globsuffix) for (k, p, s) in kindpats]
         fullregexp = _joinregexes(regexps)
 
         startidx = 0
         groupsize = 0
         for idx, r in enumerate(regexps):
             piecesize = len(r)
             if piecesize > MAX_RE_SIZE:
                 msg = _("matcher pattern is too long (%d bytes)") % piecesize
                 raise error.Abort(msg)
             elif (groupsize + piecesize) > MAX_RE_SIZE:
                 group = regexps[startidx:idx]
                 allgroups.append(_joinregexes(group))
                 startidx = idx
                 groupsize = 0
             groupsize += piecesize + 1
 
         if startidx == 0:
             matcher = _rematcher(fullregexp)
             func = lambda s: bool(matcher(s))
         else:
             group = regexps[startidx:]
             allgroups.append(_joinregexes(group))
             allmatchers = [_rematcher(g) for g in allgroups]
             func = lambda s: any(m(s) for m in allmatchers)
         return fullregexp, func
     except re.error:
         for k, p, s in kindpats:
             try:
                 _rematcher(_regex(k, p, globsuffix))
             except re.error:
                 if s:
                     raise error.Abort(_("%s: invalid pattern (%s): %s") %
                                       (s, k, p))
                 else:
                     raise error.Abort(_("invalid pattern (%s): %s") % (k, p))
         raise error.Abort(_("invalid pattern"))
 
 def _patternrootsanddirs(kindpats):
     '''Returns roots and directories corresponding to each pattern.
 
     This calculates the roots and directories exactly matching the patterns and
     returns a tuple of (roots, dirs) for each. It does not return other
     directories which may also need to be considered, like the parent
     directories.
     '''
     r = []
     d = []
     for kind, pat, source in kindpats:
         if kind in ('glob', 'rootglob'): # find the non-glob prefix
             root = []
             for p in pat.split('/'):
                 if '[' in p or '{' in p or '*' in p or '?' in p:
                     break
                 root.append(p)
             r.append('/'.join(root))
         elif kind in ('relpath', 'path'):
             if pat == '.':
                 pat = ''
             r.append(pat)
         elif kind in ('rootfilesin',):
             if pat == '.':
                 pat = ''
             d.append(pat)
         else: # relglob, re, relre
             r.append('')
     return r, d
 
 def _roots(kindpats):
     '''Returns root directories to match recursively from the given patterns.'''
     roots, dirs = _patternrootsanddirs(kindpats)
     return roots
 
 def _rootsdirsandparents(kindpats):
     '''Returns roots and exact directories from patterns.
 
     `roots` are directories to match recursively, `dirs` should
     be matched non-recursively, and `parents` are the implicitly required
     directories to walk to items in either roots or dirs.
 
     Returns a tuple of (roots, dirs, parents).
 
     >>> r = _rootsdirsandparents(
     ...     [(b'glob', b'g/h/*', b''), (b'glob', b'g/h', b''),
     ...      (b'glob', b'g*', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['g/h', 'g/h', ''], []) ['', 'g']
     >>> r = _rootsdirsandparents(
     ...     [(b'rootfilesin', b'g/h', b''), (b'rootfilesin', b'', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     ([], ['g/h', '']) ['', 'g']
     >>> r = _rootsdirsandparents(
     ...     [(b'relpath', b'r', b''), (b'path', b'p/p', b''),
     ...      (b'path', b'', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['r', 'p/p', ''], []) ['', 'p']
     >>> r = _rootsdirsandparents(
     ...     [(b'relglob', b'rg*', b''), (b're', b're/', b''),
     ...      (b'relre', b'rr', b'')])
     >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
     (['', '', ''], []) ['']
     '''
     r, d = _patternrootsanddirs(kindpats)
 
     p = set()
     # Add the parents as non-recursive/exact directories, since they must be
     # scanned to get to either the roots or the other exact directories.
     p.update(util.dirs(d))
     p.update(util.dirs(r))
 
     # FIXME: all uses of this function convert these to sets, do so before
     # returning.
     # FIXME: all uses of this function do not need anything in 'roots' and
     # 'dirs' to also be in 'parents', consider removing them before returning.
     return r, d, p
 
 def _explicitfiles(kindpats):
     '''Returns the potential explicit filenames from the patterns.
 
     >>> _explicitfiles([(b'path', b'foo/bar', b'')])
     ['foo/bar']
     >>> _explicitfiles([(b'rootfilesin', b'foo/bar', b'')])
     []
     '''
     # Keep only the pattern kinds where one can specify filenames (vs only
     # directory names).
     filable = [kp for kp in kindpats if kp[0] not in ('rootfilesin',)]
     return _roots(filable)
 
 def _prefix(kindpats):
     '''Whether all the patterns match a prefix (i.e. recursively)'''
     for kind, pat, source in kindpats:
         if kind not in ('path', 'relpath'):
             return False
     return True
 
 _commentre = None
 
 def readpatternfile(filepath, warn, sourceinfo=False):
     '''parse a pattern file, returning a list of
     patterns. These patterns should be given to compile()
     to be validated and converted into a match function.
 
     trailing white space is dropped.
     the escape character is backslash.
     comments start with #.
     empty lines are skipped.
 
     lines can be of the following formats:
 
     syntax: regexp # defaults following lines to non-rooted regexps
     syntax: glob   # defaults following lines to non-rooted globs
     re:pattern     # non-rooted regular expression
     glob:pattern   # non-rooted glob
     rootglob:pat   # rooted glob (same root as ^ in regexps)
     pattern        # pattern of the current default type
 
     if sourceinfo is set, returns a list of tuples:
     (pattern, lineno, originalline).
     This is useful to debug ignore patterns.
     '''
 
-    if rustext is not None:
-        result, warnings = rustext.filepatterns.read_pattern_file(
+    if rustmod is not None:
+        result, warnings = rustmod.read_pattern_file(
             filepath,
             bool(warn),
             sourceinfo,
         )
 
         for warning_params in warnings:
             # Can't be easily emitted from Rust, because it would require
             # a mechanism for both gettext and calling the `warn` function.
             warn(_("%s: ignoring invalid syntax '%s'\n") % warning_params)
 
         return result
 
     syntaxes = {
         're': 'relre:',
         'regexp': 'relre:',
         'glob': 'relglob:',
         'rootglob': 'rootglob:',
         'include': 'include',
         'subinclude': 'subinclude',
     }
     syntax = 'relre:'
     patterns = []
 
     fp = open(filepath, 'rb')
     for lineno, line in enumerate(util.iterfile(fp), start=1):
         if "#" in line:
             global _commentre
             if not _commentre:
                 _commentre = util.re.compile(br'((?:^|[^\\])(?:\\\\)*)#.*')
             # remove comments prefixed by an even number of escapes
             m = _commentre.search(line)
             if m:
                 line = line[:m.end(1)]
             # fixup properly escaped comments that survived the above
             line = line.replace("\\#", "#")
         line = line.rstrip()
         if not line:
             continue
 
         if line.startswith('syntax:'):
             s = line[7:].strip()
             try:
                 syntax = syntaxes[s]
             except KeyError:
                 if warn:
                     warn(_("%s: ignoring invalid syntax '%s'\n") %
                          (filepath, s))
             continue
 
         linesyntax = syntax
         for s, rels in syntaxes.iteritems():
             if line.startswith(rels):
                 linesyntax = rels
                 line = line[len(rels):]
                 break
             elif line.startswith(s+':'):
                 linesyntax = rels
                 line = line[len(s) + 1:]
                 break
         if sourceinfo:
             patterns.append((linesyntax + line, lineno, line))
         else:
             patterns.append(linesyntax + line)
     fp.close()
     return patterns

mercurial/revlog.py

@@ -1,2687 +1,2684
 # revlog.py - storage back-end for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Storage back-end for Mercurial.
 
 This provides efficient delta storage with O(1) retrieve and append
 and O(changes) merge between branches.
 """
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import errno
 import os
 import struct
 import zlib
 
 # import stuff from node for others to import from revlog
 from .node import (
     bin,
     hex,
     nullhex,
     nullid,
     nullrev,
     short,
     wdirfilenodeids,
     wdirhex,
     wdirid,
     wdirrev,
 )
 from .i18n import _
 from .revlogutils.constants import (
     FLAG_GENERALDELTA,
     FLAG_INLINE_DATA,
     REVIDX_DEFAULT_FLAGS,
     REVIDX_ELLIPSIS,
     REVIDX_EXTSTORED,
     REVIDX_FLAGS_ORDER,
     REVIDX_ISCENSORED,
     REVIDX_KNOWN_FLAGS,
     REVIDX_RAWTEXT_CHANGING_FLAGS,
     REVLOGV0,
     REVLOGV1,
     REVLOGV1_FLAGS,
     REVLOGV2,
     REVLOGV2_FLAGS,
     REVLOG_DEFAULT_FLAGS,
     REVLOG_DEFAULT_FORMAT,
     REVLOG_DEFAULT_VERSION,
 )
 from .thirdparty import (
     attr,
 )
 from . import (
     ancestor,
     dagop,
     error,
     mdiff,
     policy,
     pycompat,
     repository,
     templatefilters,
     util,
 )
 from .revlogutils import (
     deltas as deltautil,
 )
 from .utils import (
     interfaceutil,
     storageutil,
     stringutil,
 )
 
 # blanked usage of all the name to prevent pyflakes constraints
 # We need these name available in the module for extensions.
 REVLOGV0
 REVLOGV1
 REVLOGV2
 FLAG_INLINE_DATA
 FLAG_GENERALDELTA
 REVLOG_DEFAULT_FLAGS
 REVLOG_DEFAULT_FORMAT
 REVLOG_DEFAULT_VERSION
 REVLOGV1_FLAGS
 REVLOGV2_FLAGS
 REVIDX_ISCENSORED
 REVIDX_ELLIPSIS
 REVIDX_EXTSTORED
 REVIDX_DEFAULT_FLAGS
 REVIDX_FLAGS_ORDER
 REVIDX_KNOWN_FLAGS
 REVIDX_RAWTEXT_CHANGING_FLAGS
 
 parsers = policy.importmod(r'parsers')
-try:
-    from . import rustext
-    rustext.__name__  # force actual import (see hgdemandimport)
-except ImportError:
-    rustext = None
+rustancestor = policy.importrust(r'ancestor')
+rustdagop = policy.importrust(r'dagop')
 
 # Aliased for performance.
 _zlibdecompress = zlib.decompress
 
 # max size of revlog with inline data
 _maxinline = 131072
 _chunksize = 1048576
 
 # Store flag processors (cf. 'addflagprocessor()' to register)
 _flagprocessors = {
     REVIDX_ISCENSORED: None,
 }
 
 # Flag processors for REVIDX_ELLIPSIS.
 def ellipsisreadprocessor(rl, text):
     return text, False
 
 def ellipsiswriteprocessor(rl, text):
     return text, False
 
 def ellipsisrawprocessor(rl, text):
     return False
 
 ellipsisprocessor = (
     ellipsisreadprocessor,
     ellipsiswriteprocessor,
     ellipsisrawprocessor,
 )
 
 def addflagprocessor(flag, processor):
     """Register a flag processor on a revision data flag.
 
     Invariant:
     - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
       and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
     - Only one flag processor can be registered on a specific flag.
     - flagprocessors must be 3-tuples of functions (read, write, raw) with the
       following signatures:
           - (read)  f(self, rawtext) -> text, bool
           - (write) f(self, text) -> rawtext, bool
           - (raw)   f(self, rawtext) -> bool
       "text" is presented to the user. "rawtext" is stored in revlog data, not
       directly visible to the user.
       The boolean returned by these transforms is used to determine whether
       the returned text can be used for hash integrity checking. For example,
       if "write" returns False, then "text" is used to generate hash. If
       "write" returns True, that basically means "rawtext" returned by "write"
       should be used to generate hash. Usually, "write" and "read" return
       different booleans. And "raw" returns a same boolean as "write".
 
       Note: The 'raw' transform is used for changegroup generation and in some
       debug commands. In this case the transform only indicates whether the
       contents can be used for hash integrity checks.
     """
     _insertflagprocessor(flag, processor, _flagprocessors)
 
 def _insertflagprocessor(flag, processor, flagprocessors):
     if not flag & REVIDX_KNOWN_FLAGS:
         msg = _("cannot register processor on unknown flag '%#x'.") % (flag)
         raise error.ProgrammingError(msg)
     if flag not in REVIDX_FLAGS_ORDER:
         msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag)
         raise error.ProgrammingError(msg)
     if flag in flagprocessors:
         msg = _("cannot register multiple processors on flag '%#x'.") % (flag)
         raise error.Abort(msg)
     flagprocessors[flag] = processor
 
 def getoffset(q):
     return int(q >> 16)
 
 def gettype(q):
     return int(q & 0xFFFF)
 
 def offset_type(offset, type):
     if (type & ~REVIDX_KNOWN_FLAGS) != 0:
         raise ValueError('unknown revlog index flags')
     return int(int(offset) << 16 | type)
 
 @attr.s(slots=True, frozen=True)
 class _revisioninfo(object):
     """Information about a revision that allows building its fulltext
     node:       expected hash of the revision
     p1, p2:     parent revs of the revision
     btext:      built text cache consisting of a one-element list
     cachedelta: (baserev, uncompressed_delta) or None
     flags:      flags associated to the revision storage
 
     One of btext[0] or cachedelta must be set.
     """
     node = attr.ib()
     p1 = attr.ib()
     p2 = attr.ib()
     btext = attr.ib()
     textlen = attr.ib()
     cachedelta = attr.ib()
     flags = attr.ib()
 
 @interfaceutil.implementer(repository.irevisiondelta)
 @attr.s(slots=True)
 class revlogrevisiondelta(object):
     node = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     basenode = attr.ib()
     flags = attr.ib()
     baserevisionsize = attr.ib()
     revision = attr.ib()
     delta = attr.ib()
     linknode = attr.ib(default=None)
 
 @interfaceutil.implementer(repository.iverifyproblem)
 @attr.s(frozen=True)
 class revlogproblem(object):
     warning = attr.ib(default=None)
     error = attr.ib(default=None)
     node = attr.ib(default=None)
 
 # index v0:
 #  4 bytes: offset
 #  4 bytes: compressed length
 #  4 bytes: base rev
 #  4 bytes: link rev
 # 20 bytes: parent 1 nodeid
 # 20 bytes: parent 2 nodeid
 # 20 bytes: nodeid
 indexformatv0 = struct.Struct(">4l20s20s20s")
 indexformatv0_pack = indexformatv0.pack
 indexformatv0_unpack = indexformatv0.unpack
 
 class revlogoldindex(list):
     def __getitem__(self, i):
         if i == -1:
             return (0, 0, 0, -1, -1, -1, -1, nullid)
         return list.__getitem__(self, i)
 
 class revlogoldio(object):
     def __init__(self):
         self.size = indexformatv0.size
 
     def parseindex(self, data, inline):
         s = self.size
         index = []
         nodemap = {nullid: nullrev}
         n = off = 0
         l = len(data)
         while off + s <= l:
             cur = data[off:off + s]
             off += s
             e = indexformatv0_unpack(cur)
             # transform to revlogv1 format
             e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
                   nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
             index.append(e2)
             nodemap[e[6]] = n
             n += 1
 
         return revlogoldindex(index), nodemap, None
 
     def packentry(self, entry, node, version, rev):
         if gettype(entry[0]):
             raise error.RevlogError(_('index entry flags need revlog '
                                       'version 1'))
         e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
               node(entry[5]), node(entry[6]), entry[7])
         return indexformatv0_pack(*e2)
 
 # index ng:
 #  6 bytes: offset
 #  2 bytes: flags
 #  4 bytes: compressed length
 #  4 bytes: uncompressed length
 #  4 bytes: base rev
 #  4 bytes: link rev
 #  4 bytes: parent 1 rev
 #  4 bytes: parent 2 rev
 # 32 bytes: nodeid
 indexformatng = struct.Struct(">Qiiiiii20s12x")
 indexformatng_pack = indexformatng.pack
 versionformat = struct.Struct(">I")
 versionformat_pack = versionformat.pack
 versionformat_unpack = versionformat.unpack
 
 # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
 # signed integer)
 _maxentrysize = 0x7fffffff
 
 class revlogio(object):
     def __init__(self):
         self.size = indexformatng.size
 
     def parseindex(self, data, inline):
         # call the C implementation to parse the index data
         index, cache = parsers.parse_index2(data, inline)
         return index, getattr(index, 'nodemap', None), cache
 
     def packentry(self, entry, node, version, rev):
         p = indexformatng_pack(*entry)
         if rev == 0:
             p = versionformat_pack(version) + p[4:]
         return p
 
 class revlog(object):
     """
     the underlying revision storage object
 
     A revlog consists of two parts, an index and the revision data.
 
     The index is a file with a fixed record size containing
     information on each revision, including its nodeid (hash), the
     nodeids of its parents, the position and offset of its data within
     the data file, and the revision it's based on. Finally, each entry
     contains a linkrev entry that can serve as a pointer to external
     data.
 
     The revision data itself is a linear collection of data chunks.
     Each chunk represents a revision and is usually represented as a
     delta against the previous chunk. To bound lookup time, runs of
     deltas are limited to about 2 times the length of the original
     version data. This makes retrieval of a version proportional to
     its size, or O(1) relative to the number of revisions.
 
     Both pieces of the revlog are written to in an append-only
     fashion, which means we never need to rewrite a file to insert or
     remove data, and can use some simple techniques to avoid the need
     for locking while reading.
 
     If checkambig, indexfile is opened with checkambig=True at
     writing, to avoid file stat ambiguity.
 
     If mmaplargeindex is True, and an mmapindexthreshold is set, the
     index will be mmapped rather than read if it is larger than the
     configured threshold.
 
     If censorable is True, the revlog can have censored revisions.
     """
     def __init__(self, opener, indexfile, datafile=None, checkambig=False,
                  mmaplargeindex=False, censorable=False):
         """
         create a revlog object
 
         opener is a function that abstracts the file opening operation
         and can be used to implement COW semantics or the like.
         """
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + ".d")
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
         self._checkambig = checkambig
         self._mmaplargeindex = mmaplargeindex
         self._censorable = censorable
         # 3-tuple of (node, rev, text) for a raw revision.
         self._revisioncache = None
         # Maps rev to chain base rev.
         self._chainbasecache = util.lrucachedict(100)
         # 2-tuple of (offset, data) of raw data from the revlog at an offset.
         self._chunkcache = (0, '')
         # How much data to read and cache into the raw revlog data cache.
         self._chunkcachesize = 65536
         self._maxchainlen = None
         self._deltabothparents = True
         self.index = []
         # Mapping of partial identifiers to full nodes.
         self._pcache = {}
         # Mapping of revision integer to full node.
         self._nodecache = {nullid: nullrev}
         self._nodepos = None
         self._compengine = 'zlib'
         self._compengineopts = {}
         self._maxdeltachainspan = -1
         self._withsparseread = False
         self._sparserevlog = False
         self._srdensitythreshold = 0.50
         self._srmingapsize = 262144
 
         # Make copy of flag processors so each revlog instance can support
         # custom flags.
         self._flagprocessors = dict(_flagprocessors)
 
         # 2-tuple of file handles being used for active writing.
         self._writinghandles = None
 
         self._loadindex()
 
     def _loadindex(self):
         mmapindexthreshold = None
         opts = getattr(self.opener, 'options', {}) or {}
 
         if 'revlogv2' in opts:
             newversionflags = REVLOGV2 | FLAG_INLINE_DATA
         elif 'revlogv1' in opts:
             newversionflags = REVLOGV1 | FLAG_INLINE_DATA
             if 'generaldelta' in opts:
                 newversionflags |= FLAG_GENERALDELTA
         elif getattr(self.opener, 'options', None) is not None:
             # If options provided but no 'revlog*' found, the repository
             # would have no 'requires' file in it, which means we have to
             # stick to the old format.
             newversionflags = REVLOGV0
         else:
             newversionflags = REVLOG_DEFAULT_VERSION
 
         if 'chunkcachesize' in opts:
             self._chunkcachesize = opts['chunkcachesize']
         if 'maxchainlen' in opts:
             self._maxchainlen = opts['maxchainlen']
         if 'deltabothparents' in opts:
             self._deltabothparents = opts['deltabothparents']
         self._lazydelta = bool(opts.get('lazydelta', True))
         self._lazydeltabase = False
         if self._lazydelta:
             self._lazydeltabase = bool(opts.get('lazydeltabase', False))
         if 'compengine' in opts:
             self._compengine = opts['compengine']
         if 'zlib.level' in opts:
             self._compengineopts['zlib.level'] = opts['zlib.level']
         if 'zstd.level' in opts:
             self._compengineopts['zstd.level'] = opts['zstd.level']
         if 'maxdeltachainspan' in opts:
             self._maxdeltachainspan = opts['maxdeltachainspan']
         if self._mmaplargeindex and 'mmapindexthreshold' in opts:
             mmapindexthreshold = opts['mmapindexthreshold']
         self._sparserevlog = bool(opts.get('sparse-revlog', False))
         withsparseread = bool(opts.get('with-sparse-read', False))
         # sparse-revlog forces sparse-read
         self._withsparseread = self._sparserevlog or withsparseread
         if 'sparse-read-density-threshold' in opts:
             self._srdensitythreshold = opts['sparse-read-density-threshold']
         if 'sparse-read-min-gap-size' in opts:
             self._srmingapsize = opts['sparse-read-min-gap-size']
         if opts.get('enableellipsis'):
             self._flagprocessors[REVIDX_ELLIPSIS] = ellipsisprocessor
 
         # revlog v0 doesn't have flag processors
         for flag, processor in opts.get(b'flagprocessors', {}).iteritems():
             _insertflagprocessor(flag, processor, self._flagprocessors)
 
         if self._chunkcachesize <= 0:
             raise error.RevlogError(_('revlog chunk cache size %r is not '
                                       'greater than 0') % self._chunkcachesize)
         elif self._chunkcachesize & (self._chunkcachesize - 1):
             raise error.RevlogError(_('revlog chunk cache size %r is not a '
                                       'power of 2') % self._chunkcachesize)
 
         indexdata = ''
         self._initempty = True
         try:
             with self._indexfp() as f:
                 if (mmapindexthreshold is not None and
                     self.opener.fstat(f).st_size >= mmapindexthreshold):
                     # TODO: should .close() to release resources without
                     # relying on Python GC
                     indexdata = util.buffer(util.mmapread(f))
                 else:
                     indexdata = f.read()
             if len(indexdata) > 0:
                 versionflags = versionformat_unpack(indexdata[:4])[0]
                 self._initempty = False
             else:
                 versionflags = newversionflags
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
 
             versionflags = newversionflags
 
         self.version = versionflags
 
         flags = versionflags & ~0xFFFF
         fmt = versionflags & 0xFFFF
 
         if fmt == REVLOGV0:
             if flags:
                 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
                                           'revlog %s') %
                                         (flags >> 16, fmt, self.indexfile))
 
             self._inline = False
             self._generaldelta = False
 
         elif fmt == REVLOGV1:
             if flags & ~REVLOGV1_FLAGS:
                 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
                                           'revlog %s') %
                                         (flags >> 16, fmt, self.indexfile))
 
             self._inline = versionflags & FLAG_INLINE_DATA
             self._generaldelta = versionflags & FLAG_GENERALDELTA
 
         elif fmt == REVLOGV2:
             if flags & ~REVLOGV2_FLAGS:
                 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
                                           'revlog %s') %
                                         (flags >> 16, fmt, self.indexfile))
 
             self._inline = versionflags & FLAG_INLINE_DATA
             # generaldelta implied by version 2 revlogs.
             self._generaldelta = True
 
         else:
             raise error.RevlogError(_('unknown version (%d) in revlog %s') %
                                     (fmt, self.indexfile))
         # sparse-revlog can't be on without general-delta (issue6056)
         if not self._generaldelta:
             self._sparserevlog = False
 
         self._storedeltachains = True
 
         self._io = revlogio()
         if self.version == REVLOGV0:
             self._io = revlogoldio()
         try:
             d = self._io.parseindex(indexdata, self._inline)
         except (ValueError, IndexError):
             raise error.RevlogError(_("index %s is corrupted") %
                                     self.indexfile)
         self.index, nodemap, self._chunkcache = d
         if nodemap is not None:
             self.nodemap = self._nodecache = nodemap
         if not self._chunkcache:
             self._chunkclear()
         # revnum -> (chain-length, sum-delta-length)
         self._chaininfocache = {}
         # revlog header -> revlog compressor
         self._decompressors = {}
 
     @util.propertycache
     def _compressor(self):
         engine = util.compengines[self._compengine]
         return engine.revlogcompressor(self._compengineopts)
 
     def _indexfp(self, mode='r'):
         """file object for the revlog's index file"""
         args = {r'mode': mode}
         if mode != 'r':
             args[r'checkambig'] = self._checkambig
         if mode == 'w':
             args[r'atomictemp'] = True
         return self.opener(self.indexfile, **args)
 
     def _datafp(self, mode='r'):
         """file object for the revlog's data file"""
         return self.opener(self.datafile, mode=mode)
 
     @contextlib.contextmanager
     def _datareadfp(self, existingfp=None):
         """file object suitable to read data"""
         # Use explicit file handle, if given.
         if existingfp is not None:
             yield existingfp
 
         # Use a file handle being actively used for writes, if available.
         # There is some danger to doing this because reads will seek the
         # file. However, _writeentry() performs a SEEK_END before all writes,
         # so we should be safe.
         elif self._writinghandles:
             if self._inline:
                 yield self._writinghandles[0]
             else:
                 yield self._writinghandles[1]
 
         # Otherwise open a new file handle.
         else:
             if self._inline:
                 func = self._indexfp
             else:
                 func = self._datafp
             with func() as fp:
                 yield fp
 
     def tip(self):
         return self.node(len(self.index) - 1)
     def __contains__(self, rev):
         return 0 <= rev < len(self)
     def __len__(self):
         return len(self.index)
     def __iter__(self):
         return iter(pycompat.xrange(len(self)))
     def revs(self, start=0, stop=None):
         """iterate over all rev in this revlog (from start to stop)"""
         return storageutil.iterrevs(len(self), start=start, stop=stop)
 
     @util.propertycache
     def nodemap(self):
         if self.index:
             # populate mapping down to the initial node
             node0 = self.index[0][7]  # get around changelog filtering
             self.rev(node0)
         return self._nodecache
 
     def hasnode(self, node):
         try:
             self.rev(node)
             return True
         except KeyError:
             return False
 
     def candelta(self, baserev, rev):
         """whether two revisions (baserev, rev) can be delta-ed or not"""
         # Disable delta if either rev requires a content-changing flag
         # processor (ex. LFS). This is because such flag processor can alter
         # the rawtext content that the delta will be based on, and two clients
         # could have a same revlog node with different flags (i.e. different
         # rawtext contents) and the delta could be incompatible.
         if ((self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS)
             or (self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS)):
             return False
         return True
 
     def clearcaches(self):
         self._revisioncache = None
         self._chainbasecache.clear()
         self._chunkcache = (0, '')
         self._pcache = {}
 
         try:
             # If we are using the native C version, you are in a fun case
             # where self.index, self.nodemap and self._nodecaches is the same
             # object.
             self._nodecache.clearcaches()
         except AttributeError:
             self._nodecache = {nullid: nullrev}
             self._nodepos = None
 
     def rev(self, node):
         try:
             return self._nodecache[node]
         except TypeError:
             raise
         except error.RevlogError:
             # parsers.c radix tree lookup failed
             if node == wdirid or node in wdirfilenodeids:
                 raise error.WdirUnsupported
             raise error.LookupError(node, self.indexfile, _('no node'))
         except KeyError:
             # pure python cache lookup failed
             n = self._nodecache
             i = self.index
             p = self._nodepos
             if p is None:
                 p = len(i) - 1
             else:
                 assert p < len(i)
             for r in pycompat.xrange(p, -1, -1):
                 v = i[r][7]
                 n[v] = r
                 if v == node:
                     self._nodepos = r - 1
                     return r
             if node == wdirid or node in wdirfilenodeids:
                 raise error.WdirUnsupported
             raise error.LookupError(node, self.indexfile, _('no node'))
 
     # Accessors for index entries.
 
     # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes
     # are flags.
     def start(self, rev):
         return int(self.index[rev][0] >> 16)
 
     def flags(self, rev):
         return self.index[rev][0] & 0xFFFF
 
     def length(self, rev):
         return self.index[rev][1]
 
     def rawsize(self, rev):
         """return the length of the uncompressed text for a given revision"""
         l = self.index[rev][2]
         if l >= 0:
             return l
 
         t = self.revision(rev, raw=True)
         return len(t)
 
     def size(self, rev):
         """length of non-raw text (processed by a "read" flag processor)"""
         # fast path: if no "read" flag processor could change the content,
         # size is rawsize. note: ELLIPSIS is known to not change the content.
         flags = self.flags(rev)
         if flags & (REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0:
             return self.rawsize(rev)
 
         return len(self.revision(rev, raw=False))
 
     def chainbase(self, rev):
         base = self._chainbasecache.get(rev)
         if base is not None:
             return base
 
         index = self.index
         iterrev = rev
         base = index[iterrev][3]
         while base != iterrev:
             iterrev = base
             base = index[iterrev][3]
 
         self._chainbasecache[rev] = base
         return base
 
     def linkrev(self, rev):
         return self.index[rev][4]
 
     def parentrevs(self, rev):
         try:
             entry = self.index[rev]
         except IndexError:
             if rev == wdirrev:
                 raise error.WdirUnsupported
             raise
 
         return entry[5], entry[6]
 
     # fast parentrevs(rev) where rev isn't filtered
     _uncheckedparentrevs = parentrevs
 
     def node(self, rev):
         try:
             return self.index[rev][7]
         except IndexError:
             if rev == wdirrev:
                 raise error.WdirUnsupported
             raise
 
     # Derived from index values.
 
     def end(self, rev):
         return self.start(rev) + self.length(rev)
 
     def parents(self, node):
         i = self.index
         d = i[self.rev(node)]
         return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline
 
     def chainlen(self, rev):
         return self._chaininfo(rev)[0]
 
     def _chaininfo(self, rev):
         chaininfocache = self._chaininfocache
         if rev in chaininfocache:
             return chaininfocache[rev]
         index = self.index
         generaldelta = self._generaldelta
         iterrev = rev
         e = index[iterrev]
         clen = 0
         compresseddeltalen = 0
         while iterrev != e[3]:
             clen += 1
             compresseddeltalen += e[1]
             if generaldelta:
                 iterrev = e[3]
             else:
                 iterrev -= 1
             if iterrev in chaininfocache:
                 t = chaininfocache[iterrev]
                 clen += t[0]
                 compresseddeltalen += t[1]
                 break
             e = index[iterrev]
         else:
             # Add text length of base since decompressing that also takes
             # work. For cache hits the length is already included.
             compresseddeltalen += e[1]
         r = (clen, compresseddeltalen)
         chaininfocache[rev] = r
         return r
 
     def _deltachain(self, rev, stoprev=None):
         """Obtain the delta chain for a revision.
 
         ``stoprev`` specifies a revision to stop at. If not specified, we
         stop at the base of the chain.
 
         Returns a 2-tuple of (chain, stopped) where ``chain`` is a list of
         revs in ascending order and ``stopped`` is a bool indicating whether
         ``stoprev`` was hit.
         """
         # Try C implementation.
         try:
             return self.index.deltachain(rev, stoprev, self._generaldelta)
         except AttributeError:
             pass
 
         chain = []
 
         # Alias to prevent attribute lookup in tight loop.
         index = self.index
         generaldelta = self._generaldelta
 
         iterrev = rev
         e = index[iterrev]
         while iterrev != e[3] and iterrev != stoprev:
             chain.append(iterrev)
             if generaldelta:
                 iterrev = e[3]
             else:
                 iterrev -= 1
             e = index[iterrev]
 
         if iterrev == stoprev:
             stopped = True
         else:
             chain.append(iterrev)
             stopped = False
 
         chain.reverse()
         return chain, stopped
 
     def ancestors(self, revs, stoprev=0, inclusive=False):
         """Generate the ancestors of 'revs' in reverse revision order.
         Does not generate revs lower than stoprev.
 
         See the documentation for ancestor.lazyancestors for more details."""
 
         # first, make sure start revisions aren't filtered
         revs = list(revs)
         checkrev = self.node
         for r in revs:
             checkrev(r)
         # and we're sure ancestors aren't filtered as well
 
-        if rustext is not None:
-            lazyancestors = rustext.ancestor.LazyAncestors
+        if rustancestor is not None:
+            lazyancestors = rustancestor.LazyAncestors
             arg = self.index
         elif util.safehasattr(parsers, 'rustlazyancestors'):
             lazyancestors = ancestor.rustlazyancestors
             arg = self.index
         else:
             lazyancestors = ancestor.lazyancestors
             arg = self._uncheckedparentrevs
         return lazyancestors(arg, revs, stoprev=stoprev, inclusive=inclusive)
 
     def descendants(self, revs):
         return dagop.descendantrevs(revs, self.revs, self.parentrevs)
 
     def findcommonmissing(self, common=None, heads=None):
         """Return a tuple of the ancestors of common and the ancestors of heads
         that are not ancestors of common. In revset terminology, we return the
         tuple:
 
           ::common, (::heads) - (::common)
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of node IDs.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullid]
         if heads is None:
             heads = self.heads()
 
         common = [self.rev(n) for n in common]
         heads = [self.rev(n) for n in heads]
 
         # we want the ancestors, but inclusive
         class lazyset(object):
             def __init__(self, lazyvalues):
                 self.addedvalues = set()
                 self.lazyvalues = lazyvalues
 
             def __contains__(self, value):
                 return value in self.addedvalues or value in self.lazyvalues
 
             def __iter__(self):
                 added = self.addedvalues
                 for r in added:
                     yield r
                 for r in self.lazyvalues:
                     if not r in added:
                         yield r
 
             def add(self, value):
                 self.addedvalues.add(value)
 
             def update(self, values):
                 self.addedvalues.update(values)
 
         has = lazyset(self.ancestors(common))
         has.add(nullrev)
         has.update(common)
 
         # take all ancestors from heads that aren't in has
         missing = set()
         visit = collections.deque(r for r in heads if r not in has)
         while visit:
             r = visit.popleft()
             if r in missing:
                 continue
             else:
                 missing.add(r)
                 for p in self.parentrevs(r):
                     if p not in has:
                         visit.append(p)
         missing = list(missing)
         missing.sort()
         return has, [self.node(miss) for miss in missing]
 
     def incrementalmissingrevs(self, common=None):
         """Return an object that can be used to incrementally compute the
         revision numbers of the ancestors of arbitrary sets that are not
         ancestors of common. This is an ancestor.incrementalmissingancestors
         object.
 
         'common' is a list of revision numbers. If common is not supplied, uses
         nullrev.
         """
         if common is None:
             common = [nullrev]
 
-        if rustext is not None:
-            return rustext.ancestor.MissingAncestors(self.index, common)
+        if rustancestor is not None:
+            return rustancestor.MissingAncestors(self.index, common)
         return ancestor.incrementalmissingancestors(self.parentrevs, common)
 
     def findmissingrevs(self, common=None, heads=None):
         """Return the revision numbers of the ancestors of heads that
         are not ancestors of common.
 
         More specifically, return a list of revision numbers corresponding to
         nodes N such that every N satisfies the following constraints:
 
           1. N is an ancestor of some node in 'heads'
           2. N is not an ancestor of any node in 'common'
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of revision numbers.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullrev]
         if heads is None:
             heads = self.headrevs()
 
         inc = self.incrementalmissingrevs(common=common)
         return inc.missingancestors(heads)
 
     def findmissing(self, common=None, heads=None):
         """Return the ancestors of heads that are not ancestors of common.
 
         More specifically, return a list of nodes N such that every N
         satisfies the following constraints:
 
           1. N is an ancestor of some node in 'heads'
           2. N is not an ancestor of any node in 'common'
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of node IDs.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullid]
         if heads is None:
             heads = self.heads()
 
         common = [self.rev(n) for n in common]
         heads = [self.rev(n) for n in heads]
 
         inc = self.incrementalmissingrevs(common=common)
         return [self.node(r) for r in inc.missingancestors(heads)]
 
     def nodesbetween(self, roots=None, heads=None):
         """Return a topological path from 'roots' to 'heads'.
 
         Return a tuple (nodes, outroots, outheads) where 'nodes' is a
         topologically sorted list of all nodes N that satisfy both of
         these constraints:
 
           1. N is a descendant of some node in 'roots'
           2. N is an ancestor of some node in 'heads'
 
         Every node is considered to be both a descendant and an ancestor
         of itself, so every reachable node in 'roots' and 'heads' will be
         included in 'nodes'.
 
         'outroots' is the list of reachable nodes in 'roots', i.e., the
         subset of 'roots' that is returned in 'nodes'.  Likewise,
         'outheads' is the subset of 'heads' that is also in 'nodes'.
 
         'roots' and 'heads' are both lists of node IDs.  If 'roots' is
         unspecified, uses nullid as the only root.  If 'heads' is
         unspecified, uses list of all of the revlog's heads."""
         nonodes = ([], [], [])
         if roots is not None:
             roots = list(roots)
             if not roots:
                 return nonodes
             lowestrev = min([self.rev(n) for n in roots])
         else:
             roots = [nullid] # Everybody's a descendant of nullid
             lowestrev = nullrev
         if (lowestrev == nullrev) and (heads is None):
             # We want _all_ the nodes!
             return ([self.node(r) for r in self], [nullid], list(self.heads()))
         if heads is None:
             # All nodes are ancestors, so the latest ancestor is the last
             # node.
             highestrev = len(self) - 1
             # Set ancestors to None to signal that every node is an ancestor.
             ancestors = None
             # Set heads to an empty dictionary for later discovery of heads
             heads = {}
         else:
             heads = list(heads)
             if not heads:
                 return nonodes
             ancestors = set()
             # Turn heads into a dictionary so we can remove 'fake' heads.
             # Also, later we will be using it to filter out the heads we can't
             # find from roots.
             heads = dict.fromkeys(heads, False)
             # Start at the top and keep marking parents until we're done.
             nodestotag = set(heads)
             # Remember where the top was so we can use it as a limit later.
             highestrev = max([self.rev(n) for n in nodestotag])
             while nodestotag:
                 # grab a node to tag
                 n = nodestotag.pop()
                 # Never tag nullid
                 if n == nullid:
                     continue
                 # A node's revision number represents its place in a
                 # topologically sorted list of nodes.
                 r = self.rev(n)
                 if r >= lowestrev:
                     if n not in ancestors:
                         # If we are possibly a descendant of one of the roots
                         # and we haven't already been marked as an ancestor
                         ancestors.add(n) # Mark as ancestor
                         # Add non-nullid parents to list of nodes to tag.
                         nodestotag.update([p for p in self.parents(n) if
                                            p != nullid])
                     elif n in heads: # We've seen it before, is it a fake head?
                         # So it is, real heads should not be the ancestors of
                         # any other heads.
                         heads.pop(n)
             if not ancestors:
                 return nonodes
             # Now that we have our set of ancestors, we want to remove any
             # roots that are not ancestors.
 
             # If one of the roots was nullid, everything is included anyway.
             if lowestrev > nullrev:
                 # But, since we weren't, let's recompute the lowest rev to not
                 # include roots that aren't ancestors.
 
                 # Filter out roots that aren't ancestors of heads
                 roots = [root for root in roots if root in ancestors]
                 # Recompute the lowest revision
                 if roots:
                     lowestrev = min([self.rev(root) for root in roots])
                 else:
                     # No more roots?  Return empty list
                     return nonodes
             else:
                 # We are descending from nullid, and don't need to care about
                 # any other roots.
                 lowestrev = nullrev
                 roots = [nullid]
         # Transform our roots list into a set.
         descendants = set(roots)
         # Also, keep the original roots so we can filter out roots that aren't
         # 'real' roots (i.e. are descended from other roots).
         roots = descendants.copy()
         # Our topologically sorted list of output nodes.
         orderedout = []
         # Don't start at nullid since we don't want nullid in our output list,
         # and if nullid shows up in descendants, empty parents will look like
         # they're descendants.
         for r in self.revs(start=max(lowestrev, 0), stop=highestrev + 1):
             n = self.node(r)
             isdescendant = False
             if lowestrev == nullrev:  # Everybody is a descendant of nullid
                 isdescendant = True
             elif n in descendants:
                 # n is already a descendant
                 isdescendant = True
                 # This check only needs to be done here because all the roots
                 # will start being marked is descendants before the loop.
                 if n in roots:
                     # If n was a root, check if it's a 'real' root.
                     p = tuple(self.parents(n))
                     # If any of its parents are descendants, it's not a root.
                     if (p[0] in descendants) or (p[1] in descendants):
                         roots.remove(n)
             else:
                 p = tuple(self.parents(n))
                 # A node is a descendant if either of its parents are
                 # descendants.  (We seeded the dependents list with the roots
                 # up there, remember?)
                 if (p[0] in descendants) or (p[1] in descendants):
                     descendants.add(n)
                     isdescendant = True
             if isdescendant and ((ancestors is None) or (n in ancestors)):
                 # Only include nodes that are both descendants and ancestors.
                 orderedout.append(n)
                 if (ancestors is not None) and (n in heads):
                     # We're trying to figure out which heads are reachable
                     # from roots.
                     # Mark this head as having been reached
                     heads[n] = True
                 elif ancestors is None:
                     # Otherwise, we're trying to discover the heads.
                     # Assume this is a head because if it isn't, the next step
                     # will eventually remove it.
                     heads[n] = True
                     # But, obviously its parents aren't.
                     for p in self.parents(n):
                         heads.pop(p, None)
         heads = [head for head, flag in heads.iteritems() if flag]
         roots = list(roots)
         assert orderedout
         assert roots
         assert heads
         return (orderedout, roots, heads)
 
     def headrevs(self, revs=None):
         if revs is None:
             try:
                 return self.index.headrevs()
             except AttributeError:
                 return self._headrevs()
-        if rustext is not None:
-            return rustext.dagop.headrevs(self.index, revs)
+        if rustdagop is not None:
+            return rustdagop.headrevs(self.index, revs)
         return dagop.headrevs(revs, self._uncheckedparentrevs)
 
     def computephases(self, roots):
         return self.index.computephasesmapsets(roots)
 
     def _headrevs(self):
         count = len(self)
         if not count:
             return [nullrev]
         # we won't iter over filtered rev so nobody is a head at start
         ishead = [0] * (count + 1)
         index = self.index
         for r in self:
             ishead[r] = 1  # I may be an head
             e = index[r]
             ishead[e[5]] = ishead[e[6]] = 0  # my parent are not
         return [r for r, val in enumerate(ishead) if val]
 
     def heads(self, start=None, stop=None):
         """return the list of all nodes that have no children
 
         if start is specified, only heads that are descendants of
         start will be returned
         if stop is specified, it will consider all the revs from stop
         as if they had no children
         """
         if start is None and stop is None:
             if not len(self):
                 return [nullid]
             return [self.node(r) for r in self.headrevs()]
 
         if start is None:
             start = nullrev
         else:
             start = self.rev(start)
 
         stoprevs = set(self.rev(n) for n in stop or [])
 
         revs = dagop.headrevssubset(self.revs, self.parentrevs, startrev=start,
                                     stoprevs=stoprevs)
 
         return [self.node(rev) for rev in revs]
 
     def children(self, node):
         """find the children of a given node"""
         c = []
         p = self.rev(node)
         for r in self.revs(start=p + 1):
             prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
             if prevs:
                 for pr in prevs:
                     if pr == p:
                         c.append(self.node(r))
             elif p == nullrev:
                 c.append(self.node(r))
         return c
 
     def commonancestorsheads(self, a, b):
         """calculate all the heads of the common ancestors of nodes a and b"""
         a, b = self.rev(a), self.rev(b)
         ancs = self._commonancestorsheads(a, b)
         return pycompat.maplist(self.node, ancs)
 
     def _commonancestorsheads(self, *revs):
         """calculate all the heads of the common ancestors of revs"""
         try:
             ancs = self.index.commonancestorsheads(*revs)
         except (AttributeError, OverflowError): # C implementation failed
             ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
         return ancs
 
     def isancestor(self, a, b):
         """return True if node a is an ancestor of node b
 
         A revision is considered an ancestor of itself."""
         a, b = self.rev(a), self.rev(b)
         return self.isancestorrev(a, b)
 
     def isancestorrev(self, a, b):
         """return True if revision a is an ancestor of revision b
 
         A revision is considered an ancestor of itself.
 
         The implementation of this is trivial but the use of
         reachableroots is not."""
         if a == nullrev:
             return True
         elif a == b:
             return True
         elif a > b:
             return False
         return bool(self.reachableroots(a, [b], [a], includepath=False))
 
     def reachableroots(self, minroot, heads, roots, includepath=False):
         """return (heads(::<roots> and <roots>::<heads>))
 
         If includepath is True, return (<roots>::<heads>)."""
         try:
             return self.index.reachableroots2(minroot, heads, roots,
                                               includepath)
         except AttributeError:
             return dagop._reachablerootspure(self.parentrevs,
                                              minroot, roots, heads, includepath)
 
     def ancestor(self, a, b):
         """calculate the "best" common ancestor of nodes a and b"""
 
         a, b = self.rev(a), self.rev(b)
         try:
             ancs = self.index.ancestors(a, b)
         except (AttributeError, OverflowError):
             ancs = ancestor.ancestors(self.parentrevs, a, b)
         if ancs:
             # choose a consistent winner when there's a tie
             return min(map(self.node, ancs))
         return nullid
 
     def _match(self, id):
         if isinstance(id, int):
             # rev
             return self.node(id)
         if len(id) == 20:
             # possibly a binary node
             # odds of a binary node being all hex in ASCII are 1 in 10**25
             try:
                 node = id
                 self.rev(node) # quick search the index
                 return node
             except error.LookupError:
                 pass # may be partial hex id
         try:
             # str(rev)
             rev = int(id)
             if "%d" % rev != id:
                 raise ValueError
             if rev < 0:
                 rev = len(self) + rev
             if rev < 0 or rev >= len(self):
                 raise ValueError
             return self.node(rev)
         except (ValueError, OverflowError):
             pass
         if len(id) == 40:
             try:
                 # a full hex nodeid?
                 node = bin(id)
                 self.rev(node)
                 return node
             except (TypeError, error.LookupError):
                 pass
 
     def _partialmatch(self, id):
         # we don't care wdirfilenodeids as they should be always full hash
         maybewdir = wdirhex.startswith(id)
         try:
             partial = self.index.partialmatch(id)
             if partial and self.hasnode(partial):
                 if maybewdir:
                     # single 'ff...' match in radix tree, ambiguous with wdir
                     raise error.RevlogError
                 return partial
             if maybewdir:
                 # no 'ff...' match in radix tree, wdir identified
                 raise error.WdirUnsupported
             return None
         except error.RevlogError:
             # parsers.c radix tree lookup gave multiple matches
             # fast path: for unfiltered changelog, radix tree is accurate
             if not getattr(self, 'filteredrevs', None):
                 raise error.AmbiguousPrefixLookupError(
                     id, self.indexfile, _('ambiguous identifier'))
             # fall through to slow path that filters hidden revisions
         except (AttributeError, ValueError):
             # we are pure python, or key was too short to search radix tree
             pass
 
         if id in self._pcache:
             return self._pcache[id]
 
         if len(id) <= 40:
             try:
                 # hex(node)[:...]
                 l = len(id) // 2  # grab an even number of digits
                 prefix = bin(id[:l * 2])
                 nl = [e[7] for e in self.index if e[7].startswith(prefix)]
                 nl = [n for n in nl if hex(n).startswith(id) and
                       self.hasnode(n)]
                 if nullhex.startswith(id):
                     nl.append(nullid)
                 if len(nl) > 0:
                     if len(nl) == 1 and not maybewdir:
                         self._pcache[id] = nl[0]
                         return nl[0]
                     raise error.AmbiguousPrefixLookupError(
                         id, self.indexfile, _('ambiguous identifier'))
                 if maybewdir:
                     raise error.WdirUnsupported
                 return None
             except TypeError:
                 pass
 
     def lookup(self, id):
         """locate a node based on:
             - revision number or str(revision number)
             - nodeid or subset of hex nodeid
         """
         n = self._match(id)
         if n is not None:
             return n
         n = self._partialmatch(id)
         if n:
             return n
 
         raise error.LookupError(id, self.indexfile, _('no match found'))
 
     def shortest(self, node, minlength=1):
         """Find the shortest unambiguous prefix that matches node."""
         def isvalid(prefix):
             try:
                 node = self._partialmatch(prefix)
             except error.AmbiguousPrefixLookupError:
                 return False
             except error.WdirUnsupported:
                 # single 'ff...' match
                 return True
             if node is None:
                 raise error.LookupError(node, self.indexfile, _('no node'))
             return True
 
         def maybewdir(prefix):
             return all(c == 'f' for c in pycompat.iterbytestr(prefix))
 
         hexnode = hex(node)
 
         def disambiguate(hexnode, minlength):
             """Disambiguate against wdirid."""
             for length in range(minlength, 41):
                 prefix = hexnode[:length]
                 if not maybewdir(prefix):
                     return prefix
 
         if not getattr(self, 'filteredrevs', None):
             try:
                 length = max(self.index.shortest(node), minlength)
                 return disambiguate(hexnode, length)
             except error.RevlogError:
                 if node != wdirid:
                     raise error.LookupError(node, self.indexfile, _('no node'))
             except AttributeError:
                 # Fall through to pure code
                 pass
 
         if node == wdirid:
             for length in range(minlength, 41):
                 prefix = hexnode[:length]
                 if isvalid(prefix):
                     return prefix
 
         for length in range(minlength, 41):
             prefix = hexnode[:length]
             if isvalid(prefix):
                 return disambiguate(hexnode, length)
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
         p1, p2 = self.parents(node)
         return storageutil.hashrevisionsha1(text, p1, p2) != node
 
     def _cachesegment(self, offset, data):
         """Add a segment to the revlog cache.
 
         Accepts an absolute offset and the data that is at that location.
         """
         o, d = self._chunkcache
         # try to add to existing cache
         if o + len(d) == offset and len(d) + len(data) < _chunksize:
             self._chunkcache = o, d + data
         else:
             self._chunkcache = offset, data
 
     def _readsegment(self, offset, length, df=None):
         """Load a segment of raw data from the revlog.
 
         Accepts an absolute offset, length to read, and an optional existing
         file handle to read from.
 
         If an existing file handle is passed, it will be seeked and the
         original seek position will NOT be restored.
 
         Returns a str or buffer of raw byte data.
 
         Raises if the requested number of bytes could not be read.
         """
         # Cache data both forward and backward around the requested
         # data, in a fixed size window. This helps speed up operations
         # involving reading the revlog backwards.
         cachesize = self._chunkcachesize
         realoffset = offset & ~(cachesize - 1)
         reallength = (((offset + length + cachesize) & ~(cachesize - 1))
                       - realoffset)
         with self._datareadfp(df) as df:
             df.seek(realoffset)
             d = df.read(reallength)
 
         self._cachesegment(realoffset, d)
         if offset != realoffset or reallength != length:
             startoffset = offset - realoffset
             if len(d) - startoffset < length:
                 raise error.RevlogError(
                     _('partial read of revlog %s; expected %d bytes from '
                       'offset %d, got %d') %
                     (self.indexfile if self._inline else self.datafile,
                      length, realoffset, len(d) - startoffset))
 
             return util.buffer(d, startoffset, length)
 
         if len(d) < length:
             raise error.RevlogError(
                 _('partial read of revlog %s; expected %d bytes from offset '
                   '%d, got %d') %
                 (self.indexfile if self._inline else self.datafile,
                  length, offset, len(d)))
 
         return d
 
     def _getsegment(self, offset, length, df=None):
         """Obtain a segment of raw data from the revlog.
 
         Accepts an absolute offset, length of bytes to obtain, and an
         optional file handle to the already-opened revlog. If the file
         handle is used, it's original seek position will not be preserved.
 
         Requests for data may be returned from a cache.
 
         Returns a str or a buffer instance of raw byte data.
         """
         o, d = self._chunkcache
         l = len(d)
 
         # is it in the cache?
         cachestart = offset - o
         cacheend = cachestart + length
         if cachestart >= 0 and cacheend <= l:
             if cachestart == 0 and cacheend == l:
                 return d # avoid a copy
             return util.buffer(d, cachestart, cacheend - cachestart)
 
         return self._readsegment(offset, length, df=df)
 
     def _getsegmentforrevs(self, startrev, endrev, df=None):
         """Obtain a segment of raw data corresponding to a range of revisions.
 
         Accepts the start and end revisions and an optional already-open
         file handle to be used for reading. If the file handle is read, its
         seek position will not be preserved.
 
         Requests for data may be satisfied by a cache.
 
         Returns a 2-tuple of (offset, data) for the requested range of
         revisions. Offset is the integer offset from the beginning of the
         revlog and data is a str or buffer of the raw byte data.
 
         Callers will need to call ``self.start(rev)`` and ``self.length(rev)``
         to determine where each revision's data begins and ends.
         """
         # Inlined self.start(startrev) & self.end(endrev) for perf reasons
         # (functions are expensive).
         index = self.index
         istart = index[startrev]
         start = int(istart[0] >> 16)
         if startrev == endrev:
             end = start + istart[1]
         else:
             iend = index[endrev]
             end = int(iend[0] >> 16) + iend[1]
 
         if self._inline:
             start += (startrev + 1) * self._io.size
             end += (endrev + 1) * self._io.size
         length = end - start
 
         return start, self._getsegment(start, length, df=df)
 
     def _chunk(self, rev, df=None):
         """Obtain a single decompressed chunk for a revision.
 
         Accepts an integer revision and an optional already-open file handle
         to be used for reading. If used, the seek position of the file will not
         be preserved.
 
         Returns a str holding uncompressed data for the requested revision.
         """
         return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1])
 
     def _chunks(self, revs, df=None, targetsize=None):
         """Obtain decompressed chunks for the specified revisions.
 
         Accepts an iterable of numeric revisions that are assumed to be in
         ascending order. Also accepts an optional already-open file handle
         to be used for reading. If used, the seek position of the file will
         not be preserved.
 
         This function is similar to calling ``self._chunk()`` multiple times,
         but is faster.
 
         Returns a list with decompressed data for each requested revision.
         """
         if not revs:
             return []
         start = self.start
         length = self.length
         inline = self._inline
         iosize = self._io.size
         buffer = util.buffer
 
         l = []
         ladd = l.append
 
         if not self._withsparseread:
             slicedchunks = (revs,)
         else:
             slicedchunks = deltautil.slicechunk(self, revs,
                                                 targetsize=targetsize)
 
         for revschunk in slicedchunks:
             firstrev = revschunk[0]
             # Skip trailing revisions with empty diff
             for lastrev in revschunk[::-1]:
                 if length(lastrev) != 0:
                     break
 
             try:
                 offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df)
             except OverflowError:
                 # issue4215 - we can't cache a run of chunks greater than
                 # 2G on Windows
                 return [self._chunk(rev, df=df) for rev in revschunk]
 
             decomp = self.decompress
             for rev in revschunk:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(decomp(buffer(data, chunkstart - offset, chunklength)))
 
         return l
 
     def _chunkclear(self):
         """Clear the raw chunk cache."""
         self._chunkcache = (0, '')
 
     def deltaparent(self, rev):
         """return deltaparent of the given revision"""
         base = self.index[rev][3]
         if base == rev:
             return nullrev
         elif self._generaldelta:
             return base
         else:
             return rev - 1
 
     def issnapshot(self, rev):
         """tells whether rev is a snapshot
         """
         if not self._sparserevlog:
             return self.deltaparent(rev) == nullrev
         elif util.safehasattr(self.index, 'issnapshot'):
             # directly assign the method to cache the testing and access
             self.issnapshot = self.index.issnapshot
             return self.issnapshot(rev)
         if rev == nullrev:
             return True
         entry = self.index[rev]
         base = entry[3]
         if base == rev:
             return True
         if base == nullrev:
             return True
         p1 = entry[5]
         p2 = entry[6]
         if base == p1 or base == p2:
             return False
         return self.issnapshot(base)
 
     def snapshotdepth(self, rev):
         """number of snapshot in the chain before this one"""
         if not self.issnapshot(rev):
             raise error.ProgrammingError('revision %d not a snapshot')
         return len(self._deltachain(rev)[0]) - 1
 
     def revdiff(self, rev1, rev2):
         """return or calculate a delta between two revisions
 
         The delta calculated is in binary form and is intended to be written to
         revlog data directly. So this function needs raw revision data.
         """
         if rev1 != nullrev and self.deltaparent(rev2) == rev1:
             return bytes(self._chunk(rev2))
 
         return mdiff.textdiff(self.revision(rev1, raw=True),
                               self.revision(rev2, raw=True))
 
     def revision(self, nodeorrev, _df=None, raw=False):
         """return an uncompressed revision of a given node or revision
         number.
 
         _df - an existing file handle to read from. (internal-only)
         raw - an optional argument specifying if the revision data is to be
         treated as raw data when applying flag transforms. 'raw' should be set
         to True when generating changegroups or in debug commands.
         """
         if isinstance(nodeorrev, int):
             rev = nodeorrev
             node = self.node(rev)
         else:
             node = nodeorrev
             rev = None
 
         cachedrev = None
         flags = None
         rawtext = None
         if node == nullid:
             return ""
         if self._revisioncache:
             if self._revisioncache[0] == node:
                 # _cache only stores rawtext
                 if raw:
                     return self._revisioncache[2]
                 # duplicated, but good for perf
                 if rev is None:
                     rev = self.rev(node)
                 if flags is None:
                     flags = self.flags(rev)
                 # no extra flags set, no flag processor runs, text = rawtext
                 if flags == REVIDX_DEFAULT_FLAGS:
                     return self._revisioncache[2]
                 # rawtext is reusable. need to run flag processor
                 rawtext = self._revisioncache[2]
 
             cachedrev = self._revisioncache[1]
 
         # look up what we need to read
         if rawtext is None:
             if rev is None:
                 rev = self.rev(node)
 
             chain, stopped = self._deltachain(rev, stoprev=cachedrev)
             if stopped:
                 rawtext = self._revisioncache[2]
 
             # drop cache to save memory
             self._revisioncache = None
 
             targetsize = None
             rawsize = self.index[rev][2]
             if 0 <= rawsize:
                 targetsize = 4 * rawsize
 
             bins = self._chunks(chain, df=_df, targetsize=targetsize)
             if rawtext is None:
                 rawtext = bytes(bins[0])
                 bins = bins[1:]
 
             rawtext = mdiff.patches(rawtext, bins)
             self._revisioncache = (node, rev, rawtext)
 
         if flags is None:
             if rev is None:
                 rev = self.rev(node)
             flags = self.flags(rev)
 
         text, validatehash = self._processflags(rawtext, flags, 'read', raw=raw)
         if validatehash:
             self.checkhash(text, node, rev=rev)
 
         return text
 
     def hash(self, text, p1, p2):
         """Compute a node hash.
 
         Available as a function so that subclasses can replace the hash
         as needed.
         """
         return storageutil.hashrevisionsha1(text, p1, p2)
 
     def _processflags(self, text, flags, operation, raw=False):
         """Inspect revision data flags and applies transforms defined by
         registered flag processors.
 
         ``text`` - the revision data to process
         ``flags`` - the revision flags
         ``operation`` - the operation being performed (read or write)
         ``raw`` - an optional argument describing if the raw transform should be
         applied.
 
         This method processes the flags in the order (or reverse order if
         ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
         flag processors registered for present flags. The order of flags defined
         in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
 
         Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
         processed text and ``validatehash`` is a bool indicating whether the
         returned text should be checked for hash integrity.
 
         Note: If the ``raw`` argument is set, it has precedence over the
         operation and will only update the value of ``validatehash``.
         """
         # fast path: no flag processors will run
         if flags == 0:
             return text, True
         if not operation in ('read', 'write'):
             raise error.ProgrammingError(_("invalid '%s' operation") %
                                          operation)
         # Check all flags are known.
         if flags & ~REVIDX_KNOWN_FLAGS:
             raise error.RevlogError(_("incompatible revision flag '%#x'") %
                                     (flags & ~REVIDX_KNOWN_FLAGS))
         validatehash = True
         # Depending on the operation (read or write), the order might be
         # reversed due to non-commutative transforms.
         orderedflags = REVIDX_FLAGS_ORDER
         if operation == 'write':
             orderedflags = reversed(orderedflags)
 
         for flag in orderedflags:
             # If a flagprocessor has been registered for a known flag, apply the
             # related operation transform and update result tuple.
             if flag & flags:
                 vhash = True
 
                 if flag not in self._flagprocessors:
                     message = _("missing processor for flag '%#x'") % (flag)
                     raise error.RevlogError(message)
 
                 processor = self._flagprocessors[flag]
                 if processor is not None:
                     readtransform, writetransform, rawtransform = processor
 
                     if raw:
                         vhash = rawtransform(self, text)
                     elif operation == 'read':
                         text, vhash = readtransform(self, text)
                     else: # write operation
                         text, vhash = writetransform(self, text)
                 validatehash = validatehash and vhash
 
         return text, validatehash
 
     def checkhash(self, text, node, p1=None, p2=None, rev=None):
         """Check node hash integrity.
 
         Available as a function so that subclasses can extend hash mismatch
         behaviors as needed.
         """
         try:
             if p1 is None and p2 is None:
                 p1, p2 = self.parents(node)
             if node != self.hash(text, p1, p2):
                 # Clear the revision cache on hash failure. The revision cache
                 # only stores the raw revision and clearing the cache does have
                 # the side-effect that we won't have a cache hit when the raw
                 # revision data is accessed. But this case should be rare and
                 # it is extra work to teach the cache about the hash
                 # verification state.
                 if self._revisioncache and self._revisioncache[0] == node:
                     self._revisioncache = None
 
                 revornode = rev
                 if revornode is None:
                     revornode = templatefilters.short(hex(node))
                 raise error.RevlogError(_("integrity check failed on %s:%s")
                     % (self.indexfile, pycompat.bytestr(revornode)))
         except error.RevlogError:
             if self._censorable and storageutil.iscensoredtext(text):
                 raise error.CensoredNodeError(self.indexfile, node, text)
             raise
 
     def _enforceinlinesize(self, tr, fp=None):
         """Check if the revlog is too big for inline and convert if so.
 
         This should be called after revisions are added to the revlog. If the
         revlog has grown too large to be an inline revlog, it will convert it
         to use multiple index and data files.
         """
         tiprev = len(self) - 1
         if (not self._inline or
             (self.start(tiprev) + self.length(tiprev)) < _maxinline):
             return
 
         trinfo = tr.find(self.indexfile)
         if trinfo is None:
             raise error.RevlogError(_("%s not found in the transaction")
                                     % self.indexfile)
 
         trindex = trinfo[2]
         if trindex is not None:
             dataoff = self.start(trindex)
         else:
             # revlog was stripped at start of transaction, use all leftover data
             trindex = len(self) - 1
             dataoff = self.end(tiprev)
 
         tr.add(self.datafile, dataoff)
 
         if fp:
             fp.flush()
             fp.close()
             # We can't use the cached file handle after close(). So prevent
             # its usage.
             self._writinghandles = None
 
         with self._indexfp('r') as ifh, self._datafp('w') as dfh:
             for r in self:
                 dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
 
         with self._indexfp('w') as fp:
             self.version &= ~FLAG_INLINE_DATA
             self._inline = False
             io = self._io
             for i in self:
                 e = io.packentry(self.index[i], self.node, self.version, i)
                 fp.write(e)
 
             # the temp file replace the real index when we exit the context
             # manager
 
         tr.replace(self.indexfile, trindex * self._io.size)
         self._chunkclear()
 
     def _nodeduplicatecallback(self, transaction, node):
         """called when trying to add a node already stored.
         """
 
     def addrevision(self, text, transaction, link, p1, p2, cachedelta=None,
                     node=None, flags=REVIDX_DEFAULT_FLAGS, deltacomputer=None):
         """add a revision to the log
 
         text - the revision data to add
         transaction - the transaction object used for rollback
         link - the linkrev data to add
         p1, p2 - the parent nodeids of the revision
         cachedelta - an optional precomputed delta
         node - nodeid of revision; typically node is not specified, and it is
             computed by default as hash(text, p1, p2), however subclasses might
             use different hashing method (and override checkhash() in such case)
         flags - the known flags to set on the revision
         deltacomputer - an optional deltacomputer instance shared between
             multiple calls
         """
         if link == nullrev:
             raise error.RevlogError(_("attempted to add linkrev -1 to %s")
                                     % self.indexfile)
 
         if flags:
             node = node or self.hash(text, p1, p2)
 
         rawtext, validatehash = self._processflags(text, flags, 'write')
 
         # If the flag processor modifies the revision data, ignore any provided
         # cachedelta.
         if rawtext != text:
             cachedelta = None
 
         if len(rawtext) > _maxentrysize:
             raise error.RevlogError(
                 _("%s: size of %d bytes exceeds maximum revlog storage of 2GiB")
                 % (self.indexfile, len(rawtext)))
 
         node = node or self.hash(rawtext, p1, p2)
         if node in self.nodemap:
             return node
 
         if validatehash:
             self.checkhash(rawtext, node, p1=p1, p2=p2)
 
         return self.addrawrevision(rawtext, transaction, link, p1, p2, node,
                                    flags, cachedelta=cachedelta,
                                    deltacomputer=deltacomputer)
 
     def addrawrevision(self, rawtext, transaction, link, p1, p2, node, flags,
                        cachedelta=None, deltacomputer=None):
         """add a raw revision with known flags, node and parents
         useful when reusing a revision not stored in this revlog (ex: received
         over wire, or read from an external bundle).
         """
         dfh = None
         if not self._inline:
             dfh = self._datafp("a+")
         ifh = self._indexfp("a+")
         try:
             return self._addrevision(node, rawtext, transaction, link, p1, p2,
                                      flags, cachedelta, ifh, dfh,
                                      deltacomputer=deltacomputer)
         finally:
             if dfh:
                 dfh.close()
             ifh.close()
 
     def compress(self, data):
         """Generate a possibly-compressed representation of data."""
         if not data:
             return '', data
 
         compressed = self._compressor.compress(data)
 
         if compressed:
             # The revlog compressor added the header in the returned data.
             return '', compressed
 
         if data[0:1] == '\0':
             return '', data
         return 'u', data
 
     def decompress(self, data):
         """Decompress a revlog chunk.
 
         The chunk is expected to begin with a header identifying the
         format type so it can be routed to an appropriate decompressor.
         """
         if not data:
             return data
 
         # Revlogs are read much more frequently than they are written and many
         # chunks only take microseconds to decompress, so performance is
         # important here.
         #
         # We can make a few assumptions about revlogs:
         #
         # 1) the majority of chunks will be compressed (as opposed to inline
         #    raw data).
         # 2) decompressing *any* data will likely by at least 10x slower than
         #    returning raw inline data.
         # 3) we want to prioritize common and officially supported compression
         #    engines
         #
         # It follows that we want to optimize for "decompress compressed data
         # when encoded with common and officially supported compression engines"
         # case over "raw data" and "data encoded by less common or non-official
         # compression engines." That is why we have the inline lookup first
         # followed by the compengines lookup.
         #
         # According to `hg perfrevlogchunks`, this is ~0.5% faster for zlib
         # compressed chunks. And this matters for changelog and manifest reads.
         t = data[0:1]
 
         if t == 'x':
             try:
                 return _zlibdecompress(data)
             except zlib.error as e:
                 raise error.RevlogError(_('revlog decompress error: %s') %
                                         stringutil.forcebytestr(e))
         # '\0' is more common than 'u' so it goes first.
         elif t == '\0':
             return data
         elif t == 'u':
             return util.buffer(data, 1)
 
         try:
             compressor = self._decompressors[t]
         except KeyError:
             try:
                 engine = util.compengines.forrevlogheader(t)
                 compressor = engine.revlogcompressor(self._compengineopts)
                 self._decompressors[t] = compressor
             except KeyError:
                 raise error.RevlogError(_('unknown compression type %r') % t)
 
         return compressor.decompress(data)
 
     def _addrevision(self, node, rawtext, transaction, link, p1, p2, flags,
                      cachedelta, ifh, dfh, alwayscache=False,
                      deltacomputer=None):
         """internal function to add revisions to the log
 
         see addrevision for argument descriptions.
 
         note: "addrevision" takes non-raw text, "_addrevision" takes raw text.
 
         if "deltacomputer" is not provided or None, a defaultdeltacomputer will
         be used.
 
         invariants:
         - rawtext is optional (can be None); if not set, cachedelta must be set.
           if both are set, they must correspond to each other.
         """
         if node == nullid:
             raise error.RevlogError(_("%s: attempt to add null revision") %
                                     self.indexfile)
         if node == wdirid or node in wdirfilenodeids:
             raise error.RevlogError(_("%s: attempt to add wdir revision") %
                                     self.indexfile)
 
         if self._inline:
             fh = ifh
         else:
             fh = dfh
 
         btext = [rawtext]
 
         curr = len(self)
         prev = curr - 1
         offset = self.end(prev)
         p1r, p2r = self.rev(p1), self.rev(p2)
 
         # full versions are inserted when the needed deltas
         # become comparable to the uncompressed text
         if rawtext is None:
             # need rawtext size, before changed by flag processors, which is
             # the non-raw size. use revlog explicitly to avoid filelog's extra
             # logic that might remove metadata size.
             textlen = mdiff.patchedsize(revlog.size(self, cachedelta[0]),
                                         cachedelta[1])
         else:
             textlen = len(rawtext)
 
         if deltacomputer is None:
             deltacomputer = deltautil.deltacomputer(self)
 
         revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
 
         deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
 
         e = (offset_type(offset, flags), deltainfo.deltalen, textlen,
              deltainfo.base, link, p1r, p2r, node)
         self.index.append(e)
         self.nodemap[node] = curr
 
         # Reset the pure node cache start lookup offset to account for new
         # revision.
         if self._nodepos is not None:
             self._nodepos = curr
 
         entry = self._io.packentry(e, self.node, self.version, curr)
         self._writeentry(transaction, ifh, dfh, entry, deltainfo.data,
                          link, offset)
 
         rawtext = btext[0]
 
         if alwayscache and rawtext is None:
             rawtext = deltacomputer.buildtext(revinfo, fh)
 
         if type(rawtext) == bytes: # only accept immutable objects
             self._revisioncache = (node, curr, rawtext)
         self._chainbasecache[curr] = deltainfo.chainbase
         return node
 
     def _writeentry(self, transaction, ifh, dfh, entry, data, link, offset):
         # Files opened in a+ mode have inconsistent behavior on various
         # platforms. Windows requires that a file positioning call be made
         # when the file handle transitions between reads and writes. See
         # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
         # platforms, Python or the platform itself can be buggy. Some versions
         # of Solaris have been observed to not append at the end of the file
         # if the file was seeked to before the end. See issue4943 for more.
         #
         # We work around this issue by inserting a seek() before writing.
         # Note: This is likely not necessary on Python 3. However, because
         # the file handle is reused for reads and may be seeked there, we need
         # to be careful before changing this.
         ifh.seek(0, os.SEEK_END)
         if dfh:
             dfh.seek(0, os.SEEK_END)
 
         curr = len(self) - 1
         if not self._inline:
             transaction.add(self.datafile, offset)
             transaction.add(self.indexfile, curr * len(entry))
             if data[0]:
                 dfh.write(data[0])
             dfh.write(data[1])
             ifh.write(entry)
         else:
             offset += curr * self._io.size
             transaction.add(self.indexfile, offset, curr)
             ifh.write(entry)
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
         add a delta group
 
         given a set of deltas, add them to the revision log. the
         first delta is against its parent, which should be in our
         log, the rest are against the previous delta.
 
         If ``addrevisioncb`` is defined, it will be called with arguments of
         this revlog and the node that was added.
         """
 
         if self._writinghandles:
             raise error.ProgrammingError('cannot nest addgroup() calls')
 
         nodes = []
 
         r = len(self)
         end = 0
         if r:
             end = self.end(r - 1)
         ifh = self._indexfp("a+")
         isize = r * self._io.size
         if self._inline:
             transaction.add(self.indexfile, end + isize, r)
             dfh = None
         else:
             transaction.add(self.indexfile, isize, r)
             transaction.add(self.datafile, end)
             dfh = self._datafp("a+")
         def flush():
             if dfh:
                 dfh.flush()
             ifh.flush()
 
         self._writinghandles = (ifh, dfh)
 
         try:
             deltacomputer = deltautil.deltacomputer(self)
             # loop through our set of deltas
             for data in deltas:
                 node, p1, p2, linknode, deltabase, delta, flags = data
                 link = linkmapper(linknode)
                 flags = flags or REVIDX_DEFAULT_FLAGS
 
                 nodes.append(node)
 
                 if node in self.nodemap:
                     self._nodeduplicatecallback(transaction, node)
                     # this can happen if two branches make the same change
                     continue
 
                 for p in (p1, p2):
                     if p not in self.nodemap:
                         raise error.LookupError(p, self.indexfile,
                                                 _('unknown parent'))
 
                 if deltabase not in self.nodemap:
                     raise error.LookupError(deltabase, self.indexfile,
                                             _('unknown delta base'))
 
                 baserev = self.rev(deltabase)
 
                 if baserev != nullrev and self.iscensored(baserev):
                     # if base is censored, delta must be full replacement in a
                     # single patch operation
                     hlen = struct.calcsize(">lll")
                     oldlen = self.rawsize(baserev)
                     newlen = len(delta) - hlen
                     if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
                         raise error.CensoredBaseError(self.indexfile,
                                                       self.node(baserev))
 
                 if not flags and self._peek_iscensored(baserev, delta, flush):
                     flags |= REVIDX_ISCENSORED
 
                 # We assume consumers of addrevisioncb will want to retrieve
                 # the added revision, which will require a call to
                 # revision(). revision() will fast path if there is a cache
                 # hit. So, we tell _addrevision() to always cache in this case.
                 # We're only using addgroup() in the context of changegroup
                 # generation so the revision data can always be handled as raw
                 # by the flagprocessor.
                 self._addrevision(node, None, transaction, link,
                                   p1, p2, flags, (baserev, delta),
                                   ifh, dfh,
                                   alwayscache=bool(addrevisioncb),
                                   deltacomputer=deltacomputer)
 
                 if addrevisioncb:
                     addrevisioncb(self, node)
 
                 if not dfh and not self._inline:
                     # addrevision switched from inline to conventional
                     # reopen the index
                     ifh.close()
                     dfh = self._datafp("a+")
                     ifh = self._indexfp("a+")
                     self._writinghandles = (ifh, dfh)
         finally:
             self._writinghandles = None
 
             if dfh:
                 dfh.close()
             ifh.close()
 
         return nodes
 
     def iscensored(self, rev):
         """Check if a file revision is censored."""
         if not self._censorable:
             return False
 
         return self.flags(rev) & REVIDX_ISCENSORED
 
     def _peek_iscensored(self, baserev, delta, flush):
         """Quickly check if a delta produces a censored revision."""
         if not self._censorable:
             return False
 
         return storageutil.deltaiscensored(delta, baserev, self.rawsize)
 
     def getstrippoint(self, minlink):
         """find the minimum rev that must be stripped to strip the linkrev
 
         Returns a tuple containing the minimum rev and a set of all revs that
         have linkrevs that will be broken by this strip.
         """
         return storageutil.resolvestripinfo(minlink, len(self) - 1,
                                             self.headrevs(),
                                             self.linkrev, self.parentrevs)
 
     def strip(self, minlink, transaction):
         """truncate the revlog on the first revision with a linkrev >= minlink
 
         This function is called when we're stripping revision minlink and
         its descendants from the repository.
 
         We have to remove all revisions with linkrev >= minlink, because
         the equivalent changelog revisions will be renumbered after the
         strip.
 
         So we truncate the revlog on the first of these revisions, and
         trust that the caller has saved the revisions that shouldn't be
         removed and that it'll re-add them after this truncation.
         """
         if len(self) == 0:
             return
 
         rev, _ = self.getstrippoint(minlink)
         if rev == len(self):
             return
 
         # first truncate the files on disk
         end = self.start(rev)
         if not self._inline:
             transaction.add(self.datafile, end)
             end = rev * self._io.size
         else:
             end += rev * self._io.size
 
         transaction.add(self.indexfile, end)
 
         # then reset internal state in memory to forget those revisions
         self._revisioncache = None
         self._chaininfocache = {}
         self._chunkclear()
         for x in pycompat.xrange(rev, len(self)):
             del self.nodemap[self.node(x)]
 
         del self.index[rev:-1]
         self._nodepos = None
 
     def checksize(self):
         """Check size of index and data files
 
         return a (dd, di) tuple.
         - dd: extra bytes for the "data" file
         - di: extra bytes for the "index" file
 
         A healthy revlog will return (0, 0).
         """
         expected = 0
         if len(self):
             expected = max(0, self.end(len(self) - 1))
 
         try:
             with self._datafp() as f:
                 f.seek(0, 2)
                 actual = f.tell()
             dd = actual - expected
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             dd = 0
 
         try:
             f = self.opener(self.indexfile)
             f.seek(0, 2)
             actual = f.tell()
             f.close()
             s = self._io.size
             i = max(0, actual // s)
             di = actual - (i * s)
             if self._inline:
                 databytes = 0
                 for r in self:
                     databytes += max(0, self.length(r))
                 dd = 0
                 di = actual - len(self) * s - databytes
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             di = 0
 
         return (dd, di)
 
     def files(self):
         res = [self.indexfile]
         if not self._inline:
             res.append(self.datafile)
         return res
 
     def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
                       assumehaveparentrevisions=False,
                       deltamode=repository.CG_DELTAMODE_STD):
         if nodesorder not in ('nodes', 'storage', 'linear', None):
             raise error.ProgrammingError('unhandled value for nodesorder: %s' %
                                          nodesorder)
 
         if nodesorder is None and not self._generaldelta:
             nodesorder = 'storage'
 
         if (not self._storedeltachains and
                 deltamode != repository.CG_DELTAMODE_PREV):
             deltamode = repository.CG_DELTAMODE_FULL
 
         return storageutil.emitrevisions(
             self, nodes, nodesorder, revlogrevisiondelta,
             deltaparentfn=self.deltaparent,
             candeltafn=self.candelta,
             rawsizefn=self.rawsize,
             revdifffn=self.revdiff,
             flagsfn=self.flags,
             deltamode=deltamode,
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions)
 
     DELTAREUSEALWAYS = 'always'
     DELTAREUSESAMEREVS = 'samerevs'
     DELTAREUSENEVER = 'never'
 
     DELTAREUSEFULLADD = 'fulladd'
 
     DELTAREUSEALL = {'always', 'samerevs', 'never', 'fulladd'}
 
     def clone(self, tr, destrevlog, addrevisioncb=None,
               deltareuse=DELTAREUSESAMEREVS, forcedeltabothparents=None):
         """Copy this revlog to another, possibly with format changes.
 
         The destination revlog will contain the same revisions and nodes.
         However, it may not be bit-for-bit identical due to e.g. delta encoding
         differences.
 
         The ``deltareuse`` argument control how deltas from the existing revlog
         are preserved in the destination revlog. The argument can have the
         following values:
 
         DELTAREUSEALWAYS
            Deltas will always be reused (if possible), even if the destination
            revlog would not select the same revisions for the delta. This is the
            fastest mode of operation.
         DELTAREUSESAMEREVS
            Deltas will be reused if the destination revlog would pick the same
            revisions for the delta. This mode strikes a balance between speed
            and optimization.
         DELTAREUSENEVER
            Deltas will never be reused. This is the slowest mode of execution.
            This mode can be used to recompute deltas (e.g. if the diff/delta
            algorithm changes).
 
         Delta computation can be slow, so the choice of delta reuse policy can
         significantly affect run time.
 
         The default policy (``DELTAREUSESAMEREVS``) strikes a balance between
         two extremes. Deltas will be reused if they are appropriate. But if the
         delta could choose a better revision, it will do so. This means if you
         are converting a non-generaldelta revlog to a generaldelta revlog,
         deltas will be recomputed if the delta's parent isn't a parent of the
         revision.
 
         In addition to the delta policy, the ``forcedeltabothparents``
         argument controls whether to force compute deltas against both parents
         for merges. By default, the current default is used.
         """
         if deltareuse not in self.DELTAREUSEALL:
             raise ValueError(_('value for deltareuse invalid: %s') % deltareuse)
 
         if len(destrevlog):
             raise ValueError(_('destination revlog is not empty'))
 
         if getattr(self, 'filteredrevs', None):
             raise ValueError(_('source revlog has filtered revisions'))
         if getattr(destrevlog, 'filteredrevs', None):
             raise ValueError(_('destination revlog has filtered revisions'))
 
         # lazydelta and lazydeltabase controls whether to reuse a cached delta,
         # if possible.
         oldlazydelta = destrevlog._lazydelta
         oldlazydeltabase = destrevlog._lazydeltabase
         oldamd = destrevlog._deltabothparents
 
         try:
             if deltareuse == self.DELTAREUSEALWAYS:
                 destrevlog._lazydeltabase = True
                 destrevlog._lazydelta = True
             elif deltareuse == self.DELTAREUSESAMEREVS:
                 destrevlog._lazydeltabase = False
                 destrevlog._lazydelta = True
             elif deltareuse == self.DELTAREUSENEVER:
                 destrevlog._lazydeltabase = False
                 destrevlog._lazydelta = False
 
             destrevlog._deltabothparents = forcedeltabothparents or oldamd
 
             deltacomputer = deltautil.deltacomputer(destrevlog)
             index = self.index
             for rev in self:
                 entry = index[rev]
 
                 # Some classes override linkrev to take filtered revs into
                 # account. Use raw entry from index.
                 flags = entry[0] & 0xffff
                 linkrev = entry[4]
                 p1 = index[entry[5]][7]
                 p2 = index[entry[6]][7]
                 node = entry[7]
 
                 # (Possibly) reuse the delta from the revlog if allowed and
                 # the revlog chunk is a delta.
                 cachedelta = None
                 rawtext = None
                 if destrevlog._lazydelta:
                     dp = self.deltaparent(rev)
                     if dp != nullrev:
                         cachedelta = (dp, bytes(self._chunk(rev)))
 
                 if not cachedelta:
                     rawtext = self.revision(rev, raw=True)
 
 
                 if deltareuse == self.DELTAREUSEFULLADD:
                     destrevlog.addrevision(rawtext, tr, linkrev, p1, p2,
                                            cachedelta=cachedelta,
                                            node=node, flags=flags,
                                            deltacomputer=deltacomputer)
                 else:
                     ifh = destrevlog.opener(destrevlog.indexfile, 'a+',
                                             checkambig=False)
                     dfh = None
                     if not destrevlog._inline:
                         dfh = destrevlog.opener(destrevlog.datafile, 'a+')
                     try:
                         destrevlog._addrevision(node, rawtext, tr, linkrev, p1,
                                                 p2, flags, cachedelta, ifh, dfh,
                                                 deltacomputer=deltacomputer)
                     finally:
                         if dfh:
                             dfh.close()
                         ifh.close()
 
                 if addrevisioncb:
                     addrevisioncb(self, rev, node)
         finally:
             destrevlog._lazydelta = oldlazydelta
             destrevlog._lazydeltabase = oldlazydeltabase
             destrevlog._deltabothparents = oldamd
 
     def censorrevision(self, tr, censornode, tombstone=b''):
         if (self.version & 0xFFFF) == REVLOGV0:
             raise error.RevlogError(_('cannot censor with version %d revlogs') %
                                     self.version)
 
         censorrev = self.rev(censornode)
         tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
 
         if len(tombstone) > self.rawsize(censorrev):
             raise error.Abort(_('censor tombstone must be no longer than '
                                 'censored data'))
 
         # Rewriting the revlog in place is hard. Our strategy for censoring is
         # to create a new revlog, copy all revisions to it, then replace the
         # revlogs on transaction close.
 
         newindexfile = self.indexfile + b'.tmpcensored'
         newdatafile = self.datafile + b'.tmpcensored'
 
         # This is a bit dangerous. We could easily have a mismatch of state.
         newrl = revlog(self.opener, newindexfile, newdatafile,
                        censorable=True)
         newrl.version = self.version
         newrl._generaldelta = self._generaldelta
         newrl._io = self._io
 
         for rev in self.revs():
             node = self.node(rev)
             p1, p2 = self.parents(node)
 
             if rev == censorrev:
                 newrl.addrawrevision(tombstone, tr, self.linkrev(censorrev),
                                      p1, p2, censornode, REVIDX_ISCENSORED)
 
                 if newrl.deltaparent(rev) != nullrev:
                     raise error.Abort(_('censored revision stored as delta; '
                                         'cannot censor'),
                                       hint=_('censoring of revlogs is not '
                                              'fully implemented; please report '
                                              'this bug'))
                 continue
 
             if self.iscensored(rev):
                 if self.deltaparent(rev) != nullrev:
                     raise error.Abort(_('cannot censor due to censored '
                                         'revision having delta stored'))
                 rawtext = self._chunk(rev)
             else:
                 rawtext = self.revision(rev, raw=True)
 
             newrl.addrawrevision(rawtext, tr, self.linkrev(rev), p1, p2, node,
                                  self.flags(rev))
 
         tr.addbackup(self.indexfile, location='store')
         if not self._inline:
             tr.addbackup(self.datafile, location='store')
 
         self.opener.rename(newrl.indexfile, self.indexfile)
         if not self._inline:
             self.opener.rename(newrl.datafile, self.datafile)
 
         self.clearcaches()
         self._loadindex()
 
     def verifyintegrity(self, state):
         """Verifies the integrity of the revlog.
 
         Yields ``revlogproblem`` instances describing problems that are
         found.
         """
         dd, di = self.checksize()
         if dd:
             yield revlogproblem(error=_('data length off by %d bytes') % dd)
         if di:
             yield revlogproblem(error=_('index contains %d extra bytes') % di)
 
         version = self.version & 0xFFFF
 
         # The verifier tells us what version revlog we should be.
         if version != state['expectedversion']:
             yield revlogproblem(
                 warning=_("warning: '%s' uses revlog format %d; expected %d") %
                         (self.indexfile, version, state['expectedversion']))
 
         state['skipread'] = set()
 
         for rev in self:
             node = self.node(rev)
 
             # Verify contents. 4 cases to care about:
             #
             #   common: the most common case
             #   rename: with a rename
             #   meta: file content starts with b'\1\n', the metadata
             #         header defined in filelog.py, but without a rename
             #   ext: content stored externally
             #
             # More formally, their differences are shown below:
             #
             #                       | common | rename | meta  | ext
             #  -------------------------------------------------------
             #   flags()             | 0      | 0      | 0     | not 0
             #   renamed()           | False  | True   | False | ?
             #   rawtext[0:2]=='\1\n'| False  | True   | True  | ?
             #
             # "rawtext" means the raw text stored in revlog data, which
             # could be retrieved by "revision(rev, raw=True)". "text"
             # mentioned below is "revision(rev, raw=False)".
             #
             # There are 3 different lengths stored physically:
             #  1. L1: rawsize, stored in revlog index
             #  2. L2: len(rawtext), stored in revlog data
             #  3. L3: len(text), stored in revlog data if flags==0, or
             #     possibly somewhere else if flags!=0
             #
             # L1 should be equal to L2. L3 could be different from them.
             # "text" may or may not affect commit hash depending on flag
             # processors (see revlog.addflagprocessor).
             #
             #              | common  | rename | meta  | ext
             # -------------------------------------------------
             #    rawsize() | L1      | L1     | L1    | L1
             #       size() | L1      | L2-LM  | L1(*) | L1 (?)
             # len(rawtext) | L2      | L2     | L2    | L2
             #    len(text) | L2      | L2     | L2    | L3
             #  len(read()) | L2      | L2-LM  | L2-LM | L3 (?)
             #
             # LM:  length of metadata, depending on rawtext
             # (*): not ideal, see comment in filelog.size
             # (?): could be "- len(meta)" if the resolved content has
             #      rename metadata
             #
             # Checks needed to be done:
             #  1. length check: L1 == L2, in all cases.
             #  2. hash check: depending on flag processor, we may need to
             #     use either "text" (external), or "rawtext" (in revlog).
 
             try:
                 skipflags = state.get('skipflags', 0)
                 if skipflags:
                     skipflags &= self.flags(rev)
 
                 if skipflags:
                     state['skipread'].add(node)
                 else:
                     # Side-effect: read content and verify hash.
                     self.revision(node)
 
                 l1 = self.rawsize(rev)
                 l2 = len(self.revision(node, raw=True))
 
                 if l1 != l2:
                     yield revlogproblem(
                         error=_('unpacked size is %d, %d expected') % (l2, l1),
                         node=node)
 
             except error.CensoredNodeError:
                 if state['erroroncensored']:
                     yield revlogproblem(error=_('censored file data'),
                                         node=node)
                     state['skipread'].add(node)
             except Exception as e:
                 yield revlogproblem(
                     error=_('unpacking %s: %s') % (short(node),
                                                    stringutil.forcebytestr(e)),
                     node=node)
                 state['skipread'].add(node)
 
     def storageinfo(self, exclusivefiles=False, sharedfiles=False,
                     revisionscount=False, trackedsize=False,
                     storedsize=False):
         d = {}
 
         if exclusivefiles:
             d['exclusivefiles'] = [(self.opener, self.indexfile)]
             if not self._inline:
                 d['exclusivefiles'].append((self.opener, self.datafile))
 
         if sharedfiles:
             d['sharedfiles'] = []
 
         if revisionscount:
             d['revisionscount'] = len(self)
 
         if trackedsize:
             d['trackedsize'] = sum(map(self.rawsize, iter(self)))
 
         if storedsize:
             d['storedsize'] = sum(self.opener.stat(path).st_size
                                   for path in self.files())
 
         return d