revlog: move censor logic out of censor extension

The censor extension is doing very low-level things with revlogs.
It is fundamentally impossible for this logic to remain in the censor
extension while support multiple storage backends: we need each
storage backend to implement censor in its own storage-specific
way.

This commit effectively moves the revlog-specific censoring code to
be a method of revlogs themselves.

We've defined a new API on the file storage interface for censoring
an individual node. Even though the current censoring code doesn't
use it, the API requires a transaction instance because it logically
makes sense for storage backends to require an active transaction
(which implies a held write lock) in order to rewrite storage.

After this commit, the censor extension has been reduced to
boilerplate precondition checking before invoking the generic
storage API.

I tried to keep the code as similar as possible. But some minor
changes were made:

* We use self._io instead of instantiating a new revlogio instance.
* We compare self.version against REVLOGV0 instead of != REVLOGV1
  because presumably all future revlog versions will support censoring.
* We use self.opener instead of going through repo.svfs (we don't have
  a handle on the repo instance from a revlog).
* "revlog" dropped
* Replace "flog" with "self".

Differential Revision: https://phab.mercurial-scm.org/D4656

hgext/censor.py

@@ -1,187 +1,99 b''
 # Copyright (C) 2015 - Mike Edgar <[e-mail]>
 #
 # This extension enables removal of file content at a given revision,
 # rewriting the data/metadata of successive revisions to preserve revision log
 # integrity.
 
 """erase file content at a given revision
 
 The censor command instructs Mercurial to erase all content of a file at a given
 revision *without updating the changeset hash.* This allows existing history to
 remain valid while preventing future clones/pulls from receiving the erased
 data.
 
 Typical uses for censor are due to security or legal requirements, including::
 
  * Passwords, private keys, cryptographic material
  * Licensed data/code/libraries for which the license has expired
  * Personally Identifiable Information or other private data
 
 Censored nodes can interrupt mercurial's typical operation whenever the excised
 data needs to be materialized. Some commands, like ``hg cat``/``hg revert``,
 simply fail when asked to produce censored data. Others, like ``hg verify`` and
 ``hg update``, must be capable of tolerating censored data to continue to
 function in a meaningful way. Such commands only tolerate censored file
 revisions if they are allowed by the "censor.policy=ignore" config option.
 """
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial.node import short
 
 from mercurial import (
     error,
-    pycompat,
     registrar,
-    revlog,
     scmutil,
-    util,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 @command('censor',
     [('r', 'rev', '', _('censor file from specified revision'), _('REV')),
      ('t', 'tombstone', '', _('replacement tombstone data'), _('TEXT'))],
     _('-r REV [-t TEXT] [FILE]'))
 def censor(ui, repo, path, rev='', tombstone='', **opts):
     with repo.wlock(), repo.lock():
         return _docensor(ui, repo, path, rev, tombstone, **opts)
 
 def _docensor(ui, repo, path, rev='', tombstone='', **opts):
     if not path:
         raise error.Abort(_('must specify file path to censor'))
     if not rev:
         raise error.Abort(_('must specify revision to censor'))
 
     wctx = repo[None]
 
     m = scmutil.match(wctx, (path,))
     if m.anypats() or len(m.files()) != 1:
         raise error.Abort(_('can only specify an explicit filename'))
     path = m.files()[0]
     flog = repo.file(path)
     if not len(flog):
         raise error.Abort(_('cannot censor file with no history'))
 
     rev = scmutil.revsingle(repo, rev, rev).rev()
     try:
         ctx = repo[rev]
     except KeyError:
         raise error.Abort(_('invalid revision identifier %s') % rev)
 
     try:
         fctx = ctx.filectx(path)
     except error.LookupError:
         raise error.Abort(_('file does not exist at revision %s') % rev)
 
     fnode = fctx.filenode()
     heads = []
     for headnode in repo.heads():
         hc = repo[headnode]
         if path in hc and hc.filenode(path) == fnode:
             heads.append(hc)
     if heads:
         headlist = ', '.join([short(c.node()) for c in heads])
         raise error.Abort(_('cannot censor file in heads (%s)') % headlist,
             hint=_('clean/delete and commit first'))
 
     wp = wctx.parents()
     if ctx.node() in [p.node() for p in wp]:
         raise error.Abort(_('cannot censor working directory'),
             hint=_('clean/delete/update first'))
 
-    flogv = flog.version & 0xFFFF
-    if flogv != revlog.REVLOGV1:
-        raise error.Abort(
-            _('censor does not support revlog version %d') % (flogv,))
-
-    tombstone = revlog.packmeta({"censored": tombstone}, "")
-
-    crev = fctx.filerev()
-
-    if len(tombstone) > flog.rawsize(crev):
-        raise error.Abort(_(
-            'censor tombstone must be no longer than censored data'))
-
-    # Using two files instead of one makes it easy to rewrite entry-by-entry
-    idxread = repo.svfs(flog.indexfile, 'r')
-    idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True)
-    if flog.version & revlog.FLAG_INLINE_DATA:
-        dataread, datawrite = idxread, idxwrite
-    else:
-        dataread = repo.svfs(flog.datafile, 'r')
-        datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True)
-
-    # Copy all revlog data up to the entry to be censored.
-    rio = revlog.revlogio()
-    offset = flog.start(crev)
-
-    for chunk in util.filechunkiter(idxread, limit=crev * rio.size):
-        idxwrite.write(chunk)
-    for chunk in util.filechunkiter(dataread, limit=offset):
-        datawrite.write(chunk)
-
-    def rewriteindex(r, newoffs, newdata=None):
-        """Rewrite the index entry with a new data offset and optional new data.
-
-        The newdata argument, if given, is a tuple of three positive integers:
-        (new compressed, new uncompressed, added flag bits).
-        """
-        offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r]
-        flags = revlog.gettype(offlags)
-        if newdata:
-            comp, uncomp, nflags = newdata
-            flags |= nflags
-        offlags = revlog.offset_type(newoffs, flags)
-        e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
-        idxwrite.write(rio.packentry(e, None, flog.version, r))
-        idxread.seek(rio.size, 1)
-
-    def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS):
-        """Write the given full text to the filelog with the given data offset.
-
-        Returns:
-            The integer number of data bytes written, for tracking data offsets.
-        """
-        flag, compdata = flog.compress(data)
-        newcomp = len(flag) + len(compdata)
-        rewriteindex(r, offs, (newcomp, len(data), nflags))
-        datawrite.write(flag)
-        datawrite.write(compdata)
-        dataread.seek(flog.length(r), 1)
-        return newcomp
-
-    # Rewrite censored revlog entry with (padded) tombstone data.
-    pad = ' ' * (flog.rawsize(crev) - len(tombstone))
-    offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED)
-
-    # Rewrite all following filelog revisions fixing up offsets and deltas.
-    for srev in pycompat.xrange(crev + 1, len(flog)):
-        if crev in flog.parentrevs(srev):
-            # Immediate children of censored node must be re-added as fulltext.
-            try:
-                revdata = flog.revision(srev)
-            except error.CensoredNodeError as e:
-                revdata = e.tombstone
-            dlen = rewrite(srev, offset, revdata)
-        else:
-            # Copy any other revision data verbatim after fixing up the offset.
-            rewriteindex(srev, offset)
-            dlen = flog.length(srev)
-            for chunk in util.filechunkiter(dataread, limit=dlen):
-                datawrite.write(chunk)
-        offset += dlen
-
-    idxread.close()
-    idxwrite.close()
-    if dataread is not idxread:
-        dataread.close()
-        datawrite.close()
+    with repo.transaction(b'censor') as tr:
+        flog.censorrevision(tr, fnode, tombstone=tombstone)

mercurial/filelog.py

@@ -1,278 +1,281 b''
 # filelog.py - file history class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from . import (
     error,
     repository,
     revlog,
 )
 from .utils import (
     interfaceutil,
 )
 
 @interfaceutil.implementer(repository.ifilestorage)
 class filelog(object):
     def __init__(self, opener, path):
         self._revlog = revlog.revlog(opener,
                                      '/'.join(('data', path + '.i')),
                                      censorable=True)
         # full name of the user visible file, relative to the repository root
         self.filename = path
         self.index = self._revlog.index
         self.version = self._revlog.version
         self._generaldelta = self._revlog._generaldelta
 
     def __len__(self):
         return len(self._revlog)
 
     def __iter__(self):
         return self._revlog.__iter__()
 
     def revs(self, start=0, stop=None):
         return self._revlog.revs(start=start, stop=stop)
 
     def parents(self, node):
         return self._revlog.parents(node)
 
     def parentrevs(self, rev):
         return self._revlog.parentrevs(rev)
 
     def rev(self, node):
         return self._revlog.rev(node)
 
     def node(self, rev):
         return self._revlog.node(rev)
 
     def lookup(self, node):
         return self._revlog.lookup(node)
 
     def linkrev(self, rev):
         return self._revlog.linkrev(rev)
 
     def flags(self, rev):
         return self._revlog.flags(rev)
 
     def commonancestorsheads(self, node1, node2):
         return self._revlog.commonancestorsheads(node1, node2)
 
     def descendants(self, revs):
         return self._revlog.descendants(revs)
 
     def headrevs(self):
         return self._revlog.headrevs()
 
     def heads(self, start=None, stop=None):
         return self._revlog.heads(start, stop)
 
     def children(self, node):
         return self._revlog.children(node)
 
     def deltaparent(self, rev):
         return self._revlog.deltaparent(rev)
 
     def iscensored(self, rev):
         return self._revlog.iscensored(rev)
 
     def rawsize(self, rev):
         return self._revlog.rawsize(rev)
 
     def checkhash(self, text, node, p1=None, p2=None, rev=None):
         return self._revlog.checkhash(text, node, p1=p1, p2=p2, rev=rev)
 
     def revision(self, node, _df=None, raw=False):
         return self._revlog.revision(node, _df=_df, raw=raw)
 
     def revdiff(self, rev1, rev2):
         return self._revlog.revdiff(rev1, rev2)
 
     def emitrevisiondeltas(self, requests):
         return self._revlog.emitrevisiondeltas(requests)
 
     def addrevision(self, revisiondata, transaction, linkrev, p1, p2,
                     node=None, flags=revlog.REVIDX_DEFAULT_FLAGS,
                     cachedelta=None):
         return self._revlog.addrevision(revisiondata, transaction, linkrev,
                                     p1, p2, node=node, flags=flags,
                                     cachedelta=cachedelta)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         return self._revlog.addgroup(deltas, linkmapper, transaction,
                                  addrevisioncb=addrevisioncb)
 
     def getstrippoint(self, minlink):
         return self._revlog.getstrippoint(minlink)
 
     def strip(self, minlink, transaction):
         return self._revlog.strip(minlink, transaction)
 
+    def censorrevision(self, tr, node, tombstone=b''):
+        return self._revlog.censorrevision(node, tombstone=tombstone)
+
     def files(self):
         return self._revlog.files()
 
     def checksize(self):
         return self._revlog.checksize()
 
     def read(self, node):
         t = self.revision(node)
         if not t.startswith('\1\n'):
             return t
         s = t.index('\1\n', 2)
         return t[s + 2:]
 
     def add(self, text, meta, transaction, link, p1=None, p2=None):
         if meta or text.startswith('\1\n'):
             text = revlog.packmeta(meta, text)
         return self.addrevision(text, transaction, link, p1, p2)
 
     def renamed(self, node):
         if self.parents(node)[0] != revlog.nullid:
             return False
         t = self.revision(node)
         m = revlog.parsemeta(t)[0]
         # copy and copyrev occur in pairs. In rare cases due to bugs,
         # one can occur without the other.
         if m and "copy" in m and "copyrev" in m:
             return (m["copy"], revlog.bin(m["copyrev"]))
         return False
 
     def size(self, rev):
         """return the size of a given revision"""
 
         # for revisions with renames, we have to go the slow way
         node = self.node(rev)
         if self.renamed(node):
             return len(self.read(node))
         if self.iscensored(rev):
             return 0
 
         # XXX if self.read(node).startswith("\1\n"), this returns (size+4)
         return self._revlog.size(rev)
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
 
         t = text
         if text.startswith('\1\n'):
             t = '\1\n\1\n' + text
 
         samehashes = not self._revlog.cmp(node, t)
         if samehashes:
             return False
 
         # censored files compare against the empty file
         if self.iscensored(self.rev(node)):
             return text != ''
 
         # renaming a file produces a different hash, even if the data
         # remains unchanged. Check if it's the case (slow):
         if self.renamed(node):
             t2 = self.read(node)
             return t2 != text
 
         return True
 
     @property
     def filename(self):
         return self._revlog.filename
 
     @filename.setter
     def filename(self, value):
         self._revlog.filename = value
 
     # TODO these aren't part of the interface and aren't internal methods.
     # Callers should be fixed to not use them.
     @property
     def indexfile(self):
         return self._revlog.indexfile
 
     @indexfile.setter
     def indexfile(self, value):
         self._revlog.indexfile = value
 
     @property
     def datafile(self):
         return self._revlog.datafile
 
     @property
     def opener(self):
         return self._revlog.opener
 
     def clone(self, tr, destrevlog, **kwargs):
         if not isinstance(destrevlog, filelog):
             raise error.ProgrammingError('expected filelog to clone()')
 
         return self._revlog.clone(tr, destrevlog._revlog, **kwargs)
 
     def start(self, rev):
         return self._revlog.start(rev)
 
     def end(self, rev):
         return self._revlog.end(rev)
 
     def length(self, rev):
         return self._revlog.length(rev)
 
     def compress(self, data):
         return self._revlog.compress(data)
 
     def _addrevision(self, *args, **kwargs):
         return self._revlog._addrevision(*args, **kwargs)
 
 class narrowfilelog(filelog):
     """Filelog variation to be used with narrow stores."""
 
     def __init__(self, opener, path, narrowmatch):
         super(narrowfilelog, self).__init__(opener, path)
         self._narrowmatch = narrowmatch
 
     def renamed(self, node):
         res = super(narrowfilelog, self).renamed(node)
 
         # Renames that come from outside the narrowspec are problematic
         # because we may lack the base text for the rename. This can result
         # in code attempting to walk the ancestry or compute a diff
         # encountering a missing revision. We address this by silently
         # removing rename metadata if the source file is outside the
         # narrow spec.
         #
         # A better solution would be to see if the base revision is available,
         # rather than assuming it isn't.
         #
         # An even better solution would be to teach all consumers of rename
         # metadata that the base revision may not be available.
         #
         # TODO consider better ways of doing this.
         if res and not self._narrowmatch(res[0]):
             return None
 
         return res
 
     def size(self, rev):
         # Because we have a custom renamed() that may lie, we need to call
         # the base renamed() to report accurate results.
         node = self.node(rev)
         if super(narrowfilelog, self).renamed(node):
             return len(self.read(node))
         else:
             return super(narrowfilelog, self).size(rev)
 
     def cmp(self, node, text):
         different = super(narrowfilelog, self).cmp(node, text)
 
         # Because renamed() may lie, we may get false positives for
         # different content. Check for this by comparing against the original
         # renamed() implementation.
         if different:
             if super(narrowfilelog, self).renamed(node):
                 t2 = self.read(node)
                 return t2 != text
 
         return different

mercurial/repository.py

@@ -1,1585 +1,1602 b''
 # repository.py - Interfaces and base classes for repositories and peers.
 #
 # Copyright 2017 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from .i18n import _
 from . import (
     error,
 )
 from .utils import (
     interfaceutil,
 )
 
 # When narrowing is finalized and no longer subject to format changes,
 # we should move this to just "narrow" or similar.
 NARROW_REQUIREMENT = 'narrowhg-experimental'
 
 class ipeerconnection(interfaceutil.Interface):
     """Represents a "connection" to a repository.
 
     This is the base interface for representing a connection to a repository.
     It holds basic properties and methods applicable to all peer types.
 
     This is not a complete interface definition and should not be used
     outside of this module.
     """
     ui = interfaceutil.Attribute("""ui.ui instance""")
 
     def url():
         """Returns a URL string representing this peer.
 
         Currently, implementations expose the raw URL used to construct the
         instance. It may contain credentials as part of the URL. The
         expectations of the value aren't well-defined and this could lead to
         data leakage.
 
         TODO audit/clean consumers and more clearly define the contents of this
         value.
         """
 
     def local():
         """Returns a local repository instance.
 
         If the peer represents a local repository, returns an object that
         can be used to interface with it. Otherwise returns ``None``.
         """
 
     def peer():
         """Returns an object conforming to this interface.
 
         Most implementations will ``return self``.
         """
 
     def canpush():
         """Returns a boolean indicating if this peer can be pushed to."""
 
     def close():
         """Close the connection to this peer.
 
         This is called when the peer will no longer be used. Resources
         associated with the peer should be cleaned up.
         """
 
 class ipeercapabilities(interfaceutil.Interface):
     """Peer sub-interface related to capabilities."""
 
     def capable(name):
         """Determine support for a named capability.
 
         Returns ``False`` if capability not supported.
 
         Returns ``True`` if boolean capability is supported. Returns a string
         if capability support is non-boolean.
 
         Capability strings may or may not map to wire protocol capabilities.
         """
 
     def requirecap(name, purpose):
         """Require a capability to be present.
 
         Raises a ``CapabilityError`` if the capability isn't present.
         """
 
 class ipeercommands(interfaceutil.Interface):
     """Client-side interface for communicating over the wire protocol.
 
     This interface is used as a gateway to the Mercurial wire protocol.
     methods commonly call wire protocol commands of the same name.
     """
 
     def branchmap():
         """Obtain heads in named branches.
 
         Returns a dict mapping branch name to an iterable of nodes that are
         heads on that branch.
         """
 
     def capabilities():
         """Obtain capabilities of the peer.
 
         Returns a set of string capabilities.
         """
 
     def clonebundles():
         """Obtains the clone bundles manifest for the repo.
 
         Returns the manifest as unparsed bytes.
         """
 
     def debugwireargs(one, two, three=None, four=None, five=None):
         """Used to facilitate debugging of arguments passed over the wire."""
 
     def getbundle(source, **kwargs):
         """Obtain remote repository data as a bundle.
 
         This command is how the bulk of repository data is transferred from
         the peer to the local repository
 
         Returns a generator of bundle data.
         """
 
     def heads():
         """Determine all known head revisions in the peer.
 
         Returns an iterable of binary nodes.
         """
 
     def known(nodes):
         """Determine whether multiple nodes are known.
 
         Accepts an iterable of nodes whose presence to check for.
 
         Returns an iterable of booleans indicating of the corresponding node
         at that index is known to the peer.
         """
 
     def listkeys(namespace):
         """Obtain all keys in a pushkey namespace.
 
         Returns an iterable of key names.
         """
 
     def lookup(key):
         """Resolve a value to a known revision.
 
         Returns a binary node of the resolved revision on success.
         """
 
     def pushkey(namespace, key, old, new):
         """Set a value using the ``pushkey`` protocol.
 
         Arguments correspond to the pushkey namespace and key to operate on and
         the old and new values for that key.
 
         Returns a string with the peer result. The value inside varies by the
         namespace.
         """
 
     def stream_out():
         """Obtain streaming clone data.
 
         Successful result should be a generator of data chunks.
         """
 
     def unbundle(bundle, heads, url):
         """Transfer repository data to the peer.
 
         This is how the bulk of data during a push is transferred.
 
         Returns the integer number of heads added to the peer.
         """
 
 class ipeerlegacycommands(interfaceutil.Interface):
     """Interface for implementing support for legacy wire protocol commands.
 
     Wire protocol commands transition to legacy status when they are no longer
     used by modern clients. To facilitate identifying which commands are
     legacy, the interfaces are split.
     """
 
     def between(pairs):
         """Obtain nodes between pairs of nodes.
 
         ``pairs`` is an iterable of node pairs.
 
         Returns an iterable of iterables of nodes corresponding to each
         requested pair.
         """
 
     def branches(nodes):
         """Obtain ancestor changesets of specific nodes back to a branch point.
 
         For each requested node, the peer finds the first ancestor node that is
         a DAG root or is a merge.
 
         Returns an iterable of iterables with the resolved values for each node.
         """
 
     def changegroup(nodes, source):
         """Obtain a changegroup with data for descendants of specified nodes."""
 
     def changegroupsubset(bases, heads, source):
         pass
 
 class ipeercommandexecutor(interfaceutil.Interface):
     """Represents a mechanism to execute remote commands.
 
     This is the primary interface for requesting that wire protocol commands
     be executed. Instances of this interface are active in a context manager
     and have a well-defined lifetime. When the context manager exits, all
     outstanding requests are waited on.
     """
 
     def callcommand(name, args):
         """Request that a named command be executed.
 
         Receives the command name and a dictionary of command arguments.
 
         Returns a ``concurrent.futures.Future`` that will resolve to the
         result of that command request. That exact value is left up to
         the implementation and possibly varies by command.
 
         Not all commands can coexist with other commands in an executor
         instance: it depends on the underlying wire protocol transport being
         used and the command itself.
 
         Implementations MAY call ``sendcommands()`` automatically if the
         requested command can not coexist with other commands in this executor.
 
         Implementations MAY call ``sendcommands()`` automatically when the
         future's ``result()`` is called. So, consumers using multiple
         commands with an executor MUST ensure that ``result()`` is not called
         until all command requests have been issued.
         """
 
     def sendcommands():
         """Trigger submission of queued command requests.
 
         Not all transports submit commands as soon as they are requested to
         run. When called, this method forces queued command requests to be
         issued. It will no-op if all commands have already been sent.
 
         When called, no more new commands may be issued with this executor.
         """
 
     def close():
         """Signal that this command request is finished.
 
         When called, no more new commands may be issued. All outstanding
         commands that have previously been issued are waited on before
         returning. This not only includes waiting for the futures to resolve,
         but also waiting for all response data to arrive. In other words,
         calling this waits for all on-wire state for issued command requests
         to finish.
 
         When used as a context manager, this method is called when exiting the
         context manager.
 
         This method may call ``sendcommands()`` if there are buffered commands.
         """
 
 class ipeerrequests(interfaceutil.Interface):
     """Interface for executing commands on a peer."""
 
     def commandexecutor():
         """A context manager that resolves to an ipeercommandexecutor.
 
         The object this resolves to can be used to issue command requests
         to the peer.
 
         Callers should call its ``callcommand`` method to issue command
         requests.
 
         A new executor should be obtained for each distinct set of commands
         (possibly just a single command) that the consumer wants to execute
         as part of a single operation or round trip. This is because some
         peers are half-duplex and/or don't support persistent connections.
         e.g. in the case of HTTP peers, commands sent to an executor represent
         a single HTTP request. While some peers may support multiple command
         sends over the wire per executor, consumers need to code to the least
         capable peer. So it should be assumed that command executors buffer
         called commands until they are told to send them and that each
         command executor could result in a new connection or wire-level request
         being issued.
         """
 
 class ipeerbase(ipeerconnection, ipeercapabilities, ipeerrequests):
     """Unified interface for peer repositories.
 
     All peer instances must conform to this interface.
     """
 
 @interfaceutil.implementer(ipeerbase)
 class peer(object):
     """Base class for peer repositories."""
 
     def capable(self, name):
         caps = self.capabilities()
         if name in caps:
             return True
 
         name = '%s=' % name
         for cap in caps:
             if cap.startswith(name):
                 return cap[len(name):]
 
         return False
 
     def requirecap(self, name, purpose):
         if self.capable(name):
             return
 
         raise error.CapabilityError(
             _('cannot %s; remote repository does not support the %r '
               'capability') % (purpose, name))
 
 class irevisiondelta(interfaceutil.Interface):
     """Represents a delta between one revision and another.
 
     Instances convey enough information to allow a revision to be exchanged
     with another repository.
 
     Instances represent the fulltext revision data or a delta against
     another revision. Therefore the ``revision`` and ``delta`` attributes
     are mutually exclusive.
 
     Typically used for changegroup generation.
     """
 
     node = interfaceutil.Attribute(
         """20 byte node of this revision.""")
 
     p1node = interfaceutil.Attribute(
         """20 byte node of 1st parent of this revision.""")
 
     p2node = interfaceutil.Attribute(
         """20 byte node of 2nd parent of this revision.""")
 
     linknode = interfaceutil.Attribute(
         """20 byte node of the changelog revision this node is linked to.""")
 
     flags = interfaceutil.Attribute(
         """2 bytes of integer flags that apply to this revision.""")
 
     basenode = interfaceutil.Attribute(
         """20 byte node of the revision this data is a delta against.
 
         ``nullid`` indicates that the revision is a full revision and not
         a delta.
         """)
 
     baserevisionsize = interfaceutil.Attribute(
         """Size of base revision this delta is against.
 
         May be ``None`` if ``basenode`` is ``nullid``.
         """)
 
     revision = interfaceutil.Attribute(
         """Raw fulltext of revision data for this node.""")
 
     delta = interfaceutil.Attribute(
         """Delta between ``basenode`` and ``node``.
 
         Stored in the bdiff delta format.
         """)
 
 class irevisiondeltarequest(interfaceutil.Interface):
     """Represents a request to generate an ``irevisiondelta``."""
 
     node = interfaceutil.Attribute(
         """20 byte node of revision being requested.""")
 
     p1node = interfaceutil.Attribute(
         """20 byte node of 1st parent of revision.""")
 
     p2node = interfaceutil.Attribute(
         """20 byte node of 2nd parent of revision.""")
 
     linknode = interfaceutil.Attribute(
         """20 byte node to store in ``linknode`` attribute.""")
 
     basenode = interfaceutil.Attribute(
         """Base revision that delta should be generated against.
 
         If ``nullid``, the derived ``irevisiondelta`` should have its
         ``revision`` field populated and no delta should be generated.
 
         If ``None``, the delta may be generated against any revision that
         is an ancestor of this revision. Or a full revision may be used.
 
         If any other value, the delta should be produced against that
         revision.
         """)
 
     ellipsis = interfaceutil.Attribute(
         """Boolean on whether the ellipsis flag should be set.""")
 
 class ifilerevisionssequence(interfaceutil.Interface):
     """Contains index data for all revisions of a file.
 
     Types implementing this behave like lists of tuples. The index
     in the list corresponds to the revision number. The values contain
     index metadata.
 
     The *null* revision (revision number -1) is always the last item
     in the index.
     """
 
     def __len__():
         """The total number of revisions."""
 
     def __getitem__(rev):
         """Returns the object having a specific revision number.
 
         Returns an 8-tuple with the following fields:
 
         offset+flags
            Contains the offset and flags for the revision. 64-bit unsigned
            integer where first 6 bytes are the offset and the next 2 bytes
            are flags. The offset can be 0 if it is not used by the store.
         compressed size
             Size of the revision data in the store. It can be 0 if it isn't
             needed by the store.
         uncompressed size
             Fulltext size. It can be 0 if it isn't needed by the store.
         base revision
             Revision number of revision the delta for storage is encoded
             against. -1 indicates not encoded against a base revision.
         link revision
             Revision number of changelog revision this entry is related to.
         p1 revision
             Revision number of 1st parent. -1 if no 1st parent.
         p2 revision
             Revision number of 2nd parent. -1 if no 1st parent.
         node
             Binary node value for this revision number.
 
         Negative values should index off the end of the sequence. ``-1``
         should return the null revision. ``-2`` should return the most
         recent revision.
         """
 
     def __contains__(rev):
         """Whether a revision number exists."""
 
     def insert(self, i, entry):
         """Add an item to the index at specific revision."""
 
 class ifileindex(interfaceutil.Interface):
     """Storage interface for index data of a single file.
 
     File storage data is divided into index metadata and data storage.
     This interface defines the index portion of the interface.
 
     The index logically consists of:
 
     * A mapping between revision numbers and nodes.
     * DAG data (storing and querying the relationship between nodes).
     * Metadata to facilitate storage.
     """
     index = interfaceutil.Attribute(
         """An ``ifilerevisionssequence`` instance.""")
 
     def __len__():
         """Obtain the number of revisions stored for this file."""
 
     def __iter__():
         """Iterate over revision numbers for this file."""
 
     def revs(start=0, stop=None):
         """Iterate over revision numbers for this file, with control."""
 
     def parents(node):
         """Returns a 2-tuple of parent nodes for a revision.
 
         Values will be ``nullid`` if the parent is empty.
         """
 
     def parentrevs(rev):
         """Like parents() but operates on revision numbers."""
 
     def rev(node):
         """Obtain the revision number given a node.
 
         Raises ``error.LookupError`` if the node is not known.
         """
 
     def node(rev):
         """Obtain the node value given a revision number.
 
         Raises ``IndexError`` if the node is not known.
         """
 
     def lookup(node):
         """Attempt to resolve a value to a node.
 
         Value can be a binary node, hex node, revision number, or a string
         that can be converted to an integer.
 
         Raises ``error.LookupError`` if a node could not be resolved.
         """
 
     def linkrev(rev):
         """Obtain the changeset revision number a revision is linked to."""
 
     def flags(rev):
         """Obtain flags used to affect storage of a revision."""
 
     def iscensored(rev):
         """Return whether a revision's content has been censored."""
 
     def commonancestorsheads(node1, node2):
         """Obtain an iterable of nodes containing heads of common ancestors.
 
         See ``ancestor.commonancestorsheads()``.
         """
 
     def descendants(revs):
         """Obtain descendant revision numbers for a set of revision numbers.
 
         If ``nullrev`` is in the set, this is equivalent to ``revs()``.
         """
 
     def headrevs():
         """Obtain a list of revision numbers that are DAG heads.
 
         The list is sorted oldest to newest.
 
         TODO determine if sorting is required.
         """
 
     def heads(start=None, stop=None):
         """Obtain a list of nodes that are DAG heads, with control.
 
         The set of revisions examined can be limited by specifying
         ``start`` and ``stop``. ``start`` is a node. ``stop`` is an
         iterable of nodes. DAG traversal starts at earlier revision
         ``start`` and iterates forward until any node in ``stop`` is
         encountered.
         """
 
     def children(node):
         """Obtain nodes that are children of a node.
 
         Returns a list of nodes.
         """
 
     def deltaparent(rev):
         """"Return the revision that is a suitable parent to delta against."""
 
 class ifiledata(interfaceutil.Interface):
     """Storage interface for data storage of a specific file.
 
     This complements ``ifileindex`` and provides an interface for accessing
     data for a tracked file.
     """
     def rawsize(rev):
         """The size of the fulltext data for a revision as stored."""
 
     def size(rev):
         """Obtain the fulltext size of file data.
 
         Any metadata is excluded from size measurements. Use ``rawsize()`` if
         metadata size is important.
         """
 
     def checkhash(fulltext, node, p1=None, p2=None, rev=None):
         """Validate the stored hash of a given fulltext and node.
 
         Raises ``error.StorageError`` is hash validation fails.
         """
 
     def revision(node, raw=False):
         """"Obtain fulltext data for a node.
 
         By default, any storage transformations are applied before the data
         is returned. If ``raw`` is True, non-raw storage transformations
         are not applied.
 
         The fulltext data may contain a header containing metadata. Most
         consumers should use ``read()`` to obtain the actual file data.
         """
 
     def read(node):
         """Resolve file fulltext data.
 
         This is similar to ``revision()`` except any metadata in the data
         headers is stripped.
         """
 
     def renamed(node):
         """Obtain copy metadata for a node.
 
         Returns ``False`` if no copy metadata is stored or a 2-tuple of
         (path, node) from which this revision was copied.
         """
 
     def cmp(node, fulltext):
         """Compare fulltext to another revision.
 
         Returns True if the fulltext is different from what is stored.
 
         This takes copy metadata into account.
 
         TODO better document the copy metadata and censoring logic.
         """
 
     def revdiff(rev1, rev2):
         """Obtain a delta between two revision numbers.
 
         Operates on raw data in the store (``revision(node, raw=True)``).
 
         The returned data is the result of ``bdiff.bdiff`` on the raw
         revision data.
         """
 
     def emitrevisiondeltas(requests):
         """Produce ``irevisiondelta`` from ``irevisiondeltarequest``s.
 
         Given an iterable of objects conforming to the ``irevisiondeltarequest``
         interface, emits objects conforming to the ``irevisiondelta``
         interface.
 
         This method is a generator.
 
         ``irevisiondelta`` should be emitted in the same order of
         ``irevisiondeltarequest`` that was passed in.
 
         The emitted objects MUST conform by the results of
         ``irevisiondeltarequest``. Namely, they must respect any requests
         for building a delta from a specific ``basenode`` if defined.
 
         When sending deltas, implementations must take into account whether
         the client has the base delta before encoding a delta against that
         revision. A revision encountered previously in ``requests`` is
         always a suitable base revision. An example of a bad delta is a delta
         against a non-ancestor revision. Another example of a bad delta is a
         delta against a censored revision.
         """
 
 class ifilemutation(interfaceutil.Interface):
     """Storage interface for mutation events of a tracked file."""
 
     def add(filedata, meta, transaction, linkrev, p1, p2):
         """Add a new revision to the store.
 
         Takes file data, dictionary of metadata, a transaction, linkrev,
         and parent nodes.
 
         Returns the node that was added.
 
         May no-op if a revision matching the supplied data is already stored.
         """
 
     def addrevision(revisiondata, transaction, linkrev, p1, p2, node=None,
                     flags=0, cachedelta=None):
         """Add a new revision to the store.
 
         This is similar to ``add()`` except it operates at a lower level.
 
         The data passed in already contains a metadata header, if any.
 
         ``node`` and ``flags`` can be used to define the expected node and
         the flags to use with storage.
 
         ``add()`` is usually called when adding files from e.g. the working
         directory. ``addrevision()`` is often called by ``add()`` and for
         scenarios where revision data has already been computed, such as when
         applying raw data from a peer repo.
         """
 
     def addgroup(deltas, linkmapper, transaction, addrevisioncb=None):
         """Process a series of deltas for storage.
 
         ``deltas`` is an iterable of 7-tuples of
         (node, p1, p2, linknode, deltabase, delta, flags) defining revisions
         to add.
 
         The ``delta`` field contains ``mpatch`` data to apply to a base
         revision, identified by ``deltabase``. The base node can be
         ``nullid``, in which case the header from the delta can be ignored
         and the delta used as the fulltext.
 
         ``addrevisioncb`` should be called for each node as it is committed.
 
         Returns a list of nodes that were processed. A node will be in the list
         even if it existed in the store previously.
         """
 
+    def censorrevision(tr, node, tombstone=b''):
+        """Remove the content of a single revision.
+
+        The specified ``node`` will have its content purged from storage.
+        Future attempts to access the revision data for this node will
+        result in failure.
+
+        A ``tombstone`` message can optionally be stored. This message may be
+        displayed to users when they attempt to access the missing revision
+        data.
+
+        Storage backends may have stored deltas against the previous content
+        in this revision. As part of censoring a revision, these storage
+        backends are expected to rewrite any internally stored deltas such
+        that they no longer reference the deleted content.
+        """
+
     def getstrippoint(minlink):
         """Find the minimum revision that must be stripped to strip a linkrev.
 
         Returns a 2-tuple containing the minimum revision number and a set
         of all revisions numbers that would be broken by this strip.
 
         TODO this is highly revlog centric and should be abstracted into
         a higher-level deletion API. ``repair.strip()`` relies on this.
         """
 
     def strip(minlink, transaction):
         """Remove storage of items starting at a linkrev.
 
         This uses ``getstrippoint()`` to determine the first node to remove.
         Then it effectively truncates storage for all revisions after that.
 
         TODO this is highly revlog centric and should be abstracted into a
         higher-level deletion API.
         """
 
 class ifilestorage(ifileindex, ifiledata, ifilemutation):
     """Complete storage interface for a single tracked file."""
 
     version = interfaceutil.Attribute(
         """Version number of storage.
 
         TODO this feels revlog centric and could likely be removed.
         """)
 
     _generaldelta = interfaceutil.Attribute(
         """Whether deltas can be against any parent revision.
 
         TODO this is used by changegroup code and it could probably be
         folded into another API.
         """)
 
     def files():
         """Obtain paths that are backing storage for this file.
 
         TODO this is used heavily by verify code and there should probably
         be a better API for that.
         """
 
     def checksize():
         """Obtain the expected sizes of backing files.
 
         TODO this is used by verify and it should not be part of the interface.
         """
 
 class idirs(interfaceutil.Interface):
     """Interface representing a collection of directories from paths.
 
     This interface is essentially a derived data structure representing
     directories from a collection of paths.
     """
 
     def addpath(path):
         """Add a path to the collection.
 
         All directories in the path will be added to the collection.
         """
 
     def delpath(path):
         """Remove a path from the collection.
 
         If the removal was the last path in a particular directory, the
         directory is removed from the collection.
         """
 
     def __iter__():
         """Iterate over the directories in this collection of paths."""
 
     def __contains__(path):
         """Whether a specific directory is in this collection."""
 
 class imanifestdict(interfaceutil.Interface):
     """Interface representing a manifest data structure.
 
     A manifest is effectively a dict mapping paths to entries. Each entry
     consists of a binary node and extra flags affecting that entry.
     """
 
     def __getitem__(path):
         """Returns the binary node value for a path in the manifest.
 
         Raises ``KeyError`` if the path does not exist in the manifest.
 
         Equivalent to ``self.find(path)[0]``.
         """
 
     def find(path):
         """Returns the entry for a path in the manifest.
 
         Returns a 2-tuple of (node, flags).
 
         Raises ``KeyError`` if the path does not exist in the manifest.
         """
 
     def __len__():
         """Return the number of entries in the manifest."""
 
     def __nonzero__():
         """Returns True if the manifest has entries, False otherwise."""
 
     __bool__ = __nonzero__
 
     def __setitem__(path, node):
         """Define the node value for a path in the manifest.
 
         If the path is already in the manifest, its flags will be copied to
         the new entry.
         """
 
     def __contains__(path):
         """Whether a path exists in the manifest."""
 
     def __delitem__(path):
         """Remove a path from the manifest.
 
         Raises ``KeyError`` if the path is not in the manifest.
         """
 
     def __iter__():
         """Iterate over paths in the manifest."""
 
     def iterkeys():
         """Iterate over paths in the manifest."""
 
     def keys():
         """Obtain a list of paths in the manifest."""
 
     def filesnotin(other, match=None):
         """Obtain the set of paths in this manifest but not in another.
 
         ``match`` is an optional matcher function to be applied to both
         manifests.
 
         Returns a set of paths.
         """
 
     def dirs():
         """Returns an object implementing the ``idirs`` interface."""
 
     def hasdir(dir):
         """Returns a bool indicating if a directory is in this manifest."""
 
     def matches(match):
         """Generate a new manifest filtered through a matcher.
 
         Returns an object conforming to the ``imanifestdict`` interface.
         """
 
     def walk(match):
         """Generator of paths in manifest satisfying a matcher.
 
         This is equivalent to ``self.matches(match).iterkeys()`` except a new
         manifest object is not created.
 
         If the matcher has explicit files listed and they don't exist in
         the manifest, ``match.bad()`` is called for each missing file.
         """
 
     def diff(other, match=None, clean=False):
         """Find differences between this manifest and another.
 
         This manifest is compared to ``other``.
 
         If ``match`` is provided, the two manifests are filtered against this
         matcher and only entries satisfying the matcher are compared.
 
         If ``clean`` is True, unchanged files are included in the returned
         object.
 
         Returns a dict with paths as keys and values of 2-tuples of 2-tuples of
         the form ``((node1, flag1), (node2, flag2))`` where ``(node1, flag1)``
         represents the node and flags for this manifest and ``(node2, flag2)``
         are the same for the other manifest.
         """
 
     def setflag(path, flag):
         """Set the flag value for a given path.
 
         Raises ``KeyError`` if the path is not already in the manifest.
         """
 
     def get(path, default=None):
         """Obtain the node value for a path or a default value if missing."""
 
     def flags(path, default=''):
         """Return the flags value for a path or a default value if missing."""
 
     def copy():
         """Return a copy of this manifest."""
 
     def items():
         """Returns an iterable of (path, node) for items in this manifest."""
 
     def iteritems():
         """Identical to items()."""
 
     def iterentries():
         """Returns an iterable of (path, node, flags) for this manifest.
 
         Similar to ``iteritems()`` except items are a 3-tuple and include
         flags.
         """
 
     def text():
         """Obtain the raw data representation for this manifest.
 
         Result is used to create a manifest revision.
         """
 
     def fastdelta(base, changes):
         """Obtain a delta between this manifest and another given changes.
 
         ``base`` in the raw data representation for another manifest.
 
         ``changes`` is an iterable of ``(path, to_delete)``.
 
         Returns a 2-tuple containing ``bytearray(self.text())`` and the
         delta between ``base`` and this manifest.
         """
 
 class imanifestrevisionbase(interfaceutil.Interface):
     """Base interface representing a single revision of a manifest.
 
     Should not be used as a primary interface: should always be inherited
     as part of a larger interface.
     """
 
     def new():
         """Obtain a new manifest instance.
 
         Returns an object conforming to the ``imanifestrevisionwritable``
         interface. The instance will be associated with the same
         ``imanifestlog`` collection as this instance.
         """
 
     def copy():
         """Obtain a copy of this manifest instance.
 
         Returns an object conforming to the ``imanifestrevisionwritable``
         interface. The instance will be associated with the same
         ``imanifestlog`` collection as this instance.
         """
 
     def read():
         """Obtain the parsed manifest data structure.
 
         The returned object conforms to the ``imanifestdict`` interface.
         """
 
 class imanifestrevisionstored(imanifestrevisionbase):
     """Interface representing a manifest revision committed to storage."""
 
     def node():
         """The binary node for this manifest."""
 
     parents = interfaceutil.Attribute(
         """List of binary nodes that are parents for this manifest revision."""
     )
 
     def readdelta(shallow=False):
         """Obtain the manifest data structure representing changes from parent.
 
         This manifest is compared to its 1st parent. A new manifest representing
         those differences is constructed.
 
         The returned object conforms to the ``imanifestdict`` interface.
         """
 
     def readfast(shallow=False):
         """Calls either ``read()`` or ``readdelta()``.
 
         The faster of the two options is called.
         """
 
     def find(key):
         """Calls self.read().find(key)``.
 
         Returns a 2-tuple of ``(node, flags)`` or raises ``KeyError``.
         """
 
 class imanifestrevisionwritable(imanifestrevisionbase):
     """Interface representing a manifest revision that can be committed."""
 
     def write(transaction, linkrev, p1node, p2node, added, removed, match=None):
         """Add this revision to storage.
 
         Takes a transaction object, the changeset revision number it will
         be associated with, its parent nodes, and lists of added and
         removed paths.
 
         If match is provided, storage can choose not to inspect or write out
         items that do not match. Storage is still required to be able to provide
         the full manifest in the future for any directories written (these
         manifests should not be "narrowed on disk").
 
         Returns the binary node of the created revision.
         """
 
 class imanifeststorage(interfaceutil.Interface):
     """Storage interface for manifest data."""
 
     tree = interfaceutil.Attribute(
         """The path to the directory this manifest tracks.
 
         The empty bytestring represents the root manifest.
         """)
 
     index = interfaceutil.Attribute(
         """An ``ifilerevisionssequence`` instance.""")
 
     indexfile = interfaceutil.Attribute(
         """Path of revlog index file.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     opener = interfaceutil.Attribute(
         """VFS opener to use to access underlying files used for storage.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     version = interfaceutil.Attribute(
         """Revlog version number.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     _generaldelta = interfaceutil.Attribute(
         """Whether generaldelta storage is being used.
 
         TODO this is revlog specific and should not be exposed.
         """)
 
     fulltextcache = interfaceutil.Attribute(
         """Dict with cache of fulltexts.
 
         TODO this doesn't feel appropriate for the storage interface.
         """)
 
     def __len__():
         """Obtain the number of revisions stored for this manifest."""
 
     def __iter__():
         """Iterate over revision numbers for this manifest."""
 
     def rev(node):
         """Obtain the revision number given a binary node.
 
         Raises ``error.LookupError`` if the node is not known.
         """
 
     def node(rev):
         """Obtain the node value given a revision number.
 
         Raises ``error.LookupError`` if the revision is not known.
         """
 
     def lookup(value):
         """Attempt to resolve a value to a node.
 
         Value can be a binary node, hex node, revision number, or a bytes
         that can be converted to an integer.
 
         Raises ``error.LookupError`` if a ndoe could not be resolved.
 
         TODO this is only used by debug* commands and can probably be deleted
         easily.
         """
 
     def parents(node):
         """Returns a 2-tuple of parent nodes for a node.
 
         Values will be ``nullid`` if the parent is empty.
         """
 
     def parentrevs(rev):
         """Like parents() but operates on revision numbers."""
 
     def linkrev(rev):
         """Obtain the changeset revision number a revision is linked to."""
 
     def revision(node, _df=None, raw=False):
         """Obtain fulltext data for a node."""
 
     def revdiff(rev1, rev2):
         """Obtain a delta between two revision numbers.
 
         The returned data is the result of ``bdiff.bdiff()`` on the raw
         revision data.
         """
 
     def cmp(node, fulltext):
         """Compare fulltext to another revision.
 
         Returns True if the fulltext is different from what is stored.
         """
 
     def emitrevisiondeltas(requests):
         """Produce ``irevisiondelta`` from ``irevisiondeltarequest``s.
 
         See the documentation for ``ifiledata`` for more.
         """
 
     def addgroup(deltas, linkmapper, transaction, addrevisioncb=None):
         """Process a series of deltas for storage.
 
         See the documentation in ``ifilemutation`` for more.
         """
 
     def getstrippoint(minlink):
         """Find minimum revision that must be stripped to strip a linkrev.
 
         See the documentation in ``ifilemutation`` for more.
         """
 
     def strip(minlink, transaction):
         """Remove storage of items starting at a linkrev.
 
         See the documentation in ``ifilemutation`` for more.
         """
 
     def checksize():
         """Obtain the expected sizes of backing files.
 
         TODO this is used by verify and it should not be part of the interface.
         """
 
     def files():
         """Obtain paths that are backing storage for this manifest.
 
         TODO this is used by verify and there should probably be a better API
         for this functionality.
         """
 
     def deltaparent(rev):
         """Obtain the revision that a revision is delta'd against.
 
         TODO delta encoding is an implementation detail of storage and should
         not be exposed to the storage interface.
         """
 
     def clone(tr, dest, **kwargs):
         """Clone this instance to another."""
 
     def clearcaches(clear_persisted_data=False):
         """Clear any caches associated with this instance."""
 
     def dirlog(d):
         """Obtain a manifest storage instance for a tree."""
 
     def add(m, transaction, link, p1, p2, added, removed, readtree=None,
             match=None):
         """Add a revision to storage.
 
         ``m`` is an object conforming to ``imanifestdict``.
 
         ``link`` is the linkrev revision number.
 
         ``p1`` and ``p2`` are the parent revision numbers.
 
         ``added`` and ``removed`` are iterables of added and removed paths,
         respectively.
 
         ``readtree`` is a function that can be used to read the child tree(s)
         when recursively writing the full tree structure when using
         treemanifets.
 
         ``match`` is a matcher that can be used to hint to storage that not all
         paths must be inspected; this is an optimization and can be safely
         ignored. Note that the storage must still be able to reproduce a full
         manifest including files that did not match.
         """
 
 class imanifestlog(interfaceutil.Interface):
     """Interface representing a collection of manifest snapshots.
 
     Represents the root manifest in a repository.
 
     Also serves as a means to access nested tree manifests and to cache
     tree manifests.
     """
 
     def __getitem__(node):
         """Obtain a manifest instance for a given binary node.
 
         Equivalent to calling ``self.get('', node)``.
 
         The returned object conforms to the ``imanifestrevisionstored``
         interface.
         """
 
     def get(tree, node, verify=True):
         """Retrieve the manifest instance for a given directory and binary node.
 
         ``node`` always refers to the node of the root manifest (which will be
         the only manifest if flat manifests are being used).
 
         If ``tree`` is the empty string, the root manifest is returned.
         Otherwise the manifest for the specified directory will be returned
         (requires tree manifests).
 
         If ``verify`` is True, ``LookupError`` is raised if the node is not
         known.
 
         The returned object conforms to the ``imanifestrevisionstored``
         interface.
         """
 
     def getstorage(tree):
         """Retrieve an interface to storage for a particular tree.
 
         If ``tree`` is the empty bytestring, storage for the root manifest will
         be returned. Otherwise storage for a tree manifest is returned.
 
         TODO formalize interface for returned object.
         """
 
     def clearcaches():
         """Clear caches associated with this collection."""
 
     def rev(node):
         """Obtain the revision number for a binary node.
 
         Raises ``error.LookupError`` if the node is not known.
         """
 
 class ilocalrepositoryfilestorage(interfaceutil.Interface):
     """Local repository sub-interface providing access to tracked file storage.
 
     This interface defines how a repository accesses storage for a single
     tracked file path.
     """
 
     def file(f):
         """Obtain a filelog for a tracked path.
 
         The returned type conforms to the ``ifilestorage`` interface.
         """
 
 class ilocalrepositorymain(interfaceutil.Interface):
     """Main interface for local repositories.
 
     This currently captures the reality of things - not how things should be.
     """
 
     supportedformats = interfaceutil.Attribute(
         """Set of requirements that apply to stream clone.
 
         This is actually a class attribute and is shared among all instances.
         """)
 
     supported = interfaceutil.Attribute(
         """Set of requirements that this repo is capable of opening.""")
 
     requirements = interfaceutil.Attribute(
         """Set of requirements this repo uses.""")
 
     filtername = interfaceutil.Attribute(
         """Name of the repoview that is active on this repo.""")
 
     wvfs = interfaceutil.Attribute(
         """VFS used to access the working directory.""")
 
     vfs = interfaceutil.Attribute(
         """VFS rooted at the .hg directory.
 
         Used to access repository data not in the store.
         """)
 
     svfs = interfaceutil.Attribute(
         """VFS rooted at the store.
 
         Used to access repository data in the store. Typically .hg/store.
         But can point elsewhere if the store is shared.
         """)
 
     root = interfaceutil.Attribute(
         """Path to the root of the working directory.""")
 
     path = interfaceutil.Attribute(
         """Path to the .hg directory.""")
 
     origroot = interfaceutil.Attribute(
         """The filesystem path that was used to construct the repo.""")
 
     auditor = interfaceutil.Attribute(
         """A pathauditor for the working directory.
 
         This checks if a path refers to a nested repository.
 
         Operates on the filesystem.
         """)
 
     nofsauditor = interfaceutil.Attribute(
         """A pathauditor for the working directory.
 
         This is like ``auditor`` except it doesn't do filesystem checks.
         """)
 
     baseui = interfaceutil.Attribute(
         """Original ui instance passed into constructor.""")
 
     ui = interfaceutil.Attribute(
         """Main ui instance for this instance.""")
 
     sharedpath = interfaceutil.Attribute(
         """Path to the .hg directory of the repo this repo was shared from.""")
 
     store = interfaceutil.Attribute(
         """A store instance.""")
 
     spath = interfaceutil.Attribute(
         """Path to the store.""")
 
     sjoin = interfaceutil.Attribute(
         """Alias to self.store.join.""")
 
     cachevfs = interfaceutil.Attribute(
         """A VFS used to access the cache directory.
 
         Typically .hg/cache.
         """)
 
     filteredrevcache = interfaceutil.Attribute(
         """Holds sets of revisions to be filtered.""")
 
     names = interfaceutil.Attribute(
         """A ``namespaces`` instance.""")
 
     def close():
         """Close the handle on this repository."""
 
     def peer():
         """Obtain an object conforming to the ``peer`` interface."""
 
     def unfiltered():
         """Obtain an unfiltered/raw view of this repo."""
 
     def filtered(name, visibilityexceptions=None):
         """Obtain a named view of this repository."""
 
     obsstore = interfaceutil.Attribute(
         """A store of obsolescence data.""")
 
     changelog = interfaceutil.Attribute(
         """A handle on the changelog revlog.""")
 
     manifestlog = interfaceutil.Attribute(
         """An instance conforming to the ``imanifestlog`` interface.
 
         Provides access to manifests for the repository.
         """)
 
     dirstate = interfaceutil.Attribute(
         """Working directory state.""")
 
     narrowpats = interfaceutil.Attribute(
         """Matcher patterns for this repository's narrowspec.""")
 
     def narrowmatch():
         """Obtain a matcher for the narrowspec."""
 
     def setnarrowpats(newincludes, newexcludes):
         """Define the narrowspec for this repository."""
 
     def __getitem__(changeid):
         """Try to resolve a changectx."""
 
     def __contains__(changeid):
         """Whether a changeset exists."""
 
     def __nonzero__():
         """Always returns True."""
         return True
 
     __bool__ = __nonzero__
 
     def __len__():
         """Returns the number of changesets in the repo."""
 
     def __iter__():
         """Iterate over revisions in the changelog."""
 
     def revs(expr, *args):
         """Evaluate a revset.
 
         Emits revisions.
         """
 
     def set(expr, *args):
         """Evaluate a revset.
 
         Emits changectx instances.
         """
 
     def anyrevs(specs, user=False, localalias=None):
         """Find revisions matching one of the given revsets."""
 
     def url():
         """Returns a string representing the location of this repo."""
 
     def hook(name, throw=False, **args):
         """Call a hook."""
 
     def tags():
         """Return a mapping of tag to node."""
 
     def tagtype(tagname):
         """Return the type of a given tag."""
 
     def tagslist():
         """Return a list of tags ordered by revision."""
 
     def nodetags(node):
         """Return the tags associated with a node."""
 
     def nodebookmarks(node):
         """Return the list of bookmarks pointing to the specified node."""
 
     def branchmap():
         """Return a mapping of branch to heads in that branch."""
 
     def revbranchcache():
         pass
 
     def branchtip(branchtip, ignoremissing=False):
         """Return the tip node for a given branch."""
 
     def lookup(key):
         """Resolve the node for a revision."""
 
     def lookupbranch(key):
         """Look up the branch name of the given revision or branch name."""
 
     def known(nodes):
         """Determine whether a series of nodes is known.
 
         Returns a list of bools.
         """
 
     def local():
         """Whether the repository is local."""
         return True
 
     def publishing():
         """Whether the repository is a publishing repository."""
 
     def cancopy():
         pass
 
     def shared():
         """The type of shared repository or None."""
 
     def wjoin(f, *insidef):
         """Calls self.vfs.reljoin(self.root, f, *insidef)"""
 
     def setparents(p1, p2):
         """Set the parent nodes of the working directory."""
 
     def filectx(path, changeid=None, fileid=None):
         """Obtain a filectx for the given file revision."""
 
     def getcwd():
         """Obtain the current working directory from the dirstate."""
 
     def pathto(f, cwd=None):
         """Obtain the relative path to a file."""
 
     def adddatafilter(name, fltr):
         pass
 
     def wread(filename):
         """Read a file from wvfs, using data filters."""
 
     def wwrite(filename, data, flags, backgroundclose=False, **kwargs):
         """Write data to a file in the wvfs, using data filters."""
 
     def wwritedata(filename, data):
         """Resolve data for writing to the wvfs, using data filters."""
 
     def currenttransaction():
         """Obtain the current transaction instance or None."""
 
     def transaction(desc, report=None):
         """Open a new transaction to write to the repository."""
 
     def undofiles():
         """Returns a list of (vfs, path) for files to undo transactions."""
 
     def recover():
         """Roll back an interrupted transaction."""
 
     def rollback(dryrun=False, force=False):
         """Undo the last transaction.
 
         DANGEROUS.
         """
 
     def updatecaches(tr=None, full=False):
         """Warm repo caches."""
 
     def invalidatecaches():
         """Invalidate cached data due to the repository mutating."""
 
     def invalidatevolatilesets():
         pass
 
     def invalidatedirstate():
         """Invalidate the dirstate."""
 
     def invalidate(clearfilecache=False):
         pass
 
     def invalidateall():
         pass
 
     def lock(wait=True):
         """Lock the repository store and return a lock instance."""
 
     def wlock(wait=True):
         """Lock the non-store parts of the repository."""
 
     def currentwlock():
         """Return the wlock if it's held or None."""
 
     def checkcommitpatterns(wctx, vdirs, match, status, fail):
         pass
 
     def commit(text='', user=None, date=None, match=None, force=False,
                editor=False, extra=None):
         """Add a new revision to the repository."""
 
     def commitctx(ctx, error=False):
         """Commit a commitctx instance to the repository."""
 
     def destroying():
         """Inform the repository that nodes are about to be destroyed."""
 
     def destroyed():
         """Inform the repository that nodes have been destroyed."""
 
     def status(node1='.', node2=None, match=None, ignored=False,
                clean=False, unknown=False, listsubrepos=False):
         """Convenience method to call repo[x].status()."""
 
     def addpostdsstatus(ps):
         pass
 
     def postdsstatus():
         pass
 
     def clearpostdsstatus():
         pass
 
     def heads(start=None):
         """Obtain list of nodes that are DAG heads."""
 
     def branchheads(branch=None, start=None, closed=False):
         pass
 
     def branches(nodes):
         pass
 
     def between(pairs):
         pass
 
     def checkpush(pushop):
         pass
 
     prepushoutgoinghooks = interfaceutil.Attribute(
         """util.hooks instance.""")
 
     def pushkey(namespace, key, old, new):
         pass
 
     def listkeys(namespace):
         pass
 
     def debugwireargs(one, two, three=None, four=None, five=None):
         pass
 
     def savecommitmessage(text):
         pass
 
 class completelocalrepository(ilocalrepositorymain,
                               ilocalrepositoryfilestorage):
     """Complete interface for a local repository."""

mercurial/revlog.py

@@ -1,2494 +1,2583 b''
 # revlog.py - storage back-end for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Storage back-end for Mercurial.
 
 This provides efficient delta storage with O(1) retrieve and append
 and O(changes) merge between branches.
 """
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import errno
 import hashlib
 import os
 import re
 import struct
 import zlib
 
 # import stuff from node for others to import from revlog
 from .node import (
     bin,
     hex,
     nullhex,
     nullid,
     nullrev,
     wdirfilenodeids,
     wdirhex,
     wdirid,
     wdirrev,
 )
 from .i18n import _
 from .revlogutils.constants import (
     FLAG_GENERALDELTA,
     FLAG_INLINE_DATA,
     REVIDX_DEFAULT_FLAGS,
     REVIDX_ELLIPSIS,
     REVIDX_EXTSTORED,
     REVIDX_FLAGS_ORDER,
     REVIDX_ISCENSORED,
     REVIDX_KNOWN_FLAGS,
     REVIDX_RAWTEXT_CHANGING_FLAGS,
     REVLOGV0,
     REVLOGV1,
     REVLOGV1_FLAGS,
     REVLOGV2,
     REVLOGV2_FLAGS,
     REVLOG_DEFAULT_FLAGS,
     REVLOG_DEFAULT_FORMAT,
     REVLOG_DEFAULT_VERSION,
 )
 from .thirdparty import (
     attr,
 )
 from . import (
     ancestor,
     error,
     mdiff,
     policy,
     pycompat,
     repository,
     templatefilters,
     util,
 )
 from .revlogutils import (
     deltas as deltautil,
 )
 from .utils import (
     interfaceutil,
     stringutil,
 )
 
 # blanked usage of all the name to prevent pyflakes constraints
 # We need these name available in the module for extensions.
 REVLOGV0
 REVLOGV1
 REVLOGV2
 FLAG_INLINE_DATA
 FLAG_GENERALDELTA
 REVLOG_DEFAULT_FLAGS
 REVLOG_DEFAULT_FORMAT
 REVLOG_DEFAULT_VERSION
 REVLOGV1_FLAGS
 REVLOGV2_FLAGS
 REVIDX_ISCENSORED
 REVIDX_ELLIPSIS
 REVIDX_EXTSTORED
 REVIDX_DEFAULT_FLAGS
 REVIDX_FLAGS_ORDER
 REVIDX_KNOWN_FLAGS
 REVIDX_RAWTEXT_CHANGING_FLAGS
 
 parsers = policy.importmod(r'parsers')
 
 # Aliased for performance.
 _zlibdecompress = zlib.decompress
 
 # max size of revlog with inline data
 _maxinline = 131072
 _chunksize = 1048576
 
 # Store flag processors (cf. 'addflagprocessor()' to register)
 _flagprocessors = {
     REVIDX_ISCENSORED: None,
 }
 
 # Flag processors for REVIDX_ELLIPSIS.
 def ellipsisreadprocessor(rl, text):
     return text, False
 
 def ellipsiswriteprocessor(rl, text):
     return text, False
 
 def ellipsisrawprocessor(rl, text):
     return False
 
 ellipsisprocessor = (
     ellipsisreadprocessor,
     ellipsiswriteprocessor,
     ellipsisrawprocessor,
 )
 
 _mdre = re.compile('\1\n')
 def parsemeta(text):
     """return (metadatadict, metadatasize)"""
     # text can be buffer, so we can't use .startswith or .index
     if text[:2] != '\1\n':
         return None, None
     s = _mdre.search(text, 2).start()
     mtext = text[2:s]
     meta = {}
     for l in mtext.splitlines():
         k, v = l.split(": ", 1)
         meta[k] = v
     return meta, (s + 2)
 
 def packmeta(meta, text):
     keys = sorted(meta)
     metatext = "".join("%s: %s\n" % (k, meta[k]) for k in keys)
     return "\1\n%s\1\n%s" % (metatext, text)
 
 def _censoredtext(text):
     m, offs = parsemeta(text)
     return m and "censored" in m
 
 def addflagprocessor(flag, processor):
     """Register a flag processor on a revision data flag.
 
     Invariant:
     - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
       and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
     - Only one flag processor can be registered on a specific flag.
     - flagprocessors must be 3-tuples of functions (read, write, raw) with the
       following signatures:
           - (read)  f(self, rawtext) -> text, bool
           - (write) f(self, text) -> rawtext, bool
           - (raw)   f(self, rawtext) -> bool
       "text" is presented to the user. "rawtext" is stored in revlog data, not
       directly visible to the user.
       The boolean returned by these transforms is used to determine whether
       the returned text can be used for hash integrity checking. For example,
       if "write" returns False, then "text" is used to generate hash. If
       "write" returns True, that basically means "rawtext" returned by "write"
       should be used to generate hash. Usually, "write" and "read" return
       different booleans. And "raw" returns a same boolean as "write".
 
       Note: The 'raw' transform is used for changegroup generation and in some
       debug commands. In this case the transform only indicates whether the
       contents can be used for hash integrity checks.
     """
     if not flag & REVIDX_KNOWN_FLAGS:
         msg = _("cannot register processor on unknown flag '%#x'.") % (flag)
         raise error.ProgrammingError(msg)
     if flag not in REVIDX_FLAGS_ORDER:
         msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag)
         raise error.ProgrammingError(msg)
     if flag in _flagprocessors:
         msg = _("cannot register multiple processors on flag '%#x'.") % (flag)
         raise error.Abort(msg)
     _flagprocessors[flag] = processor
 
 def getoffset(q):
     return int(q >> 16)
 
 def gettype(q):
     return int(q & 0xFFFF)
 
 def offset_type(offset, type):
     if (type & ~REVIDX_KNOWN_FLAGS) != 0:
         raise ValueError('unknown revlog index flags')
     return int(int(offset) << 16 | type)
 
 _nullhash = hashlib.sha1(nullid)
 
 def hash(text, p1, p2):
     """generate a hash from the given text and its parent hashes
 
     This hash combines both the current file contents and its history
     in a manner that makes it easy to distinguish nodes with the same
     content in the revision graph.
     """
     # As of now, if one of the parent node is null, p2 is null
     if p2 == nullid:
         # deep copy of a hash is faster than creating one
         s = _nullhash.copy()
         s.update(p1)
     else:
         # none of the parent nodes are nullid
         if p1 < p2:
             a = p1
             b = p2
         else:
             a = p2
             b = p1
         s = hashlib.sha1(a)
         s.update(b)
     s.update(text)
     return s.digest()
 
 @attr.s(slots=True, frozen=True)
 class _revisioninfo(object):
     """Information about a revision that allows building its fulltext
     node:       expected hash of the revision
     p1, p2:     parent revs of the revision
     btext:      built text cache consisting of a one-element list
     cachedelta: (baserev, uncompressed_delta) or None
     flags:      flags associated to the revision storage
 
     One of btext[0] or cachedelta must be set.
     """
     node = attr.ib()
     p1 = attr.ib()
     p2 = attr.ib()
     btext = attr.ib()
     textlen = attr.ib()
     cachedelta = attr.ib()
     flags = attr.ib()
 
 @interfaceutil.implementer(repository.irevisiondelta)
 @attr.s(slots=True, frozen=True)
 class revlogrevisiondelta(object):
     node = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     basenode = attr.ib()
     linknode = attr.ib()
     flags = attr.ib()
     baserevisionsize = attr.ib()
     revision = attr.ib()
     delta = attr.ib()
 
 # index v0:
 #  4 bytes: offset
 #  4 bytes: compressed length
 #  4 bytes: base rev
 #  4 bytes: link rev
 # 20 bytes: parent 1 nodeid
 # 20 bytes: parent 2 nodeid
 # 20 bytes: nodeid
 indexformatv0 = struct.Struct(">4l20s20s20s")
 indexformatv0_pack = indexformatv0.pack
 indexformatv0_unpack = indexformatv0.unpack
 
 class revlogoldindex(list):
     def __getitem__(self, i):
         if i == -1:
             return (0, 0, 0, -1, -1, -1, -1, nullid)
         return list.__getitem__(self, i)
 
 class revlogoldio(object):
     def __init__(self):
         self.size = indexformatv0.size
 
     def parseindex(self, data, inline):
         s = self.size
         index = []
         nodemap = {nullid: nullrev}
         n = off = 0
         l = len(data)
         while off + s <= l:
             cur = data[off:off + s]
             off += s
             e = indexformatv0_unpack(cur)
             # transform to revlogv1 format
             e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
                   nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
             index.append(e2)
             nodemap[e[6]] = n
             n += 1
 
         return revlogoldindex(index), nodemap, None
 
     def packentry(self, entry, node, version, rev):
         if gettype(entry[0]):
             raise error.RevlogError(_('index entry flags need revlog '
                                       'version 1'))
         e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
               node(entry[5]), node(entry[6]), entry[7])
         return indexformatv0_pack(*e2)
 
 # index ng:
 #  6 bytes: offset
 #  2 bytes: flags
 #  4 bytes: compressed length
 #  4 bytes: uncompressed length
 #  4 bytes: base rev
 #  4 bytes: link rev
 #  4 bytes: parent 1 rev
 #  4 bytes: parent 2 rev
 # 32 bytes: nodeid
 indexformatng = struct.Struct(">Qiiiiii20s12x")
 indexformatng_pack = indexformatng.pack
 versionformat = struct.Struct(">I")
 versionformat_pack = versionformat.pack
 versionformat_unpack = versionformat.unpack
 
 # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
 # signed integer)
 _maxentrysize = 0x7fffffff
 
 class revlogio(object):
     def __init__(self):
         self.size = indexformatng.size
 
     def parseindex(self, data, inline):
         # call the C implementation to parse the index data
         index, cache = parsers.parse_index2(data, inline)
         return index, getattr(index, 'nodemap', None), cache
 
     def packentry(self, entry, node, version, rev):
         p = indexformatng_pack(*entry)
         if rev == 0:
             p = versionformat_pack(version) + p[4:]
         return p
 
 class revlog(object):
     """
     the underlying revision storage object
 
     A revlog consists of two parts, an index and the revision data.
 
     The index is a file with a fixed record size containing
     information on each revision, including its nodeid (hash), the
     nodeids of its parents, the position and offset of its data within
     the data file, and the revision it's based on. Finally, each entry
     contains a linkrev entry that can serve as a pointer to external
     data.
 
     The revision data itself is a linear collection of data chunks.
     Each chunk represents a revision and is usually represented as a
     delta against the previous chunk. To bound lookup time, runs of
     deltas are limited to about 2 times the length of the original
     version data. This makes retrieval of a version proportional to
     its size, or O(1) relative to the number of revisions.
 
     Both pieces of the revlog are written to in an append-only
     fashion, which means we never need to rewrite a file to insert or
     remove data, and can use some simple techniques to avoid the need
     for locking while reading.
 
     If checkambig, indexfile is opened with checkambig=True at
     writing, to avoid file stat ambiguity.
 
     If mmaplargeindex is True, and an mmapindexthreshold is set, the
     index will be mmapped rather than read if it is larger than the
     configured threshold.
 
     If censorable is True, the revlog can have censored revisions.
     """
     def __init__(self, opener, indexfile, datafile=None, checkambig=False,
                  mmaplargeindex=False, censorable=False):
         """
         create a revlog object
 
         opener is a function that abstracts the file opening operation
         and can be used to implement COW semantics or the like.
         """
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + ".d")
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
         self._checkambig = checkambig
         self._censorable = censorable
         # 3-tuple of (node, rev, text) for a raw revision.
         self._cache = None
         # Maps rev to chain base rev.
         self._chainbasecache = util.lrucachedict(100)
         # 2-tuple of (offset, data) of raw data from the revlog at an offset.
         self._chunkcache = (0, '')
         # How much data to read and cache into the raw revlog data cache.
         self._chunkcachesize = 65536
         self._maxchainlen = None
         self._deltabothparents = True
         self.index = []
         # Mapping of partial identifiers to full nodes.
         self._pcache = {}
         # Mapping of revision integer to full node.
         self._nodecache = {nullid: nullrev}
         self._nodepos = None
         self._compengine = 'zlib'
         self._maxdeltachainspan = -1
         self._withsparseread = False
         self._sparserevlog = False
         self._srdensitythreshold = 0.50
         self._srmingapsize = 262144
 
         # Make copy of flag processors so each revlog instance can support
         # custom flags.
         self._flagprocessors = dict(_flagprocessors)
 
         mmapindexthreshold = None
         v = REVLOG_DEFAULT_VERSION
         opts = getattr(opener, 'options', None)
         if opts is not None:
             if 'revlogv2' in opts:
                 # version 2 revlogs always use generaldelta.
                 v = REVLOGV2 | FLAG_GENERALDELTA | FLAG_INLINE_DATA
             elif 'revlogv1' in opts:
                 if 'generaldelta' in opts:
                     v |= FLAG_GENERALDELTA
             else:
                 v = 0
             if 'chunkcachesize' in opts:
                 self._chunkcachesize = opts['chunkcachesize']
             if 'maxchainlen' in opts:
                 self._maxchainlen = opts['maxchainlen']
             if 'deltabothparents' in opts:
                 self._deltabothparents = opts['deltabothparents']
             self._lazydeltabase = bool(opts.get('lazydeltabase', False))
             if 'compengine' in opts:
                 self._compengine = opts['compengine']
             if 'maxdeltachainspan' in opts:
                 self._maxdeltachainspan = opts['maxdeltachainspan']
             if mmaplargeindex and 'mmapindexthreshold' in opts:
                 mmapindexthreshold = opts['mmapindexthreshold']
             self._sparserevlog = bool(opts.get('sparse-revlog', False))
             withsparseread = bool(opts.get('with-sparse-read', False))
             # sparse-revlog forces sparse-read
             self._withsparseread = self._sparserevlog or withsparseread
             if 'sparse-read-density-threshold' in opts:
                 self._srdensitythreshold = opts['sparse-read-density-threshold']
             if 'sparse-read-min-gap-size' in opts:
                 self._srmingapsize = opts['sparse-read-min-gap-size']
             if opts.get('enableellipsis'):
                 self._flagprocessors[REVIDX_ELLIPSIS] = ellipsisprocessor
 
         if self._chunkcachesize <= 0:
             raise error.RevlogError(_('revlog chunk cache size %r is not '
                                       'greater than 0') % self._chunkcachesize)
         elif self._chunkcachesize & (self._chunkcachesize - 1):
             raise error.RevlogError(_('revlog chunk cache size %r is not a '
                                       'power of 2') % self._chunkcachesize)
 
         indexdata = ''
         self._initempty = True
         try:
             with self._indexfp() as f:
                 if (mmapindexthreshold is not None and
                         self.opener.fstat(f).st_size >= mmapindexthreshold):
                     indexdata = util.buffer(util.mmapread(f))
                 else:
                     indexdata = f.read()
             if len(indexdata) > 0:
                 v = versionformat_unpack(indexdata[:4])[0]
                 self._initempty = False
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
 
         self.version = v
         self._inline = v & FLAG_INLINE_DATA
         self._generaldelta = v & FLAG_GENERALDELTA
         flags = v & ~0xFFFF
         fmt = v & 0xFFFF
         if fmt == REVLOGV0:
             if flags:
                 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
                                           'revlog %s') %
                                         (flags >> 16, fmt, self.indexfile))
         elif fmt == REVLOGV1:
             if flags & ~REVLOGV1_FLAGS:
                 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
                                           'revlog %s') %
                                         (flags >> 16, fmt, self.indexfile))
         elif fmt == REVLOGV2:
             if flags & ~REVLOGV2_FLAGS:
                 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
                                           'revlog %s') %
                                         (flags >> 16, fmt, self.indexfile))
         else:
             raise error.RevlogError(_('unknown version (%d) in revlog %s') %
                                     (fmt, self.indexfile))
 
         self._storedeltachains = True
 
         self._io = revlogio()
         if self.version == REVLOGV0:
             self._io = revlogoldio()
         try:
             d = self._io.parseindex(indexdata, self._inline)
         except (ValueError, IndexError):
             raise error.RevlogError(_("index %s is corrupted") %
                                     self.indexfile)
         self.index, nodemap, self._chunkcache = d
         if nodemap is not None:
             self.nodemap = self._nodecache = nodemap
         if not self._chunkcache:
             self._chunkclear()
         # revnum -> (chain-length, sum-delta-length)
         self._chaininfocache = {}
         # revlog header -> revlog compressor
         self._decompressors = {}
 
     @util.propertycache
     def _compressor(self):
         return util.compengines[self._compengine].revlogcompressor()
 
     def _indexfp(self, mode='r'):
         """file object for the revlog's index file"""
         args = {r'mode': mode}
         if mode != 'r':
             args[r'checkambig'] = self._checkambig
         if mode == 'w':
             args[r'atomictemp'] = True
         return self.opener(self.indexfile, **args)
 
     def _datafp(self, mode='r'):
         """file object for the revlog's data file"""
         return self.opener(self.datafile, mode=mode)
 
     @contextlib.contextmanager
     def _datareadfp(self, existingfp=None):
         """file object suitable to read data"""
         if existingfp is not None:
             yield existingfp
         else:
             if self._inline:
                 func = self._indexfp
             else:
                 func = self._datafp
             with func() as fp:
                 yield fp
 
     def tip(self):
         return self.node(len(self.index) - 1)
     def __contains__(self, rev):
         return 0 <= rev < len(self)
     def __len__(self):
         return len(self.index)
     def __iter__(self):
         return iter(pycompat.xrange(len(self)))
     def revs(self, start=0, stop=None):
         """iterate over all rev in this revlog (from start to stop)"""
         step = 1
         length = len(self)
         if stop is not None:
             if start > stop:
                 step = -1
             stop += step
             if stop > length:
                 stop = length
         else:
             stop = length
         return pycompat.xrange(start, stop, step)
 
     @util.propertycache
     def nodemap(self):
         if self.index:
             # populate mapping down to the initial node
             node0 = self.index[0][7]  # get around changelog filtering
             self.rev(node0)
         return self._nodecache
 
     def hasnode(self, node):
         try:
             self.rev(node)
             return True
         except KeyError:
             return False
 
     def candelta(self, baserev, rev):
         """whether two revisions (baserev, rev) can be delta-ed or not"""
         # Disable delta if either rev requires a content-changing flag
         # processor (ex. LFS). This is because such flag processor can alter
         # the rawtext content that the delta will be based on, and two clients
         # could have a same revlog node with different flags (i.e. different
         # rawtext contents) and the delta could be incompatible.
         if ((self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS)
             or (self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS)):
             return False
         return True
 
     def clearcaches(self):
         self._cache = None
         self._chainbasecache.clear()
         self._chunkcache = (0, '')
         self._pcache = {}
 
         try:
             self._nodecache.clearcaches()
         except AttributeError:
             self._nodecache = {nullid: nullrev}
             self._nodepos = None
 
     def rev(self, node):
         try:
             return self._nodecache[node]
         except TypeError:
             raise
         except error.RevlogError:
             # parsers.c radix tree lookup failed
             if node == wdirid or node in wdirfilenodeids:
                 raise error.WdirUnsupported
             raise error.LookupError(node, self.indexfile, _('no node'))
         except KeyError:
             # pure python cache lookup failed
             n = self._nodecache
             i = self.index
             p = self._nodepos
             if p is None:
                 p = len(i) - 1
             else:
                 assert p < len(i)
             for r in pycompat.xrange(p, -1, -1):
                 v = i[r][7]
                 n[v] = r
                 if v == node:
                     self._nodepos = r - 1
                     return r
             if node == wdirid or node in wdirfilenodeids:
                 raise error.WdirUnsupported
             raise error.LookupError(node, self.indexfile, _('no node'))
 
     # Accessors for index entries.
 
     # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes
     # are flags.
     def start(self, rev):
         return int(self.index[rev][0] >> 16)
 
     def flags(self, rev):
         return self.index[rev][0] & 0xFFFF
 
     def length(self, rev):
         return self.index[rev][1]
 
     def rawsize(self, rev):
         """return the length of the uncompressed text for a given revision"""
         l = self.index[rev][2]
         if l >= 0:
             return l
 
         t = self.revision(rev, raw=True)
         return len(t)
 
     def size(self, rev):
         """length of non-raw text (processed by a "read" flag processor)"""
         # fast path: if no "read" flag processor could change the content,
         # size is rawsize. note: ELLIPSIS is known to not change the content.
         flags = self.flags(rev)
         if flags & (REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0:
             return self.rawsize(rev)
 
         return len(self.revision(rev, raw=False))
 
     def chainbase(self, rev):
         base = self._chainbasecache.get(rev)
         if base is not None:
             return base
 
         index = self.index
         iterrev = rev
         base = index[iterrev][3]
         while base != iterrev:
             iterrev = base
             base = index[iterrev][3]
 
         self._chainbasecache[rev] = base
         return base
 
     def linkrev(self, rev):
         return self.index[rev][4]
 
     def parentrevs(self, rev):
         try:
             entry = self.index[rev]
         except IndexError:
             if rev == wdirrev:
                 raise error.WdirUnsupported
             raise
 
         return entry[5], entry[6]
 
     def node(self, rev):
         try:
             return self.index[rev][7]
         except IndexError:
             if rev == wdirrev:
                 raise error.WdirUnsupported
             raise
 
     # Derived from index values.
 
     def end(self, rev):
         return self.start(rev) + self.length(rev)
 
     def parents(self, node):
         i = self.index
         d = i[self.rev(node)]
         return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline
 
     def chainlen(self, rev):
         return self._chaininfo(rev)[0]
 
     def _chaininfo(self, rev):
         chaininfocache = self._chaininfocache
         if rev in chaininfocache:
             return chaininfocache[rev]
         index = self.index
         generaldelta = self._generaldelta
         iterrev = rev
         e = index[iterrev]
         clen = 0
         compresseddeltalen = 0
         while iterrev != e[3]:
             clen += 1
             compresseddeltalen += e[1]
             if generaldelta:
                 iterrev = e[3]
             else:
                 iterrev -= 1
             if iterrev in chaininfocache:
                 t = chaininfocache[iterrev]
                 clen += t[0]
                 compresseddeltalen += t[1]
                 break
             e = index[iterrev]
         else:
             # Add text length of base since decompressing that also takes
             # work. For cache hits the length is already included.
             compresseddeltalen += e[1]
         r = (clen, compresseddeltalen)
         chaininfocache[rev] = r
         return r
 
     def _deltachain(self, rev, stoprev=None):
         """Obtain the delta chain for a revision.
 
         ``stoprev`` specifies a revision to stop at. If not specified, we
         stop at the base of the chain.
 
         Returns a 2-tuple of (chain, stopped) where ``chain`` is a list of
         revs in ascending order and ``stopped`` is a bool indicating whether
         ``stoprev`` was hit.
         """
         # Try C implementation.
         try:
             return self.index.deltachain(rev, stoprev, self._generaldelta)
         except AttributeError:
             pass
 
         chain = []
 
         # Alias to prevent attribute lookup in tight loop.
         index = self.index
         generaldelta = self._generaldelta
 
         iterrev = rev
         e = index[iterrev]
         while iterrev != e[3] and iterrev != stoprev:
             chain.append(iterrev)
             if generaldelta:
                 iterrev = e[3]
             else:
                 iterrev -= 1
             e = index[iterrev]
 
         if iterrev == stoprev:
             stopped = True
         else:
             chain.append(iterrev)
             stopped = False
 
         chain.reverse()
         return chain, stopped
 
     def ancestors(self, revs, stoprev=0, inclusive=False):
         """Generate the ancestors of 'revs' in reverse topological order.
         Does not generate revs lower than stoprev.
 
         See the documentation for ancestor.lazyancestors for more details."""
 
         return ancestor.lazyancestors(self.parentrevs, revs, stoprev=stoprev,
                                       inclusive=inclusive)
 
     def descendants(self, revs):
         """Generate the descendants of 'revs' in revision order.
 
         Yield a sequence of revision numbers starting with a child of
         some rev in revs, i.e., each revision is *not* considered a
         descendant of itself.  Results are ordered by revision number (a
         topological sort)."""
         first = min(revs)
         if first == nullrev:
             for i in self:
                 yield i
             return
 
         seen = set(revs)
         for i in self.revs(start=first + 1):
             for x in self.parentrevs(i):
                 if x != nullrev and x in seen:
                     seen.add(i)
                     yield i
                     break
 
     def findcommonmissing(self, common=None, heads=None):
         """Return a tuple of the ancestors of common and the ancestors of heads
         that are not ancestors of common. In revset terminology, we return the
         tuple:
 
           ::common, (::heads) - (::common)
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of node IDs.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullid]
         if heads is None:
             heads = self.heads()
 
         common = [self.rev(n) for n in common]
         heads = [self.rev(n) for n in heads]
 
         # we want the ancestors, but inclusive
         class lazyset(object):
             def __init__(self, lazyvalues):
                 self.addedvalues = set()
                 self.lazyvalues = lazyvalues
 
             def __contains__(self, value):
                 return value in self.addedvalues or value in self.lazyvalues
 
             def __iter__(self):
                 added = self.addedvalues
                 for r in added:
                     yield r
                 for r in self.lazyvalues:
                     if not r in added:
                         yield r
 
             def add(self, value):
                 self.addedvalues.add(value)
 
             def update(self, values):
                 self.addedvalues.update(values)
 
         has = lazyset(self.ancestors(common))
         has.add(nullrev)
         has.update(common)
 
         # take all ancestors from heads that aren't in has
         missing = set()
         visit = collections.deque(r for r in heads if r not in has)
         while visit:
             r = visit.popleft()
             if r in missing:
                 continue
             else:
                 missing.add(r)
                 for p in self.parentrevs(r):
                     if p not in has:
                         visit.append(p)
         missing = list(missing)
         missing.sort()
         return has, [self.node(miss) for miss in missing]
 
     def incrementalmissingrevs(self, common=None):
         """Return an object that can be used to incrementally compute the
         revision numbers of the ancestors of arbitrary sets that are not
         ancestors of common. This is an ancestor.incrementalmissingancestors
         object.
 
         'common' is a list of revision numbers. If common is not supplied, uses
         nullrev.
         """
         if common is None:
             common = [nullrev]
 
         return ancestor.incrementalmissingancestors(self.parentrevs, common)
 
     def findmissingrevs(self, common=None, heads=None):
         """Return the revision numbers of the ancestors of heads that
         are not ancestors of common.
 
         More specifically, return a list of revision numbers corresponding to
         nodes N such that every N satisfies the following constraints:
 
           1. N is an ancestor of some node in 'heads'
           2. N is not an ancestor of any node in 'common'
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of revision numbers.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullrev]
         if heads is None:
             heads = self.headrevs()
 
         inc = self.incrementalmissingrevs(common=common)
         return inc.missingancestors(heads)
 
     def findmissing(self, common=None, heads=None):
         """Return the ancestors of heads that are not ancestors of common.
 
         More specifically, return a list of nodes N such that every N
         satisfies the following constraints:
 
           1. N is an ancestor of some node in 'heads'
           2. N is not an ancestor of any node in 'common'
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of node IDs.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullid]
         if heads is None:
             heads = self.heads()
 
         common = [self.rev(n) for n in common]
         heads = [self.rev(n) for n in heads]
 
         inc = self.incrementalmissingrevs(common=common)
         return [self.node(r) for r in inc.missingancestors(heads)]
 
     def nodesbetween(self, roots=None, heads=None):
         """Return a topological path from 'roots' to 'heads'.
 
         Return a tuple (nodes, outroots, outheads) where 'nodes' is a
         topologically sorted list of all nodes N that satisfy both of
         these constraints:
 
           1. N is a descendant of some node in 'roots'
           2. N is an ancestor of some node in 'heads'
 
         Every node is considered to be both a descendant and an ancestor
         of itself, so every reachable node in 'roots' and 'heads' will be
         included in 'nodes'.
 
         'outroots' is the list of reachable nodes in 'roots', i.e., the
         subset of 'roots' that is returned in 'nodes'.  Likewise,
         'outheads' is the subset of 'heads' that is also in 'nodes'.
 
         'roots' and 'heads' are both lists of node IDs.  If 'roots' is
         unspecified, uses nullid as the only root.  If 'heads' is
         unspecified, uses list of all of the revlog's heads."""
         nonodes = ([], [], [])
         if roots is not None:
             roots = list(roots)
             if not roots:
                 return nonodes
             lowestrev = min([self.rev(n) for n in roots])
         else:
             roots = [nullid] # Everybody's a descendant of nullid
             lowestrev = nullrev
         if (lowestrev == nullrev) and (heads is None):
             # We want _all_ the nodes!
             return ([self.node(r) for r in self], [nullid], list(self.heads()))
         if heads is None:
             # All nodes are ancestors, so the latest ancestor is the last
             # node.
             highestrev = len(self) - 1
             # Set ancestors to None to signal that every node is an ancestor.
             ancestors = None
             # Set heads to an empty dictionary for later discovery of heads
             heads = {}
         else:
             heads = list(heads)
             if not heads:
                 return nonodes
             ancestors = set()
             # Turn heads into a dictionary so we can remove 'fake' heads.
             # Also, later we will be using it to filter out the heads we can't
             # find from roots.
             heads = dict.fromkeys(heads, False)
             # Start at the top and keep marking parents until we're done.
             nodestotag = set(heads)
             # Remember where the top was so we can use it as a limit later.
             highestrev = max([self.rev(n) for n in nodestotag])
             while nodestotag:
                 # grab a node to tag
                 n = nodestotag.pop()
                 # Never tag nullid
                 if n == nullid:
                     continue
                 # A node's revision number represents its place in a
                 # topologically sorted list of nodes.
                 r = self.rev(n)
                 if r >= lowestrev:
                     if n not in ancestors:
                         # If we are possibly a descendant of one of the roots
                         # and we haven't already been marked as an ancestor
                         ancestors.add(n) # Mark as ancestor
                         # Add non-nullid parents to list of nodes to tag.
                         nodestotag.update([p for p in self.parents(n) if
                                            p != nullid])
                     elif n in heads: # We've seen it before, is it a fake head?
                         # So it is, real heads should not be the ancestors of
                         # any other heads.
                         heads.pop(n)
             if not ancestors:
                 return nonodes
             # Now that we have our set of ancestors, we want to remove any
             # roots that are not ancestors.
 
             # If one of the roots was nullid, everything is included anyway.
             if lowestrev > nullrev:
                 # But, since we weren't, let's recompute the lowest rev to not
                 # include roots that aren't ancestors.
 
                 # Filter out roots that aren't ancestors of heads
                 roots = [root for root in roots if root in ancestors]
                 # Recompute the lowest revision
                 if roots:
                     lowestrev = min([self.rev(root) for root in roots])
                 else:
                     # No more roots?  Return empty list
                     return nonodes
             else:
                 # We are descending from nullid, and don't need to care about
                 # any other roots.
                 lowestrev = nullrev
                 roots = [nullid]
         # Transform our roots list into a set.
         descendants = set(roots)
         # Also, keep the original roots so we can filter out roots that aren't
         # 'real' roots (i.e. are descended from other roots).
         roots = descendants.copy()
         # Our topologically sorted list of output nodes.
         orderedout = []
         # Don't start at nullid since we don't want nullid in our output list,
         # and if nullid shows up in descendants, empty parents will look like
         # they're descendants.
         for r in self.revs(start=max(lowestrev, 0), stop=highestrev + 1):
             n = self.node(r)
             isdescendant = False
             if lowestrev == nullrev:  # Everybody is a descendant of nullid
                 isdescendant = True
             elif n in descendants:
                 # n is already a descendant
                 isdescendant = True
                 # This check only needs to be done here because all the roots
                 # will start being marked is descendants before the loop.
                 if n in roots:
                     # If n was a root, check if it's a 'real' root.
                     p = tuple(self.parents(n))
                     # If any of its parents are descendants, it's not a root.
                     if (p[0] in descendants) or (p[1] in descendants):
                         roots.remove(n)
             else:
                 p = tuple(self.parents(n))
                 # A node is a descendant if either of its parents are
                 # descendants.  (We seeded the dependents list with the roots
                 # up there, remember?)
                 if (p[0] in descendants) or (p[1] in descendants):
                     descendants.add(n)
                     isdescendant = True
             if isdescendant and ((ancestors is None) or (n in ancestors)):
                 # Only include nodes that are both descendants and ancestors.
                 orderedout.append(n)
                 if (ancestors is not None) and (n in heads):
                     # We're trying to figure out which heads are reachable
                     # from roots.
                     # Mark this head as having been reached
                     heads[n] = True
                 elif ancestors is None:
                     # Otherwise, we're trying to discover the heads.
                     # Assume this is a head because if it isn't, the next step
                     # will eventually remove it.
                     heads[n] = True
                     # But, obviously its parents aren't.
                     for p in self.parents(n):
                         heads.pop(p, None)
         heads = [head for head, flag in heads.iteritems() if flag]
         roots = list(roots)
         assert orderedout
         assert roots
         assert heads
         return (orderedout, roots, heads)
 
     def headrevs(self):
         try:
             return self.index.headrevs()
         except AttributeError:
             return self._headrevs()
 
     def computephases(self, roots):
         return self.index.computephasesmapsets(roots)
 
     def _headrevs(self):
         count = len(self)
         if not count:
             return [nullrev]
         # we won't iter over filtered rev so nobody is a head at start
         ishead = [0] * (count + 1)
         index = self.index
         for r in self:
             ishead[r] = 1  # I may be an head
             e = index[r]
             ishead[e[5]] = ishead[e[6]] = 0  # my parent are not
         return [r for r, val in enumerate(ishead) if val]
 
     def heads(self, start=None, stop=None):
         """return the list of all nodes that have no children
 
         if start is specified, only heads that are descendants of
         start will be returned
         if stop is specified, it will consider all the revs from stop
         as if they had no children
         """
         if start is None and stop is None:
             if not len(self):
                 return [nullid]
             return [self.node(r) for r in self.headrevs()]
 
         if start is None:
             start = nullid
         if stop is None:
             stop = []
         stoprevs = set([self.rev(n) for n in stop])
         startrev = self.rev(start)
         reachable = {startrev}
         heads = {startrev}
 
         parentrevs = self.parentrevs
         for r in self.revs(start=startrev + 1):
             for p in parentrevs(r):
                 if p in reachable:
                     if r not in stoprevs:
                         reachable.add(r)
                     heads.add(r)
                 if p in heads and p not in stoprevs:
                     heads.remove(p)
 
         return [self.node(r) for r in heads]
 
     def children(self, node):
         """find the children of a given node"""
         c = []
         p = self.rev(node)
         for r in self.revs(start=p + 1):
             prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
             if prevs:
                 for pr in prevs:
                     if pr == p:
                         c.append(self.node(r))
             elif p == nullrev:
                 c.append(self.node(r))
         return c
 
     def commonancestorsheads(self, a, b):
         """calculate all the heads of the common ancestors of nodes a and b"""
         a, b = self.rev(a), self.rev(b)
         ancs = self._commonancestorsheads(a, b)
         return pycompat.maplist(self.node, ancs)
 
     def _commonancestorsheads(self, *revs):
         """calculate all the heads of the common ancestors of revs"""
         try:
             ancs = self.index.commonancestorsheads(*revs)
         except (AttributeError, OverflowError): # C implementation failed
             ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
         return ancs
 
     def isancestor(self, a, b):
         """return True if node a is an ancestor of node b
 
         A revision is considered an ancestor of itself."""
         a, b = self.rev(a), self.rev(b)
         return self.isancestorrev(a, b)
 
     def isancestorrev(self, a, b):
         """return True if revision a is an ancestor of revision b
 
         A revision is considered an ancestor of itself.
 
         The implementation of this is trivial but the use of
         commonancestorsheads is not."""
         if a == nullrev:
             return True
         elif a == b:
             return True
         elif a > b:
             return False
         return a in self._commonancestorsheads(a, b)
 
     def ancestor(self, a, b):
         """calculate the "best" common ancestor of nodes a and b"""
 
         a, b = self.rev(a), self.rev(b)
         try:
             ancs = self.index.ancestors(a, b)
         except (AttributeError, OverflowError):
             ancs = ancestor.ancestors(self.parentrevs, a, b)
         if ancs:
             # choose a consistent winner when there's a tie
             return min(map(self.node, ancs))
         return nullid
 
     def _match(self, id):
         if isinstance(id, int):
             # rev
             return self.node(id)
         if len(id) == 20:
             # possibly a binary node
             # odds of a binary node being all hex in ASCII are 1 in 10**25
             try:
                 node = id
                 self.rev(node) # quick search the index
                 return node
             except error.LookupError:
                 pass # may be partial hex id
         try:
             # str(rev)
             rev = int(id)
             if "%d" % rev != id:
                 raise ValueError
             if rev < 0:
                 rev = len(self) + rev
             if rev < 0 or rev >= len(self):
                 raise ValueError
             return self.node(rev)
         except (ValueError, OverflowError):
             pass
         if len(id) == 40:
             try:
                 # a full hex nodeid?
                 node = bin(id)
                 self.rev(node)
                 return node
             except (TypeError, error.LookupError):
                 pass
 
     def _partialmatch(self, id):
         # we don't care wdirfilenodeids as they should be always full hash
         maybewdir = wdirhex.startswith(id)
         try:
             partial = self.index.partialmatch(id)
             if partial and self.hasnode(partial):
                 if maybewdir:
                     # single 'ff...' match in radix tree, ambiguous with wdir
                     raise error.RevlogError
                 return partial
             if maybewdir:
                 # no 'ff...' match in radix tree, wdir identified
                 raise error.WdirUnsupported
             return None
         except error.RevlogError:
             # parsers.c radix tree lookup gave multiple matches
             # fast path: for unfiltered changelog, radix tree is accurate
             if not getattr(self, 'filteredrevs', None):
                 raise error.AmbiguousPrefixLookupError(
                     id, self.indexfile, _('ambiguous identifier'))
             # fall through to slow path that filters hidden revisions
         except (AttributeError, ValueError):
             # we are pure python, or key was too short to search radix tree
             pass
 
         if id in self._pcache:
             return self._pcache[id]
 
         if len(id) <= 40:
             try:
                 # hex(node)[:...]
                 l = len(id) // 2  # grab an even number of digits
                 prefix = bin(id[:l * 2])
                 nl = [e[7] for e in self.index if e[7].startswith(prefix)]
                 nl = [n for n in nl if hex(n).startswith(id) and
                       self.hasnode(n)]
                 if nullhex.startswith(id):
                     nl.append(nullid)
                 if len(nl) > 0:
                     if len(nl) == 1 and not maybewdir:
                         self._pcache[id] = nl[0]
                         return nl[0]
                     raise error.AmbiguousPrefixLookupError(
                         id, self.indexfile, _('ambiguous identifier'))
                 if maybewdir:
                     raise error.WdirUnsupported
                 return None
             except TypeError:
                 pass
 
     def lookup(self, id):
         """locate a node based on:
             - revision number or str(revision number)
             - nodeid or subset of hex nodeid
         """
         n = self._match(id)
         if n is not None:
             return n
         n = self._partialmatch(id)
         if n:
             return n
 
         raise error.LookupError(id, self.indexfile, _('no match found'))
 
     def shortest(self, node, minlength=1):
         """Find the shortest unambiguous prefix that matches node."""
         def isvalid(prefix):
             try:
                 node = self._partialmatch(prefix)
             except error.RevlogError:
                 return False
             except error.WdirUnsupported:
                 # single 'ff...' match
                 return True
             if node is None:
                 raise error.LookupError(node, self.indexfile, _('no node'))
             return True
 
         def maybewdir(prefix):
             return all(c == 'f' for c in prefix)
 
         hexnode = hex(node)
 
         def disambiguate(hexnode, minlength):
             """Disambiguate against wdirid."""
             for length in range(minlength, 41):
                 prefix = hexnode[:length]
                 if not maybewdir(prefix):
                     return prefix
 
         if not getattr(self, 'filteredrevs', None):
             try:
                 length = max(self.index.shortest(node), minlength)
                 return disambiguate(hexnode, length)
             except error.RevlogError:
                 if node != wdirid:
                     raise error.LookupError(node, self.indexfile, _('no node'))
             except AttributeError:
                 # Fall through to pure code
                 pass
 
         if node == wdirid:
             for length in range(minlength, 41):
                 prefix = hexnode[:length]
                 if isvalid(prefix):
                     return prefix
 
         for length in range(minlength, 41):
             prefix = hexnode[:length]
             if isvalid(prefix):
                 return disambiguate(hexnode, length)
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
         p1, p2 = self.parents(node)
         return hash(text, p1, p2) != node
 
     def _cachesegment(self, offset, data):
         """Add a segment to the revlog cache.
 
         Accepts an absolute offset and the data that is at that location.
         """
         o, d = self._chunkcache
         # try to add to existing cache
         if o + len(d) == offset and len(d) + len(data) < _chunksize:
             self._chunkcache = o, d + data
         else:
             self._chunkcache = offset, data
 
     def _readsegment(self, offset, length, df=None):
         """Load a segment of raw data from the revlog.
 
         Accepts an absolute offset, length to read, and an optional existing
         file handle to read from.
 
         If an existing file handle is passed, it will be seeked and the
         original seek position will NOT be restored.
 
         Returns a str or buffer of raw byte data.
         """
         # Cache data both forward and backward around the requested
         # data, in a fixed size window. This helps speed up operations
         # involving reading the revlog backwards.
         cachesize = self._chunkcachesize
         realoffset = offset & ~(cachesize - 1)
         reallength = (((offset + length + cachesize) & ~(cachesize - 1))
                       - realoffset)
         with self._datareadfp(df) as df:
             df.seek(realoffset)
             d = df.read(reallength)
         self._cachesegment(realoffset, d)
         if offset != realoffset or reallength != length:
             return util.buffer(d, offset - realoffset, length)
         return d
 
     def _getsegment(self, offset, length, df=None):
         """Obtain a segment of raw data from the revlog.
 
         Accepts an absolute offset, length of bytes to obtain, and an
         optional file handle to the already-opened revlog. If the file
         handle is used, it's original seek position will not be preserved.
 
         Requests for data may be returned from a cache.
 
         Returns a str or a buffer instance of raw byte data.
         """
         o, d = self._chunkcache
         l = len(d)
 
         # is it in the cache?
         cachestart = offset - o
         cacheend = cachestart + length
         if cachestart >= 0 and cacheend <= l:
             if cachestart == 0 and cacheend == l:
                 return d # avoid a copy
             return util.buffer(d, cachestart, cacheend - cachestart)
 
         return self._readsegment(offset, length, df=df)
 
     def _getsegmentforrevs(self, startrev, endrev, df=None):
         """Obtain a segment of raw data corresponding to a range of revisions.
 
         Accepts the start and end revisions and an optional already-open
         file handle to be used for reading. If the file handle is read, its
         seek position will not be preserved.
 
         Requests for data may be satisfied by a cache.
 
         Returns a 2-tuple of (offset, data) for the requested range of
         revisions. Offset is the integer offset from the beginning of the
         revlog and data is a str or buffer of the raw byte data.
 
         Callers will need to call ``self.start(rev)`` and ``self.length(rev)``
         to determine where each revision's data begins and ends.
         """
         # Inlined self.start(startrev) & self.end(endrev) for perf reasons
         # (functions are expensive).
         index = self.index
         istart = index[startrev]
         start = int(istart[0] >> 16)
         if startrev == endrev:
             end = start + istart[1]
         else:
             iend = index[endrev]
             end = int(iend[0] >> 16) + iend[1]
 
         if self._inline:
             start += (startrev + 1) * self._io.size
             end += (endrev + 1) * self._io.size
         length = end - start
 
         return start, self._getsegment(start, length, df=df)
 
     def _chunk(self, rev, df=None):
         """Obtain a single decompressed chunk for a revision.
 
         Accepts an integer revision and an optional already-open file handle
         to be used for reading. If used, the seek position of the file will not
         be preserved.
 
         Returns a str holding uncompressed data for the requested revision.
         """
         return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1])
 
     def _chunks(self, revs, df=None, targetsize=None):
         """Obtain decompressed chunks for the specified revisions.
 
         Accepts an iterable of numeric revisions that are assumed to be in
         ascending order. Also accepts an optional already-open file handle
         to be used for reading. If used, the seek position of the file will
         not be preserved.
 
         This function is similar to calling ``self._chunk()`` multiple times,
         but is faster.
 
         Returns a list with decompressed data for each requested revision.
         """
         if not revs:
             return []
         start = self.start
         length = self.length
         inline = self._inline
         iosize = self._io.size
         buffer = util.buffer
 
         l = []
         ladd = l.append
 
         if not self._withsparseread:
             slicedchunks = (revs,)
         else:
             slicedchunks = deltautil.slicechunk(self, revs,
                                                 targetsize=targetsize)
 
         for revschunk in slicedchunks:
             firstrev = revschunk[0]
             # Skip trailing revisions with empty diff
             for lastrev in revschunk[::-1]:
                 if length(lastrev) != 0:
                     break
 
             try:
                 offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df)
             except OverflowError:
                 # issue4215 - we can't cache a run of chunks greater than
                 # 2G on Windows
                 return [self._chunk(rev, df=df) for rev in revschunk]
 
             decomp = self.decompress
             for rev in revschunk:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(decomp(buffer(data, chunkstart - offset, chunklength)))
 
         return l
 
     def _chunkclear(self):
         """Clear the raw chunk cache."""
         self._chunkcache = (0, '')
 
     def deltaparent(self, rev):
         """return deltaparent of the given revision"""
         base = self.index[rev][3]
         if base == rev:
             return nullrev
         elif self._generaldelta:
             return base
         else:
             return rev - 1
 
     def issnapshot(self, rev):
         """tells whether rev is a snapshot
         """
         if rev == nullrev:
             return True
         deltap = self.deltaparent(rev)
         if deltap == nullrev:
             return True
         p1, p2 = self.parentrevs(rev)
         if deltap in (p1, p2):
             return False
         return self.issnapshot(deltap)
 
     def snapshotdepth(self, rev):
         """number of snapshot in the chain before this one"""
         if not self.issnapshot(rev):
             raise error.ProgrammingError('revision %d not a snapshot')
         return len(self._deltachain(rev)[0]) - 1
 
     def revdiff(self, rev1, rev2):
         """return or calculate a delta between two revisions
 
         The delta calculated is in binary form and is intended to be written to
         revlog data directly. So this function needs raw revision data.
         """
         if rev1 != nullrev and self.deltaparent(rev2) == rev1:
             return bytes(self._chunk(rev2))
 
         return mdiff.textdiff(self.revision(rev1, raw=True),
                               self.revision(rev2, raw=True))
 
     def revision(self, nodeorrev, _df=None, raw=False):
         """return an uncompressed revision of a given node or revision
         number.
 
         _df - an existing file handle to read from. (internal-only)
         raw - an optional argument specifying if the revision data is to be
         treated as raw data when applying flag transforms. 'raw' should be set
         to True when generating changegroups or in debug commands.
         """
         if isinstance(nodeorrev, int):
             rev = nodeorrev
             node = self.node(rev)
         else:
             node = nodeorrev
             rev = None
 
         cachedrev = None
         flags = None
         rawtext = None
         if node == nullid:
             return ""
         if self._cache:
             if self._cache[0] == node:
                 # _cache only stores rawtext
                 if raw:
                     return self._cache[2]
                 # duplicated, but good for perf
                 if rev is None:
                     rev = self.rev(node)
                 if flags is None:
                     flags = self.flags(rev)
                 # no extra flags set, no flag processor runs, text = rawtext
                 if flags == REVIDX_DEFAULT_FLAGS:
                     return self._cache[2]
                 # rawtext is reusable. need to run flag processor
                 rawtext = self._cache[2]
 
             cachedrev = self._cache[1]
 
         # look up what we need to read
         if rawtext is None:
             if rev is None:
                 rev = self.rev(node)
 
             chain, stopped = self._deltachain(rev, stoprev=cachedrev)
             if stopped:
                 rawtext = self._cache[2]
 
             # drop cache to save memory
             self._cache = None
 
             targetsize = None
             rawsize = self.index[rev][2]
             if 0 <= rawsize:
                 targetsize = 4 * rawsize
 
             bins = self._chunks(chain, df=_df, targetsize=targetsize)
             if rawtext is None:
                 rawtext = bytes(bins[0])
                 bins = bins[1:]
 
             rawtext = mdiff.patches(rawtext, bins)
             self._cache = (node, rev, rawtext)
 
         if flags is None:
             if rev is None:
                 rev = self.rev(node)
             flags = self.flags(rev)
 
         text, validatehash = self._processflags(rawtext, flags, 'read', raw=raw)
         if validatehash:
             self.checkhash(text, node, rev=rev)
 
         return text
 
     def hash(self, text, p1, p2):
         """Compute a node hash.
 
         Available as a function so that subclasses can replace the hash
         as needed.
         """
         return hash(text, p1, p2)
 
     def _processflags(self, text, flags, operation, raw=False):
         """Inspect revision data flags and applies transforms defined by
         registered flag processors.
 
         ``text`` - the revision data to process
         ``flags`` - the revision flags
         ``operation`` - the operation being performed (read or write)
         ``raw`` - an optional argument describing if the raw transform should be
         applied.
 
         This method processes the flags in the order (or reverse order if
         ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
         flag processors registered for present flags. The order of flags defined
         in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
 
         Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
         processed text and ``validatehash`` is a bool indicating whether the
         returned text should be checked for hash integrity.
 
         Note: If the ``raw`` argument is set, it has precedence over the
         operation and will only update the value of ``validatehash``.
         """
         # fast path: no flag processors will run
         if flags == 0:
             return text, True
         if not operation in ('read', 'write'):
             raise error.ProgrammingError(_("invalid '%s' operation") %
                                          operation)
         # Check all flags are known.
         if flags & ~REVIDX_KNOWN_FLAGS:
             raise error.RevlogError(_("incompatible revision flag '%#x'") %
                                     (flags & ~REVIDX_KNOWN_FLAGS))
         validatehash = True
         # Depending on the operation (read or write), the order might be
         # reversed due to non-commutative transforms.
         orderedflags = REVIDX_FLAGS_ORDER
         if operation == 'write':
             orderedflags = reversed(orderedflags)
 
         for flag in orderedflags:
             # If a flagprocessor has been registered for a known flag, apply the
             # related operation transform and update result tuple.
             if flag & flags:
                 vhash = True
 
                 if flag not in self._flagprocessors:
                     message = _("missing processor for flag '%#x'") % (flag)
                     raise error.RevlogError(message)
 
                 processor = self._flagprocessors[flag]
                 if processor is not None:
                     readtransform, writetransform, rawtransform = processor
 
                     if raw:
                         vhash = rawtransform(self, text)
                     elif operation == 'read':
                         text, vhash = readtransform(self, text)
                     else: # write operation
                         text, vhash = writetransform(self, text)
                 validatehash = validatehash and vhash
 
         return text, validatehash
 
     def checkhash(self, text, node, p1=None, p2=None, rev=None):
         """Check node hash integrity.
 
         Available as a function so that subclasses can extend hash mismatch
         behaviors as needed.
         """
         try:
             if p1 is None and p2 is None:
                 p1, p2 = self.parents(node)
             if node != self.hash(text, p1, p2):
                 revornode = rev
                 if revornode is None:
                     revornode = templatefilters.short(hex(node))
                 raise error.RevlogError(_("integrity check failed on %s:%s")
                     % (self.indexfile, pycompat.bytestr(revornode)))
         except error.RevlogError:
             if self._censorable and _censoredtext(text):
                 raise error.CensoredNodeError(self.indexfile, node, text)
             raise
 
     def _enforceinlinesize(self, tr, fp=None):
         """Check if the revlog is too big for inline and convert if so.
 
         This should be called after revisions are added to the revlog. If the
         revlog has grown too large to be an inline revlog, it will convert it
         to use multiple index and data files.
         """
         tiprev = len(self) - 1
         if (not self._inline or
             (self.start(tiprev) + self.length(tiprev)) < _maxinline):
             return
 
         trinfo = tr.find(self.indexfile)
         if trinfo is None:
             raise error.RevlogError(_("%s not found in the transaction")
                                     % self.indexfile)
 
         trindex = trinfo[2]
         if trindex is not None:
             dataoff = self.start(trindex)
         else:
             # revlog was stripped at start of transaction, use all leftover data
             trindex = len(self) - 1
             dataoff = self.end(tiprev)
 
         tr.add(self.datafile, dataoff)
 
         if fp:
             fp.flush()
             fp.close()
 
         with self._datafp('w') as df:
             for r in self:
                 df.write(self._getsegmentforrevs(r, r)[1])
 
         with self._indexfp('w') as fp:
             self.version &= ~FLAG_INLINE_DATA
             self._inline = False
             io = self._io
             for i in self:
                 e = io.packentry(self.index[i], self.node, self.version, i)
                 fp.write(e)
 
             # the temp file replace the real index when we exit the context
             # manager
 
         tr.replace(self.indexfile, trindex * self._io.size)
         self._chunkclear()
 
     def addrevision(self, text, transaction, link, p1, p2, cachedelta=None,
                     node=None, flags=REVIDX_DEFAULT_FLAGS, deltacomputer=None):
         """add a revision to the log
 
         text - the revision data to add
         transaction - the transaction object used for rollback
         link - the linkrev data to add
         p1, p2 - the parent nodeids of the revision
         cachedelta - an optional precomputed delta
         node - nodeid of revision; typically node is not specified, and it is
             computed by default as hash(text, p1, p2), however subclasses might
             use different hashing method (and override checkhash() in such case)
         flags - the known flags to set on the revision
         deltacomputer - an optional deltacomputer instance shared between
             multiple calls
         """
         if link == nullrev:
             raise error.RevlogError(_("attempted to add linkrev -1 to %s")
                                     % self.indexfile)
 
         if flags:
             node = node or self.hash(text, p1, p2)
 
         rawtext, validatehash = self._processflags(text, flags, 'write')
 
         # If the flag processor modifies the revision data, ignore any provided
         # cachedelta.
         if rawtext != text:
             cachedelta = None
 
         if len(rawtext) > _maxentrysize:
             raise error.RevlogError(
                 _("%s: size of %d bytes exceeds maximum revlog storage of 2GiB")
                 % (self.indexfile, len(rawtext)))
 
         node = node or self.hash(rawtext, p1, p2)
         if node in self.nodemap:
             return node
 
         if validatehash:
             self.checkhash(rawtext, node, p1=p1, p2=p2)
 
         return self.addrawrevision(rawtext, transaction, link, p1, p2, node,
                                    flags, cachedelta=cachedelta,
                                    deltacomputer=deltacomputer)
 
     def addrawrevision(self, rawtext, transaction, link, p1, p2, node, flags,
                        cachedelta=None, deltacomputer=None):
         """add a raw revision with known flags, node and parents
         useful when reusing a revision not stored in this revlog (ex: received
         over wire, or read from an external bundle).
         """
         dfh = None
         if not self._inline:
             dfh = self._datafp("a+")
         ifh = self._indexfp("a+")
         try:
             return self._addrevision(node, rawtext, transaction, link, p1, p2,
                                      flags, cachedelta, ifh, dfh,
                                      deltacomputer=deltacomputer)
         finally:
             if dfh:
                 dfh.close()
             ifh.close()
 
     def compress(self, data):
         """Generate a possibly-compressed representation of data."""
         if not data:
             return '', data
 
         compressed = self._compressor.compress(data)
 
         if compressed:
             # The revlog compressor added the header in the returned data.
             return '', compressed
 
         if data[0:1] == '\0':
             return '', data
         return 'u', data
 
     def decompress(self, data):
         """Decompress a revlog chunk.
 
         The chunk is expected to begin with a header identifying the
         format type so it can be routed to an appropriate decompressor.
         """
         if not data:
             return data
 
         # Revlogs are read much more frequently than they are written and many
         # chunks only take microseconds to decompress, so performance is
         # important here.
         #
         # We can make a few assumptions about revlogs:
         #
         # 1) the majority of chunks will be compressed (as opposed to inline
         #    raw data).
         # 2) decompressing *any* data will likely by at least 10x slower than
         #    returning raw inline data.
         # 3) we want to prioritize common and officially supported compression
         #    engines
         #
         # It follows that we want to optimize for "decompress compressed data
         # when encoded with common and officially supported compression engines"
         # case over "raw data" and "data encoded by less common or non-official
         # compression engines." That is why we have the inline lookup first
         # followed by the compengines lookup.
         #
         # According to `hg perfrevlogchunks`, this is ~0.5% faster for zlib
         # compressed chunks. And this matters for changelog and manifest reads.
         t = data[0:1]
 
         if t == 'x':
             try:
                 return _zlibdecompress(data)
             except zlib.error as e:
                 raise error.RevlogError(_('revlog decompress error: %s') %
                                         stringutil.forcebytestr(e))
         # '\0' is more common than 'u' so it goes first.
         elif t == '\0':
             return data
         elif t == 'u':
             return util.buffer(data, 1)
 
         try:
             compressor = self._decompressors[t]
         except KeyError:
             try:
                 engine = util.compengines.forrevlogheader(t)
                 compressor = engine.revlogcompressor()
                 self._decompressors[t] = compressor
             except KeyError:
                 raise error.RevlogError(_('unknown compression type %r') % t)
 
         return compressor.decompress(data)
 
     def _addrevision(self, node, rawtext, transaction, link, p1, p2, flags,
                      cachedelta, ifh, dfh, alwayscache=False,
                      deltacomputer=None):
         """internal function to add revisions to the log
 
         see addrevision for argument descriptions.
 
         note: "addrevision" takes non-raw text, "_addrevision" takes raw text.
 
         if "deltacomputer" is not provided or None, a defaultdeltacomputer will
         be used.
 
         invariants:
         - rawtext is optional (can be None); if not set, cachedelta must be set.
           if both are set, they must correspond to each other.
         """
         if node == nullid:
             raise error.RevlogError(_("%s: attempt to add null revision") %
                                     self.indexfile)
         if node == wdirid or node in wdirfilenodeids:
             raise error.RevlogError(_("%s: attempt to add wdir revision") %
                                     self.indexfile)
 
         if self._inline:
             fh = ifh
         else:
             fh = dfh
 
         btext = [rawtext]
 
         curr = len(self)
         prev = curr - 1
         offset = self.end(prev)
         p1r, p2r = self.rev(p1), self.rev(p2)
 
         # full versions are inserted when the needed deltas
         # become comparable to the uncompressed text
         if rawtext is None:
             # need rawtext size, before changed by flag processors, which is
             # the non-raw size. use revlog explicitly to avoid filelog's extra
             # logic that might remove metadata size.
             textlen = mdiff.patchedsize(revlog.size(self, cachedelta[0]),
                                         cachedelta[1])
         else:
             textlen = len(rawtext)
 
         if deltacomputer is None:
             deltacomputer = deltautil.deltacomputer(self)
 
         revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
 
         deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
 
         e = (offset_type(offset, flags), deltainfo.deltalen, textlen,
              deltainfo.base, link, p1r, p2r, node)
         self.index.append(e)
         self.nodemap[node] = curr
 
         entry = self._io.packentry(e, self.node, self.version, curr)
         self._writeentry(transaction, ifh, dfh, entry, deltainfo.data,
                          link, offset)
 
         rawtext = btext[0]
 
         if alwayscache and rawtext is None:
             rawtext = deltacomputer.buildtext(revinfo, fh)
 
         if type(rawtext) == bytes: # only accept immutable objects
             self._cache = (node, curr, rawtext)
         self._chainbasecache[curr] = deltainfo.chainbase
         return node
 
     def _writeentry(self, transaction, ifh, dfh, entry, data, link, offset):
         # Files opened in a+ mode have inconsistent behavior on various
         # platforms. Windows requires that a file positioning call be made
         # when the file handle transitions between reads and writes. See
         # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
         # platforms, Python or the platform itself can be buggy. Some versions
         # of Solaris have been observed to not append at the end of the file
         # if the file was seeked to before the end. See issue4943 for more.
         #
         # We work around this issue by inserting a seek() before writing.
         # Note: This is likely not necessary on Python 3.
         ifh.seek(0, os.SEEK_END)
         if dfh:
             dfh.seek(0, os.SEEK_END)
 
         curr = len(self) - 1
         if not self._inline:
             transaction.add(self.datafile, offset)
             transaction.add(self.indexfile, curr * len(entry))
             if data[0]:
                 dfh.write(data[0])
             dfh.write(data[1])
             ifh.write(entry)
         else:
             offset += curr * self._io.size
             transaction.add(self.indexfile, offset, curr)
             ifh.write(entry)
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
         add a delta group
 
         given a set of deltas, add them to the revision log. the
         first delta is against its parent, which should be in our
         log, the rest are against the previous delta.
 
         If ``addrevisioncb`` is defined, it will be called with arguments of
         this revlog and the node that was added.
         """
 
         nodes = []
 
         r = len(self)
         end = 0
         if r:
             end = self.end(r - 1)
         ifh = self._indexfp("a+")
         isize = r * self._io.size
         if self._inline:
             transaction.add(self.indexfile, end + isize, r)
             dfh = None
         else:
             transaction.add(self.indexfile, isize, r)
             transaction.add(self.datafile, end)
             dfh = self._datafp("a+")
         def flush():
             if dfh:
                 dfh.flush()
             ifh.flush()
         try:
             deltacomputer = deltautil.deltacomputer(self)
             # loop through our set of deltas
             for data in deltas:
                 node, p1, p2, linknode, deltabase, delta, flags = data
                 link = linkmapper(linknode)
                 flags = flags or REVIDX_DEFAULT_FLAGS
 
                 nodes.append(node)
 
                 if node in self.nodemap:
                     # this can happen if two branches make the same change
                     continue
 
                 for p in (p1, p2):
                     if p not in self.nodemap:
                         raise error.LookupError(p, self.indexfile,
                                                 _('unknown parent'))
 
                 if deltabase not in self.nodemap:
                     raise error.LookupError(deltabase, self.indexfile,
                                             _('unknown delta base'))
 
                 baserev = self.rev(deltabase)
 
                 if baserev != nullrev and self.iscensored(baserev):
                     # if base is censored, delta must be full replacement in a
                     # single patch operation
                     hlen = struct.calcsize(">lll")
                     oldlen = self.rawsize(baserev)
                     newlen = len(delta) - hlen
                     if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
                         raise error.CensoredBaseError(self.indexfile,
                                                       self.node(baserev))
 
                 if not flags and self._peek_iscensored(baserev, delta, flush):
                     flags |= REVIDX_ISCENSORED
 
                 # We assume consumers of addrevisioncb will want to retrieve
                 # the added revision, which will require a call to
                 # revision(). revision() will fast path if there is a cache
                 # hit. So, we tell _addrevision() to always cache in this case.
                 # We're only using addgroup() in the context of changegroup
                 # generation so the revision data can always be handled as raw
                 # by the flagprocessor.
                 self._addrevision(node, None, transaction, link,
                                   p1, p2, flags, (baserev, delta),
                                   ifh, dfh,
                                   alwayscache=bool(addrevisioncb),
                                   deltacomputer=deltacomputer)
 
                 if addrevisioncb:
                     addrevisioncb(self, node)
 
                 if not dfh and not self._inline:
                     # addrevision switched from inline to conventional
                     # reopen the index
                     ifh.close()
                     dfh = self._datafp("a+")
                     ifh = self._indexfp("a+")
         finally:
             if dfh:
                 dfh.close()
             ifh.close()
 
         return nodes
 
     def iscensored(self, rev):
         """Check if a file revision is censored."""
         if not self._censorable:
             return False
 
         return self.flags(rev) & REVIDX_ISCENSORED
 
     def _peek_iscensored(self, baserev, delta, flush):
         """Quickly check if a delta produces a censored revision."""
         if not self._censorable:
             return False
 
         # Fragile heuristic: unless new file meta keys are added alphabetically
         # preceding "censored", all censored revisions are prefixed by
         # "\1\ncensored:". A delta producing such a censored revision must be a
         # full-replacement delta, so we inspect the first and only patch in the
         # delta for this prefix.
         hlen = struct.calcsize(">lll")
         if len(delta) <= hlen:
             return False
 
         oldlen = self.rawsize(baserev)
         newlen = len(delta) - hlen
         if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
             return False
 
         add = "\1\ncensored:"
         addlen = len(add)
         return newlen >= addlen and delta[hlen:hlen + addlen] == add
 
     def getstrippoint(self, minlink):
         """find the minimum rev that must be stripped to strip the linkrev
 
         Returns a tuple containing the minimum rev and a set of all revs that
         have linkrevs that will be broken by this strip.
         """
         brokenrevs = set()
         strippoint = len(self)
 
         heads = {}
         futurelargelinkrevs = set()
         for head in self.headrevs():
             headlinkrev = self.linkrev(head)
             heads[head] = headlinkrev
             if headlinkrev >= minlink:
                 futurelargelinkrevs.add(headlinkrev)
 
         # This algorithm involves walking down the rev graph, starting at the
         # heads. Since the revs are topologically sorted according to linkrev,
         # once all head linkrevs are below the minlink, we know there are
         # no more revs that could have a linkrev greater than minlink.
         # So we can stop walking.
         while futurelargelinkrevs:
             strippoint -= 1
             linkrev = heads.pop(strippoint)
 
             if linkrev < minlink:
                 brokenrevs.add(strippoint)
             else:
                 futurelargelinkrevs.remove(linkrev)
 
             for p in self.parentrevs(strippoint):
                 if p != nullrev:
                     plinkrev = self.linkrev(p)
                     heads[p] = plinkrev
                     if plinkrev >= minlink:
                         futurelargelinkrevs.add(plinkrev)
 
         return strippoint, brokenrevs
 
     def strip(self, minlink, transaction):
         """truncate the revlog on the first revision with a linkrev >= minlink
 
         This function is called when we're stripping revision minlink and
         its descendants from the repository.
 
         We have to remove all revisions with linkrev >= minlink, because
         the equivalent changelog revisions will be renumbered after the
         strip.
 
         So we truncate the revlog on the first of these revisions, and
         trust that the caller has saved the revisions that shouldn't be
         removed and that it'll re-add them after this truncation.
         """
         if len(self) == 0:
             return
 
         rev, _ = self.getstrippoint(minlink)
         if rev == len(self):
             return
 
         # first truncate the files on disk
         end = self.start(rev)
         if not self._inline:
             transaction.add(self.datafile, end)
             end = rev * self._io.size
         else:
             end += rev * self._io.size
 
         transaction.add(self.indexfile, end)
 
         # then reset internal state in memory to forget those revisions
         self._cache = None
         self._chaininfocache = {}
         self._chunkclear()
         for x in pycompat.xrange(rev, len(self)):
             del self.nodemap[self.node(x)]
 
         del self.index[rev:-1]
         self._nodepos = None
 
     def checksize(self):
         expected = 0
         if len(self):
             expected = max(0, self.end(len(self) - 1))
 
         try:
             with self._datafp() as f:
                 f.seek(0, 2)
                 actual = f.tell()
             dd = actual - expected
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             dd = 0
 
         try:
             f = self.opener(self.indexfile)
             f.seek(0, 2)
             actual = f.tell()
             f.close()
             s = self._io.size
             i = max(0, actual // s)
             di = actual - (i * s)
             if self._inline:
                 databytes = 0
                 for r in self:
                     databytes += max(0, self.length(r))
                 dd = 0
                 di = actual - len(self) * s - databytes
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             di = 0
 
         return (dd, di)
 
     def files(self):
         res = [self.indexfile]
         if not self._inline:
             res.append(self.datafile)
         return res
 
     def emitrevisiondeltas(self, requests):
         frev = self.rev
 
         prevrev = None
         for request in requests:
             node = request.node
             rev = frev(node)
 
             if prevrev is None:
                 prevrev = self.index[rev][5]
 
             # Requesting a full revision.
             if request.basenode == nullid:
                 baserev = nullrev
             # Requesting an explicit revision.
             elif request.basenode is not None:
                 baserev = frev(request.basenode)
             # Allowing us to choose.
             else:
                 p1rev, p2rev = self.parentrevs(rev)
                 deltaparentrev = self.deltaparent(rev)
 
                 # Avoid sending full revisions when delta parent is null. Pick
                 # prev in that case. It's tempting to pick p1 in this case, as
                 # p1 will be smaller in the common case. However, computing a
                 # delta against p1 may require resolving the raw text of p1,
                 # which could be expensive. The revlog caches should have prev
                 # cached, meaning less CPU for delta generation. There is
                 # likely room to add a flag and/or config option to control this
                 # behavior.
                 if deltaparentrev == nullrev and self._storedeltachains:
                     baserev = prevrev
 
                 # Revlog is configured to use full snapshot for a reason.
                 # Stick to full snapshot.
                 elif deltaparentrev == nullrev:
                     baserev = nullrev
 
                 # Pick previous when we can't be sure the base is available
                 # on consumer.
                 elif deltaparentrev not in (p1rev, p2rev, prevrev):
                     baserev = prevrev
                 else:
                     baserev = deltaparentrev
 
                 if baserev != nullrev and not self.candelta(baserev, rev):
                     baserev = nullrev
 
             revision = None
             delta = None
             baserevisionsize = None
 
             if self.iscensored(baserev) or self.iscensored(rev):
                 try:
                     revision = self.revision(node, raw=True)
                 except error.CensoredNodeError as e:
                     revision = e.tombstone
 
                 if baserev != nullrev:
                     baserevisionsize = self.rawsize(baserev)
 
             elif baserev == nullrev:
                 revision = self.revision(node, raw=True)
             else:
                 delta = self.revdiff(baserev, rev)
 
             extraflags = REVIDX_ELLIPSIS if request.ellipsis else 0
 
             yield revlogrevisiondelta(
                 node=node,
                 p1node=request.p1node,
                 p2node=request.p2node,
                 linknode=request.linknode,
                 basenode=self.node(baserev),
                 flags=self.flags(rev) | extraflags,
                 baserevisionsize=baserevisionsize,
                 revision=revision,
                 delta=delta)
 
             prevrev = rev
 
     DELTAREUSEALWAYS = 'always'
     DELTAREUSESAMEREVS = 'samerevs'
     DELTAREUSENEVER = 'never'
 
     DELTAREUSEFULLADD = 'fulladd'
 
     DELTAREUSEALL = {'always', 'samerevs', 'never', 'fulladd'}
 
     def clone(self, tr, destrevlog, addrevisioncb=None,
               deltareuse=DELTAREUSESAMEREVS, deltabothparents=None):
         """Copy this revlog to another, possibly with format changes.
 
         The destination revlog will contain the same revisions and nodes.
         However, it may not be bit-for-bit identical due to e.g. delta encoding
         differences.
 
         The ``deltareuse`` argument control how deltas from the existing revlog
         are preserved in the destination revlog. The argument can have the
         following values:
 
         DELTAREUSEALWAYS
            Deltas will always be reused (if possible), even if the destination
            revlog would not select the same revisions for the delta. This is the
            fastest mode of operation.
         DELTAREUSESAMEREVS
            Deltas will be reused if the destination revlog would pick the same
            revisions for the delta. This mode strikes a balance between speed
            and optimization.
         DELTAREUSENEVER
            Deltas will never be reused. This is the slowest mode of execution.
            This mode can be used to recompute deltas (e.g. if the diff/delta
            algorithm changes).
 
         Delta computation can be slow, so the choice of delta reuse policy can
         significantly affect run time.
 
         The default policy (``DELTAREUSESAMEREVS``) strikes a balance between
         two extremes. Deltas will be reused if they are appropriate. But if the
         delta could choose a better revision, it will do so. This means if you
         are converting a non-generaldelta revlog to a generaldelta revlog,
         deltas will be recomputed if the delta's parent isn't a parent of the
         revision.
 
         In addition to the delta policy, the ``deltabothparents`` argument
         controls whether to compute deltas against both parents for merges.
         By default, the current default is used.
         """
         if deltareuse not in self.DELTAREUSEALL:
             raise ValueError(_('value for deltareuse invalid: %s') % deltareuse)
 
         if len(destrevlog):
             raise ValueError(_('destination revlog is not empty'))
 
         if getattr(self, 'filteredrevs', None):
             raise ValueError(_('source revlog has filtered revisions'))
         if getattr(destrevlog, 'filteredrevs', None):
             raise ValueError(_('destination revlog has filtered revisions'))
 
         # lazydeltabase controls whether to reuse a cached delta, if possible.
         oldlazydeltabase = destrevlog._lazydeltabase
         oldamd = destrevlog._deltabothparents
 
         try:
             if deltareuse == self.DELTAREUSEALWAYS:
                 destrevlog._lazydeltabase = True
             elif deltareuse == self.DELTAREUSESAMEREVS:
                 destrevlog._lazydeltabase = False
 
             destrevlog._deltabothparents = deltabothparents or oldamd
 
             populatecachedelta = deltareuse in (self.DELTAREUSEALWAYS,
                                                 self.DELTAREUSESAMEREVS)
 
             deltacomputer = deltautil.deltacomputer(destrevlog)
             index = self.index
             for rev in self:
                 entry = index[rev]
 
                 # Some classes override linkrev to take filtered revs into
                 # account. Use raw entry from index.
                 flags = entry[0] & 0xffff
                 linkrev = entry[4]
                 p1 = index[entry[5]][7]
                 p2 = index[entry[6]][7]
                 node = entry[7]
 
                 # (Possibly) reuse the delta from the revlog if allowed and
                 # the revlog chunk is a delta.
                 cachedelta = None
                 rawtext = None
                 if populatecachedelta:
                     dp = self.deltaparent(rev)
                     if dp != nullrev:
                         cachedelta = (dp, bytes(self._chunk(rev)))
 
                 if not cachedelta:
                     rawtext = self.revision(rev, raw=True)
 
 
                 if deltareuse == self.DELTAREUSEFULLADD:
                     destrevlog.addrevision(rawtext, tr, linkrev, p1, p2,
                                            cachedelta=cachedelta,
                                            node=node, flags=flags,
                                            deltacomputer=deltacomputer)
                 else:
                     ifh = destrevlog.opener(destrevlog.indexfile, 'a+',
                                             checkambig=False)
                     dfh = None
                     if not destrevlog._inline:
                         dfh = destrevlog.opener(destrevlog.datafile, 'a+')
                     try:
                         destrevlog._addrevision(node, rawtext, tr, linkrev, p1,
                                                 p2, flags, cachedelta, ifh, dfh,
                                                 deltacomputer=deltacomputer)
                     finally:
                         if dfh:
                             dfh.close()
                         ifh.close()
 
                 if addrevisioncb:
                     addrevisioncb(self, rev, node)
         finally:
             destrevlog._lazydeltabase = oldlazydeltabase
             destrevlog._deltabothparents = oldamd
+
+    def censorrevision(self, node, tombstone=b''):
+        if (self.version & 0xFFFF) == REVLOGV0:
+            raise error.RevlogError(_('cannot censor with version %d revlogs') %
+                                    self.version)
+
+        rev = self.rev(node)
+        tombstone = packmeta({b'censored': tombstone}, b'')
+
+        if len(tombstone) > self.rawsize(rev):
+            raise error.Abort(_('censor tombstone must be no longer than '
+                                'censored data'))
+
+        # Using two files instead of one makes it easy to rewrite entry-by-entry
+        idxread = self.opener(self.indexfile, 'r')
+        idxwrite = self.opener(self.indexfile, 'wb', atomictemp=True)
+        if self.version & FLAG_INLINE_DATA:
+            dataread, datawrite = idxread, idxwrite
+        else:
+            dataread = self.opener(self.datafile, 'r')
+            datawrite = self.opener(self.datafile, 'wb', atomictemp=True)
+
+        # Copy all revlog data up to the entry to be censored.
+        offset = self.start(rev)
+
+        for chunk in util.filechunkiter(idxread, limit=rev * self._io.size):
+            idxwrite.write(chunk)
+        for chunk in util.filechunkiter(dataread, limit=offset):
+            datawrite.write(chunk)
+
+        def rewriteindex(r, newoffs, newdata=None):
+            """Rewrite the index entry with a new data offset and new data.
+
+            The newdata argument, if given, is a tuple of three positive
+            integers: (new compressed, new uncompressed, added flag bits).
+            """
+            offlags, comp, uncomp, base, link, p1, p2, nodeid = self.index[r]
+            flags = gettype(offlags)
+            if newdata:
+                comp, uncomp, nflags = newdata
+                flags |= nflags
+            offlags = offset_type(newoffs, flags)
+            e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
+            idxwrite.write(self._io.packentry(e, None, self.version, r))
+            idxread.seek(self._io.size, 1)
+
+        def rewrite(r, offs, data, nflags=REVIDX_DEFAULT_FLAGS):
+            """Write the given fulltext with the given data offset.
+
+            Returns:
+                The integer number of data bytes written, for tracking data
+                offsets.
+            """
+            flag, compdata = self.compress(data)
+            newcomp = len(flag) + len(compdata)
+            rewriteindex(r, offs, (newcomp, len(data), nflags))
+            datawrite.write(flag)
+            datawrite.write(compdata)
+            dataread.seek(self.length(r), 1)
+            return newcomp
+
+        # Rewrite censored entry with (padded) tombstone data.
+        pad = ' ' * (self.rawsize(rev) - len(tombstone))
+        offset += rewrite(rev, offset, tombstone + pad, REVIDX_ISCENSORED)
+
+        # Rewrite all following filelog revisions fixing up offsets and deltas.
+        for srev in pycompat.xrange(rev + 1, len(self)):
+            if rev in self.parentrevs(srev):
+                # Immediate children of censored node must be re-added as
+                # fulltext.
+                try:
+                    revdata = self.revision(srev)
+                except error.CensoredNodeError as e:
+                    revdata = e.tombstone
+                dlen = rewrite(srev, offset, revdata)
+            else:
+                # Copy any other revision data verbatim after fixing up the
+                # offset.
+                rewriteindex(srev, offset)
+                dlen = self.length(srev)
+                for chunk in util.filechunkiter(dataread, limit=dlen):
+                    datawrite.write(chunk)
+            offset += dlen
+
+        idxread.close()
+        idxwrite.close()
+        if dataread is not idxread:
+            dataread.close()
+            datawrite.close()