censor: make various path forms available like other Mercurial commands

Before this patch, censored file should be exactly "a path relative to
repository root" regardless of current working directory, because "hg
censor" passes "path" to "repo.file()" directly without any
preparations.

To make various path forms available like other Mercurial commands,
this patch gets a target file path in the way of "hg parents FILE".

Getting "wctx" is relocated to reuse "wctx" for efficiency.

hgext/censor.py

@@ -1,165 +1,170 b''
 # Copyright (C) 2015 - Mike Edgar <[e-mail]>
 #
 # This extension enables removal of file content at a given revision,
 # rewriting the data/metadata of successive revisions to preserve revision log
 # integrity.
 
 """erase file content at a given revision
 
 The censor command instructs Mercurial to erase all content of a file at a given
 revision *without updating the changeset hash.* This allows existing history to
 remain valid while preventing future clones/pulls from receiving the erased
 data.
 
 Typical uses for censor are due to security or legal requirements, including::
 
  * Passwords, private keys, crytographic material
  * Licensed data/code/libraries for which the license has expired
  * Personally Identifiable Information or other private data
 
 Censored nodes can interrupt mercurial's typical operation whenever the excised
 data needs to be materialized. Some commands, like ``hg cat``/``hg revert``,
 simply fail when asked to produce censored data. Others, like ``hg verify`` and
 ``hg update``, must be capable of tolerating censored data to continue to
 function in a meaningful way. Such commands only tolerate censored file
 revisions if they are allowed by the "censor.policy=ignore" config option.
 """
 
 from mercurial.node import short
 from mercurial import cmdutil, error, filelog, revlog, scmutil, util
 from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'internal'
 
 @command('censor',
     [('r', 'rev', '', _('censor file from specified revision'), _('REV')),
      ('t', 'tombstone', '', _('replacement tombstone data'), _('TEXT'))],
     _('-r REV [-t TEXT] [FILE]'))
 def censor(ui, repo, path, rev='', tombstone='', **opts):
     if not path:
         raise util.Abort(_('must specify file path to censor'))
     if not rev:
         raise util.Abort(_('must specify revision to censor'))
 
+    wctx = repo[None]
+
+    m = scmutil.match(wctx, (path,))
+    if m.anypats() or len(m.files()) != 1:
+        raise util.Abort(_('can only specify an explicit filename'))
+    path = m.files()[0]
     flog = repo.file(path)
     if not len(flog):
         raise util.Abort(_('cannot censor file with no history'))
 
     rev = scmutil.revsingle(repo, rev, rev).rev()
     try:
         ctx = repo[rev]
     except KeyError:
         raise util.Abort(_('invalid revision identifier %s') % rev)
 
     try:
         fctx = ctx.filectx(path)
     except error.LookupError:
         raise util.Abort(_('file does not exist at revision %s') % rev)
 
     fnode = fctx.filenode()
     headctxs = [repo[c] for c in repo.heads()]
     heads = [c for c in headctxs if path in c and c.filenode(path) == fnode]
     if heads:
         headlist = ', '.join([short(c.node()) for c in heads])
         raise util.Abort(_('cannot censor file in heads (%s)') % headlist,
             hint=_('clean/delete and commit first'))
 
-    wctx = repo[None]
     wp = wctx.parents()
     if ctx.node() in [p.node() for p in wp]:
         raise util.Abort(_('cannot censor working directory'),
             hint=_('clean/delete/update first'))
 
     flogv = flog.version & 0xFFFF
     if flogv != revlog.REVLOGNG:
         raise util.Abort(
             _('censor does not support revlog version %d') % (flogv,))
 
     tombstone = filelog.packmeta({"censored": tombstone}, "")
 
     crev = fctx.filerev()
 
     if len(tombstone) > flog.rawsize(crev):
         raise util.Abort(_(
             'censor tombstone must be no longer than censored data'))
 
     # Using two files instead of one makes it easy to rewrite entry-by-entry
     idxread = repo.svfs(flog.indexfile, 'r')
     idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True)
     if flog.version & revlog.REVLOGNGINLINEDATA:
         dataread, datawrite = idxread, idxwrite
     else:
         dataread = repo.svfs(flog.datafile, 'r')
         datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True)
 
     # Copy all revlog data up to the entry to be censored.
     rio = revlog.revlogio()
     offset = flog.start(crev)
 
     for chunk in util.filechunkiter(idxread, limit=crev * rio.size):
         idxwrite.write(chunk)
     for chunk in util.filechunkiter(dataread, limit=offset):
         datawrite.write(chunk)
 
     def rewriteindex(r, newoffs, newdata=None):
         """Rewrite the index entry with a new data offset and optional new data.
 
         The newdata argument, if given, is a tuple of three positive integers:
         (new compressed, new uncompressed, added flag bits).
         """
         offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r]
         flags = revlog.gettype(offlags)
         if newdata:
             comp, uncomp, nflags = newdata
             flags |= nflags
         offlags = revlog.offset_type(newoffs, flags)
         e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
         idxwrite.write(rio.packentry(e, None, flog.version, r))
         idxread.seek(rio.size, 1)
 
     def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS):
         """Write the given full text to the filelog with the given data offset.
 
         Returns:
             The integer number of data bytes written, for tracking data offsets.
         """
         flag, compdata = flog.compress(data)
         newcomp = len(flag) + len(compdata)
         rewriteindex(r, offs, (newcomp, len(data), nflags))
         datawrite.write(flag)
         datawrite.write(compdata)
         dataread.seek(flog.length(r), 1)
         return newcomp
 
     # Rewrite censored revlog entry with (padded) tombstone data.
     pad = ' ' * (flog.rawsize(crev) - len(tombstone))
     offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED)
 
     # Rewrite all following filelog revisions fixing up offsets and deltas.
     for srev in xrange(crev + 1, len(flog)):
         if crev in flog.parentrevs(srev):
             # Immediate children of censored node must be re-added as fulltext.
             try:
                 revdata = flog.revision(srev)
             except error.CensoredNodeError as e:
                 revdata = e.tombstone
             dlen = rewrite(srev, offset, revdata)
         else:
             # Copy any other revision data verbatim after fixing up the offset.
             rewriteindex(srev, offset)
             dlen = flog.length(srev)
             for chunk in util.filechunkiter(dataread, limit=dlen):
                 datawrite.write(chunk)
         offset += dlen
 
     idxread.close()
     idxwrite.close()
     if dataread is not idxread:
         dataread.close()
         datawrite.close()

tests/test-censor.t

@@ -1,480 +1,485 b''
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > censor=
   > EOF
   $ cp $HGRCPATH $HGRCPATH.orig
 
 Create repo with unimpeachable content
 
   $ hg init r
   $ cd r
   $ echo 'Initially untainted file' > target
   $ echo 'Normal file here' > bystander
   $ hg add target bystander
   $ hg ci -m init
 
 Clone repo so we can test pull later
 
   $ cd ..
   $ hg clone r rpull
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd r
 
 Introduce content which will ultimately require censorship. Name the first
 censored node C1, second C2, and so on
 
   $ echo 'Tainted file' > target
   $ echo 'Passwords: hunter2' >> target
   $ hg ci -m taint target
   $ C1=`hg id --debug -i`
 
   $ echo 'hunter3' >> target
   $ echo 'Normal file v2' > bystander
   $ hg ci -m moretaint target bystander
   $ C2=`hg id --debug -i`
 
 Add a new sanitized versions to correct our mistake. Name the first head H1,
 the second head H2, and so on
 
   $ echo 'Tainted file is now sanitized' > target
   $ hg ci -m sanitized target
   $ H1=`hg id --debug -i`
 
   $ hg update -r $C2
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo 'Tainted file now super sanitized' > target
   $ hg ci -m 'super sanitized' target
   created new head
   $ H2=`hg id --debug -i`
 
 Verify target contents before censorship at each revision
 
   $ hg cat -r $H1 target
   Tainted file is now sanitized
   $ hg cat -r $H2 target
   Tainted file now super sanitized
   $ hg cat -r $C2 target
   Tainted file
   Passwords: hunter2
   hunter3
   $ hg cat -r $C1 target
   Tainted file
   Passwords: hunter2
   $ hg cat -r 0 target
   Initially untainted file
 
 Try to censor revision with too large of a tombstone message
 
   $ hg censor -r $C1 -t 'blah blah blah blah blah blah blah blah bla' target
   abort: censor tombstone must be no longer than censored data
   [255]
 
 Censor revision with 2 offenses
 
-  $ hg censor -r $C2 -t "remove password" target
+(this also tests file pattern matching: path relative to cwd case)
+
+  $ mkdir -p foo/bar/baz
+  $ hg --cwd foo/bar/baz censor -r $C2 -t "remove password" ../../../target
   $ hg cat -r $H1 target
   Tainted file is now sanitized
   $ hg cat -r $H2 target
   Tainted file now super sanitized
   $ hg cat -r $C2 target
   abort: censored node: 1e0247a9a4b7
   (set censor.policy to ignore errors)
   [255]
   $ hg cat -r $C1 target
   Tainted file
   Passwords: hunter2
   $ hg cat -r 0 target
   Initially untainted file
 
 Censor revision with 1 offense
 
-  $ hg censor -r $C1 target
+(this also tests file pattern matching: with 'path:' scheme)
+
+  $ hg --cwd foo/bar/baz censor -r $C1 path:target
   $ hg cat -r $H1 target
   Tainted file is now sanitized
   $ hg cat -r $H2 target
   Tainted file now super sanitized
   $ hg cat -r $C2 target
   abort: censored node: 1e0247a9a4b7
   (set censor.policy to ignore errors)
   [255]
   $ hg cat -r $C1 target
   abort: censored node: 613bc869fceb
   (set censor.policy to ignore errors)
   [255]
   $ hg cat -r 0 target
   Initially untainted file
 
 Can only checkout target at uncensored revisions, -X is workaround for --all
 
   $ hg revert -r $C2 target
   abort: censored node: 1e0247a9a4b7
   (set censor.policy to ignore errors)
   [255]
   $ hg revert -r $C1 target
   abort: censored node: 613bc869fceb
   (set censor.policy to ignore errors)
   [255]
   $ hg revert -r $C1 --all
   reverting bystander
   reverting target
   abort: censored node: 613bc869fceb
   (set censor.policy to ignore errors)
   [255]
   $ hg revert -r $C1 --all -X target
   $ cat target
   Tainted file now super sanitized
   $ hg revert -r 0 --all
   reverting target
   $ cat target
   Initially untainted file
   $ hg revert -r $H2 --all
   reverting bystander
   reverting target
   $ cat target
   Tainted file now super sanitized
 
 Uncensored file can be viewed at any revision
 
   $ hg cat -r $H1 bystander
   Normal file v2
   $ hg cat -r $C2 bystander
   Normal file v2
   $ hg cat -r $C1 bystander
   Normal file here
   $ hg cat -r 0 bystander
   Normal file here
 
 Can update to children of censored revision
 
   $ hg update -r $H1
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Tainted file is now sanitized
   $ hg update -r $H2
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Tainted file now super sanitized
 
 Set censor policy to abort in trusted $HGRC so hg verify fails
 
   $ cp $HGRCPATH.orig $HGRCPATH
   $ cat >> $HGRCPATH <<EOF
   > [censor]
   > policy = abort
   > EOF
 
 Repo fails verification due to censorship
 
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
    target@1: censored file data
    target@2: censored file data
   2 files, 5 changesets, 7 total revisions
   2 integrity errors encountered!
   (first damaged changeset appears to be 1)
   [1]
 
 Cannot update to revision with censored data
 
   $ hg update -r $C2
   abort: censored node: 1e0247a9a4b7
   (set censor.policy to ignore errors)
   [255]
   $ hg update -r $C1
   abort: censored node: 613bc869fceb
   (set censor.policy to ignore errors)
   [255]
   $ hg update -r 0
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg update -r $H2
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 Set censor policy to ignore in trusted $HGRC so hg verify passes
 
   $ cp $HGRCPATH.orig $HGRCPATH
   $ cat >> $HGRCPATH <<EOF
   > [censor]
   > policy = ignore
   > EOF
 
 Repo passes verification with warnings with explicit config
 
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 5 changesets, 7 total revisions
 
 May update to revision with censored data with explicit config
 
   $ hg update -r $C2
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   $ hg update -r $C1
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   $ hg update -r 0
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Initially untainted file
   $ hg update -r $H2
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Tainted file now super sanitized
 
 Can merge in revision with censored data. Test requires one branch of history
 with the file censored, but we can't censor at a head, so advance H1.
 
   $ hg update -r $H1
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ C3=$H1
   $ echo 'advanced head H1' > target
   $ hg ci -m 'advance head H1' target
   $ H1=`hg id --debug -i`
   $ hg censor -r $C3 target
   $ hg update -r $H2
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg merge -r $C3
   merging target
   0 files updated, 1 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 
 Revisions present in repository heads may not be censored
 
   $ hg update -C -r $H2
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg censor -r $H2 target
   abort: cannot censor file in heads (78a8fc215e79)
   (clean/delete and commit first)
   [255]
   $ echo 'twiddling thumbs' > bystander
   $ hg ci -m 'bystander commit'
   $ H2=`hg id --debug -i`
   $ hg censor -r "$H2^" target
   abort: cannot censor file in heads (efbe78065929)
   (clean/delete and commit first)
   [255]
 
 Cannot censor working directory
 
   $ echo 'seriously no passwords' > target
   $ hg ci -m 'extend second head arbitrarily' target
   $ H2=`hg id --debug -i`
   $ hg update -r "$H2^"
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg censor -r . target
   abort: cannot censor working directory
   (clean/delete/update first)
   [255]
   $ hg update -r $H2
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 Can re-add file after being deleted + censored
 
   $ C4=$H2
   $ hg rm target
   $ hg ci -m 'delete target so it may be censored'
   $ H2=`hg id --debug -i`
   $ hg censor -r $C4 target
   $ hg cat -r $C4 target
   $ hg cat -r "$H2^^" target
   Tainted file now super sanitized
   $ echo 'fresh start' > target
   $ hg add target
   $ hg ci -m reincarnated target
   $ H2=`hg id --debug -i`
   $ hg cat -r $H2 target
   fresh start
   $ hg cat -r "$H2^" target
   target: no such file in rev 452ec1762369
   [1]
   $ hg cat -r $C4 target
   $ hg cat -r "$H2^^^" target
   Tainted file now super sanitized
 
 Can censor after revlog has expanded to no longer permit inline storage
 
   $ for x in `python $TESTDIR/seq.py 0 50000`
   > do
   >   echo "Password: hunter$x" >> target
   > done
   $ hg ci -m 'add 100k passwords'
   $ H2=`hg id --debug -i`
   $ C5=$H2
   $ hg revert -r "$H2^" target
   $ hg ci -m 'cleaned 100k passwords'
   $ H2=`hg id --debug -i`
   $ hg censor -r $C5 target
   $ hg cat -r $C5 target
   $ hg cat -r $H2 target
   fresh start
 
 Repo with censored nodes can be cloned and cloned nodes are censored
 
   $ cd ..
   $ hg clone r rclone
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd rclone
   $ hg cat -r $H1 target
   advanced head H1
   $ hg cat -r $H2~5 target
   Tainted file now super sanitized
   $ hg cat -r $C2 target
   $ hg cat -r $C1 target
   $ hg cat -r 0 target
   Initially untainted file
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 12 changesets, 13 total revisions
 
 Repo cloned before tainted content introduced can pull censored nodes
 
   $ cd ../rpull
   $ hg cat -r tip target
   Initially untainted file
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 1 changesets, 2 total revisions
   $ hg pull -r $H1 -r $H2
   pulling from $TESTTMP/r (glob)
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 11 changesets with 11 changes to 2 files (+1 heads)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   $ hg update 4
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Tainted file now super sanitized
   $ hg cat -r $H1 target
   advanced head H1
   $ hg cat -r $H2~5 target
   Tainted file now super sanitized
   $ hg cat -r $C2 target
   $ hg cat -r $C1 target
   $ hg cat -r 0 target
   Initially untainted file
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 12 changesets, 13 total revisions
 
 Censored nodes can be pushed if they censor previously unexchanged nodes
 
   $ echo 'Passwords: hunter2hunter2' > target
   $ hg ci -m 're-add password from clone' target
   created new head
   $ H3=`hg id --debug -i`
   $ REV=$H3
   $ echo 'Re-sanitized; nothing to see here' > target
   $ hg ci -m 're-sanitized' target
   $ H2=`hg id --debug -i`
   $ CLEANREV=$H2
   $ hg cat -r $REV target
   Passwords: hunter2hunter2
   $ hg censor -r $REV target
   $ hg cat -r $REV target
   $ hg cat -r $CLEANREV target
   Re-sanitized; nothing to see here
   $ hg push -f -r $H2
   pushing to $TESTTMP/r (glob)
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 2 changes to 1 files (+1 heads)
 
   $ cd ../r
   $ hg cat -r $REV target
   $ hg cat -r $CLEANREV target
   Re-sanitized; nothing to see here
   $ hg update $CLEANREV
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Re-sanitized; nothing to see here
 
 Censored nodes can be bundled up and unbundled in another repo
 
   $ hg bundle --base 0 ../pwbundle
   13 changesets found
   $ cd ../rclone
   $ hg unbundle ../pwbundle
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 2 changes to 2 files (+1 heads)
   (run 'hg heads .' to see heads, 'hg merge' to merge)
   $ hg cat -r $REV target
   $ hg cat -r $CLEANREV target
   Re-sanitized; nothing to see here
   $ hg update $CLEANREV
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Re-sanitized; nothing to see here
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 14 changesets, 15 total revisions
 
 Censored nodes can be imported on top of censored nodes, consecutively
 
   $ hg init ../rimport
   $ hg bundle --base 1 ../rimport/splitbundle
   12 changesets found
   $ cd ../rimport
   $ hg pull -r $H1 -r $H2 ../r
   pulling from ../r
   adding changesets
   adding manifests
   adding file changes
   added 8 changesets with 10 changes to 2 files (+1 heads)
   (run 'hg heads' to see heads, 'hg merge' to merge)
   $ hg unbundle splitbundle
   adding changesets
   adding manifests
   adding file changes
   added 6 changesets with 5 changes to 2 files (+1 heads)
   (run 'hg heads .' to see heads, 'hg merge' to merge)
   $ hg update $H2
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cat target
   Re-sanitized; nothing to see here
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 14 changesets, 15 total revisions
   $ cd ../r
 
 Can import bundle where first revision of a file is censored
 
   $ hg init ../rinit
   $ hg censor -r 0 target
   $ hg bundle -r 0 --base null ../rinit/initbundle
   1 changesets found
   $ cd ../rinit
   $ hg unbundle initbundle
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 2 changes to 2 files
   (run 'hg update' to get a working copy)
   $ hg cat -r 0 target