py3: use pickle directly

pycompat.pickle abstracted over the different pickle modules in
Python 2 and 3. Now that we're Python 3 only, it is safe to use the
`pickle` module directly. So this commit does that.

As part of this we remove the rules from check-code.py that were
forbidden direct pickle module use.

We retain the `util.pickle` symbol for backwards compatibility, just
in case some extensions were using it.

Differential Revision: https://phab.mercurial-scm.org/D12249

contrib/check-code.py

@@ -1,1126 +1,1124 b''
 #!/usr/bin/env python3
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """style and portability checker for Mercurial
 
 when a rule triggers wrong, do one of the following (prefer one from top):
  * do the work-around the rule suggests
  * doublecheck that it is a false match
  * improve the rule pattern
  * add an ignore pattern to the rule (3rd arg) which matches your good line
    (you can append a short comment and match this, like: #re-raises)
  * change the pattern to a warning and list the exception in test-check-code-hg
  * ONLY use no--check-code for skipping entire files from external sources
 """
 
 from __future__ import absolute_import, print_function
 import glob
 import keyword
 import optparse
 import os
 import re
 import sys
 
 if sys.version_info[0] < 3:
     opentext = open
 else:
 
     def opentext(f):
         return open(f, encoding='latin1')
 
 
 try:
     xrange
 except NameError:
     xrange = range
 try:
     import re2
 except ImportError:
     re2 = None
 
 import testparseutil
 
 
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
     if re2:
         try:
             return re2.compile(pat)
         except re2.error:
             pass
     return re.compile(pat)
 
 
 # check "rules depending on implementation of repquote()" in each
 # patterns (especially pypats), before changing around repquote()
 _repquotefixedmap = {
     ' ': ' ',
     '\n': '\n',
     '.': 'p',
     ':': 'q',
     '%': '%',
     '\\': 'b',
     '*': 'A',
     '+': 'P',
     '-': 'M',
 }
 
 
 def _repquoteencodechr(i):
     if i > 255:
         return 'u'
     c = chr(i)
     if c in _repquotefixedmap:
         return _repquotefixedmap[c]
     if c.isalpha():
         return 'x'
     if c.isdigit():
         return 'n'
     return 'o'
 
 
 _repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256))
 
 
 def repquote(m):
     t = m.group('text')
     t = t.translate(_repquotett)
     return m.group('quote') + t + m.group('quote')
 
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         l = len(comment.rstrip())
         return "#" * l + comment[l:]
     return repquote(m)
 
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
     [
         (r'\b(push|pop)d\b', "don't use 'pushd' or 'popd', use 'cd'"),
         (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
         (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
         (r'(?<!hg )grep.* -a', "don't use 'grep -a', use in-line python"),
         (r'sed.*-i', "don't use 'sed -i', use a temporary file"),
         (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"),
         (r'echo -n', "don't use 'echo -n', use printf"),
         (r'(^|\|\s*)\bwc\b[^|]*$\n(?!.*\(re\))', "filter wc output"),
         (r'head -c', "don't use 'head -c', use 'dd'"),
         (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
         (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
         (r'\bls\b.*-\w*R', "don't use 'ls -R', use 'find'"),
         (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"),
         (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
         (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
         (
             r'\[[^\]]+==',
             '[ foo == bar ] is a bashism, use [ foo = bar ] instead',
         ),
         (
             r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
             "use egrep for extended grep syntax",
         ),
         (r'(^|\|\s*)e?grep .*\\S', "don't use \\S in regular expression"),
         (r'(?<!!)/bin/', "don't use explicit paths for tools"),
         (r'#!.*/bash', "don't use bash in shebang, use sh"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'export .*=', "don't export and assign at once"),
         (r'^source\b', "don't use 'source', use '.'"),
         (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
         (r'\bls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
         (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
         (r'^stop\(\)', "don't use 'stop' as a shell function name"),
         (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
         (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"),
         (r'^alias\b.*=', "don't use alias, use a function"),
         (r'if\s*!', "don't use '!' to negate exit status"),
         (r'/dev/u?random', "don't use entropy, use /dev/zero"),
         (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"),
         (
             r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)',
             "put a backslash-escaped newline after sed 'i' command",
         ),
         (r'^diff *-\w*[uU].*$\n(^  \$ |^$)', "prefix diff -u/-U with cmp"),
         (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"),
         (r'[\s="`\']python\s(?!bindings)', "don't use 'python', use '$PYTHON'"),
         (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (r'\|&', "don't use |&, use 2>&1"),
         (r'\w =  +\w', "only one space after = allowed"),
         (
             r'\bsed\b.*[^\\]\\n',
             "don't use 'sed ... \\n', use a \\ and a newline",
         ),
         (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"),
         (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"),
         (r'grep.* -[ABC]', "don't use grep's context flags"),
         (
             r'find.*-printf',
             "don't use 'find -printf', it doesn't exist on BSD find(1)",
         ),
         (r'\$RANDOM ', "don't use bash-only $RANDOM to generate random values"),
     ],
     # warnings
     [
         (r'^function', "don't use 'function', use old style"),
         (r'^diff.*-\w*N', "don't use 'diff -N'"),
         (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`", "no-pwd-check"),
         (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"),
         (r'kill (`|\$\()', "don't use kill, use killdaemons.py"),
     ],
 ]
 
 testfilters = [
     (r"( *)(#([^!][^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 uprefix = r"^  \$ "
 utestpats = [
     [
         (r'^(\S.*||  [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"),
         (
             uprefix + r'.*\|\s*sed[^|>\n]*\n',
             "use regex test output patterns instead of sed",
         ),
         (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
         (
             uprefix + r'.*\|\| echo.*(fail|error)',
             "explicit exit code checks unnecessary",
         ),
         (uprefix + r'set -e', "don't use set -e"),
         (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"),
         (
             uprefix + r'.*:\.\S*/',
             "x:.y in a path does not work on msys, rewrite "
             "as x://.y, or see `hg log -k msys` for alternatives",
             r'-\S+:\.|' '# no-msys',  # -Rxxx
         ),  # in test-pull.t which is skipped on windows
         (
             r'^  [^$>].*27\.0\.0\.1',
             'use $LOCALIP not an explicit loopback address',
         ),
         (
             r'^  (?![>$] ).*\$LOCALIP.*[^)]$',
             'mark $LOCALIP output lines with (glob) to help tests in BSD jails',
         ),
         (
             r'^  (cat|find): .*: \$ENOENT\$',
             'use test -f to test for file existence',
         ),
         (
             r'^  diff -[^ -]*p',
             "don't use (external) diff with -p for portability",
         ),
         (r' readlink ', 'use readlink.py instead of readlink'),
         (
             r'^  [-+][-+][-+] .* [-+]0000 \(glob\)',
             "glob timezone field in diff output for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+,[0-9]+ @@',
             "use '@@ -N* +N,n @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+,[0-9]+ [+][0-9]+ @@',
             "use '@@ -N,n +N* @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+ @@',
             "use '@@ -N* +N* @@ (glob)' style chunk header for portability",
         ),
         (
             uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff'
             r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$',
             "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)",
         ),
     ],
     # warnings
     [
         (
             r'^  (?!.*\$LOCALIP)[^*?/\n]* \(glob\)$',
             "glob match with no glob string (?, *, /, and $LOCALIP)",
         ),
     ],
 ]
 
 # transform plain test rules to unified test's
 for i in [0, 1]:
     for tp in testpats[i]:
         p = tp[0]
         m = tp[1]
         if p.startswith('^'):
             p = "^  [$>] (%s)" % p[1:]
         else:
             p = "^  [$>] .*(%s)" % p
         utestpats[i].append((p, m) + tp[2:])
 
 # don't transform the following rules:
 # "  > \t" and "  \t" should be allowed in unified tests
 testpats[0].append((r'^( *)\t', "don't use tabs to indent"))
 utestpats[0].append((r'^( ?)\t', "don't use tabs to indent"))
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
     (r"( +)(#([^!][^\n]*\S)?)", repcomment),
 ]
 
 # common patterns to check *.py
 commonpypats = [
     [
         (r'\\$', 'Use () to wrap long lines in Python, not \\'),
         (
             r'^\s*def\s*\w+\s*\(.*,\s*\(',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (
             r'lambda\s*\(.*,.*\)',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
         (r'(?<!\.)\breduce\s*\(.*', "reduce is not available in Python 3+"),
         (
             r'\bdict\(.*=',
             'dict() is different in Py2 and 3 and is slower than {}',
             'dict-from-generator',
         ),
         (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
         (r'\s<>\s', '<> operator is not available in Python 3+, use !='),
         (r'^\s*\t', "don't use tabs"),
         (r'\S;\s*\n', "semicolon"),
         (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"),
         (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"),
         (r'(\w|\)),\w', "missing whitespace after ,"),
         (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (
             (
                 # a line ending with a colon, potentially with trailing comments
                 r':([ \t]*#[^\n]*)?\n'
                 # one that is not a pass and not only a comment
                 r'(?P<indent>[ \t]+)[^#][^\n]+\n'
                 # more lines at the same indent level
                 r'((?P=indent)[^\n]+\n)*'
                 # a pass at the same indent level, which is bogus
                 r'(?P=indent)pass[ \t\n#]'
             ),
             'omit superfluous pass',
         ),
         (r'[^\n]\Z', "no trailing newline"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (
             r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
             "linebreak after :",
         ),
         (
             r'class\s[^( \n]+:',
             "old-style class, use class foo(object)",
             r'#.*old-style',
         ),
         (
             r'class\s[^( \n]+\(\):',
             "class foo() creates old style object, use class foo(object)",
             r'#.*old-style',
         ),
         (
             r'\b(%s)\('
             % '|'.join(k for k in keyword.kwlist if k not in ('print', 'exec')),
             "Python keyword is not a function",
         ),
         #    (r'class\s[A-Z][^\(]*\((?!Exception)',
         #     "don't capitalize non-exception classes"),
         #    (r'in range\(', "use xrange"),
         #    (r'^\s*print\s+', "avoid using print in core and extensions"),
         (r'[\x80-\xff]', "non-ASCII character literal"),
         (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"),
         (
             r'([\(\[][ \t]\S)|(\S[ \t][\)\]])',
             "gratuitous whitespace in () or []",
         ),
         #    (r'\s\s=', "gratuitous whitespace before ="),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s',
             "missing whitespace around operator",
         ),
         (
             r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', "wrong whitespace around ="),
         (
             r'\([^()]*( =[^=]|[^<>!=]= )',
             "no whitespace around = for named parameters",
         ),
         (
             r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
             "don't use old-style two-argument raise, use Exception(message)",
         ),
         (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
         (
             r' [=!]=\s+(True|False|None)',
             "comparison with singleton, use 'is' or 'is not' instead",
         ),
         (
             r'^\s*(while|if) [01]:',
             "use True/False for constant Boolean expression",
         ),
         (r'^\s*if False(:| +and)', 'Remove code instead of using `if False`'),
         (
             r'(?:(?<!def)\s+|\()hasattr\(',
             'hasattr(foo, bar) is broken on py2, use util.safehasattr(foo, bar) '
             'instead',
             r'#.*hasattr-py3-only',
         ),
         (r'opener\([^)]*\).read\(', "use opener.read() instead"),
         (r'opener\([^)]*\).write\(', "use opener.write() instead"),
         (r'(?i)descend[e]nt', "the proper spelling is descendAnt"),
         (r'\.debug\(\_', "don't mark debug messages for translation"),
         (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
         (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
         (
             r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
             'legacy exception syntax; use "as" instead of ","',
         ),
         (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
         (r'\bdef\s+__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
         (
             r'os\.path\.join\(.*, *(""|\'\')\)',
             "use pathutil.normasprefix(path) instead of os.path.join(path, '')",
         ),
         (r'\s0[0-7]+\b', 'legacy octal syntax; use "0o" prefix instead of "0"'),
         # XXX only catch mutable arguments on the first line of the definition
         (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (
             r'^@(\w*\.)?cachefunc',
             "module-level @cachefunc is risky, please avoid",
         ),
         (
             r'^import Queue',
             "don't use Queue, use pycompat.queue.Queue + "
             "pycompat.queue.Empty",
         ),
         (
             r'^import cStringIO',
             "don't use cStringIO.StringIO, use util.stringio",
         ),
         (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"),
         (
             r'^import SocketServer',
             "don't use SockerServer, use util.socketserver",
         ),
         (r'^import urlparse', "don't use urlparse, use util.urlreq"),
         (r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"),
-        (r'^import cPickle', "don't use cPickle, use util.pickle"),
-        (r'^import pickle', "don't use pickle, use util.pickle"),
         (r'^import httplib', "don't use httplib, use util.httplib"),
         (r'^import BaseHTTPServer', "use util.httpserver instead"),
         (
             r'^(from|import) mercurial\.(cext|pure|cffi)',
             "use mercurial.policy.importmod instead",
         ),
         (r'\.next\(\)', "don't use .next(), use next(...)"),
         (
             r'([a-z]*).revision\(\1\.node\(',
             "don't convert rev to node before passing to revision(nodeorrev)",
         ),
         (r'platform\.system\(\)', "don't use platform.system(), use pycompat"),
     ],
     # warnings
     [],
 ]
 
 # patterns to check normal *.py files
 pypats = [
     [
         # Ideally, these should be placed in "commonpypats" for
         # consistency of coding rules in Mercurial source tree.
         # But on the other hand, these are not so seriously required for
         # python code fragments embedded in test scripts. Fixing test
         # scripts for these patterns requires many changes, and has less
         # profit than effort.
         (r'raise Exception', "don't raise generic exceptions"),
         (r'[\s\(](open|file)\([^)]*\)\.read\(', "use util.readfile() instead"),
         (
             r'[\s\(](open|file)\([^)]*\)\.write\(',
             "use util.writefile() instead",
         ),
         (
             r'^[\s\(]*(open(er)?|file)\([^)]*\)(?!\.close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (
             r'[\s\(](open|file)\([^)]*\)\.(?!close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
         (r'^import atexit', "don't use atexit, use ui.atexit"),
         # rules depending on implementation of repquote()
         (
             r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
             'string join across lines with no space',
         ),
         (
             r'''(?x)ui\.(status|progress|write|note|warn)\(
          [ \t\n#]*
          (?# any strings/comments might precede a string, which
            # contains translatable message)
          b?((['"]|\'\'\'|""")[ \npq%bAPMxno]*(['"]|\'\'\'|""")[ \t\n#]+)*
          (?# sequence consisting of below might precede translatable message
            # - formatting string: "% 10s", "%05d", "% -3.2f", "%*s", "%%" ...
            # - escaped character: "\\", "\n", "\0" ...
            # - character other than '%', 'b' as '\', and 'x' as alphabet)
          (['"]|\'\'\'|""")
          ((%([ n]?[PM]?([np]+|A))?x)|%%|b[bnx]|[ \nnpqAPMo])*x
          (?# this regexp can't use [^...] style,
            # because _preparepats forcibly adds "\n" into [^...],
            # even though this regexp wants match it against "\n")''',
             "missing _() in ui message (use () to hide false-positives)",
         ),
     ]
     + commonpypats[0],
     # warnings
     [
         # rules depending on implementation of repquote()
         (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
     ]
     + commonpypats[1],
 ]
 
 # patterns to check *.py for embedded ones in test script
 embeddedpypats = [
     [] + commonpypats[0],
     # warnings
     [] + commonpypats[1],
 ]
 
 # common filters to convert *.py
 commonpyfilters = [
     (
         r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""",
         reppython,
     ),
 ]
 
 # pattern only for mercurial and extensions
 core_py_pats = [
     [
         # Windows tend to get confused about capitalization of the drive letter
         #
         # see mercurial.windows.abspath for details
         (
             r'os\.path\.abspath',
             "use util.abspath instead (windows)",
             r'#.*re-exports',
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert normal *.py files
 pyfilters = [] + commonpyfilters
 
 # non-filter patterns
 pynfpats = [
     [
         (r'pycompat\.osname\s*[=!]=\s*[\'"]nt[\'"]', "use pycompat.iswindows"),
         (r'pycompat\.osname\s*[=!]=\s*[\'"]posix[\'"]', "use pycompat.isposix"),
         (
             r'pycompat\.sysplatform\s*[!=]=\s*[\'"]darwin[\'"]',
             "use pycompat.isdarwin",
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert *.py for embedded ones in test script
 embeddedpyfilters = [] + commonpyfilters
 
 # extension non-filter patterns
 pyextnfpats = [
     [(r'^"""\n?[A-Z]', "don't capitalize docstring title")],
     # warnings
     [],
 ]
 
 txtfilters = []
 
 txtpats = [
     [
         (r'\s$', 'trailing whitespace'),
         ('.. note::[ \n][^\n]', 'add two newlines after note::'),
     ],
     [],
 ]
 
 cpats = [
     [
         (r'//', "don't use //-style comments"),
         (r'\S\t', "don't use tabs except for indent"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (r'(while|if|do|for)\(', "use space after while/if/do/for"),
         (r'return\(', "return is not a function"),
         (r' ;', "no space before ;"),
         (r'[^;] \)', "no space before )"),
         (r'[)][{]', "space between ) and {"),
         (r'\w+\* \w+', "use int *foo, not int* foo"),
         (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
         (r'\w+ (\+\+|--)', "use foo++, not foo ++"),
         (r'\w,\w', "missing whitespace after ,"),
         (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (r'^#\s+\w', "use #foo, not # foo"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'^\s*#import\b', "use only #include in standard C code"),
         (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"),
         (r'strcat\(', "don't use strcat"),
         # rules depending on implementation of repquote()
     ],
     # warnings
     [
         # rules depending on implementation of repquote()
     ],
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
     [
         (r'\bui\.', "don't use ui in util"),
     ],
     # warnings
     [],
 ]
 
 inrevlogpats = [
     [
         (r'\brepo\.', "don't use repo in revlog"),
     ],
     # warnings
     [],
 ]
 
 webtemplatefilters = []
 
 webtemplatepats = [
     [],
     [
         (
             r'{desc(\|(?!websub|firstline)[^\|]*)+}',
             'follow desc keyword with either firstline or websub',
         ),
     ],
 ]
 
 allfilesfilters = []
 
 allfilespats = [
     [
         (
             r'(http|https)://[a-zA-Z0-9./]*selenic.com/',
             'use mercurial-scm.org domain URL',
         ),
         (
             r'mercurial@selenic\.com',
             'use mercurial-scm.org domain for mercurial ML address',
         ),
         (
             r'mercurial-devel@selenic\.com',
             'use mercurial-scm.org domain for mercurial-devel ML address',
         ),
     ],
     # warnings
     [],
 ]
 
 py3pats = [
     [
         (
             r'os\.environ',
             "use encoding.environ instead (py3)",
             r'#.*re-exports',
         ),
         (r'os\.name', "use pycompat.osname instead (py3)"),
         (r'os\.getcwd', "use encoding.getcwd instead (py3)", r'#.*re-exports'),
         (r'os\.sep', "use pycompat.ossep instead (py3)"),
         (r'os\.pathsep', "use pycompat.ospathsep instead (py3)"),
         (r'os\.altsep', "use pycompat.osaltsep instead (py3)"),
         (r'sys\.platform', "use pycompat.sysplatform instead (py3)"),
         (r'getopt\.getopt', "use pycompat.getoptb instead (py3)"),
         (r'os\.getenv', "use encoding.environ.get instead"),
         (r'os\.setenv', "modifying the environ dict is not preferred"),
         (r'(?<!pycompat\.)xrange', "use pycompat.xrange instead (py3)"),
     ],
     # warnings
     [],
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', pyfilters, pypats),
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', [], pynfpats),
     ('python', r'.*hgext.*\.py$', '', [], pyextnfpats),
     (
         'python 3',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         py3pats,
     ),
     (
         'core files',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         core_py_pats,
     ),
     ('test script', r'(.*/)?test-[^.~]*$', '', testfilters, testpats),
     ('c', r'.*\.[ch]$', '', cfilters, cpats),
     ('unified test', r'.*\.t$', '', utestfilters, utestpats),
     (
         'layering violation repo in revlog',
         r'mercurial/revlog\.py',
         '',
         pyfilters,
         inrevlogpats,
     ),
     (
         'layering violation ui in util',
         r'mercurial/util\.py',
         '',
         pyfilters,
         inutilpats,
     ),
     ('txt', r'.*\.txt$', '', txtfilters, txtpats),
     (
         'web template',
         r'mercurial/templates/.*\.tmpl',
         '',
         webtemplatefilters,
         webtemplatepats,
     ),
     ('all except for .po', r'.*(?<!\.po)$', '', allfilesfilters, allfilespats),
 ]
 
 # (desc,
 #  func to pick up embedded code fragments,
 #  list of patterns to convert target files
 #  list of patterns to detect errors/warnings)
 embeddedchecks = [
     (
         'embedded python',
         testparseutil.pyembedded,
         embeddedpyfilters,
         embeddedpypats,
     )
 ]
 
 
 def _preparepats():
     def preparefailandwarn(failandwarn):
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
                 p = pseq[0]
                 # \s doesn't match \n (done in two steps)
                 # first, we replace \s that appears in a set already
                 p = re.sub(r'\[\\s', r'[ \\t', p)
                 # now we replace other \s instances.
                 p = re.sub(r'(?<!(\\|\[))\\s', r'[ \\t]', p)
                 # [^...] doesn't match newline
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
 
     def preparefilters(filters):
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
 
     for cs in (checks, embeddedchecks):
         for c in cs:
             failandwarn = c[-1]
             preparefailandwarn(failandwarn)
 
             filters = c[-2]
             preparefilters(filters)
 
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print("%s:%d (%s):" % (fname, lineno, blame))
             else:
                 print("%s:%d:" % (fname, lineno))
             print(" > %s" % line)
             self._lastseen = msgid
         print(" " + msg)
 
 
 _defaultlogger = norepeatlogger()
 
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 
 def checkfile(
     f,
     logfunc=_defaultlogger.log,
     maxerr=None,
     warnings=False,
     blame=False,
     debug=False,
     lineno=True,
 ):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting.
              Set to false (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     result = True
 
     try:
         with opentext(f) as fp:
             try:
                 pre = fp.read()
             except UnicodeDecodeError as e:
                 print("%s while reading %s" % (e, f))
                 return result
     except IOError as e:
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
 
     # context information shared while single checkfile() invocation
     context = {'blamecache': None}
 
     for name, match, magic, filters, pats in checks:
         if debug:
             print(name, f)
         if not (re.match(match, f) or (magic and re.search(magic, pre))):
             if debug:
                 print(
                     "Skipping %s for %s it doesn't match %s" % (name, match, f)
                 )
             continue
         if "no-" "check-code" in pre:
             # If you're looking at this line, it's because a file has:
             # no- check- code
             # but the reason to output skipping is to make life for
             # tests easier. So, instead of writing it with a normal
             # spelling, we write it with the expected spelling from
             # tests/test-check-code.t
             print("Skipping %s it has no-che?k-code (glob)" % f)
             return "Skip"  # skip checking this file
 
         fc = _checkfiledata(
             name,
             f,
             pre,
             filters,
             pats,
             context,
             logfunc,
             maxerr,
             warnings,
             blame,
             debug,
             lineno,
         )
         if fc:
             result = False
 
     if f.endswith('.t') and "no-" "check-code" not in pre:
         if debug:
             print("Checking embedded code in %s" % f)
 
         prelines = pre.splitlines()
         embeddederros = []
         for name, embedded, filters, pats in embeddedchecks:
             # "reset curmax at each repetition" treats maxerr as "max
             # nubmer of errors in an actual file per entry of
             # (embedded)checks"
             curmaxerr = maxerr
 
             for found in embedded(f, prelines, embeddederros):
                 filename, starts, ends, code = found
                 fc = _checkfiledata(
                     name,
                     f,
                     code,
                     filters,
                     pats,
                     context,
                     logfunc,
                     curmaxerr,
                     warnings,
                     blame,
                     debug,
                     lineno,
                     offset=starts - 1,
                 )
                 if fc:
                     result = False
                     if curmaxerr:
                         if fc >= curmaxerr:
                             break
                         curmaxerr -= fc
 
     return result
 
 
 def _checkfiledata(
     name,
     f,
     filedata,
     filters,
     pats,
     context,
     logfunc,
     maxerr,
     warnings,
     blame,
     debug,
     lineno,
     offset=None,
 ):
     """Execute actual error check for file data
 
     :name: of the checking category
     :f: filepath
     :filedata: content of a file
     :filters: to be applied before checking
     :pats: to detect errors
     :context: a dict of information shared while single checkfile() invocation
               Valid keys: 'blamecache'.
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting, or False to
              report all errors
     :warnings: whether warning level checks should be applied
     :blame: whether blame information should be displayed at error reporting
     :debug: whether debug information should be displayed
     :lineno: whether lineno should be displayed at error reporting
     :offset: line number offset of 'filedata' in 'f' for checking
              an embedded code fragment, or None (offset=0 is different
              from offset=None)
 
     returns number of detected errors.
     """
     blamecache = context['blamecache']
     if offset is None:
         lineoffset = 0
     else:
         lineoffset = offset
 
     fc = 0
     pre = post = filedata
 
     if True:  # TODO: get rid of this redundant 'if' block
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0])  # nerr elements are errors
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
 
         if debug:
             print("Checking %s for %s" % (name, f))
 
         prelines = None
         errors = []
         for i, pat in enumerate(pats):
             if len(pat) == 3:
                 p, msg, ignore = pat
             else:
                 p, msg = pat
                 ignore = None
             if i >= nerrs:
                 msg = "warning: " + msg
 
             pos = 0
             n = 0
             for m in p.finditer(post):
                 if prelines is None:
                     prelines = pre.splitlines()
                     postlines = post.splitlines(True)
 
                 start = m.start()
                 while n < len(postlines):
                     step = len(postlines[n])
                     if pos + step > start:
                         break
                     pos += step
                     n += 1
                 l = prelines[n]
 
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print(
                             "Skipping %s for %s:%s (ignore pattern)"
                             % (name, f, (n + lineoffset))
                         )
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
                     if blamecache is None:
                         blamecache = getblame(f)
                         context['blamecache'] = blamecache
                     if (n + lineoffset) < len(blamecache):
                         bl, bu, br = blamecache[(n + lineoffset)]
                         if offset is None and bl == l:
                             bd = '%s@%s' % (bu, br)
                         elif offset is not None and bl.endswith(l):
                             # "offset is not None" means "checking
                             # embedded code fragment". In this case,
                             # "l" does not have information about the
                             # beginning of an *original* line in the
                             # file (e.g. '  > ').
                             # Therefore, use "str.endswith()", and
                             # show "maybe" for a little loose
                             # examination.
                             bd = '%s@%s, maybe' % (bu, br)
 
                 errors.append((f, lineno and (n + lineoffset + 1), l, msg, bd))
 
         errors.sort()
         for e in errors:
             logfunc(*e)
             fc += 1
             if maxerr and fc >= maxerr:
                 print(" (too many errors, giving up)")
                 break
 
     return fc
 
 
 def main():
     parser = optparse.OptionParser("%prog [options] [files | -]")
     parser.add_option(
         "-w",
         "--warnings",
         action="store_true",
         help="include warning-level checks",
     )
     parser.add_option(
         "-p", "--per-file", type="int", help="max warnings per file"
     )
     parser.add_option(
         "-b",
         "--blame",
         action="store_true",
         help="use annotate to generate blame info",
     )
     parser.add_option(
         "", "--debug", action="store_true", help="show debug information"
     )
     parser.add_option(
         "",
         "--nolineno",
         action="store_false",
         dest='lineno',
         help="don't show line numbers",
     )
 
     parser.set_defaults(
         per_file=15, warnings=False, blame=False, debug=False, lineno=True
     )
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     elif args == ['-']:
         # read file list from stdin
         check = sys.stdin.read().splitlines()
     else:
         check = args
 
     _preparepats()
 
     ret = 0
     for f in check:
         if not checkfile(
             f,
             maxerr=options.per_file,
             warnings=options.warnings,
             blame=options.blame,
             debug=options.debug,
             lineno=options.lineno,
         ):
             ret = 1
     return ret
 
 
 if __name__ == "__main__":
     sys.exit(main())

hgext/convert/common.py

@@ -1,598 +1,598 b''
 # common.py - common code for the convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import base64
 import datetime
 import errno
 import os
+import pickle
 import re
 import shlex
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     phases,
     pycompat,
     util,
 )
 from mercurial.utils import procutil
 
-pickle = util.pickle
 propertycache = util.propertycache
 
 
 def _encodeornone(d):
     if d is None:
         return
     return d.encode('latin1')
 
 
 class _shlexpy3proxy(object):
     def __init__(self, l):
         self._l = l
 
     def __iter__(self):
         return (_encodeornone(v) for v in self._l)
 
     def get_token(self):
         return _encodeornone(self._l.get_token())
 
     @property
     def infile(self):
         return self._l.infile or b'<unknown>'
 
     @property
     def lineno(self):
         return self._l.lineno
 
 
 def shlexer(data=None, filepath=None, wordchars=None, whitespace=None):
     if data is None:
         if pycompat.ispy3:
             data = open(filepath, b'r', encoding='latin1')
         else:
             data = open(filepath, b'r')
     else:
         if filepath is not None:
             raise error.ProgrammingError(
                 b'shlexer only accepts data or filepath, not both'
             )
         if pycompat.ispy3:
             data = data.decode('latin1')
     l = shlex.shlex(data, infile=filepath, posix=True)
     if whitespace is not None:
         l.whitespace_split = True
         if pycompat.ispy3:
             l.whitespace += whitespace.decode('latin1')
         else:
             l.whitespace += whitespace
     if wordchars is not None:
         if pycompat.ispy3:
             l.wordchars += wordchars.decode('latin1')
         else:
             l.wordchars += wordchars
     if pycompat.ispy3:
         return _shlexpy3proxy(l)
     return l
 
 
 if pycompat.ispy3:
     base64_encodebytes = base64.encodebytes
     base64_decodebytes = base64.decodebytes
 else:
     base64_encodebytes = base64.encodestring
     base64_decodebytes = base64.decodestring
 
 
 def encodeargs(args):
     def encodearg(s):
         lines = base64_encodebytes(s)
         lines = [l.splitlines()[0] for l in pycompat.iterbytestr(lines)]
         return b''.join(lines)
 
     s = pickle.dumps(args)
     return encodearg(s)
 
 
 def decodeargs(s):
     s = base64_decodebytes(s)
     return pickle.loads(s)
 
 
 class MissingTool(Exception):
     pass
 
 
 def checktool(exe, name=None, abort=True):
     name = name or exe
     if not procutil.findexe(exe):
         if abort:
             exc = error.Abort
         else:
             exc = MissingTool
         raise exc(_(b'cannot find required "%s" tool') % name)
 
 
 class NoRepo(Exception):
     pass
 
 
 SKIPREV = b'SKIP'
 
 
 class commit(object):
     def __init__(
         self,
         author,
         date,
         desc,
         parents,
         branch=None,
         rev=None,
         extra=None,
         sortkey=None,
         saverev=True,
         phase=phases.draft,
         optparents=None,
         ctx=None,
     ):
         self.author = author or b'unknown'
         self.date = date or b'0 0'
         self.desc = desc
         self.parents = parents  # will be converted and used as parents
         self.optparents = optparents or []  # will be used if already converted
         self.branch = branch
         self.rev = rev
         self.extra = extra or {}
         self.sortkey = sortkey
         self.saverev = saverev
         self.phase = phase
         self.ctx = ctx  # for hg to hg conversions
 
 
 class converter_source(object):
     """Conversion source interface"""
 
     def __init__(self, ui, repotype, path=None, revs=None):
         """Initialize conversion source (or raise NoRepo("message")
         exception if path is not a valid repository)"""
         self.ui = ui
         self.path = path
         self.revs = revs
         self.repotype = repotype
 
         self.encoding = b'utf-8'
 
     def checkhexformat(self, revstr, mapname=b'splicemap'):
         """fails if revstr is not a 40 byte hex. mercurial and git both uses
         such format for their revision numbering
         """
         if not re.match(br'[0-9a-fA-F]{40,40}$', revstr):
             raise error.Abort(
                 _(b'%s entry %s is not a valid revision identifier')
                 % (mapname, revstr)
             )
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def targetfilebelongstosource(self, targetfilename):
         """Returns true if the given targetfile belongs to the source repo. This
         is useful when only a subdirectory of the target belongs to the source
         repo."""
         # For normal full repo converts, this is always True.
         return True
 
     def setrevmap(self, revmap):
         """set the map of already-converted revisions"""
 
     def getheads(self):
         """Return a list of this repository's heads"""
         raise NotImplementedError
 
     def getfile(self, name, rev):
         """Return a pair (data, mode) where data is the file content
         as a string and mode one of '', 'x' or 'l'. rev is the
         identifier returned by a previous call to getchanges().
         Data is None if file is missing/deleted in rev.
         """
         raise NotImplementedError
 
     def getchanges(self, version, full):
         """Returns a tuple of (files, copies, cleanp2).
 
         files is a sorted list of (filename, id) tuples for all files
         changed between version and its first parent returned by
         getcommit(). If full, all files in that revision is returned.
         id is the source revision id of the file.
 
         copies is a dictionary of dest: source
 
         cleanp2 is the set of files filenames that are clean against p2.
         (Files that are clean against p1 are already not in files (unless
         full). This makes it possible to handle p2 clean files similarly.)
         """
         raise NotImplementedError
 
     def getcommit(self, version):
         """Return the commit object for version"""
         raise NotImplementedError
 
     def numcommits(self):
         """Return the number of commits in this source.
 
         If unknown, return None.
         """
         return None
 
     def gettags(self):
         """Return the tags as a dictionary of name: revision
 
         Tag names must be UTF-8 strings.
         """
         raise NotImplementedError
 
     def recode(self, s, encoding=None):
         if not encoding:
             encoding = self.encoding or b'utf-8'
 
         if isinstance(s, pycompat.unicode):
             return s.encode("utf-8")
         try:
             return s.decode(pycompat.sysstr(encoding)).encode("utf-8")
         except UnicodeError:
             try:
                 return s.decode("latin-1").encode("utf-8")
             except UnicodeError:
                 return s.decode(pycompat.sysstr(encoding), "replace").encode(
                     "utf-8"
                 )
 
     def getchangedfiles(self, rev, i):
         """Return the files changed by rev compared to parent[i].
 
         i is an index selecting one of the parents of rev.  The return
         value should be the list of files that are different in rev and
         this parent.
 
         If rev has no parents, i is None.
 
         This function is only needed to support --filemap
         """
         raise NotImplementedError
 
     def converted(self, rev, sinkrev):
         '''Notify the source that a revision has been converted.'''
 
     def hasnativeorder(self):
         """Return true if this source has a meaningful, native revision
         order. For instance, Mercurial revisions are store sequentially
         while there is no such global ordering with Darcs.
         """
         return False
 
     def hasnativeclose(self):
         """Return true if this source has ability to close branch."""
         return False
 
     def lookuprev(self, rev):
         """If rev is a meaningful revision reference in source, return
         the referenced identifier in the same format used by getcommit().
         return None otherwise.
         """
         return None
 
     def getbookmarks(self):
         """Return the bookmarks as a dictionary of name: revision
 
         Bookmark names are to be UTF-8 strings.
         """
         return {}
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """revstr is a string that describes a revision in the given
         source control system.  Return true if revstr has correct
         format.
         """
         return True
 
 
 class converter_sink(object):
     """Conversion sink (target) interface"""
 
     def __init__(self, ui, repotype, path):
         """Initialize conversion sink (or raise NoRepo("message")
         exception if path is not a valid repository)
 
         created is a list of paths to remove if a fatal error occurs
         later"""
         self.ui = ui
         self.path = path
         self.created = []
         self.repotype = repotype
 
     def revmapfile(self):
         """Path to a file that will contain lines
         source_rev_id sink_rev_id
         mapping equivalent revision identifiers for each system."""
         raise NotImplementedError
 
     def authorfile(self):
         """Path to a file that will contain lines
         srcauthor=dstauthor
         mapping equivalent authors identifiers for each system."""
         return None
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         """Create a revision with all changed files listed in 'files'
         and having listed parents. 'commit' is a commit object
         containing at a minimum the author, date, and message for this
         changeset.  'files' is a list of (path, version) tuples,
         'copies' is a dictionary mapping destinations to sources,
         'source' is the source repository, and 'revmap' is a mapfile
         of source revisions to converted revisions. Only getfile() and
         lookuprev() should be called on 'source'. 'full' means that 'files'
         is complete and all other files should be removed.
         'cleanp2' is a set of the filenames that are unchanged from p2
         (only in the common merge case where there two parents).
 
         Note that the sink repository is not told to update itself to
         a particular revision (or even what that revision would be)
         before it receives the file data.
         """
         raise NotImplementedError
 
     def puttags(self, tags):
         """Put tags into sink.
 
         tags: {tagname: sink_rev_id, ...} where tagname is an UTF-8 string.
         Return a pair (tag_revision, tag_parent_revision), or (None, None)
         if nothing was changed.
         """
         raise NotImplementedError
 
     def setbranch(self, branch, pbranches):
         """Set the current branch name. Called before the first putcommit
         on the branch.
         branch: branch name for subsequent commits
         pbranches: (converted parent revision, parent branch) tuples"""
 
     def setfilemapmode(self, active):
         """Tell the destination that we're using a filemap
 
         Some converter_sources (svn in particular) can claim that a file
         was changed in a revision, even if there was no change.  This method
         tells the destination that we're using a filemap and that it should
         filter empty revisions.
         """
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def putbookmarks(self, bookmarks):
         """Put bookmarks into sink.
 
         bookmarks: {bookmarkname: sink_rev_id, ...}
         where bookmarkname is an UTF-8 string.
         """
 
     def hascommitfrommap(self, rev):
         """Return False if a rev mentioned in a filemap is known to not be
         present."""
         raise NotImplementedError
 
     def hascommitforsplicemap(self, rev):
         """This method is for the special needs for splicemap handling and not
         for general use. Returns True if the sink contains rev, aborts on some
         special cases."""
         raise NotImplementedError
 
 
 class commandline(object):
     def __init__(self, ui, command):
         self.ui = ui
         self.command = command
 
     def prerun(self):
         pass
 
     def postrun(self):
         pass
 
     def _cmdline(self, cmd, *args, **kwargs):
         kwargs = pycompat.byteskwargs(kwargs)
         cmdline = [self.command, cmd] + list(args)
         for k, v in pycompat.iteritems(kwargs):
             if len(k) == 1:
                 cmdline.append(b'-' + k)
             else:
                 cmdline.append(b'--' + k.replace(b'_', b'-'))
             try:
                 if len(k) == 1:
                     cmdline.append(b'' + v)
                 else:
                     cmdline[-1] += b'=' + v
             except TypeError:
                 pass
         cmdline = [procutil.shellquote(arg) for arg in cmdline]
         if not self.ui.debugflag:
             cmdline += [b'2>', pycompat.bytestr(os.devnull)]
         cmdline = b' '.join(cmdline)
         return cmdline
 
     def _run(self, cmd, *args, **kwargs):
         def popen(cmdline):
             p = subprocess.Popen(
                 procutil.tonativestr(cmdline),
                 shell=True,
                 bufsize=-1,
                 close_fds=procutil.closefds,
                 stdout=subprocess.PIPE,
             )
             return p
 
         return self._dorun(popen, cmd, *args, **kwargs)
 
     def _run2(self, cmd, *args, **kwargs):
         return self._dorun(procutil.popen2, cmd, *args, **kwargs)
 
     def _run3(self, cmd, *args, **kwargs):
         return self._dorun(procutil.popen3, cmd, *args, **kwargs)
 
     def _dorun(self, openfunc, cmd, *args, **kwargs):
         cmdline = self._cmdline(cmd, *args, **kwargs)
         self.ui.debug(b'running: %s\n' % (cmdline,))
         self.prerun()
         try:
             return openfunc(cmdline)
         finally:
             self.postrun()
 
     def run(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.communicate()[0]
         self.ui.debug(output)
         return output, p.returncode
 
     def runlines(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.stdout.readlines()
         p.wait()
         self.ui.debug(b''.join(output))
         return output, p.returncode
 
     def checkexit(self, status, output=b''):
         if status:
             if output:
                 self.ui.warn(_(b'%s error:\n') % self.command)
                 self.ui.warn(output)
             msg = procutil.explainexit(status)
             raise error.Abort(b'%s %s' % (self.command, msg))
 
     def run0(self, cmd, *args, **kwargs):
         output, status = self.run(cmd, *args, **kwargs)
         self.checkexit(status, output)
         return output
 
     def runlines0(self, cmd, *args, **kwargs):
         output, status = self.runlines(cmd, *args, **kwargs)
         self.checkexit(status, b''.join(output))
         return output
 
     @propertycache
     def argmax(self):
         # POSIX requires at least 4096 bytes for ARG_MAX
         argmax = 4096
         try:
             argmax = os.sysconf("SC_ARG_MAX")
         except (AttributeError, ValueError):
             pass
 
         # Windows shells impose their own limits on command line length,
         # down to 2047 bytes for cmd.exe under Windows NT/2k and 2500 bytes
         # for older 4nt.exe. See http://support.microsoft.com/kb/830473 for
         # details about cmd.exe limitations.
 
         # Since ARG_MAX is for command line _and_ environment, lower our limit
         # (and make happy Windows shells while doing this).
         return argmax // 2 - 1
 
     def _limit_arglist(self, arglist, cmd, *args, **kwargs):
         cmdlen = len(self._cmdline(cmd, *args, **kwargs))
         limit = self.argmax - cmdlen
         numbytes = 0
         fl = []
         for fn in arglist:
             b = len(fn) + 3
             if numbytes + b < limit or len(fl) == 0:
                 fl.append(fn)
                 numbytes += b
             else:
                 yield fl
                 fl = [fn]
                 numbytes = b
         if fl:
             yield fl
 
     def xargs(self, arglist, cmd, *args, **kwargs):
         for l in self._limit_arglist(arglist, cmd, *args, **kwargs):
             self.run0(cmd, *(list(args) + l), **kwargs)
 
 
 class mapfile(dict):
     def __init__(self, ui, path):
         super(mapfile, self).__init__()
         self.ui = ui
         self.path = path
         self.fp = None
         self.order = []
         self._read()
 
     def _read(self):
         if not self.path:
             return
         try:
             fp = open(self.path, b'rb')
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         for i, line in enumerate(util.iterfile(fp)):
             line = line.splitlines()[0].rstrip()
             if not line:
                 # Ignore blank lines
                 continue
             try:
                 key, value = line.rsplit(b' ', 1)
             except ValueError:
                 raise error.Abort(
                     _(b'syntax error in %s(%d): key/value pair expected')
                     % (self.path, i + 1)
                 )
             if key not in self:
                 self.order.append(key)
             super(mapfile, self).__setitem__(key, value)
         fp.close()
 
     def __setitem__(self, key, value):
         if self.fp is None:
             try:
                 self.fp = open(self.path, b'ab')
             except IOError as err:
                 raise error.Abort(
                     _(b'could not open map file %r: %s')
                     % (self.path, encoding.strtolocal(err.strerror))
                 )
         self.fp.write(util.tonativeeol(b'%s %s\n' % (key, value)))
         self.fp.flush()
         super(mapfile, self).__setitem__(key, value)
 
     def close(self):
         if self.fp:
             self.fp.close()
             self.fp = None
 
 
 def makedatetimestamp(t):
     """Like dateutil.makedate() but for time t instead of current time"""
     delta = datetime.datetime.utcfromtimestamp(
         t
     ) - datetime.datetime.fromtimestamp(t)
     tz = delta.days * 86400 + delta.seconds
     return t, tz

hgext/convert/cvsps.py

@@ -1,1070 +1,1069 b''
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import functools
 import os
+import pickle
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     hook,
     pycompat,
     util,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
-pickle = util.pickle
-
 
 class logentry(object):
     """Class logentry has the following attributes:
     .author    - author name as CVS knows it
     .branch    - name of branch this revision is on
     .branches  - revision tuple of branches starting at this revision
     .comment   - commit message
     .commitid  - CVS commitid or None
     .date      - the commit date as a (time, tz) tuple
     .dead      - true if file revision is dead
     .file      - Name of file
     .lines     - a tuple (+lines, -lines) or None
     .parent    - Previous revision of this entry
     .rcs       - name of file as returned from CVS
     .revision  - revision number as tuple
     .tags      - list of tags on the file
     .synthetic - is this a synthetic "file ... added on ..." revision?
     .mergepoint - the branch that has been merged from (if present in
                   rlog output) or None
     .branchpoints - the branches that start at the current entry or empty
     """
 
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r" % (k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)" % (type(self).__name__, ", ".join(items))
 
 
 class logerror(Exception):
     pass
 
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath(b'/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'user@server/path/to/repository')
     '/path/to/repository'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # CVSpath is splitted into parts and then position of the first occurrence
     # of the '/' char after the '@' is located. The solution is the rest of the
     # string after that '/' sign including it
 
     parts = cvspath.split(b':')
     atposition = parts[-1].find(b'@')
     start = 0
 
     if atposition != -1:
         start = atposition
 
     repopath = parts[-1][parts[-1].find(b'/', start) :]
     return repopath
 
 
 def createlog(ui, directory=None, root=b"", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
 
     def scache(s):
         """return a shared version of a string"""
         return _scache.setdefault(s, s)
 
     ui.status(_(b'collecting CVS rlog\n'))
 
     log = []  # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile(b'RCS file: (.+)$')
     re_01 = re.compile(b'cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile(b'cvs (r?log|server): (.+)\n$')
     re_03 = re.compile(
         b"(Cannot access.+CVSROOT)|(can't create temporary directory.+)$"
     )
     re_10 = re.compile(b'Working file: (.+)$')
     re_20 = re.compile(b'symbolic names:')
     re_30 = re.compile(b'\t(.+): ([\\d.]+)$')
     re_31 = re.compile(b'----------------------------$')
     re_32 = re.compile(
         b'======================================='
         b'======================================$'
     )
     re_50 = re.compile(br'revision ([\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(
         br'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
         br'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
         br'(\s+commitid:\s+([^;]+);)?'
         br'(.*mergepoint:\s+([^;]+);)?'
     )
     re_70 = re.compile(b'branches: (.+);$')
 
     file_added_re = re.compile(br'file [^/]+ was (initially )?added on branch')
 
     prefix = b''  # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             with open(os.path.join(b'CVS', b'Repository'), b'rb') as f:
                 prefix = f.read().strip()
             directory = prefix
             if prefix == b".":
                 prefix = b""
         except IOError:
             raise logerror(_(b'not a CVS sandbox'))
 
         if prefix and not prefix.endswith(pycompat.ossep):
             prefix += pycompat.ossep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join(b'CVS', b'Root'), b'rb').read().strip()
         except IOError:
             pass
 
     if not root:
         root = encoding.environ.get(b'CVSROOT', b'')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser(b'~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumeric characters, concatenated in a way that does not
         # mix up the various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(b":") + [directory, b"cache"]
         cachefile = [b'-'.join(re.findall(br'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(
             cachedir, b'.'.join([s for s in cachefile if s])
         )
 
     if cache == b'update':
         try:
             ui.note(_(b'reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile, b'rb'))
             for e in oldlog:
                 if not (
                     util.safehasattr(e, b'branchpoints')
                     and util.safehasattr(e, b'commitid')
                     and util.safehasattr(e, b'mergepoint')
                 ):
                     ui.status(_(b'ignoring old cache\n'))
                     oldlog = []
                     break
 
             ui.note(_(b'cache has %d log entries\n') % len(oldlog))
         except Exception as e:
             ui.note(_(b'error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date  # last commit date as a (time,tz) tuple
             date = dateutil.datestr(date, b'%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = [b'cvs', b'-q']
     if root:
         cmd.append(b'-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith(b'/'):
             p += b'/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append([b'log', b'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append(b'-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}  # dictionary of revisions on current file with their tags
     branchmap = {}  # mapping between branch names and revision numbers
     rcsmap = {}
     state = 0
     store = False  # set when a new record can be appended
 
     cmd = [procutil.shellquote(arg) for arg in cmd]
     ui.note(_(b"running %s\n") % (b' '.join(cmd)))
     ui.debug(b"prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = procutil.popen(b' '.join(cmd), b'rb')
     peek = util.fromnativeeol(pfp.readline())
     while True:
         line = peek
         if line == b'':
             break
         peek = util.fromnativeeol(pfp.readline())
         if line.endswith(b'\n'):
             line = line[:-1]
         # ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix) :]
                     if filename.startswith(b'/'):
                         filename = filename[1:]
                     if filename.startswith(b'Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace(b'/Attic/', b'/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _(b'RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split(b'.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _(
                     b'must have at least some revisions'
                 )
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _(b'expected revision number')
             e = logentry(
                 rcs=scache(rcs),
                 file=scache(filename),
                 revision=tuple([int(x) for x in match.group(1).split(b'.')]),
                 branches=[],
                 parent=None,
                 commitid=None,
                 mergepoint=None,
                 branchpoints=set(),
             )
 
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _(b'revision must be followed by date line')
             d = match.group(1)
             if d[2] == b'/':
                 # Y2K
                 d = b'19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + b' UTC'
             e.date = dateutil.parsedate(
                 d,
                 [
                     b'%y/%m/%d %H:%M:%S',
                     b'%Y/%m/%d %H:%M:%S',
                     b'%Y-%m-%d %H:%M:%S',
                 ],
             )
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == b'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7):  # cvs 1.12 commitid
                 e.commitid = match.group(8)
 
             if match.group(9):  # cvsnt mergepoint
                 myrev = match.group(10).split(b'.')
                 if len(myrev) == 2:  # head
                     e.mergepoint = b'HEAD'
                 else:
                     myrev = b'.'.join(myrev[:-2] + [b'0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, (
                         b'unknown branch: %s' % e.mergepoint
                     )
                     e.mergepoint = branches[0]
 
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [
                     tuple([int(y) for y in x.strip().split(b'.')])
                     for x in m.group(1).split(b';')
                 ]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
                 cpeek = peek
                 if cpeek.endswith(b'\n'):
                     cpeek = cpeek[:-1]
                 if re_50.match(cpeek):
                     state = 5
                     store = True
                 else:
                     e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (
             store
             and e.dead
             and e.revision[-1] == 1
             and len(e.comment) == 1  # 1.1 or 1.1.x.1
             and file_added_re.match(e.comment[0])
         ):
             ui.debug(
                 b'found synthetic revision in %s: %r\n' % (e.rcs, e.comment[0])
             )
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache(b'\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
             for branch, revision in pycompat.iteritems(branchmap):
                 revparts = tuple([int(i) for i in revision.split(b'.')])
                 if len(revparts) < 2:  # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1):  # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             rcsmap[e.rcs.replace(b'/Attic/', b'/')] = e.rcs
 
             if len(log) % 100 == 0:
                 ui.status(
                     stringutil.ellipsis(b'%d %s' % (len(log), e.file), 80)
                     + b'\n'
                 )
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in sorted(oldlog, key=lambda x: (x.rcs, x.revision)):
         rcs = e.rcs.replace(b'/Attic/', b'/')
         if rcs in rcsmap:
             e.rcs = rcsmap[rcs]
         branch = e.revision[:-1]
         versions[(e.rcs, branch)] = e.revision
 
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(
                     _(
                         b'log cache overlaps with new log entries,'
                         b' re-run without cache.'
                     )
                 )
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_(b'writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, b'wb'))
         else:
             log = oldlog
 
     ui.status(_(b'%d log entries\n') % len(log))
 
     encodings = ui.configlist(b'convert', b'cvsps.logencoding')
     if encodings:
 
         def revstr(r):
             # this is needed, because logentry.revision is a tuple of "int"
             # (e.g. (1, 2) for "1.2")
             return b'.'.join(pycompat.maplist(pycompat.bytestr, r))
 
         for entry in log:
             comment = entry.comment
             for e in encodings:
                 try:
                     entry.comment = comment.decode(pycompat.sysstr(e)).encode(
                         'utf-8'
                     )
                     if ui.debugflag:
                         ui.debug(
                             b"transcoding by %s: %s of %s\n"
                             % (e, revstr(entry.revision), entry.file)
                         )
                     break
                 except UnicodeDecodeError:
                     pass  # try next encoding
                 except LookupError as inst:  # unknown encoding, maybe
                     raise error.Abort(
                         pycompat.bytestr(inst),
                         hint=_(
                             b'check convert.cvsps.logencoding configuration'
                         ),
                     )
             else:
                 raise error.Abort(
                     _(
                         b"no encoding can transcode"
                         b" CVS log message for %s of %s"
                     )
                     % (revstr(entry.revision), entry.file),
                     hint=_(b'check convert.cvsps.logencoding configuration'),
                 )
 
     hook.hook(ui, None, b"cvslog", True, log=log)
 
     return log
 
 
 class changeset(object):
     """Class changeset has the following attributes:
     .id        - integer identifying this changeset (list index)
     .author    - author name as CVS knows it
     .branch    - name of branch this changeset is on, or None
     .comment   - commit message
     .commitid  - CVS commitid or None
     .date      - the commit date as a (time,tz) tuple
     .entries   - list of logentry objects in this changeset
     .parents   - list of one or two parent changesets
     .tags      - list of tags on this changeset
     .synthetic - from synthetic revision "file ... added on branch ..."
     .mergepoint- the branch that has been merged from or None
     .branchpoints- the branches that start at the current entry or empty
     """
 
     def __init__(self, **entries):
         self.id = None
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = (
             b"%s=%r" % (k, self.__dict__[k]) for k in sorted(self.__dict__)
         )
         return b"%s(%s)" % (type(self).__name__, b", ".join(items))
 
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_(b'creating changesets\n'))
 
     # try to order commitids by date
     mindate = {}
     for e in log:
         if e.commitid:
             if e.commitid not in mindate:
                 mindate[e.commitid] = e.date
             else:
                 mindate[e.commitid] = min(e.date, mindate[e.commitid])
 
     # Merge changesets
     log.sort(
         key=lambda x: (
             mindate.get(x.commitid, (-1, 0)),
             x.commitid or b'',
             x.comment,
             x.author,
             x.branch or b'',
             x.date,
             x.branchpoints,
         )
     )
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file-centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (
             c
             and e.branchpoints == c.branchpoints
             and (  # cvs commitids
                 (e.commitid is not None and e.commitid == c.commitid)
                 or (  # no commitids, use fuzzy commit detection
                     (e.commitid is None or c.commitid is None)
                     and e.comment == c.comment
                     and e.author == c.author
                     and e.branch == c.branch
                     and (
                         (c.date[0] + c.date[1])
                         <= (e.date[0] + e.date[1])
                         <= (c.date[0] + c.date[1]) + fuzz
                     )
                     and e.file not in files
                 )
             )
         ):
             c = changeset(
                 comment=e.comment,
                 author=e.author,
                 branch=e.branch,
                 date=e.date,
                 entries=[],
                 mergepoint=e.mergepoint,
                 branchpoints=e.branchpoints,
                 commitid=e.commitid,
             )
             changesets.append(c)
 
             files = set()
             if len(changesets) % 100 == 0:
                 t = b'%d %s' % (len(changesets), repr(e.comment)[1:-1])
                 ui.status(stringutil.ellipsis(t, 80) + b'\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date  # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     def entitycompare(l, r):
         """Mimic cvsps sorting order"""
         l = l.file.split(b'/')
         r = r.file.split(b'/')
         nl = len(l)
         nr = len(r)
         n = min(nl, nr)
         for i in range(n):
             if i + 1 == nl and nl < nr:
                 return -1
             elif i + 1 == nr and nl > nr:
                 return +1
             elif l[i] < r[i]:
                 return -1
             elif l[i] > r[i]:
                 return +1
         return 0
 
     for c in changesets:
         c.entries.sort(key=functools.cmp_to_key(entitycompare))
 
     # Sort changesets by date
 
     odd = set()
 
     def cscmp(l, r):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 if d:
                     odd.add((l, r))
                 d = -1
                 break
         # By this point, the changesets are sufficiently compared that
         # we don't really care about ordering. However, this leaves
         # some race conditions in the tests, so we compare on the
         # number of files modified, the files contained in each
         # changeset, and the branchpoints in the change to ensure test
         # output remains stable.
 
         # recommended replacement for cmp from
         # https://docs.python.org/3.0/whatsnew/3.0.html
         c = lambda x, y: (x > y) - (x < y)
         # Sort bigger changes first.
         if not d:
             d = c(len(l.entries), len(r.entries))
         # Try sorting by filename in the change.
         if not d:
             d = c([e.file for e in l.entries], [e.file for e in r.entries])
         # Try and put changes without a branch point before ones with
         # a branch point.
         if not d:
             d = c(len(l.branchpoints), len(r.branchpoints))
         return d
 
     changesets.sort(key=functools.cmp_to_key(cscmp))
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = br'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = br'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}  # changeset index where we saw any particular file version
     branches = {}  # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in pycompat.xrange(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, _(
                     b'synthetic changeset cannot have multiple parents'
                 )
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == b'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == b'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(
                         _(
                             b"warning: CVS commit message references "
                             b"non-existent branch %r:\n%s\n"
                         )
                         % (pycompat.bytestr(m), c.comment)
                     )
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 if m.groups():
                     m = m.group(1)
                     if m == b'HEAD':
                         m = None
                 else:
                     m = None  # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author,
                         branch=m,
                         date=c.date,
                         comment=b'convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[],
                         tags=[],
                         parents=[changesets[branches[m]], c],
                     )
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     if odd:
         for l, r in odd:
             if l.id is not None and r.id is not None:
                 ui.warn(
                     _(b'changeset %d is both before and after %d\n')
                     % (l.id, r.id)
                 )
 
     ui.status(_(b'%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, b"cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     """Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     """
     opts = pycompat.byteskwargs(opts)
     if opts[b"new_cache"]:
         cache = b"write"
     elif opts[b"update_cache"]:
         cache = b"update"
     else:
         cache = None
 
     revisions = opts[b"revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts[b"root"], cache=cache)
         else:
             log = createlog(ui, root=opts[b"root"], cache=cache)
     except logerror as e:
         ui.write(b"%r\n" % e)
         return
 
     changesets = createchangeset(ui, log, opts[b"fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}  # latest version number in each branch
     ancestors = {}  # parent branch
     for cs in changesets:
 
         if opts[b"ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (
                     changesets[cs.parents[0].id - 1].branch,
                     cs.parents[0].id,
                 )
             branches[cs.branch] = cs.id
 
         # limit by branches
         if (
             opts[b"branches"]
             and (cs.branch or b'HEAD') not in opts[b"branches"]
         ):
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write(b'---------------------\n')
             ui.write((b'PatchSet %d \n' % cs.id))
             ui.write(
                 (
                     b'Date: %s\n'
                     % dateutil.datestr(cs.date, b'%Y/%m/%d %H:%M:%S %1%2')
                 )
             )
             ui.write((b'Author: %s\n' % cs.author))
             ui.write((b'Branch: %s\n' % (cs.branch or b'HEAD')))
             ui.write(
                 (
                     b'Tag%s: %s \n'
                     % (
                         [b'', b's'][len(cs.tags) > 1],
                         b','.join(cs.tags) or b'(none)',
                     )
                 )
             )
             if cs.branchpoints:
                 ui.writenoi18n(
                     b'Branchpoints: %s \n' % b', '.join(sorted(cs.branchpoints))
                 )
             if opts[b"parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write(
                         (
                             b'Parents: %s\n'
                             % (b','.join([(b"%d" % p.id) for p in cs.parents]))
                         )
                     )
                 else:
                     ui.write((b'Parent: %d\n' % cs.parents[0].id))
 
             if opts[b"ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append(b'%s:%d:%d' % (b or b"HEAD", c, branches[b]))
                 if r:
                     ui.write((b'Ancestors: %s\n' % (b','.join(r))))
 
             ui.writenoi18n(b'Log:\n')
             ui.write(b'%s\n\n' % cs.comment)
             ui.writenoi18n(b'Members: \n')
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts[b"prefix"]):
                     fn = fn[len(opts[b"prefix"]) :]
                 ui.write(
                     b'\t%s:%s->%s%s \n'
                     % (
                         fn,
                         b'.'.join([b"%d" % x for x in f.parent]) or b'INITIAL',
                         b'.'.join([(b"%d" % x) for x in f.revision]),
                         [b'', b'(DEAD)'][f.dead],
                     )
                 )
             ui.write(b'\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == (b"%d" % cs.id) or revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == (b"%d" % cs.id) or revisions[1] in cs.tags:
                 break

hgext/convert/subversion.py

@@ -1,1741 +1,1741 b''
 # Subversion 1.4/1.5 Python API backend
 #
 # Copyright(C) 2007 Daniel Holth et al
 from __future__ import absolute_import
 
 import codecs
 import locale
 import os
+import pickle
 import re
 import xml.dom.minidom
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     pycompat,
     util,
     vfs as vfsmod,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
 from . import common
 
-pickle = util.pickle
 stringio = util.stringio
 propertycache = util.propertycache
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 commandline = common.commandline
 commit = common.commit
 converter_sink = common.converter_sink
 converter_source = common.converter_source
 decodeargs = common.decodeargs
 encodeargs = common.encodeargs
 makedatetimestamp = common.makedatetimestamp
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 
 # Subversion stuff. Works best with very recent Python SVN bindings
 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
 # these bindings.
 
 try:
     import svn
     import svn.client
     import svn.core
     import svn.ra
     import svn.delta
     from . import transport
     import warnings
 
     warnings.filterwarnings(
         'ignore', module='svn.core', category=DeprecationWarning
     )
     svn.core.SubversionException  # trigger import to catch error
 
 except ImportError:
     svn = None
 
 
 # In Subversion, paths and URLs are Unicode (encoded as UTF-8), which
 # Subversion converts from / to native strings when interfacing with the OS.
 # When passing paths and URLs to Subversion, we have to recode them such that
 # it roundstrips with what Subversion is doing.
 
 fsencoding = None
 
 
 def init_fsencoding():
     global fsencoding, fsencoding_is_utf8
     if fsencoding is not None:
         return
     if pycompat.iswindows:
         # On Windows, filenames are Unicode, but we store them using the MBCS
         # encoding.
         fsencoding = 'mbcs'
     else:
         # This is the encoding used to convert UTF-8 back to natively-encoded
         # strings in Subversion 1.14.0 or earlier with APR 1.7.0 or earlier.
         with util.with_lc_ctype():
             fsencoding = locale.nl_langinfo(locale.CODESET) or 'ISO-8859-1'
     fsencoding = codecs.lookup(fsencoding).name
     fsencoding_is_utf8 = fsencoding == codecs.lookup('utf-8').name
 
 
 def fs2svn(s):
     if fsencoding_is_utf8:
         return s
     else:
         return s.decode(fsencoding).encode('utf-8')
 
 
 def formatsvndate(date):
     return dateutil.datestr(date, b'%Y-%m-%dT%H:%M:%S.000000Z')
 
 
 def parsesvndate(s):
     # Example SVN datetime. Includes microseconds.
     # ISO-8601 conformant
     # '2007-01-04T17:35:00.902377Z'
     return dateutil.parsedate(s[:19] + b' UTC', [b'%Y-%m-%dT%H:%M:%S'])
 
 
 class SvnPathNotFound(Exception):
     pass
 
 
 def revsplit(rev):
     """Parse a revision string and return (uuid, path, revnum).
     >>> revsplit(b'svn:a2147622-4a9f-4db4-a8d3-13562ff547b2'
     ...          b'/proj%20B/mytrunk/mytrunk@1')
     ('a2147622-4a9f-4db4-a8d3-13562ff547b2', '/proj%20B/mytrunk/mytrunk', 1)
     >>> revsplit(b'svn:8af66a51-67f5-4354-b62c-98d67cc7be1d@1')
     ('', '', 1)
     >>> revsplit(b'@7')
     ('', '', 7)
     >>> revsplit(b'7')
     ('', '', 0)
     >>> revsplit(b'bad')
     ('', '', 0)
     """
     parts = rev.rsplit(b'@', 1)
     revnum = 0
     if len(parts) > 1:
         revnum = int(parts[1])
     parts = parts[0].split(b'/', 1)
     uuid = b''
     mod = b''
     if len(parts) > 1 and parts[0].startswith(b'svn:'):
         uuid = parts[0][4:]
         mod = b'/' + parts[1]
     return uuid, mod, revnum
 
 
 def quote(s):
     # As of svn 1.7, many svn calls expect "canonical" paths. In
     # theory, we should call svn.core.*canonicalize() on all paths
     # before passing them to the API.  Instead, we assume the base url
     # is canonical and copy the behaviour of svn URL encoding function
     # so we can extend it safely with new components. The "safe"
     # characters were taken from the "svn_uri__char_validity" table in
     # libsvn_subr/path.c.
     return urlreq.quote(s, b"!$&'()*+,-./:=@_~")
 
 
 def geturl(path):
     """Convert path or URL to a SVN URL, encoded in UTF-8.
 
     This can raise UnicodeDecodeError if the path or URL can't be converted to
     unicode using `fsencoding`.
     """
     try:
         return svn.client.url_from_path(
             svn.core.svn_path_canonicalize(fs2svn(path))
         )
     except svn.core.SubversionException:
         # svn.client.url_from_path() fails with local repositories
         pass
     if os.path.isdir(path):
         path = os.path.normpath(util.abspath(path))
         if pycompat.iswindows:
             path = b'/' + util.normpath(path)
         # Module URL is later compared with the repository URL returned
         # by svn API, which is UTF-8.
         path = fs2svn(path)
         path = b'file://%s' % quote(path)
     return svn.core.svn_path_canonicalize(path)
 
 
 def optrev(number):
     optrev = svn.core.svn_opt_revision_t()
     optrev.kind = svn.core.svn_opt_revision_number
     optrev.value.number = number
     return optrev
 
 
 class changedpath(object):
     def __init__(self, p):
         self.copyfrom_path = p.copyfrom_path
         self.copyfrom_rev = p.copyfrom_rev
         self.action = p.action
 
 
 def get_log_child(
     fp,
     url,
     paths,
     start,
     end,
     limit=0,
     discover_changed_paths=True,
     strict_node_history=False,
 ):
     protocol = -1
 
     def receiver(orig_paths, revnum, author, date, message, pool):
         paths = {}
         if orig_paths is not None:
             for k, v in pycompat.iteritems(orig_paths):
                 paths[k] = changedpath(v)
         pickle.dump((paths, revnum, author, date, message), fp, protocol)
 
     try:
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(
             t.ra,
             paths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
             receiver,
         )
     except IOError:
         # Caller may interrupt the iteration
         pickle.dump(None, fp, protocol)
     except Exception as inst:
         pickle.dump(stringutil.forcebytestr(inst), fp, protocol)
     else:
         pickle.dump(None, fp, protocol)
     fp.flush()
     # With large history, cleanup process goes crazy and suddenly
     # consumes *huge* amount of memory. The output file being closed,
     # there is no need for clean termination.
     os._exit(0)
 
 
 def debugsvnlog(ui, **opts):
     """Fetch SVN log in a subprocess and channel them back to parent to
     avoid memory collection issues.
     """
     with util.with_lc_ctype():
         if svn is None:
             raise error.Abort(
                 _(b'debugsvnlog could not load Subversion python bindings')
             )
 
         args = decodeargs(ui.fin.read())
         get_log_child(ui.fout, *args)
 
 
 class logstream(object):
     """Interruptible revision log iterator."""
 
     def __init__(self, stdout):
         self._stdout = stdout
 
     def __iter__(self):
         while True:
             try:
                 entry = pickle.load(self._stdout)
             except EOFError:
                 raise error.Abort(
                     _(
                         b'Mercurial failed to run itself, check'
                         b' hg executable is in PATH'
                     )
                 )
             try:
                 orig_paths, revnum, author, date, message = entry
             except (TypeError, ValueError):
                 if entry is None:
                     break
                 raise error.Abort(_(b"log stream exception '%s'") % entry)
             yield entry
 
     def close(self):
         if self._stdout:
             self._stdout.close()
             self._stdout = None
 
 
 class directlogstream(list):
     """Direct revision log iterator.
     This can be used for debugging and development but it will probably leak
     memory and is not suitable for real conversions."""
 
     def __init__(
         self,
         url,
         paths,
         start,
         end,
         limit=0,
         discover_changed_paths=True,
         strict_node_history=False,
     ):
         def receiver(orig_paths, revnum, author, date, message, pool):
             paths = {}
             if orig_paths is not None:
                 for k, v in pycompat.iteritems(orig_paths):
                     paths[k] = changedpath(v)
             self.append((paths, revnum, author, date, message))
 
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(
             t.ra,
             paths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
             receiver,
         )
 
     def close(self):
         pass
 
 
 # Check to see if the given path is a local Subversion repo. Verify this by
 # looking for several svn-specific files and directories in the given
 # directory.
 def filecheck(ui, path, proto):
     for x in (b'locks', b'hooks', b'format', b'db'):
         if not os.path.exists(os.path.join(path, x)):
             return False
     return True
 
 
 # Check to see if a given path is the root of an svn repo over http. We verify
 # this by requesting a version-controlled URL we know can't exist and looking
 # for the svn-specific "not found" XML.
 def httpcheck(ui, path, proto):
     try:
         opener = urlreq.buildopener()
         rsp = opener.open(
             pycompat.strurl(b'%s://%s/!svn/ver/0/.svn' % (proto, path)), b'rb'
         )
         data = rsp.read()
     except urlerr.httperror as inst:
         if inst.code != 404:
             # Except for 404 we cannot know for sure this is not an svn repo
             ui.warn(
                 _(
                     b'svn: cannot probe remote repository, assume it could '
                     b'be a subversion repository. Use --source-type if you '
                     b'know better.\n'
                 )
             )
             return True
         data = inst.fp.read()
     except Exception:
         # Could be urlerr.urlerror if the URL is invalid or anything else.
         return False
     return b'<m:human-readable errcode="160013">' in data
 
 
 protomap = {
     b'http': httpcheck,
     b'https': httpcheck,
     b'file': filecheck,
 }
 
 
 class NonUtf8PercentEncodedBytes(Exception):
     pass
 
 
 # Subversion paths are Unicode. Since the percent-decoding is done on
 # UTF-8-encoded strings, percent-encoded bytes are interpreted as UTF-8.
 def url2pathname_like_subversion(unicodepath):
     if pycompat.ispy3:
         # On Python 3, we have to pass unicode to urlreq.url2pathname().
         # Percent-decoded bytes get decoded using UTF-8 and the 'replace' error
         # handler.
         unicodepath = urlreq.url2pathname(unicodepath)
         if u'\N{REPLACEMENT CHARACTER}' in unicodepath:
             raise NonUtf8PercentEncodedBytes
         else:
             return unicodepath
     else:
         # If we passed unicode on Python 2, it would be converted using the
         # latin-1 encoding. Therefore, we pass UTF-8-encoded bytes.
         unicodepath = urlreq.url2pathname(unicodepath.encode('utf-8'))
         try:
             return unicodepath.decode('utf-8')
         except UnicodeDecodeError:
             raise NonUtf8PercentEncodedBytes
 
 
 def issvnurl(ui, url):
     try:
         proto, path = url.split(b'://', 1)
         if proto == b'file':
             if (
                 pycompat.iswindows
                 and path[:1] == b'/'
                 and path[1:2].isalpha()
                 and path[2:6].lower() == b'%3a/'
             ):
                 path = path[:2] + b':/' + path[6:]
             try:
                 unicodepath = path.decode(fsencoding)
             except UnicodeDecodeError:
                 ui.warn(
                     _(
                         b'Subversion requires that file URLs can be converted '
                         b'to Unicode using the current locale encoding (%s)\n'
                     )
                     % pycompat.sysbytes(fsencoding)
                 )
                 return False
             try:
                 unicodepath = url2pathname_like_subversion(unicodepath)
             except NonUtf8PercentEncodedBytes:
                 ui.warn(
                     _(
                         b'Subversion does not support non-UTF-8 '
                         b'percent-encoded bytes in file URLs\n'
                     )
                 )
                 return False
             # Below, we approximate how Subversion checks the path. On Unix, we
             # should therefore convert the path to bytes using `fsencoding`
             # (like Subversion does). On Windows, the right thing would
             # actually be to leave the path as unicode. For now, we restrict
             # the path to MBCS.
             path = unicodepath.encode(fsencoding)
     except ValueError:
         proto = b'file'
         path = util.abspath(url)
         try:
             path.decode(fsencoding)
         except UnicodeDecodeError:
             ui.warn(
                 _(
                     b'Subversion requires that paths can be converted to '
                     b'Unicode using the current locale encoding (%s)\n'
                 )
                 % pycompat.sysbytes(fsencoding)
             )
             return False
     if proto == b'file':
         path = util.pconvert(path)
     elif proto in (b'http', 'https'):
         if not encoding.isasciistr(path):
             ui.warn(
                 _(
                     b"Subversion sources don't support non-ASCII characters in "
                     b"HTTP(S) URLs. Please percent-encode them.\n"
                 )
             )
             return False
     check = protomap.get(proto, lambda *args: False)
     while b'/' in path:
         if check(ui, path, proto):
             return True
         path = path.rsplit(b'/', 1)[0]
     return False
 
 
 # SVN conversion code stolen from bzr-svn and tailor
 #
 # Subversion looks like a versioned filesystem, branches structures
 # are defined by conventions and not enforced by the tool. First,
 # we define the potential branches (modules) as "trunk" and "branches"
 # children directories. Revisions are then identified by their
 # module and revision number (and a repository identifier).
 #
 # The revision graph is really a tree (or a forest). By default, a
 # revision parent is the previous revision in the same module. If the
 # module directory is copied/moved from another module then the
 # revision is the module root and its parent the source revision in
 # the parent module. A revision has at most one parent.
 #
 class svn_source(converter_source):
     def __init__(self, ui, repotype, url, revs=None):
         super(svn_source, self).__init__(ui, repotype, url, revs=revs)
 
         init_fsencoding()
         if not (
             url.startswith(b'svn://')
             or url.startswith(b'svn+ssh://')
             or (
                 os.path.exists(url)
                 and os.path.exists(os.path.join(url, b'.svn'))
             )
             or issvnurl(ui, url)
         ):
             raise NoRepo(
                 _(b"%s does not look like a Subversion repository") % url
             )
         if svn is None:
             raise MissingTool(_(b'could not load Subversion python bindings'))
 
         try:
             version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
             if version < (1, 4):
                 raise MissingTool(
                     _(
                         b'Subversion python bindings %d.%d found, '
                         b'1.4 or later required'
                     )
                     % version
                 )
         except AttributeError:
             raise MissingTool(
                 _(
                     b'Subversion python bindings are too old, 1.4 '
                     b'or later required'
                 )
             )
 
         self.lastrevs = {}
 
         latest = None
         try:
             # Support file://path@rev syntax. Useful e.g. to convert
             # deleted branches.
             at = url.rfind(b'@')
             if at >= 0:
                 latest = int(url[at + 1 :])
                 url = url[:at]
         except ValueError:
             pass
         self.url = geturl(url)
         self.encoding = b'UTF-8'  # Subversion is always nominal UTF-8
         try:
             with util.with_lc_ctype():
                 self.transport = transport.SvnRaTransport(url=self.url)
                 self.ra = self.transport.ra
                 self.ctx = self.transport.client
                 self.baseurl = svn.ra.get_repos_root(self.ra)
                 # Module is either empty or a repository path starting with
                 # a slash and not ending with a slash.
                 self.module = urlreq.unquote(self.url[len(self.baseurl) :])
                 self.prevmodule = None
                 self.rootmodule = self.module
                 self.commits = {}
                 self.paths = {}
                 self.uuid = svn.ra.get_uuid(self.ra)
         except svn.core.SubversionException:
             ui.traceback()
             svnversion = b'%d.%d.%d' % (
                 svn.core.SVN_VER_MAJOR,
                 svn.core.SVN_VER_MINOR,
                 svn.core.SVN_VER_MICRO,
             )
             raise NoRepo(
                 _(
                     b"%s does not look like a Subversion repository "
                     b"to libsvn version %s"
                 )
                 % (self.url, svnversion)
             )
 
         if revs:
             if len(revs) > 1:
                 raise error.Abort(
                     _(
                         b'subversion source does not support '
                         b'specifying multiple revisions'
                     )
                 )
             try:
                 latest = int(revs[0])
             except ValueError:
                 raise error.Abort(
                     _(b'svn: revision %s is not an integer') % revs[0]
                 )
 
         trunkcfg = self.ui.config(b'convert', b'svn.trunk')
         if trunkcfg is None:
             trunkcfg = b'trunk'
         self.trunkname = trunkcfg.strip(b'/')
         self.startrev = self.ui.config(b'convert', b'svn.startrev')
         try:
             self.startrev = int(self.startrev)
             if self.startrev < 0:
                 self.startrev = 0
         except ValueError:
             raise error.Abort(
                 _(b'svn: start revision %s is not an integer') % self.startrev
             )
 
         try:
             with util.with_lc_ctype():
                 self.head = self.latest(self.module, latest)
         except SvnPathNotFound:
             self.head = None
         if not self.head:
             raise error.Abort(
                 _(b'no revision found in module %s') % self.module
             )
         self.last_changed = self.revnum(self.head)
 
         self._changescache = (None, None)
 
         if os.path.exists(os.path.join(url, b'.svn/entries')):
             self.wc = url
         else:
             self.wc = None
         self.convertfp = None
 
     def before(self):
         self.with_lc_ctype = util.with_lc_ctype()
         self.with_lc_ctype.__enter__()
 
     def after(self):
         self.with_lc_ctype.__exit__(None, None, None)
 
     def setrevmap(self, revmap):
         lastrevs = {}
         for revid in revmap:
             uuid, module, revnum = revsplit(revid)
             lastrevnum = lastrevs.setdefault(module, revnum)
             if revnum > lastrevnum:
                 lastrevs[module] = revnum
         self.lastrevs = lastrevs
 
     def exists(self, path, optrev):
         try:
             svn.client.ls(
                 self.url.rstrip(b'/') + b'/' + quote(path),
                 optrev,
                 False,
                 self.ctx,
             )
             return True
         except svn.core.SubversionException:
             return False
 
     def getheads(self):
         def isdir(path, revnum):
             kind = self._checkpath(path, revnum)
             return kind == svn.core.svn_node_dir
 
         def getcfgpath(name, rev):
             cfgpath = self.ui.config(b'convert', b'svn.' + name)
             if cfgpath is not None and cfgpath.strip() == b'':
                 return None
             path = (cfgpath or name).strip(b'/')
             if not self.exists(path, rev):
                 if self.module.endswith(path) and name == b'trunk':
                     # we are converting from inside this directory
                     return None
                 if cfgpath:
                     raise error.Abort(
                         _(b'expected %s to be at %r, but not found')
                         % (name, path)
                     )
                 return None
             self.ui.note(
                 _(b'found %s at %r\n') % (name, pycompat.bytestr(path))
             )
             return path
 
         rev = optrev(self.last_changed)
         oldmodule = b''
         trunk = getcfgpath(b'trunk', rev)
         self.tags = getcfgpath(b'tags', rev)
         branches = getcfgpath(b'branches', rev)
 
         # If the project has a trunk or branches, we will extract heads
         # from them. We keep the project root otherwise.
         if trunk:
             oldmodule = self.module or b''
             self.module += b'/' + trunk
             self.head = self.latest(self.module, self.last_changed)
             if not self.head:
                 raise error.Abort(
                     _(b'no revision found in module %s') % self.module
                 )
 
         # First head in the list is the module's head
         self.heads = [self.head]
         if self.tags is not None:
             self.tags = b'%s/%s' % (oldmodule, (self.tags or b'tags'))
 
         # Check if branches bring a few more heads to the list
         if branches:
             rpath = self.url.strip(b'/')
             branchnames = svn.client.ls(
                 rpath + b'/' + quote(branches), rev, False, self.ctx
             )
             for branch in sorted(branchnames):
                 module = b'%s/%s/%s' % (oldmodule, branches, branch)
                 if not isdir(module, self.last_changed):
                     continue
                 brevid = self.latest(module, self.last_changed)
                 if not brevid:
                     self.ui.note(_(b'ignoring empty branch %s\n') % branch)
                     continue
                 self.ui.note(
                     _(b'found branch %s at %d\n')
                     % (branch, self.revnum(brevid))
                 )
                 self.heads.append(brevid)
 
         if self.startrev and self.heads:
             if len(self.heads) > 1:
                 raise error.Abort(
                     _(
                         b'svn: start revision is not supported '
                         b'with more than one branch'
                     )
                 )
             revnum = self.revnum(self.heads[0])
             if revnum < self.startrev:
                 raise error.Abort(
                     _(b'svn: no revision found after start revision %d')
                     % self.startrev
                 )
 
         return self.heads
 
     def _getchanges(self, rev, full):
         (paths, parents) = self.paths[rev]
         copies = {}
         if parents:
             files, self.removed, copies = self.expandpaths(rev, paths, parents)
         if full or not parents:
             # Perform a full checkout on roots
             uuid, module, revnum = revsplit(rev)
             entries = svn.client.ls(
                 self.baseurl + quote(module), optrev(revnum), True, self.ctx
             )
             files = [
                 n
                 for n, e in pycompat.iteritems(entries)
                 if e.kind == svn.core.svn_node_file
             ]
             self.removed = set()
 
         files.sort()
         files = pycompat.ziplist(files, [rev] * len(files))
         return (files, copies)
 
     def getchanges(self, rev, full):
         # reuse cache from getchangedfiles
         if self._changescache[0] == rev and not full:
             (files, copies) = self._changescache[1]
         else:
             (files, copies) = self._getchanges(rev, full)
             # caller caches the result, so free it here to release memory
             del self.paths[rev]
         return (files, copies, set())
 
     def getchangedfiles(self, rev, i):
         # called from filemap - cache computed values for reuse in getchanges
         (files, copies) = self._getchanges(rev, False)
         self._changescache = (rev, (files, copies))
         return [f[0] for f in files]
 
     def getcommit(self, rev):
         if rev not in self.commits:
             uuid, module, revnum = revsplit(rev)
             self.module = module
             self.reparent(module)
             # We assume that:
             # - requests for revisions after "stop" come from the
             # revision graph backward traversal. Cache all of them
             # down to stop, they will be used eventually.
             # - requests for revisions before "stop" come to get
             # isolated branches parents. Just fetch what is needed.
             stop = self.lastrevs.get(module, 0)
             if revnum < stop:
                 stop = revnum + 1
             self._fetch_revisions(revnum, stop)
             if rev not in self.commits:
                 raise error.Abort(_(b'svn: revision %s not found') % revnum)
         revcommit = self.commits[rev]
         # caller caches the result, so free it here to release memory
         del self.commits[rev]
         return revcommit
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """fails if revision format does not match the correct format"""
         if not re.match(
             br'svn:[0-9a-f]{8,8}-[0-9a-f]{4,4}-'
             br'[0-9a-f]{4,4}-[0-9a-f]{4,4}-[0-9a-f]'
             br'{12,12}(.*)@[0-9]+$',
             revstr,
         ):
             raise error.Abort(
                 _(b'%s entry %s is not a valid revision identifier')
                 % (mapname, revstr)
             )
 
     def numcommits(self):
         return int(self.head.rsplit(b'@', 1)[1]) - self.startrev
 
     def gettags(self):
         tags = {}
         if self.tags is None:
             return tags
 
         # svn tags are just a convention, project branches left in a
         # 'tags' directory. There is no other relationship than
         # ancestry, which is expensive to discover and makes them hard
         # to update incrementally.  Worse, past revisions may be
         # referenced by tags far away in the future, requiring a deep
         # history traversal on every calculation.  Current code
         # performs a single backward traversal, tracking moves within
         # the tags directory (tag renaming) and recording a new tag
         # everytime a project is copied from outside the tags
         # directory. It also lists deleted tags, this behaviour may
         # change in the future.
         pendings = []
         tagspath = self.tags
         start = svn.ra.get_latest_revnum(self.ra)
         stream = self._getlog([self.tags], start, self.startrev)
         try:
             for entry in stream:
                 origpaths, revnum, author, date, message = entry
                 if not origpaths:
                     origpaths = []
                 copies = [
                     (e.copyfrom_path, e.copyfrom_rev, p)
                     for p, e in pycompat.iteritems(origpaths)
                     if e.copyfrom_path
                 ]
                 # Apply moves/copies from more specific to general
                 copies.sort(reverse=True)
 
                 srctagspath = tagspath
                 if copies and copies[-1][2] == tagspath:
                     # Track tags directory moves
                     srctagspath = copies.pop()[0]
 
                 for source, sourcerev, dest in copies:
                     if not dest.startswith(tagspath + b'/'):
                         continue
                     for tag in pendings:
                         if tag[0].startswith(dest):
                             tagpath = source + tag[0][len(dest) :]
                             tag[:2] = [tagpath, sourcerev]
                             break
                     else:
                         pendings.append([source, sourcerev, dest])
 
                 # Filter out tags with children coming from different
                 # parts of the repository like:
                 # /tags/tag.1 (from /trunk:10)
                 # /tags/tag.1/foo (from /branches/foo:12)
                 # Here/tags/tag.1 discarded as well as its children.
                 # It happens with tools like cvs2svn. Such tags cannot
                 # be represented in mercurial.
                 addeds = {
                     p: e.copyfrom_path
                     for p, e in pycompat.iteritems(origpaths)
                     if e.action == b'A' and e.copyfrom_path
                 }
                 badroots = set()
                 for destroot in addeds:
                     for source, sourcerev, dest in pendings:
                         if not dest.startswith(
                             destroot + b'/'
                         ) or source.startswith(addeds[destroot] + b'/'):
                             continue
                         badroots.add(destroot)
                         break
 
                 for badroot in badroots:
                     pendings = [
                         p
                         for p in pendings
                         if p[2] != badroot
                         and not p[2].startswith(badroot + b'/')
                     ]
 
                 # Tell tag renamings from tag creations
                 renamings = []
                 for source, sourcerev, dest in pendings:
                     tagname = dest.split(b'/')[-1]
                     if source.startswith(srctagspath):
                         renamings.append([source, sourcerev, tagname])
                         continue
                     if tagname in tags:
                         # Keep the latest tag value
                         continue
                     # From revision may be fake, get one with changes
                     try:
                         tagid = self.latest(source, sourcerev)
                         if tagid and tagname not in tags:
                             tags[tagname] = tagid
                     except SvnPathNotFound:
                         # It happens when we are following directories
                         # we assumed were copied with their parents
                         # but were really created in the tag
                         # directory.
                         pass
                 pendings = renamings
                 tagspath = srctagspath
         finally:
             stream.close()
         return tags
 
     def converted(self, rev, destrev):
         if not self.wc:
             return
         if self.convertfp is None:
             self.convertfp = open(
                 os.path.join(self.wc, b'.svn', b'hg-shamap'), b'ab'
             )
         self.convertfp.write(
             util.tonativeeol(b'%s %d\n' % (destrev, self.revnum(rev)))
         )
         self.convertfp.flush()
 
     def revid(self, revnum, module=None):
         return b'svn:%s%s@%d' % (self.uuid, module or self.module, revnum)
 
     def revnum(self, rev):
         return int(rev.split(b'@')[-1])
 
     def latest(self, path, stop=None):
         """Find the latest revid affecting path, up to stop revision
         number. If stop is None, default to repository latest
         revision. It may return a revision in a different module,
         since a branch may be moved without a change being
         reported. Return None if computed module does not belong to
         rootmodule subtree.
         """
 
         def findchanges(path, start, stop=None):
             stream = self._getlog([path], start, stop or 1)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if stop is None and paths:
                         # We do not know the latest changed revision,
                         # keep the first one with changed paths.
                         break
                     if stop is not None and revnum <= stop:
                         break
 
                     for p in paths:
                         if not path.startswith(p) or not paths[p].copyfrom_path:
                             continue
                         newpath = paths[p].copyfrom_path + path[len(p) :]
                         self.ui.debug(
                             b"branch renamed from %s to %s at %d\n"
                             % (path, newpath, revnum)
                         )
                         path = newpath
                         break
                 if not paths:
                     revnum = None
                 return revnum, path
             finally:
                 stream.close()
 
         if not path.startswith(self.rootmodule):
             # Requests on foreign branches may be forbidden at server level
             self.ui.debug(b'ignoring foreign branch %r\n' % path)
             return None
 
         if stop is None:
             stop = svn.ra.get_latest_revnum(self.ra)
         try:
             prevmodule = self.reparent(b'')
             dirent = svn.ra.stat(self.ra, path.strip(b'/'), stop)
             self.reparent(prevmodule)
         except svn.core.SubversionException:
             dirent = None
         if not dirent:
             raise SvnPathNotFound(
                 _(b'%s not found up to revision %d') % (path, stop)
             )
 
         # stat() gives us the previous revision on this line of
         # development, but it might be in *another module*. Fetch the
         # log and detect renames down to the latest revision.
         revnum, realpath = findchanges(path, stop, dirent.created_rev)
         if revnum is None:
             # Tools like svnsync can create empty revision, when
             # synchronizing only a subtree for instance. These empty
             # revisions created_rev still have their original values
             # despite all changes having disappeared and can be
             # returned by ra.stat(), at least when stating the root
             # module. In that case, do not trust created_rev and scan
             # the whole history.
             revnum, realpath = findchanges(path, stop)
             if revnum is None:
                 self.ui.debug(b'ignoring empty branch %r\n' % realpath)
                 return None
 
         if not realpath.startswith(self.rootmodule):
             self.ui.debug(b'ignoring foreign branch %r\n' % realpath)
             return None
         return self.revid(revnum, realpath)
 
     def reparent(self, module):
         """Reparent the svn transport and return the previous parent."""
         if self.prevmodule == module:
             return module
         svnurl = self.baseurl + quote(module)
         prevmodule = self.prevmodule
         if prevmodule is None:
             prevmodule = b''
         self.ui.debug(b"reparent to %s\n" % svnurl)
         svn.ra.reparent(self.ra, svnurl)
         self.prevmodule = module
         return prevmodule
 
     def expandpaths(self, rev, paths, parents):
         changed, removed = set(), set()
         copies = {}
 
         new_module, revnum = revsplit(rev)[1:]
         if new_module != self.module:
             self.module = new_module
             self.reparent(self.module)
 
         progress = self.ui.makeprogress(
             _(b'scanning paths'), unit=_(b'paths'), total=len(paths)
         )
         for i, (path, ent) in enumerate(paths):
             progress.update(i, item=path)
             entrypath = self.getrelpath(path)
 
             kind = self._checkpath(entrypath, revnum)
             if kind == svn.core.svn_node_file:
                 changed.add(self.recode(entrypath))
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrom_path:
                     continue
                 self.ui.debug(
                     b"copied to %s from %s@%d\n"
                     % (entrypath, copyfrom_path, ent.copyfrom_rev)
                 )
                 copies[self.recode(entrypath)] = self.recode(copyfrom_path)
             elif kind == 0:  # gone, but had better be a deleted *file*
                 self.ui.debug(b"gone from %d\n" % ent.copyfrom_rev)
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 parentpath = pmodule + b"/" + entrypath
                 fromkind = self._checkpath(entrypath, prevnum, pmodule)
 
                 if fromkind == svn.core.svn_node_file:
                     removed.add(self.recode(entrypath))
                 elif fromkind == svn.core.svn_node_dir:
                     oroot = parentpath.strip(b'/')
                     nroot = path.strip(b'/')
                     children = self._iterfiles(oroot, prevnum)
                     for childpath in children:
                         childpath = childpath.replace(oroot, nroot)
                         childpath = self.getrelpath(b"/" + childpath, pmodule)
                         if childpath:
                             removed.add(self.recode(childpath))
                 else:
                     self.ui.debug(
                         b'unknown path in revision %d: %s\n' % (revnum, path)
                     )
             elif kind == svn.core.svn_node_dir:
                 if ent.action == b'M':
                     # If the directory just had a prop change,
                     # then we shouldn't need to look for its children.
                     continue
                 if ent.action == b'R' and parents:
                     # If a directory is replacing a file, mark the previous
                     # file as deleted
                     pmodule, prevnum = revsplit(parents[0])[1:]
                     pkind = self._checkpath(entrypath, prevnum, pmodule)
                     if pkind == svn.core.svn_node_file:
                         removed.add(self.recode(entrypath))
                     elif pkind == svn.core.svn_node_dir:
                         # We do not know what files were kept or removed,
                         # mark them all as changed.
                         for childpath in self._iterfiles(pmodule, prevnum):
                             childpath = self.getrelpath(b"/" + childpath)
                             if childpath:
                                 changed.add(self.recode(childpath))
 
                 for childpath in self._iterfiles(path, revnum):
                     childpath = self.getrelpath(b"/" + childpath)
                     if childpath:
                         changed.add(self.recode(childpath))
 
                 # Handle directory copies
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrompath:
                     continue
                 self.ui.debug(
                     b"mark %s came from %s:%d\n"
                     % (path, copyfrompath, ent.copyfrom_rev)
                 )
                 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
                 for childpath in children:
                     childpath = self.getrelpath(b"/" + childpath, pmodule)
                     if not childpath:
                         continue
                     copytopath = path + childpath[len(copyfrompath) :]
                     copytopath = self.getrelpath(copytopath)
                     copies[self.recode(copytopath)] = self.recode(childpath)
 
         progress.complete()
         changed.update(removed)
         return (list(changed), removed, copies)
 
     def _fetch_revisions(self, from_revnum, to_revnum):
         if from_revnum < to_revnum:
             from_revnum, to_revnum = to_revnum, from_revnum
 
         self.child_cset = None
 
         def parselogentry(orig_paths, revnum, author, date, message):
             """Return the parsed commit object or None, and True if
             the revision is a branch root.
             """
             self.ui.debug(
                 b"parsing revision %d (%d changes)\n"
                 % (revnum, len(orig_paths))
             )
 
             branched = False
             rev = self.revid(revnum)
             # branch log might return entries for a parent we already have
 
             if rev in self.commits or revnum < to_revnum:
                 return None, branched
 
             parents = []
             # check whether this revision is the start of a branch or part
             # of a branch renaming
             orig_paths = sorted(pycompat.iteritems(orig_paths))
             root_paths = [
                 (p, e) for p, e in orig_paths if self.module.startswith(p)
             ]
             if root_paths:
                 path, ent = root_paths[-1]
                 if ent.copyfrom_path:
                     branched = True
                     newpath = ent.copyfrom_path + self.module[len(path) :]
                     # ent.copyfrom_rev may not be the actual last revision
                     previd = self.latest(newpath, ent.copyfrom_rev)
                     if previd is not None:
                         prevmodule, prevnum = revsplit(previd)[1:]
                         if prevnum >= self.startrev:
                             parents = [previd]
                             self.ui.note(
                                 _(b'found parent of branch %s at %d: %s\n')
                                 % (self.module, prevnum, prevmodule)
                             )
                 else:
                     self.ui.debug(b"no copyfrom path, don't know what to do.\n")
 
             paths = []
             # filter out unrelated paths
             for path, ent in orig_paths:
                 if self.getrelpath(path) is None:
                     continue
                 paths.append((path, ent))
 
             date = parsesvndate(date)
             if self.ui.configbool(b'convert', b'localtimezone'):
                 date = makedatetimestamp(date[0])
 
             if message:
                 log = self.recode(message)
             else:
                 log = b''
 
             if author:
                 author = self.recode(author)
             else:
                 author = b''
 
             try:
                 branch = self.module.split(b"/")[-1]
                 if branch == self.trunkname:
                     branch = None
             except IndexError:
                 branch = None
 
             cset = commit(
                 author=author,
                 date=dateutil.datestr(date, b'%Y-%m-%d %H:%M:%S %1%2'),
                 desc=log,
                 parents=parents,
                 branch=branch,
                 rev=rev,
             )
 
             self.commits[rev] = cset
             # The parents list is *shared* among self.paths and the
             # commit object. Both will be updated below.
             self.paths[rev] = (paths, cset.parents)
             if self.child_cset and not self.child_cset.parents:
                 self.child_cset.parents[:] = [rev]
             self.child_cset = cset
             return cset, branched
 
         self.ui.note(
             _(b'fetching revision log for "%s" from %d to %d\n')
             % (self.module, from_revnum, to_revnum)
         )
 
         try:
             firstcset = None
             lastonbranch = False
             stream = self._getlog([self.module], from_revnum, to_revnum)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if revnum < self.startrev:
                         lastonbranch = True
                         break
                     if not paths:
                         self.ui.debug(b'revision %d has no entries\n' % revnum)
                         # If we ever leave the loop on an empty
                         # revision, do not try to get a parent branch
                         lastonbranch = lastonbranch or revnum == 0
                         continue
                     cset, lastonbranch = parselogentry(
                         paths, revnum, author, date, message
                     )
                     if cset:
                         firstcset = cset
                     if lastonbranch:
                         break
             finally:
                 stream.close()
 
             if not lastonbranch and firstcset and not firstcset.parents:
                 # The first revision of the sequence (the last fetched one)
                 # has invalid parents if not a branch root. Find the parent
                 # revision now, if any.
                 try:
                     firstrevnum = self.revnum(firstcset.rev)
                     if firstrevnum > 1:
                         latest = self.latest(self.module, firstrevnum - 1)
                         if latest:
                             firstcset.parents.append(latest)
                 except SvnPathNotFound:
                     pass
         except svn.core.SubversionException as xxx_todo_changeme:
             (inst, num) = xxx_todo_changeme.args
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise error.Abort(
                     _(b'svn: branch has no revision %s') % to_revnum
                 )
             raise
 
     def getfile(self, file, rev):
         # TODO: ra.get_file transmits the whole file instead of diffs.
         if file in self.removed:
             return None, None
         try:
             new_module, revnum = revsplit(rev)[1:]
             if self.module != new_module:
                 self.module = new_module
                 self.reparent(self.module)
             io = stringio()
             info = svn.ra.get_file(self.ra, file, revnum, io)
             data = io.getvalue()
             # ra.get_file() seems to keep a reference on the input buffer
             # preventing collection. Release it explicitly.
             io.close()
             if isinstance(info, list):
                 info = info[-1]
             mode = (b"svn:executable" in info) and b'x' or b''
             mode = (b"svn:special" in info) and b'l' or mode
         except svn.core.SubversionException as e:
             notfound = (
                 svn.core.SVN_ERR_FS_NOT_FOUND,
                 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND,
             )
             if e.apr_err in notfound:  # File not found
                 return None, None
             raise
         if mode == b'l':
             link_prefix = b"link "
             if data.startswith(link_prefix):
                 data = data[len(link_prefix) :]
         return data, mode
 
     def _iterfiles(self, path, revnum):
         """Enumerate all files in path at revnum, recursively."""
         path = path.strip(b'/')
         pool = svn.core.Pool()
         rpath = b'/'.join([self.baseurl, quote(path)]).strip(b'/')
         entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)
         if path:
             path += b'/'
         return (
             (path + p)
             for p, e in pycompat.iteritems(entries)
             if e.kind == svn.core.svn_node_file
         )
 
     def getrelpath(self, path, module=None):
         if module is None:
             module = self.module
         # Given the repository url of this wc, say
         #   "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
         # extract the "entry" portion (a relative path) from what
         # svn log --xml says, i.e.
         #   "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
         # that is to say "tests/PloneTestCase.py"
         if path.startswith(module):
             relative = path.rstrip(b'/')[len(module) :]
             if relative.startswith(b'/'):
                 return relative[1:]
             elif relative == b'':
                 return relative
 
         # The path is outside our tracked tree...
         self.ui.debug(
             b'%r is not under %r, ignoring\n'
             % (pycompat.bytestr(path), pycompat.bytestr(module))
         )
         return None
 
     def _checkpath(self, path, revnum, module=None):
         if module is not None:
             prevmodule = self.reparent(b'')
             path = module + b'/' + path
         try:
             # ra.check_path does not like leading slashes very much, it leads
             # to PROPFIND subversion errors
             return svn.ra.check_path(self.ra, path.strip(b'/'), revnum)
         finally:
             if module is not None:
                 self.reparent(prevmodule)
 
     def _getlog(
         self,
         paths,
         start,
         end,
         limit=0,
         discover_changed_paths=True,
         strict_node_history=False,
     ):
         # Normalize path names, svn >= 1.5 only wants paths relative to
         # supplied URL
         relpaths = []
         for p in paths:
             if not p.startswith(b'/'):
                 p = self.module + b'/' + p
             relpaths.append(p.strip(b'/'))
         args = [
             self.baseurl,
             relpaths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
         ]
         # developer config: convert.svn.debugsvnlog
         if not self.ui.configbool(b'convert', b'svn.debugsvnlog'):
             return directlogstream(*args)
         arg = encodeargs(args)
         hgexe = procutil.hgexecutable()
         cmd = b'%s debugsvnlog' % procutil.shellquote(hgexe)
         stdin, stdout = procutil.popen2(cmd)
         stdin.write(arg)
         try:
             stdin.close()
         except IOError:
             raise error.Abort(
                 _(
                     b'Mercurial failed to run itself, check'
                     b' hg executable is in PATH'
                 )
             )
         return logstream(stdout)
 
 
 pre_revprop_change_template = b'''#!/bin/sh
 
 REPOS="$1"
 REV="$2"
 USER="$3"
 PROPNAME="$4"
 ACTION="$5"
 
 %(rules)s
 
 echo "Changing prohibited revision property" >&2
 exit 1
 '''
 
 
 def gen_pre_revprop_change_hook(prop_actions_allowed):
     rules = []
     for action, propname in prop_actions_allowed:
         rules.append(
             (
                 b'if [ "$ACTION" = "%s" -a "$PROPNAME" = "%s" ]; '
                 b'then exit 0; fi'
             )
             % (action, propname)
         )
     return pre_revprop_change_template % {b'rules': b'\n'.join(rules)}
 
 
 class svn_sink(converter_sink, commandline):
     commit_re = re.compile(br'Committed revision (\d+).', re.M)
     uuid_re = re.compile(br'Repository UUID:\s*(\S+)', re.M)
 
     def prerun(self):
         if self.wc:
             os.chdir(self.wc)
 
     def postrun(self):
         if self.wc:
             os.chdir(self.cwd)
 
     def join(self, name):
         return os.path.join(self.wc, b'.svn', name)
 
     def revmapfile(self):
         return self.join(b'hg-shamap')
 
     def authorfile(self):
         return self.join(b'hg-authormap')
 
     def __init__(self, ui, repotype, path):
 
         converter_sink.__init__(self, ui, repotype, path)
         commandline.__init__(self, ui, b'svn')
         self.delete = []
         self.setexec = []
         self.delexec = []
         self.copies = []
         self.wc = None
         self.cwd = encoding.getcwd()
 
         created = False
         if os.path.isfile(os.path.join(path, b'.svn', b'entries')):
             self.wc = os.path.realpath(path)
             self.run0(b'update')
         else:
             if not re.search(br'^(file|http|https|svn|svn\+ssh)://', path):
                 path = os.path.realpath(path)
                 if os.path.isdir(os.path.dirname(path)):
                     if not os.path.exists(
                         os.path.join(path, b'db', b'fs-type')
                     ):
                         ui.status(
                             _(b"initializing svn repository '%s'\n")
                             % os.path.basename(path)
                         )
                         commandline(ui, b'svnadmin').run0(b'create', path)
                         created = path
                     path = util.normpath(path)
                     if not path.startswith(b'/'):
                         path = b'/' + path
                     path = b'file://' + path
 
             wcpath = os.path.join(
                 encoding.getcwd(), os.path.basename(path) + b'-wc'
             )
             ui.status(
                 _(b"initializing svn working copy '%s'\n")
                 % os.path.basename(wcpath)
             )
             self.run0(b'checkout', path, wcpath)
 
             self.wc = wcpath
         self.opener = vfsmod.vfs(self.wc)
         self.wopener = vfsmod.vfs(self.wc)
         self.childmap = mapfile(ui, self.join(b'hg-childmap'))
         if util.checkexec(self.wc):
             self.is_exec = util.isexec
         else:
             self.is_exec = None
 
         if created:
             prop_actions_allowed = [
                 (b'M', b'svn:log'),
                 (b'A', b'hg:convert-branch'),
                 (b'A', b'hg:convert-rev'),
             ]
 
             if self.ui.configbool(
                 b'convert', b'svn.dangerous-set-commit-dates'
             ):
                 prop_actions_allowed.append((b'M', b'svn:date'))
 
             hook = os.path.join(created, b'hooks', b'pre-revprop-change')
             fp = open(hook, b'wb')
             fp.write(gen_pre_revprop_change_hook(prop_actions_allowed))
             fp.close()
             util.setflags(hook, False, True)
 
         output = self.run0(b'info')
         self.uuid = self.uuid_re.search(output).group(1).strip()
 
     def wjoin(self, *names):
         return os.path.join(self.wc, *names)
 
     @propertycache
     def manifest(self):
         # As of svn 1.7, the "add" command fails when receiving
         # already tracked entries, so we have to track and filter them
         # ourselves.
         m = set()
         output = self.run0(b'ls', recursive=True, xml=True)
         doc = xml.dom.minidom.parseString(output)
         for e in doc.getElementsByTagName('entry'):
             for n in e.childNodes:
                 if n.nodeType != n.ELEMENT_NODE or n.tagName != 'name':
                     continue
                 name = ''.join(
                     c.data for c in n.childNodes if c.nodeType == c.TEXT_NODE
                 )
                 # Entries are compared with names coming from
                 # mercurial, so bytes with undefined encoding. Our
                 # best bet is to assume they are in local
                 # encoding. They will be passed to command line calls
                 # later anyway, so they better be.
                 m.add(encoding.unitolocal(name))
                 break
         return m
 
     def putfile(self, filename, flags, data):
         if b'l' in flags:
             self.wopener.symlink(data, filename)
         else:
             try:
                 if os.path.islink(self.wjoin(filename)):
                     os.unlink(filename)
             except OSError:
                 pass
 
             if self.is_exec:
                 # We need to check executability of the file before the change,
                 # because `vfs.write` is able to reset exec bit.
                 wasexec = False
                 if os.path.exists(self.wjoin(filename)):
                     wasexec = self.is_exec(self.wjoin(filename))
 
             self.wopener.write(filename, data)
 
             if self.is_exec:
                 if wasexec:
                     if b'x' not in flags:
                         self.delexec.append(filename)
                 else:
                     if b'x' in flags:
                         self.setexec.append(filename)
                 util.setflags(self.wjoin(filename), False, b'x' in flags)
 
     def _copyfile(self, source, dest):
         # SVN's copy command pukes if the destination file exists, but
         # our copyfile method expects to record a copy that has
         # already occurred.  Cross the semantic gap.
         wdest = self.wjoin(dest)
         exists = os.path.lexists(wdest)
         if exists:
             fd, tempname = pycompat.mkstemp(
                 prefix=b'hg-copy-', dir=os.path.dirname(wdest)
             )
             os.close(fd)
             os.unlink(tempname)
             os.rename(wdest, tempname)
         try:
             self.run0(b'copy', source, dest)
         finally:
             self.manifest.add(dest)
             if exists:
                 try:
                     os.unlink(wdest)
                 except OSError:
                     pass
                 os.rename(tempname, wdest)
 
     def dirs_of(self, files):
         dirs = set()
         for f in files:
             if os.path.isdir(self.wjoin(f)):
                 dirs.add(f)
             i = len(f)
             for i in iter(lambda: f.rfind(b'/', 0, i), -1):
                 dirs.add(f[:i])
         return dirs
 
     def add_dirs(self, files):
         add_dirs = [
             d for d in sorted(self.dirs_of(files)) if d not in self.manifest
         ]
         if add_dirs:
             self.manifest.update(add_dirs)
             self.xargs(add_dirs, b'add', non_recursive=True, quiet=True)
         return add_dirs
 
     def add_files(self, files):
         files = [f for f in files if f not in self.manifest]
         if files:
             self.manifest.update(files)
             self.xargs(files, b'add', quiet=True)
         return files
 
     def addchild(self, parent, child):
         self.childmap[parent] = child
 
     def revid(self, rev):
         return b"svn:%s@%s" % (self.uuid, rev)
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         for parent in parents:
             try:
                 return self.revid(self.childmap[parent])
             except KeyError:
                 pass
 
         # Apply changes to working copy
         for f, v in files:
             data, mode = source.getfile(f, v)
             if data is None:
                 self.delete.append(f)
             else:
                 self.putfile(f, mode, data)
                 if f in copies:
                     self.copies.append([copies[f], f])
         if full:
             self.delete.extend(sorted(self.manifest.difference(files)))
         files = [f[0] for f in files]
 
         entries = set(self.delete)
         files = frozenset(files)
         entries.update(self.add_dirs(files.difference(entries)))
         if self.copies:
             for s, d in self.copies:
                 self._copyfile(s, d)
             self.copies = []
         if self.delete:
             self.xargs(self.delete, b'delete')
             for f in self.delete:
                 self.manifest.remove(f)
             self.delete = []
         entries.update(self.add_files(files.difference(entries)))
         if self.delexec:
             self.xargs(self.delexec, b'propdel', b'svn:executable')
             self.delexec = []
         if self.setexec:
             self.xargs(self.setexec, b'propset', b'svn:executable', b'*')
             self.setexec = []
 
         fd, messagefile = pycompat.mkstemp(prefix=b'hg-convert-')
         fp = os.fdopen(fd, 'wb')
         fp.write(util.tonativeeol(commit.desc))
         fp.close()
         try:
             output = self.run0(
                 b'commit',
                 username=stringutil.shortuser(commit.author),
                 file=messagefile,
                 encoding=b'utf-8',
             )
             try:
                 rev = self.commit_re.search(output).group(1)
             except AttributeError:
                 if not files:
                     return parents[0] if parents else b'None'
                 self.ui.warn(_(b'unexpected svn output:\n'))
                 self.ui.warn(output)
                 raise error.Abort(_(b'unable to cope with svn output'))
             if commit.rev:
                 self.run(
                     b'propset',
                     b'hg:convert-rev',
                     commit.rev,
                     revprop=True,
                     revision=rev,
                 )
             if commit.branch and commit.branch != b'default':
                 self.run(
                     b'propset',
                     b'hg:convert-branch',
                     commit.branch,
                     revprop=True,
                     revision=rev,
                 )
 
             if self.ui.configbool(
                 b'convert', b'svn.dangerous-set-commit-dates'
             ):
                 # Subverson always uses UTC to represent date and time
                 date = dateutil.parsedate(commit.date)
                 date = (date[0], 0)
 
                 # The only way to set date and time for svn commit is to use propset after commit is done
                 self.run(
                     b'propset',
                     b'svn:date',
                     formatsvndate(date),
                     revprop=True,
                     revision=rev,
                 )
 
             for parent in parents:
                 self.addchild(parent, rev)
             return self.revid(rev)
         finally:
             os.unlink(messagefile)
 
     def puttags(self, tags):
         self.ui.warn(_(b'writing Subversion tags is not yet implemented\n'))
         return None, None
 
     def hascommitfrommap(self, rev):
         # We trust that revisions referenced in a map still is present
         # TODO: implement something better if necessary and feasible
         return True
 
     def hascommitforsplicemap(self, rev):
         # This is not correct as one can convert to an existing subversion
         # repository and childmap would not list all revisions. Too bad.
         if rev in self.childmap:
             return True
         raise error.Abort(
             _(
                 b'splice map revision %s not found in subversion '
                 b'child map (revision lookups are not implemented)'
             )
             % rev
         )

hgext/histedit.py

@@ -1,2690 +1,2690 b''
 # histedit.py - interactive history editing for mercurial
 #
 # Copyright 2009 Augie Fackler <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """interactive history editing
 
 With this extension installed, Mercurial gains one new command: histedit. Usage
 is as follows, assuming the following history::
 
  @  3[tip]   7c2fd3b9020c   2009-04-27 18:04 -0500   durin42
  |    Add delta
  |
  o  2   030b686bedc4   2009-04-27 18:04 -0500   durin42
  |    Add gamma
  |
  o  1   c561b4e977df   2009-04-27 18:04 -0500   durin42
  |    Add beta
  |
  o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
       Add alpha
 
 If you were to run ``hg histedit c561b4e977df``, you would see the following
 file open in your editor::
 
  pick c561b4e977df Add beta
  pick 030b686bedc4 Add gamma
  pick 7c2fd3b9020c Add delta
 
  # Edit history between c561b4e977df and 7c2fd3b9020c
  #
  # Commits are listed from least to most recent
  #
  # Commands:
  #  p, pick = use commit
  #  e, edit = use commit, but allow edits before making new commit
  #  f, fold = use commit, but combine it with the one above
  #  r, roll = like fold, but discard this commit's description and date
  #  d, drop = remove commit from history
  #  m, mess = edit commit message without changing commit content
  #  b, base = checkout changeset and apply further changesets from there
  #
 
 In this file, lines beginning with ``#`` are ignored. You must specify a rule
 for each revision in your history. For example, if you had meant to add gamma
 before beta, and then wanted to add delta in the same revision as beta, you
 would reorganize the file to look like this::
 
  pick 030b686bedc4 Add gamma
  pick c561b4e977df Add beta
  fold 7c2fd3b9020c Add delta
 
  # Edit history between c561b4e977df and 7c2fd3b9020c
  #
  # Commits are listed from least to most recent
  #
  # Commands:
  #  p, pick = use commit
  #  e, edit = use commit, but allow edits before making new commit
  #  f, fold = use commit, but combine it with the one above
  #  r, roll = like fold, but discard this commit's description and date
  #  d, drop = remove commit from history
  #  m, mess = edit commit message without changing commit content
  #  b, base = checkout changeset and apply further changesets from there
  #
 
 At which point you close the editor and ``histedit`` starts working. When you
 specify a ``fold`` operation, ``histedit`` will open an editor when it folds
 those revisions together, offering you a chance to clean up the commit message::
 
  Add beta
  ***
  Add delta
 
 Edit the commit message to your liking, then close the editor. The date used
 for the commit will be the later of the two commits' dates. For this example,
 let's assume that the commit message was changed to ``Add beta and delta.``
 After histedit has run and had a chance to remove any old or temporary
 revisions it needed, the history looks like this::
 
  @  2[tip]   989b4d060121   2009-04-27 18:04 -0500   durin42
  |    Add beta and delta.
  |
  o  1   081603921c3f   2009-04-27 18:04 -0500   durin42
  |    Add gamma
  |
  o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
       Add alpha
 
 Note that ``histedit`` does *not* remove any revisions (even its own temporary
 ones) until after it has completed all the editing operations, so it will
 probably perform several strip operations when it's done. For the above example,
 it had to run strip twice. Strip can be slow depending on a variety of factors,
 so you might need to be a little patient. You can choose to keep the original
 revisions by passing the ``--keep`` flag.
 
 The ``edit`` operation will drop you back to a command prompt,
 allowing you to edit files freely, or even use ``hg record`` to commit
 some changes as a separate commit. When you're done, any remaining
 uncommitted changes will be committed as well. When done, run ``hg
 histedit --continue`` to finish this step. If there are uncommitted
 changes, you'll be prompted for a new commit message, but the default
 commit message will be the original message for the ``edit`` ed
 revision, and the date of the original commit will be preserved.
 
 The ``message`` operation will give you a chance to revise a commit
 message without changing the contents. It's a shortcut for doing
 ``edit`` immediately followed by `hg histedit --continue``.
 
 If ``histedit`` encounters a conflict when moving a revision (while
 handling ``pick`` or ``fold``), it'll stop in a similar manner to
 ``edit`` with the difference that it won't prompt you for a commit
 message when done. If you decide at this point that you don't like how
 much work it will be to rearrange history, or that you made a mistake,
 you can use ``hg histedit --abort`` to abandon the new changes you
 have made and return to the state before you attempted to edit your
 history.
 
 If we clone the histedit-ed example repository above and add four more
 changes, such that we have the following history::
 
    @  6[tip]   038383181893   2009-04-27 18:04 -0500   stefan
    |    Add theta
    |
    o  5   140988835471   2009-04-27 18:04 -0500   stefan
    |    Add eta
    |
    o  4   122930637314   2009-04-27 18:04 -0500   stefan
    |    Add zeta
    |
    o  3   836302820282   2009-04-27 18:04 -0500   stefan
    |    Add epsilon
    |
    o  2   989b4d060121   2009-04-27 18:04 -0500   durin42
    |    Add beta and delta.
    |
    o  1   081603921c3f   2009-04-27 18:04 -0500   durin42
    |    Add gamma
    |
    o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
         Add alpha
 
 If you run ``hg histedit --outgoing`` on the clone then it is the same
 as running ``hg histedit 836302820282``. If you need plan to push to a
 repository that Mercurial does not detect to be related to the source
 repo, you can add a ``--force`` option.
 
 Config
 ------
 
 Histedit rule lines are truncated to 80 characters by default. You
 can customize this behavior by setting a different length in your
 configuration file::
 
   [histedit]
   linelen = 120      # truncate rule lines at 120 characters
 
 The summary of a change can be customized as well::
 
   [histedit]
   summary-template = '{rev} {bookmarks} {desc|firstline}'
 
 The customized summary should be kept short enough that rule lines
 will fit in the configured line length. See above if that requires
 customization.
 
 ``hg histedit`` attempts to automatically choose an appropriate base
 revision to use. To change which base revision is used, define a
 revset in your configuration file::
 
   [histedit]
   defaultrev = only(.) & draft()
 
 By default each edited revision needs to be present in histedit commands.
 To remove revision you need to use ``drop`` operation. You can configure
 the drop to be implicit for missing commits by adding::
 
   [histedit]
   dropmissing = True
 
 By default, histedit will close the transaction after each action. For
 performance purposes, you can configure histedit to use a single transaction
 across the entire histedit. WARNING: This setting introduces a significant risk
 of losing the work you've done in a histedit if the histedit aborts
 unexpectedly::
 
   [histedit]
   singletransaction = True
 
 """
 
 from __future__ import absolute_import
 
 # chistedit dependencies that are not available everywhere
 try:
     import fcntl
     import termios
 except ImportError:
     fcntl = None
     termios = None
 
 import functools
 import os
+import pickle
 import struct
 
 from mercurial.i18n import _
 from mercurial.pycompat import (
     getattr,
     open,
 )
 from mercurial.node import (
     bin,
     hex,
     short,
 )
 from mercurial import (
     bundle2,
     cmdutil,
     context,
     copies,
     destutil,
     discovery,
     encoding,
     error,
     exchange,
     extensions,
     hg,
     logcmdutil,
     merge as mergemod,
     mergestate as mergestatemod,
     mergeutil,
     obsolete,
     pycompat,
     registrar,
     repair,
     rewriteutil,
     scmutil,
     state as statemod,
     util,
 )
 from mercurial.utils import (
     dateutil,
     stringutil,
     urlutil,
 )
 
-pickle = util.pickle
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 configitem(
     b'experimental',
     b'histedit.autoverb',
     default=False,
 )
 configitem(
     b'histedit',
     b'defaultrev',
     default=None,
 )
 configitem(
     b'histedit',
     b'dropmissing',
     default=False,
 )
 configitem(
     b'histedit',
     b'linelen',
     default=80,
 )
 configitem(
     b'histedit',
     b'singletransaction',
     default=False,
 )
 configitem(
     b'ui',
     b'interface.histedit',
     default=None,
 )
 configitem(b'histedit', b'summary-template', default=b'{rev} {desc|firstline}')
 # TODO: Teach the text-based histedit interface to respect this config option
 # before we make it non-experimental.
 configitem(
     b'histedit', b'later-commits-first', default=False, experimental=True
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 actiontable = {}
 primaryactions = set()
 secondaryactions = set()
 tertiaryactions = set()
 internalactions = set()
 
 
 def geteditcomment(ui, first, last):
     """construct the editor comment
     The comment includes::
      - an intro
      - sorted primary commands
      - sorted short commands
      - sorted long commands
      - additional hints
 
     Commands are only included once.
     """
     intro = _(
         b"""Edit history between %s and %s
 
 Commits are listed from least to most recent
 
 You can reorder changesets by reordering the lines
 
 Commands:
 """
     )
     actions = []
 
     def addverb(v):
         a = actiontable[v]
         lines = a.message.split(b"\n")
         if len(a.verbs):
             v = b', '.join(sorted(a.verbs, key=lambda v: len(v)))
         actions.append(b" %s = %s" % (v, lines[0]))
         actions.extend([b'  %s'] * (len(lines) - 1))
 
     for v in (
         sorted(primaryactions)
         + sorted(secondaryactions)
         + sorted(tertiaryactions)
     ):
         addverb(v)
     actions.append(b'')
 
     hints = []
     if ui.configbool(b'histedit', b'dropmissing'):
         hints.append(
             b"Deleting a changeset from the list "
             b"will DISCARD it from the edited history!"
         )
 
     lines = (intro % (first, last)).split(b'\n') + actions + hints
 
     return b''.join([b'# %s\n' % l if l else b'#\n' for l in lines])
 
 
 class histeditstate(object):
     def __init__(self, repo):
         self.repo = repo
         self.actions = None
         self.keep = None
         self.topmost = None
         self.parentctxnode = None
         self.lock = None
         self.wlock = None
         self.backupfile = None
         self.stateobj = statemod.cmdstate(repo, b'histedit-state')
         self.replacements = []
 
     def read(self):
         """Load histedit state from disk and set fields appropriately."""
         if not self.stateobj.exists():
             cmdutil.wrongtooltocontinue(self.repo, _(b'histedit'))
 
         data = self._read()
 
         self.parentctxnode = data[b'parentctxnode']
         actions = parserules(data[b'rules'], self)
         self.actions = actions
         self.keep = data[b'keep']
         self.topmost = data[b'topmost']
         self.replacements = data[b'replacements']
         self.backupfile = data[b'backupfile']
 
     def _read(self):
         fp = self.repo.vfs.read(b'histedit-state')
         if fp.startswith(b'v1\n'):
             data = self._load()
             parentctxnode, rules, keep, topmost, replacements, backupfile = data
         else:
             data = pickle.loads(fp)
             parentctxnode, rules, keep, topmost, replacements = data
             backupfile = None
         rules = b"\n".join([b"%s %s" % (verb, rest) for [verb, rest] in rules])
 
         return {
             b'parentctxnode': parentctxnode,
             b"rules": rules,
             b"keep": keep,
             b"topmost": topmost,
             b"replacements": replacements,
             b"backupfile": backupfile,
         }
 
     def write(self, tr=None):
         if tr:
             tr.addfilegenerator(
                 b'histedit-state',
                 (b'histedit-state',),
                 self._write,
                 location=b'plain',
             )
         else:
             with self.repo.vfs(b"histedit-state", b"w") as f:
                 self._write(f)
 
     def _write(self, fp):
         fp.write(b'v1\n')
         fp.write(b'%s\n' % hex(self.parentctxnode))
         fp.write(b'%s\n' % hex(self.topmost))
         fp.write(b'%s\n' % (b'True' if self.keep else b'False'))
         fp.write(b'%d\n' % len(self.actions))
         for action in self.actions:
             fp.write(b'%s\n' % action.tostate())
         fp.write(b'%d\n' % len(self.replacements))
         for replacement in self.replacements:
             fp.write(
                 b'%s%s\n'
                 % (
                     hex(replacement[0]),
                     b''.join(hex(r) for r in replacement[1]),
                 )
             )
         backupfile = self.backupfile
         if not backupfile:
             backupfile = b''
         fp.write(b'%s\n' % backupfile)
 
     def _load(self):
         fp = self.repo.vfs(b'histedit-state', b'r')
         lines = [l[:-1] for l in fp.readlines()]
 
         index = 0
         lines[index]  # version number
         index += 1
 
         parentctxnode = bin(lines[index])
         index += 1
 
         topmost = bin(lines[index])
         index += 1
 
         keep = lines[index] == b'True'
         index += 1
 
         # Rules
         rules = []
         rulelen = int(lines[index])
         index += 1
         for i in pycompat.xrange(rulelen):
             ruleaction = lines[index]
             index += 1
             rule = lines[index]
             index += 1
             rules.append((ruleaction, rule))
 
         # Replacements
         replacements = []
         replacementlen = int(lines[index])
         index += 1
         for i in pycompat.xrange(replacementlen):
             replacement = lines[index]
             original = bin(replacement[:40])
             succ = [
                 bin(replacement[i : i + 40])
                 for i in range(40, len(replacement), 40)
             ]
             replacements.append((original, succ))
             index += 1
 
         backupfile = lines[index]
         index += 1
 
         fp.close()
 
         return parentctxnode, rules, keep, topmost, replacements, backupfile
 
     def clear(self):
         if self.inprogress():
             self.repo.vfs.unlink(b'histedit-state')
 
     def inprogress(self):
         return self.repo.vfs.exists(b'histedit-state')
 
 
 class histeditaction(object):
     def __init__(self, state, node):
         self.state = state
         self.repo = state.repo
         self.node = node
 
     @classmethod
     def fromrule(cls, state, rule):
         """Parses the given rule, returning an instance of the histeditaction."""
         ruleid = rule.strip().split(b' ', 1)[0]
         # ruleid can be anything from rev numbers, hashes, "bookmarks" etc
         # Check for validation of rule ids and get the rulehash
         try:
             rev = bin(ruleid)
         except TypeError:
             try:
                 _ctx = scmutil.revsingle(state.repo, ruleid)
                 rulehash = _ctx.hex()
                 rev = bin(rulehash)
             except error.RepoLookupError:
                 raise error.ParseError(_(b"invalid changeset %s") % ruleid)
         return cls(state, rev)
 
     def verify(self, prev, expected, seen):
         """Verifies semantic correctness of the rule"""
         repo = self.repo
         ha = hex(self.node)
         self.node = scmutil.resolvehexnodeidprefix(repo, ha)
         if self.node is None:
             raise error.ParseError(_(b'unknown changeset %s listed') % ha[:12])
         self._verifynodeconstraints(prev, expected, seen)
 
     def _verifynodeconstraints(self, prev, expected, seen):
         # by default command need a node in the edited list
         if self.node not in expected:
             raise error.ParseError(
                 _(b'%s "%s" changeset was not a candidate')
                 % (self.verb, short(self.node)),
                 hint=_(b'only use listed changesets'),
             )
         # and only one command per node
         if self.node in seen:
             raise error.ParseError(
                 _(b'duplicated command for changeset %s') % short(self.node)
             )
 
     def torule(self):
         """build a histedit rule line for an action
 
         by default lines are in the form:
         <hash> <rev> <summary>
         """
         ctx = self.repo[self.node]
         ui = self.repo.ui
         # We don't want color codes in the commit message template, so
         # disable the label() template function while we render it.
         with ui.configoverride(
             {(b'templatealias', b'label(l,x)'): b"x"}, b'histedit'
         ):
             summary = cmdutil.rendertemplate(
                 ctx, ui.config(b'histedit', b'summary-template')
             )
         # Handle the fact that `''.splitlines() => []`
         summary = summary.splitlines()[0] if summary else b''
         line = b'%s %s %s' % (self.verb, ctx, summary)
         # trim to 75 columns by default so it's not stupidly wide in my editor
         # (the 5 more are left for verb)
         maxlen = self.repo.ui.configint(b'histedit', b'linelen')
         maxlen = max(maxlen, 22)  # avoid truncating hash
         return stringutil.ellipsis(line, maxlen)
 
     def tostate(self):
         """Print an action in format used by histedit state files
         (the first line is a verb, the remainder is the second)
         """
         return b"%s\n%s" % (self.verb, hex(self.node))
 
     def run(self):
         """Runs the action. The default behavior is simply apply the action's
         rulectx onto the current parentctx."""
         self.applychange()
         self.continuedirty()
         return self.continueclean()
 
     def applychange(self):
         """Applies the changes from this action's rulectx onto the current
         parentctx, but does not commit them."""
         repo = self.repo
         rulectx = repo[self.node]
         with repo.ui.silent():
             hg.update(repo, self.state.parentctxnode, quietempty=True)
         stats = applychanges(repo.ui, repo, rulectx, {})
         repo.dirstate.setbranch(rulectx.branch())
         if stats.unresolvedcount:
             raise error.InterventionRequired(
                 _(b'Fix up the change (%s %s)') % (self.verb, short(self.node)),
                 hint=_(b'hg histedit --continue to resume'),
             )
 
     def continuedirty(self):
         """Continues the action when changes have been applied to the working
         copy. The default behavior is to commit the dirty changes."""
         repo = self.repo
         rulectx = repo[self.node]
 
         editor = self.commiteditor()
         commit = commitfuncfor(repo, rulectx)
         if repo.ui.configbool(b'rewrite', b'update-timestamp'):
             date = dateutil.makedate()
         else:
             date = rulectx.date()
         commit(
             text=rulectx.description(),
             user=rulectx.user(),
             date=date,
             extra=rulectx.extra(),
             editor=editor,
         )
 
     def commiteditor(self):
         """The editor to be used to edit the commit message."""
         return False
 
     def continueclean(self):
         """Continues the action when the working copy is clean. The default
         behavior is to accept the current commit as the new version of the
         rulectx."""
         ctx = self.repo[b'.']
         if ctx.node() == self.state.parentctxnode:
             self.repo.ui.warn(
                 _(b'%s: skipping changeset (no changes)\n') % short(self.node)
             )
             return ctx, [(self.node, tuple())]
         if ctx.node() == self.node:
             # Nothing changed
             return ctx, []
         return ctx, [(self.node, (ctx.node(),))]
 
 
 def commitfuncfor(repo, src):
     """Build a commit function for the replacement of <src>
 
     This function ensure we apply the same treatment to all changesets.
 
     - Add a 'histedit_source' entry in extra.
 
     Note that fold has its own separated logic because its handling is a bit
     different and not easily factored out of the fold method.
     """
     phasemin = src.phase()
 
     def commitfunc(**kwargs):
         overrides = {(b'phases', b'new-commit'): phasemin}
         with repo.ui.configoverride(overrides, b'histedit'):
             extra = kwargs.get('extra', {}).copy()
             extra[b'histedit_source'] = src.hex()
             kwargs['extra'] = extra
             return repo.commit(**kwargs)
 
     return commitfunc
 
 
 def applychanges(ui, repo, ctx, opts):
     """Merge changeset from ctx (only) in the current working directory"""
     if ctx.p1().node() == repo.dirstate.p1():
         # edits are "in place" we do not need to make any merge,
         # just applies changes on parent for editing
         with ui.silent():
             cmdutil.revert(ui, repo, ctx, all=True)
             stats = mergemod.updateresult(0, 0, 0, 0)
     else:
         try:
             # ui.forcemerge is an internal variable, do not document
             repo.ui.setconfig(
                 b'ui', b'forcemerge', opts.get(b'tool', b''), b'histedit'
             )
             stats = mergemod.graft(
                 repo,
                 ctx,
                 labels=[
                     b'already edited',
                     b'current change',
                     b'parent of current change',
                 ],
             )
         finally:
             repo.ui.setconfig(b'ui', b'forcemerge', b'', b'histedit')
     return stats
 
 
 def collapse(repo, firstctx, lastctx, commitopts, skipprompt=False):
     """collapse the set of revisions from first to last as new one.
 
     Expected commit options are:
         - message
         - date
         - username
     Commit message is edited in all cases.
 
     This function works in memory."""
     ctxs = list(repo.set(b'%d::%d', firstctx.rev(), lastctx.rev()))
     if not ctxs:
         return None
     for c in ctxs:
         if not c.mutable():
             raise error.ParseError(
                 _(b"cannot fold into public change %s") % short(c.node())
             )
     base = firstctx.p1()
 
     # commit a new version of the old changeset, including the update
     # collect all files which might be affected
     files = set()
     for ctx in ctxs:
         files.update(ctx.files())
 
     # Recompute copies (avoid recording a -> b -> a)
     copied = copies.pathcopies(base, lastctx)
 
     # prune files which were reverted by the updates
     files = [f for f in files if not cmdutil.samefile(f, lastctx, base)]
     # commit version of these files as defined by head
     headmf = lastctx.manifest()
 
     def filectxfn(repo, ctx, path):
         if path in headmf:
             fctx = lastctx[path]
             flags = fctx.flags()
             mctx = context.memfilectx(
                 repo,
                 ctx,
                 fctx.path(),
                 fctx.data(),
                 islink=b'l' in flags,
                 isexec=b'x' in flags,
                 copysource=copied.get(path),
             )
             return mctx
         return None
 
     if commitopts.get(b'message'):
         message = commitopts[b'message']
     else:
         message = firstctx.description()
     user = commitopts.get(b'user')
     date = commitopts.get(b'date')
     extra = commitopts.get(b'extra')
 
     parents = (firstctx.p1().node(), firstctx.p2().node())
     editor = None
     if not skipprompt:
         editor = cmdutil.getcommiteditor(edit=True, editform=b'histedit.fold')
     new = context.memctx(
         repo,
         parents=parents,
         text=message,
         files=files,
         filectxfn=filectxfn,
         user=user,
         date=date,
         extra=extra,
         editor=editor,
     )
     return repo.commitctx(new)
 
 
 def _isdirtywc(repo):
     return repo[None].dirty(missing=True)
 
 
 def abortdirty():
     raise error.StateError(
         _(b'working copy has pending changes'),
         hint=_(
             b'amend, commit, or revert them and run histedit '
             b'--continue, or abort with histedit --abort'
         ),
     )
 
 
 def action(verbs, message, priority=False, internal=False):
     def wrap(cls):
         assert not priority or not internal
         verb = verbs[0]
         if priority:
             primaryactions.add(verb)
         elif internal:
             internalactions.add(verb)
         elif len(verbs) > 1:
             secondaryactions.add(verb)
         else:
             tertiaryactions.add(verb)
 
         cls.verb = verb
         cls.verbs = verbs
         cls.message = message
         for verb in verbs:
             actiontable[verb] = cls
         return cls
 
     return wrap
 
 
 @action([b'pick', b'p'], _(b'use commit'), priority=True)
 class pick(histeditaction):
     def run(self):
         rulectx = self.repo[self.node]
         if rulectx.p1().node() == self.state.parentctxnode:
             self.repo.ui.debug(b'node %s unchanged\n' % short(self.node))
             return rulectx, []
 
         return super(pick, self).run()
 
 
 @action(
     [b'edit', b'e'],
     _(b'use commit, but allow edits before making new commit'),
     priority=True,
 )
 class edit(histeditaction):
     def run(self):
         repo = self.repo
         rulectx = repo[self.node]
         hg.update(repo, self.state.parentctxnode, quietempty=True)
         applychanges(repo.ui, repo, rulectx, {})
         hint = _(b'to edit %s, `hg histedit --continue` after making changes')
         raise error.InterventionRequired(
             _(b'Editing (%s), commit as needed now to split the change')
             % short(self.node),
             hint=hint % short(self.node),
         )
 
     def commiteditor(self):
         return cmdutil.getcommiteditor(edit=True, editform=b'histedit.edit')
 
 
 @action([b'fold', b'f'], _(b'use commit, but combine it with the one above'))
 class fold(histeditaction):
     def verify(self, prev, expected, seen):
         """Verifies semantic correctness of the fold rule"""
         super(fold, self).verify(prev, expected, seen)
         repo = self.repo
         if not prev:
             c = repo[self.node].p1()
         elif not prev.verb in (b'pick', b'base'):
             return
         else:
             c = repo[prev.node]
         if not c.mutable():
             raise error.ParseError(
                 _(b"cannot fold into public change %s") % short(c.node())
             )
 
     def continuedirty(self):
         repo = self.repo
         rulectx = repo[self.node]
 
         commit = commitfuncfor(repo, rulectx)
         commit(
             text=b'fold-temp-revision %s' % short(self.node),
             user=rulectx.user(),
             date=rulectx.date(),
             extra=rulectx.extra(),
         )
 
     def continueclean(self):
         repo = self.repo
         ctx = repo[b'.']
         rulectx = repo[self.node]
         parentctxnode = self.state.parentctxnode
         if ctx.node() == parentctxnode:
             repo.ui.warn(_(b'%s: empty changeset\n') % short(self.node))
             return ctx, [(self.node, (parentctxnode,))]
 
         parentctx = repo[parentctxnode]
         newcommits = {
             c.node()
             for c in repo.set(b'(%d::. - %d)', parentctx.rev(), parentctx.rev())
         }
         if not newcommits:
             repo.ui.warn(
                 _(
                     b'%s: cannot fold - working copy is not a '
                     b'descendant of previous commit %s\n'
                 )
                 % (short(self.node), short(parentctxnode))
             )
             return ctx, [(self.node, (ctx.node(),))]
 
         middlecommits = newcommits.copy()
         middlecommits.discard(ctx.node())
 
         return self.finishfold(
             repo.ui, repo, parentctx, rulectx, ctx.node(), middlecommits
         )
 
     def skipprompt(self):
         """Returns true if the rule should skip the message editor.
 
         For example, 'fold' wants to show an editor, but 'rollup'
         doesn't want to.
         """
         return False
 
     def mergedescs(self):
         """Returns true if the rule should merge messages of multiple changes.
 
         This exists mainly so that 'rollup' rules can be a subclass of
         'fold'.
         """
         return True
 
     def firstdate(self):
         """Returns true if the rule should preserve the date of the first
         change.
 
         This exists mainly so that 'rollup' rules can be a subclass of
         'fold'.
         """
         return False
 
     def finishfold(self, ui, repo, ctx, oldctx, newnode, internalchanges):
         mergemod.update(ctx.p1())
         ### prepare new commit data
         commitopts = {}
         commitopts[b'user'] = ctx.user()
         # commit message
         if not self.mergedescs():
             newmessage = ctx.description()
         else:
             newmessage = (
                 b'\n***\n'.join(
                     [ctx.description()]
                     + [repo[r].description() for r in internalchanges]
                     + [oldctx.description()]
                 )
                 + b'\n'
             )
         commitopts[b'message'] = newmessage
         # date
         if self.firstdate():
             commitopts[b'date'] = ctx.date()
         else:
             commitopts[b'date'] = max(ctx.date(), oldctx.date())
         # if date is to be updated to current
         if ui.configbool(b'rewrite', b'update-timestamp'):
             commitopts[b'date'] = dateutil.makedate()
 
         extra = ctx.extra().copy()
         # histedit_source
         # note: ctx is likely a temporary commit but that the best we can do
         #       here. This is sufficient to solve issue3681 anyway.
         extra[b'histedit_source'] = b'%s,%s' % (ctx.hex(), oldctx.hex())
         commitopts[b'extra'] = extra
         phasemin = max(ctx.phase(), oldctx.phase())
         overrides = {(b'phases', b'new-commit'): phasemin}
         with repo.ui.configoverride(overrides, b'histedit'):
             n = collapse(
                 repo,
                 ctx,
                 repo[newnode],
                 commitopts,
                 skipprompt=self.skipprompt(),
             )
         if n is None:
             return ctx, []
         mergemod.update(repo[n])
         replacements = [
             (oldctx.node(), (newnode,)),
             (ctx.node(), (n,)),
             (newnode, (n,)),
         ]
         for ich in internalchanges:
             replacements.append((ich, (n,)))
         return repo[n], replacements
 
 
 @action(
     [b'base', b'b'],
     _(b'checkout changeset and apply further changesets from there'),
 )
 class base(histeditaction):
     def run(self):
         if self.repo[b'.'].node() != self.node:
             mergemod.clean_update(self.repo[self.node])
         return self.continueclean()
 
     def continuedirty(self):
         abortdirty()
 
     def continueclean(self):
         basectx = self.repo[b'.']
         return basectx, []
 
     def _verifynodeconstraints(self, prev, expected, seen):
         # base can only be use with a node not in the edited set
         if self.node in expected:
             msg = _(b'%s "%s" changeset was an edited list candidate')
             raise error.ParseError(
                 msg % (self.verb, short(self.node)),
                 hint=_(b'base must only use unlisted changesets'),
             )
 
 
 @action(
     [b'_multifold'],
     _(
         """fold subclass used for when multiple folds happen in a row
 
     We only want to fire the editor for the folded message once when
     (say) four changes are folded down into a single change. This is
     similar to rollup, but we should preserve both messages so that
     when the last fold operation runs we can show the user all the
     commit messages in their editor.
     """
     ),
     internal=True,
 )
 class _multifold(fold):
     def skipprompt(self):
         return True
 
 
 @action(
     [b"roll", b"r"],
     _(b"like fold, but discard this commit's description and date"),
 )
 class rollup(fold):
     def mergedescs(self):
         return False
 
     def skipprompt(self):
         return True
 
     def firstdate(self):
         return True
 
 
 @action([b"drop", b"d"], _(b'remove commit from history'))
 class drop(histeditaction):
     def run(self):
         parentctx = self.repo[self.state.parentctxnode]
         return parentctx, [(self.node, tuple())]
 
 
 @action(
     [b"mess", b"m"],
     _(b'edit commit message without changing commit content'),
     priority=True,
 )
 class message(histeditaction):
     def commiteditor(self):
         return cmdutil.getcommiteditor(edit=True, editform=b'histedit.mess')
 
 
 def findoutgoing(ui, repo, remote=None, force=False, opts=None):
     """utility function to find the first outgoing changeset
 
     Used by initialization code"""
     if opts is None:
         opts = {}
     path = urlutil.get_unique_push_path(b'histedit', repo, ui, remote)
     dest = path.pushloc or path.loc
 
     ui.status(_(b'comparing with %s\n') % urlutil.hidepassword(dest))
 
     revs, checkout = hg.addbranchrevs(repo, repo, (path.branch, []), None)
     other = hg.peer(repo, opts, dest)
 
     if revs:
         revs = [repo.lookup(rev) for rev in revs]
 
     outgoing = discovery.findcommonoutgoing(repo, other, revs, force=force)
     if not outgoing.missing:
         raise error.StateError(_(b'no outgoing ancestors'))
     roots = list(repo.revs(b"roots(%ln)", outgoing.missing))
     if len(roots) > 1:
         msg = _(b'there are ambiguous outgoing revisions')
         hint = _(b"see 'hg help histedit' for more detail")
         raise error.StateError(msg, hint=hint)
     return repo[roots[0]].node()
 
 
 # Curses Support
 try:
     import curses
 except ImportError:
     curses = None
 
 KEY_LIST = [b'pick', b'edit', b'fold', b'drop', b'mess', b'roll']
 ACTION_LABELS = {
     b'fold': b'^fold',
     b'roll': b'^roll',
 }
 
 COLOR_HELP, COLOR_SELECTED, COLOR_OK, COLOR_WARN, COLOR_CURRENT = 1, 2, 3, 4, 5
 COLOR_DIFF_ADD_LINE, COLOR_DIFF_DEL_LINE, COLOR_DIFF_OFFSET = 6, 7, 8
 COLOR_ROLL, COLOR_ROLL_CURRENT, COLOR_ROLL_SELECTED = 9, 10, 11
 
 E_QUIT, E_HISTEDIT = 1, 2
 E_PAGEDOWN, E_PAGEUP, E_LINEUP, E_LINEDOWN, E_RESIZE = 3, 4, 5, 6, 7
 MODE_INIT, MODE_PATCH, MODE_RULES, MODE_HELP = 0, 1, 2, 3
 
 KEYTABLE = {
     b'global': {
         b'h': b'next-action',
         b'KEY_RIGHT': b'next-action',
         b'l': b'prev-action',
         b'KEY_LEFT': b'prev-action',
         b'q': b'quit',
         b'c': b'histedit',
         b'C': b'histedit',
         b'v': b'showpatch',
         b'?': b'help',
     },
     MODE_RULES: {
         b'd': b'action-drop',
         b'e': b'action-edit',
         b'f': b'action-fold',
         b'm': b'action-mess',
         b'p': b'action-pick',
         b'r': b'action-roll',
         b' ': b'select',
         b'j': b'down',
         b'k': b'up',
         b'KEY_DOWN': b'down',
         b'KEY_UP': b'up',
         b'J': b'move-down',
         b'K': b'move-up',
         b'KEY_NPAGE': b'move-down',
         b'KEY_PPAGE': b'move-up',
         b'0': b'goto',  # Used for 0..9
     },
     MODE_PATCH: {
         b' ': b'page-down',
         b'KEY_NPAGE': b'page-down',
         b'KEY_PPAGE': b'page-up',
         b'j': b'line-down',
         b'k': b'line-up',
         b'KEY_DOWN': b'line-down',
         b'KEY_UP': b'line-up',
         b'J': b'down',
         b'K': b'up',
     },
     MODE_HELP: {},
 }
 
 
 def screen_size():
     return struct.unpack(b'hh', fcntl.ioctl(1, termios.TIOCGWINSZ, b'    '))
 
 
 class histeditrule(object):
     def __init__(self, ui, ctx, pos, action=b'pick'):
         self.ui = ui
         self.ctx = ctx
         self.action = action
         self.origpos = pos
         self.pos = pos
         self.conflicts = []
 
     def __bytes__(self):
         # Example display of several histeditrules:
         #
         #  #10 pick   316392:06a16c25c053   add option to skip tests
         #  #11 ^roll  316393:71313c964cc5   <RED>oops a fixup commit</RED>
         #  #12 pick   316394:ab31f3973b0d   include mfbt for mozilla-config.h
         #  #13 ^fold  316395:14ce5803f4c3   fix warnings
         #
         # The carets point to the changeset being folded into ("roll this
         # changeset into the changeset above").
         return b'%s%s' % (self.prefix, self.desc)
 
     __str__ = encoding.strmethod(__bytes__)
 
     @property
     def prefix(self):
         # Some actions ('fold' and 'roll') combine a patch with a
         # previous one. Add a marker showing which patch they apply
         # to.
         action = ACTION_LABELS.get(self.action, self.action)
 
         h = self.ctx.hex()[0:12]
         r = self.ctx.rev()
 
         return b"#%s %s %d:%s   " % (
             (b'%d' % self.origpos).ljust(2),
             action.ljust(6),
             r,
             h,
         )
 
     @util.propertycache
     def desc(self):
         summary = cmdutil.rendertemplate(
             self.ctx, self.ui.config(b'histedit', b'summary-template')
         )
         if summary:
             return summary
         # This is split off from the prefix property so that we can
         # separately make the description for 'roll' red (since it
         # will get discarded).
         return self.ctx.description().splitlines()[0].strip()
 
     def checkconflicts(self, other):
         if other.pos > self.pos and other.origpos <= self.origpos:
             if set(other.ctx.files()) & set(self.ctx.files()) != set():
                 self.conflicts.append(other)
                 return self.conflicts
 
         if other in self.conflicts:
             self.conflicts.remove(other)
         return self.conflicts
 
 
 def makecommands(rules):
     """Returns a list of commands consumable by histedit --commands based on
     our list of rules"""
     commands = []
     for rules in rules:
         commands.append(b'%s %s\n' % (rules.action, rules.ctx))
     return commands
 
 
 def addln(win, y, x, line, color=None):
     """Add a line to the given window left padding but 100% filled with
     whitespace characters, so that the color appears on the whole line"""
     maxy, maxx = win.getmaxyx()
     length = maxx - 1 - x
     line = bytes(line).ljust(length)[:length]
     if y < 0:
         y = maxy + y
     if x < 0:
         x = maxx + x
     if color:
         win.addstr(y, x, line, color)
     else:
         win.addstr(y, x, line)
 
 
 def _trunc_head(line, n):
     if len(line) <= n:
         return line
     return b'> ' + line[-(n - 2) :]
 
 
 def _trunc_tail(line, n):
     if len(line) <= n:
         return line
     return line[: n - 2] + b' >'
 
 
 class _chistedit_state(object):
     def __init__(
         self,
         repo,
         rules,
         stdscr,
     ):
         self.repo = repo
         self.rules = rules
         self.stdscr = stdscr
         self.later_on_top = repo.ui.configbool(
             b'histedit', b'later-commits-first'
         )
         # The current item in display order, initialized to point to the top
         # of the screen.
         self.pos = 0
         self.selected = None
         self.mode = (MODE_INIT, MODE_INIT)
         self.page_height = None
         self.modes = {
             MODE_RULES: {
                 b'line_offset': 0,
             },
             MODE_PATCH: {
                 b'line_offset': 0,
             },
         }
 
     def render_commit(self, win):
         """Renders the commit window that shows the log of the current selected
         commit"""
         rule = self.rules[self.display_pos_to_rule_pos(self.pos)]
 
         ctx = rule.ctx
         win.box()
 
         maxy, maxx = win.getmaxyx()
         length = maxx - 3
 
         line = b"changeset: %d:%s" % (ctx.rev(), ctx.hex()[:12])
         win.addstr(1, 1, line[:length])
 
         line = b"user:      %s" % ctx.user()
         win.addstr(2, 1, line[:length])
 
         bms = self.repo.nodebookmarks(ctx.node())
         line = b"bookmark:  %s" % b' '.join(bms)
         win.addstr(3, 1, line[:length])
 
         line = b"summary:   %s" % (ctx.description().splitlines()[0])
         win.addstr(4, 1, line[:length])
 
         line = b"files:     "
         win.addstr(5, 1, line)
         fnx = 1 + len(line)
         fnmaxx = length - fnx + 1
         y = 5
         fnmaxn = maxy - (1 + y) - 1
         files = ctx.files()
         for i, line1 in enumerate(files):
             if len(files) > fnmaxn and i == fnmaxn - 1:
                 win.addstr(y, fnx, _trunc_tail(b','.join(files[i:]), fnmaxx))
                 y = y + 1
                 break
             win.addstr(y, fnx, _trunc_head(line1, fnmaxx))
             y = y + 1
 
         conflicts = rule.conflicts
         if len(conflicts) > 0:
             conflictstr = b','.join(map(lambda r: r.ctx.hex()[:12], conflicts))
             conflictstr = b"changed files overlap with %s" % conflictstr
         else:
             conflictstr = b'no overlap'
 
         win.addstr(y, 1, conflictstr[:length])
         win.noutrefresh()
 
     def helplines(self):
         if self.mode[0] == MODE_PATCH:
             help = b"""\
 ?: help, k/up: line up, j/down: line down, v: stop viewing patch
 pgup: prev page, space/pgdn: next page, c: commit, q: abort
 """
         else:
             help = b"""\
 ?: help, k/up: move up, j/down: move down, space: select, v: view patch
 d: drop, e: edit, f: fold, m: mess, p: pick, r: roll
 pgup/K: move patch up, pgdn/J: move patch down, c: commit, q: abort
 """
             if self.later_on_top:
                 help += b"Newer commits are shown above older commits.\n"
             else:
                 help += b"Older commits are shown above newer commits.\n"
         return help.splitlines()
 
     def render_help(self, win):
         maxy, maxx = win.getmaxyx()
         for y, line in enumerate(self.helplines()):
             if y >= maxy:
                 break
             addln(win, y, 0, line, curses.color_pair(COLOR_HELP))
         win.noutrefresh()
 
     def layout(self):
         maxy, maxx = self.stdscr.getmaxyx()
         helplen = len(self.helplines())
         mainlen = maxy - helplen - 12
         if mainlen < 1:
             raise error.Abort(
                 _(b"terminal dimensions %d by %d too small for curses histedit")
                 % (maxy, maxx),
                 hint=_(
                     b"enlarge your terminal or use --config ui.interface=text"
                 ),
             )
         return {
             b'commit': (12, maxx),
             b'help': (helplen, maxx),
             b'main': (mainlen, maxx),
         }
 
     def display_pos_to_rule_pos(self, display_pos):
         """Converts a position in display order to rule order.
 
         The `display_pos` is the order from the top in display order, not
         considering which items are currently visible on the screen. Thus,
         `display_pos=0` is the item at the top (possibly after scrolling to
         the top)
         """
         if self.later_on_top:
             return len(self.rules) - 1 - display_pos
         else:
             return display_pos
 
     def render_rules(self, rulesscr):
         start = self.modes[MODE_RULES][b'line_offset']
 
         conflicts = [r.ctx for r in self.rules if r.conflicts]
         if len(conflicts) > 0:
             line = b"potential conflict in %s" % b','.join(
                 map(pycompat.bytestr, conflicts)
             )
             addln(rulesscr, -1, 0, line, curses.color_pair(COLOR_WARN))
 
         for display_pos in range(start, len(self.rules)):
             y = display_pos - start
             if y < 0 or y >= self.page_height:
                 continue
             rule_pos = self.display_pos_to_rule_pos(display_pos)
             rule = self.rules[rule_pos]
             if len(rule.conflicts) > 0:
                 rulesscr.addstr(y, 0, b" ", curses.color_pair(COLOR_WARN))
             else:
                 rulesscr.addstr(y, 0, b" ", curses.COLOR_BLACK)
 
             if display_pos == self.selected:
                 rollcolor = COLOR_ROLL_SELECTED
                 addln(rulesscr, y, 2, rule, curses.color_pair(COLOR_SELECTED))
             elif display_pos == self.pos:
                 rollcolor = COLOR_ROLL_CURRENT
                 addln(
                     rulesscr,
                     y,
                     2,
                     rule,
                     curses.color_pair(COLOR_CURRENT) | curses.A_BOLD,
                 )
             else:
                 rollcolor = COLOR_ROLL
                 addln(rulesscr, y, 2, rule)
 
             if rule.action == b'roll':
                 rulesscr.addstr(
                     y,
                     2 + len(rule.prefix),
                     rule.desc,
                     curses.color_pair(rollcolor),
                 )
 
         rulesscr.noutrefresh()
 
     def render_string(self, win, output, diffcolors=False):
         maxy, maxx = win.getmaxyx()
         length = min(maxy - 1, len(output))
         for y in range(0, length):
             line = output[y]
             if diffcolors:
                 if line and line[0] == b'+':
                     win.addstr(
                         y, 0, line, curses.color_pair(COLOR_DIFF_ADD_LINE)
                     )
                 elif line and line[0] == b'-':
                     win.addstr(
                         y, 0, line, curses.color_pair(COLOR_DIFF_DEL_LINE)
                     )
                 elif line.startswith(b'@@ '):
                     win.addstr(y, 0, line, curses.color_pair(COLOR_DIFF_OFFSET))
                 else:
                     win.addstr(y, 0, line)
             else:
                 win.addstr(y, 0, line)
         win.noutrefresh()
 
     def render_patch(self, win):
         start = self.modes[MODE_PATCH][b'line_offset']
         content = self.modes[MODE_PATCH][b'patchcontents']
         self.render_string(win, content[start:], diffcolors=True)
 
     def event(self, ch):
         """Change state based on the current character input
 
         This takes the current state and based on the current character input from
         the user we change the state.
         """
         oldpos = self.pos
 
         if ch in (curses.KEY_RESIZE, b"KEY_RESIZE"):
             return E_RESIZE
 
         lookup_ch = ch
         if ch is not None and b'0' <= ch <= b'9':
             lookup_ch = b'0'
 
         curmode, prevmode = self.mode
         action = KEYTABLE[curmode].get(
             lookup_ch, KEYTABLE[b'global'].get(lookup_ch)
         )
         if action is None:
             return
         if action in (b'down', b'move-down'):
             newpos = min(oldpos + 1, len(self.rules) - 1)
             self.move_cursor(oldpos, newpos)
             if self.selected is not None or action == b'move-down':
                 self.swap(oldpos, newpos)
         elif action in (b'up', b'move-up'):
             newpos = max(0, oldpos - 1)
             self.move_cursor(oldpos, newpos)
             if self.selected is not None or action == b'move-up':
                 self.swap(oldpos, newpos)
         elif action == b'next-action':
             self.cycle_action(oldpos, next=True)
         elif action == b'prev-action':
             self.cycle_action(oldpos, next=False)
         elif action == b'select':
             self.selected = oldpos if self.selected is None else None
             self.make_selection(self.selected)
         elif action == b'goto' and int(ch) < len(self.rules) <= 10:
             newrule = next((r for r in self.rules if r.origpos == int(ch)))
             self.move_cursor(oldpos, newrule.pos)
             if self.selected is not None:
                 self.swap(oldpos, newrule.pos)
         elif action.startswith(b'action-'):
             self.change_action(oldpos, action[7:])
         elif action == b'showpatch':
             self.change_mode(MODE_PATCH if curmode != MODE_PATCH else prevmode)
         elif action == b'help':
             self.change_mode(MODE_HELP if curmode != MODE_HELP else prevmode)
         elif action == b'quit':
             return E_QUIT
         elif action == b'histedit':
             return E_HISTEDIT
         elif action == b'page-down':
             return E_PAGEDOWN
         elif action == b'page-up':
             return E_PAGEUP
         elif action == b'line-down':
             return E_LINEDOWN
         elif action == b'line-up':
             return E_LINEUP
 
     def patch_contents(self):
         repo = self.repo
         rule = self.rules[self.display_pos_to_rule_pos(self.pos)]
         displayer = logcmdutil.changesetdisplayer(
             repo.ui,
             repo,
             {b"patch": True, b"template": b"status"},
             buffered=True,
         )
         overrides = {(b'ui', b'verbose'): True}
         with repo.ui.configoverride(overrides, source=b'histedit'):
             displayer.show(rule.ctx)
             displayer.close()
         return displayer.hunk[rule.ctx.rev()].splitlines()
 
     def move_cursor(self, oldpos, newpos):
         """Change the rule/changeset that the cursor is pointing to, regardless of
         current mode (you can switch between patches from the view patch window)."""
         self.pos = newpos
 
         mode, _ = self.mode
         if mode == MODE_RULES:
             # Scroll through the list by updating the view for MODE_RULES, so that
             # even if we are not currently viewing the rules, switching back will
             # result in the cursor's rule being visible.
             modestate = self.modes[MODE_RULES]
             if newpos < modestate[b'line_offset']:
                 modestate[b'line_offset'] = newpos
             elif newpos > modestate[b'line_offset'] + self.page_height - 1:
                 modestate[b'line_offset'] = newpos - self.page_height + 1
 
         # Reset the patch view region to the top of the new patch.
         self.modes[MODE_PATCH][b'line_offset'] = 0
 
     def change_mode(self, mode):
         curmode, _ = self.mode
         self.mode = (mode, curmode)
         if mode == MODE_PATCH:
             self.modes[MODE_PATCH][b'patchcontents'] = self.patch_contents()
 
     def make_selection(self, pos):
         self.selected = pos
 
     def swap(self, oldpos, newpos):
         """Swap two positions and calculate necessary conflicts in
         O(|newpos-oldpos|) time"""
         old_rule_pos = self.display_pos_to_rule_pos(oldpos)
         new_rule_pos = self.display_pos_to_rule_pos(newpos)
 
         rules = self.rules
         assert 0 <= old_rule_pos < len(rules) and 0 <= new_rule_pos < len(rules)
 
         rules[old_rule_pos], rules[new_rule_pos] = (
             rules[new_rule_pos],
             rules[old_rule_pos],
         )
 
         # TODO: swap should not know about histeditrule's internals
         rules[new_rule_pos].pos = new_rule_pos
         rules[old_rule_pos].pos = old_rule_pos
 
         start = min(old_rule_pos, new_rule_pos)
         end = max(old_rule_pos, new_rule_pos)
         for r in pycompat.xrange(start, end + 1):
             rules[new_rule_pos].checkconflicts(rules[r])
             rules[old_rule_pos].checkconflicts(rules[r])
 
         if self.selected:
             self.make_selection(newpos)
 
     def change_action(self, pos, action):
         """Change the action state on the given position to the new action"""
         assert 0 <= pos < len(self.rules)
         self.rules[pos].action = action
 
     def cycle_action(self, pos, next=False):
         """Changes the action state the next or the previous action from
         the action list"""
         assert 0 <= pos < len(self.rules)
         current = self.rules[pos].action
 
         assert current in KEY_LIST
 
         index = KEY_LIST.index(current)
         if next:
             index += 1
         else:
             index -= 1
         self.change_action(pos, KEY_LIST[index % len(KEY_LIST)])
 
     def change_view(self, delta, unit):
         """Change the region of whatever is being viewed (a patch or the list of
         changesets). 'delta' is an amount (+/- 1) and 'unit' is 'page' or 'line'."""
         mode, _ = self.mode
         if mode != MODE_PATCH:
             return
         mode_state = self.modes[mode]
         num_lines = len(mode_state[b'patchcontents'])
         page_height = self.page_height
         unit = page_height if unit == b'page' else 1
         num_pages = 1 + (num_lines - 1) // page_height
         max_offset = (num_pages - 1) * page_height
         newline = mode_state[b'line_offset'] + delta * unit
         mode_state[b'line_offset'] = max(0, min(max_offset, newline))
 
 
 def _chisteditmain(repo, rules, stdscr):
     try:
         curses.use_default_colors()
     except curses.error:
         pass
 
     # initialize color pattern
     curses.init_pair(COLOR_HELP, curses.COLOR_WHITE, curses.COLOR_BLUE)
     curses.init_pair(COLOR_SELECTED, curses.COLOR_BLACK, curses.COLOR_WHITE)
     curses.init_pair(COLOR_WARN, curses.COLOR_BLACK, curses.COLOR_YELLOW)
     curses.init_pair(COLOR_OK, curses.COLOR_BLACK, curses.COLOR_GREEN)
     curses.init_pair(COLOR_CURRENT, curses.COLOR_WHITE, curses.COLOR_MAGENTA)
     curses.init_pair(COLOR_DIFF_ADD_LINE, curses.COLOR_GREEN, -1)
     curses.init_pair(COLOR_DIFF_DEL_LINE, curses.COLOR_RED, -1)
     curses.init_pair(COLOR_DIFF_OFFSET, curses.COLOR_MAGENTA, -1)
     curses.init_pair(COLOR_ROLL, curses.COLOR_RED, -1)
     curses.init_pair(
         COLOR_ROLL_CURRENT, curses.COLOR_BLACK, curses.COLOR_MAGENTA
     )
     curses.init_pair(COLOR_ROLL_SELECTED, curses.COLOR_RED, curses.COLOR_WHITE)
 
     # don't display the cursor
     try:
         curses.curs_set(0)
     except curses.error:
         pass
 
     def drawvertwin(size, y, x):
         win = curses.newwin(size[0], size[1], y, x)
         y += size[0]
         return win, y, x
 
     state = _chistedit_state(repo, rules, stdscr)
 
     # eventloop
     ch = None
     stdscr.clear()
     stdscr.refresh()
     while True:
         oldmode, unused = state.mode
         if oldmode == MODE_INIT:
             state.change_mode(MODE_RULES)
         e = state.event(ch)
 
         if e == E_QUIT:
             return False
         if e == E_HISTEDIT:
             return state.rules
         else:
             if e == E_RESIZE:
                 size = screen_size()
                 if size != stdscr.getmaxyx():
                     curses.resizeterm(*size)
 
             sizes = state.layout()
             curmode, unused = state.mode
             if curmode != oldmode:
                 state.page_height = sizes[b'main'][0]
                 # Adjust the view to fit the current screen size.
                 state.move_cursor(state.pos, state.pos)
 
             # Pack the windows against the top, each pane spread across the
             # full width of the screen.
             y, x = (0, 0)
             helpwin, y, x = drawvertwin(sizes[b'help'], y, x)
             mainwin, y, x = drawvertwin(sizes[b'main'], y, x)
             commitwin, y, x = drawvertwin(sizes[b'commit'], y, x)
 
             if e in (E_PAGEDOWN, E_PAGEUP, E_LINEDOWN, E_LINEUP):
                 if e == E_PAGEDOWN:
                     state.change_view(+1, b'page')
                 elif e == E_PAGEUP:
                     state.change_view(-1, b'page')
                 elif e == E_LINEDOWN:
                     state.change_view(+1, b'line')
                 elif e == E_LINEUP:
                     state.change_view(-1, b'line')
 
             # start rendering
             commitwin.erase()
             helpwin.erase()
             mainwin.erase()
             if curmode == MODE_PATCH:
                 state.render_patch(mainwin)
             elif curmode == MODE_HELP:
                 state.render_string(mainwin, __doc__.strip().splitlines())
             else:
                 state.render_rules(mainwin)
                 state.render_commit(commitwin)
             state.render_help(helpwin)
             curses.doupdate()
             # done rendering
             ch = encoding.strtolocal(stdscr.getkey())
 
 
 def _chistedit(ui, repo, freeargs, opts):
     """interactively edit changeset history via a curses interface
 
     Provides a ncurses interface to histedit. Press ? in chistedit mode
     to see an extensive help. Requires python-curses to be installed."""
 
     if curses is None:
         raise error.Abort(_(b"Python curses library required"))
 
     # disable color
     ui._colormode = None
 
     try:
         keep = opts.get(b'keep')
         revs = opts.get(b'rev', [])[:]
         cmdutil.checkunfinished(repo)
         cmdutil.bailifchanged(repo)
 
         revs.extend(freeargs)
         if not revs:
             defaultrev = destutil.desthistedit(ui, repo)
             if defaultrev is not None:
                 revs.append(defaultrev)
         if len(revs) != 1:
             raise error.InputError(
                 _(b'histedit requires exactly one ancestor revision')
             )
 
         rr = list(repo.set(b'roots(%ld)', logcmdutil.revrange(repo, revs)))
         if len(rr) != 1:
             raise error.InputError(
                 _(
                     b'The specified revisions must have '
                     b'exactly one common root'
                 )
             )
         root = rr[0].node()
 
         topmost = repo.dirstate.p1()
         revs = between(repo, root, topmost, keep)
         if not revs:
             raise error.InputError(
                 _(b'%s is not an ancestor of working directory') % short(root)
             )
 
         rules = []
         for i, r in enumerate(revs):
             rules.append(histeditrule(ui, repo[r], i))
         with util.with_lc_ctype():
             rc = curses.wrapper(functools.partial(_chisteditmain, repo, rules))
         curses.echo()
         curses.endwin()
         if rc is False:
             ui.write(_(b"histedit aborted\n"))
             return 0
         if type(rc) is list:
             ui.status(_(b"performing changes\n"))
             rules = makecommands(rc)
             with repo.vfs(b'chistedit', b'w+') as fp:
                 for r in rules:
                     fp.write(r)
                 opts[b'commands'] = fp.name
             return _texthistedit(ui, repo, freeargs, opts)
     except KeyboardInterrupt:
         pass
     return -1
 
 
 @command(
     b'histedit',
     [
         (
             b'',
             b'commands',
             b'',
             _(b'read history edits from the specified file'),
             _(b'FILE'),
         ),
         (b'c', b'continue', False, _(b'continue an edit already in progress')),
         (b'', b'edit-plan', False, _(b'edit remaining actions list')),
         (
             b'k',
             b'keep',
             False,
             _(b"don't strip old nodes after edit is complete"),
         ),
         (b'', b'abort', False, _(b'abort an edit in progress')),
         (b'o', b'outgoing', False, _(b'changesets not found in destination')),
         (
             b'f',
             b'force',
             False,
             _(b'force outgoing even for unrelated repositories'),
         ),
         (b'r', b'rev', [], _(b'first revision to be edited'), _(b'REV')),
     ]
     + cmdutil.formatteropts,
     _(b"[OPTIONS] ([ANCESTOR] | --outgoing [URL])"),
     helpcategory=command.CATEGORY_CHANGE_MANAGEMENT,
 )
 def histedit(ui, repo, *freeargs, **opts):
     """interactively edit changeset history
 
     This command lets you edit a linear series of changesets (up to
     and including the working directory, which should be clean).
     You can:
 
     - `pick` to [re]order a changeset
 
     - `drop` to omit changeset
 
     - `mess` to reword the changeset commit message
 
     - `fold` to combine it with the preceding changeset (using the later date)
 
     - `roll` like fold, but discarding this commit's description and date
 
     - `edit` to edit this changeset (preserving date)
 
     - `base` to checkout changeset and apply further changesets from there
 
     There are a number of ways to select the root changeset:
 
     - Specify ANCESTOR directly
 
     - Use --outgoing -- it will be the first linear changeset not
       included in destination. (See :hg:`help config.paths.default-push`)
 
     - Otherwise, the value from the "histedit.defaultrev" config option
       is used as a revset to select the base revision when ANCESTOR is not
       specified. The first revision returned by the revset is used. By
       default, this selects the editable history that is unique to the
       ancestry of the working directory.
 
     .. container:: verbose
 
        If you use --outgoing, this command will abort if there are ambiguous
        outgoing revisions. For example, if there are multiple branches
        containing outgoing revisions.
 
        Use "min(outgoing() and ::.)" or similar revset specification
        instead of --outgoing to specify edit target revision exactly in
        such ambiguous situation. See :hg:`help revsets` for detail about
        selecting revisions.
 
     .. container:: verbose
 
        Examples:
 
          - A number of changes have been made.
            Revision 3 is no longer needed.
 
            Start history editing from revision 3::
 
              hg histedit -r 3
 
            An editor opens, containing the list of revisions,
            with specific actions specified::
 
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 0a9639fcda9d 5 Morgify the cromulancy
 
            Additional information about the possible actions
            to take appears below the list of revisions.
 
            To remove revision 3 from the history,
            its action (at the beginning of the relevant line)
            is changed to 'drop'::
 
              drop 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 0a9639fcda9d 5 Morgify the cromulancy
 
          - A number of changes have been made.
            Revision 2 and 4 need to be swapped.
 
            Start history editing from revision 2::
 
              hg histedit -r 2
 
            An editor opens, containing the list of revisions,
            with specific actions specified::
 
              pick 252a1af424ad 2 Blorb a morgwazzle
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
 
            To swap revision 2 and 4, its lines are swapped
            in the editor::
 
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 252a1af424ad 2 Blorb a morgwazzle
 
     Returns 0 on success, 1 if user intervention is required (not only
     for intentional "edit" command, but also for resolving unexpected
     conflicts).
     """
     opts = pycompat.byteskwargs(opts)
 
     # kludge: _chistedit only works for starting an edit, not aborting
     # or continuing, so fall back to regular _texthistedit for those
     # operations.
     if ui.interface(b'histedit') == b'curses' and _getgoal(opts) == goalnew:
         return _chistedit(ui, repo, freeargs, opts)
     return _texthistedit(ui, repo, freeargs, opts)
 
 
 def _texthistedit(ui, repo, freeargs, opts):
     state = histeditstate(repo)
     with repo.wlock() as wlock, repo.lock() as lock:
         state.wlock = wlock
         state.lock = lock
         _histedit(ui, repo, state, freeargs, opts)
 
 
 goalcontinue = b'continue'
 goalabort = b'abort'
 goaleditplan = b'edit-plan'
 goalnew = b'new'
 
 
 def _getgoal(opts):
     if opts.get(b'continue'):
         return goalcontinue
     if opts.get(b'abort'):
         return goalabort
     if opts.get(b'edit_plan'):
         return goaleditplan
     return goalnew
 
 
 def _readfile(ui, path):
     if path == b'-':
         with ui.timeblockedsection(b'histedit'):
             return ui.fin.read()
     else:
         with open(path, b'rb') as f:
             return f.read()
 
 
 def _validateargs(ui, repo, freeargs, opts, goal, rules, revs):
     # TODO only abort if we try to histedit mq patches, not just
     # blanket if mq patches are applied somewhere
     mq = getattr(repo, 'mq', None)
     if mq and mq.applied:
         raise error.StateError(_(b'source has mq patches applied'))
 
     # basic argument incompatibility processing
     outg = opts.get(b'outgoing')
     editplan = opts.get(b'edit_plan')
     abort = opts.get(b'abort')
     force = opts.get(b'force')
     if force and not outg:
         raise error.InputError(_(b'--force only allowed with --outgoing'))
     if goal == b'continue':
         if any((outg, abort, revs, freeargs, rules, editplan)):
             raise error.InputError(_(b'no arguments allowed with --continue'))
     elif goal == b'abort':
         if any((outg, revs, freeargs, rules, editplan)):
             raise error.InputError(_(b'no arguments allowed with --abort'))
     elif goal == b'edit-plan':
         if any((outg, revs, freeargs)):
             raise error.InputError(
                 _(b'only --commands argument allowed with --edit-plan')
             )
     else:
         if outg:
             if revs:
                 raise error.InputError(
                     _(b'no revisions allowed with --outgoing')
                 )
             if len(freeargs) > 1:
                 raise error.InputError(
                     _(b'only one repo argument allowed with --outgoing')
                 )
         else:
             revs.extend(freeargs)
             if len(revs) == 0:
                 defaultrev = destutil.desthistedit(ui, repo)
                 if defaultrev is not None:
                     revs.append(defaultrev)
 
             if len(revs) != 1:
                 raise error.InputError(
                     _(b'histedit requires exactly one ancestor revision')
                 )
 
 
 def _histedit(ui, repo, state, freeargs, opts):
     fm = ui.formatter(b'histedit', opts)
     fm.startitem()
     goal = _getgoal(opts)
     revs = opts.get(b'rev', [])
     nobackup = not ui.configbool(b'rewrite', b'backup-bundle')
     rules = opts.get(b'commands', b'')
     state.keep = opts.get(b'keep', False)
 
     _validateargs(ui, repo, freeargs, opts, goal, rules, revs)
 
     hastags = False
     if revs:
         revs = logcmdutil.revrange(repo, revs)
         ctxs = [repo[rev] for rev in revs]
         for ctx in ctxs:
             tags = [tag for tag in ctx.tags() if tag != b'tip']
             if not hastags:
                 hastags = len(tags)
     if hastags:
         if ui.promptchoice(
             _(
                 b'warning: tags associated with the given'
                 b' changeset will be lost after histedit.\n'
                 b'do you want to continue (yN)? $$ &Yes $$ &No'
             ),
             default=1,
         ):
             raise error.CanceledError(_(b'histedit cancelled\n'))
     # rebuild state
     if goal == goalcontinue:
         state.read()
         state = bootstrapcontinue(ui, state, opts)
     elif goal == goaleditplan:
         _edithisteditplan(ui, repo, state, rules)
         return
     elif goal == goalabort:
         _aborthistedit(ui, repo, state, nobackup=nobackup)
         return
     else:
         # goal == goalnew
         _newhistedit(ui, repo, state, revs, freeargs, opts)
 
     _continuehistedit(ui, repo, state)
     _finishhistedit(ui, repo, state, fm)
     fm.end()
 
 
 def _continuehistedit(ui, repo, state):
     """This function runs after either:
     - bootstrapcontinue (if the goal is 'continue')
     - _newhistedit (if the goal is 'new')
     """
     # preprocess rules so that we can hide inner folds from the user
     # and only show one editor
     actions = state.actions[:]
     for idx, (action, nextact) in enumerate(zip(actions, actions[1:] + [None])):
         if action.verb == b'fold' and nextact and nextact.verb == b'fold':
             state.actions[idx].__class__ = _multifold
 
     # Force an initial state file write, so the user can run --abort/continue
     # even if there's an exception before the first transaction serialize.
     state.write()
 
     tr = None
     # Don't use singletransaction by default since it rolls the entire
     # transaction back if an unexpected exception happens (like a
     # pretxncommit hook throws, or the user aborts the commit msg editor).
     if ui.configbool(b"histedit", b"singletransaction"):
         # Don't use a 'with' for the transaction, since actions may close
         # and reopen a transaction. For example, if the action executes an
         # external process it may choose to commit the transaction first.
         tr = repo.transaction(b'histedit')
     progress = ui.makeprogress(
         _(b"editing"), unit=_(b'changes'), total=len(state.actions)
     )
     with progress, util.acceptintervention(tr):
         while state.actions:
             state.write(tr=tr)
             actobj = state.actions[0]
             progress.increment(item=actobj.torule())
             ui.debug(
                 b'histedit: processing %s %s\n' % (actobj.verb, actobj.torule())
             )
             parentctx, replacement_ = actobj.run()
             state.parentctxnode = parentctx.node()
             state.replacements.extend(replacement_)
             state.actions.pop(0)
 
     state.write()
 
 
 def _finishhistedit(ui, repo, state, fm):
     """This action runs when histedit is finishing its session"""
     mergemod.update(repo[state.parentctxnode])
 
     mapping, tmpnodes, created, ntm = processreplacement(state)
     if mapping:
         for prec, succs in pycompat.iteritems(mapping):
             if not succs:
                 ui.debug(b'histedit: %s is dropped\n' % short(prec))
             else:
                 ui.debug(
                     b'histedit: %s is replaced by %s\n'
                     % (short(prec), short(succs[0]))
                 )
                 if len(succs) > 1:
                     m = b'histedit:                            %s'
                     for n in succs[1:]:
                         ui.debug(m % short(n))
 
     if not state.keep:
         if mapping:
             movetopmostbookmarks(repo, state.topmost, ntm)
             # TODO update mq state
     else:
         mapping = {}
 
     for n in tmpnodes:
         if n in repo:
             mapping[n] = ()
 
     # remove entries about unknown nodes
     has_node = repo.unfiltered().changelog.index.has_node
     mapping = {
         k: v
         for k, v in mapping.items()
         if has_node(k) and all(has_node(n) for n in v)
     }
     scmutil.cleanupnodes(repo, mapping, b'histedit')
     hf = fm.hexfunc
     fl = fm.formatlist
     fd = fm.formatdict
     nodechanges = fd(
         {
             hf(oldn): fl([hf(n) for n in newn], name=b'node')
             for oldn, newn in pycompat.iteritems(mapping)
         },
         key=b"oldnode",
         value=b"newnodes",
     )
     fm.data(nodechanges=nodechanges)
 
     state.clear()
     if os.path.exists(repo.sjoin(b'undo')):
         os.unlink(repo.sjoin(b'undo'))
     if repo.vfs.exists(b'histedit-last-edit.txt'):
         repo.vfs.unlink(b'histedit-last-edit.txt')
 
 
 def _aborthistedit(ui, repo, state, nobackup=False):
     try:
         state.read()
         __, leafs, tmpnodes, __ = processreplacement(state)
         ui.debug(b'restore wc to old parent %s\n' % short(state.topmost))
 
         # Recover our old commits if necessary
         if not state.topmost in repo and state.backupfile:
             backupfile = repo.vfs.join(state.backupfile)
             f = hg.openpath(ui, backupfile)
             gen = exchange.readbundle(ui, f, backupfile)
             with repo.transaction(b'histedit.abort') as tr:
                 bundle2.applybundle(
                     repo,
                     gen,
                     tr,
                     source=b'histedit',
                     url=b'bundle:' + backupfile,
                 )
 
             os.remove(backupfile)
 
         # check whether we should update away
         if repo.unfiltered().revs(
             b'parents() and (%n  or %ln::)',
             state.parentctxnode,
             leafs | tmpnodes,
         ):
             hg.clean(repo, state.topmost, show_stats=True, quietempty=True)
         cleanupnode(ui, repo, tmpnodes, nobackup=nobackup)
         cleanupnode(ui, repo, leafs, nobackup=nobackup)
     except Exception:
         if state.inprogress():
             ui.warn(
                 _(
                     b'warning: encountered an exception during histedit '
                     b'--abort; the repository may not have been completely '
                     b'cleaned up\n'
                 )
             )
         raise
     finally:
         state.clear()
 
 
 def hgaborthistedit(ui, repo):
     state = histeditstate(repo)
     nobackup = not ui.configbool(b'rewrite', b'backup-bundle')
     with repo.wlock() as wlock, repo.lock() as lock:
         state.wlock = wlock
         state.lock = lock
         _aborthistedit(ui, repo, state, nobackup=nobackup)
 
 
 def _edithisteditplan(ui, repo, state, rules):
     state.read()
     if not rules:
         comment = geteditcomment(
             ui, short(state.parentctxnode), short(state.topmost)
         )
         rules = ruleeditor(repo, ui, state.actions, comment)
     else:
         rules = _readfile(ui, rules)
     actions = parserules(rules, state)
     ctxs = [repo[act.node] for act in state.actions if act.node]
     warnverifyactions(ui, repo, actions, state, ctxs)
     state.actions = actions
     state.write()
 
 
 def _newhistedit(ui, repo, state, revs, freeargs, opts):
     outg = opts.get(b'outgoing')
     rules = opts.get(b'commands', b'')
     force = opts.get(b'force')
 
     cmdutil.checkunfinished(repo)
     cmdutil.bailifchanged(repo)
 
     topmost = repo.dirstate.p1()
     if outg:
         if freeargs:
             remote = freeargs[0]
         else:
             remote = None
         root = findoutgoing(ui, repo, remote, force, opts)
     else:
         rr = list(repo.set(b'roots(%ld)', logcmdutil.revrange(repo, revs)))
         if len(rr) != 1:
             raise error.InputError(
                 _(
                     b'The specified revisions must have '
                     b'exactly one common root'
                 )
             )
         root = rr[0].node()
 
     revs = between(repo, root, topmost, state.keep)
     if not revs:
         raise error.InputError(
             _(b'%s is not an ancestor of working directory') % short(root)
         )
 
     ctxs = [repo[r] for r in revs]
 
     wctx = repo[None]
     # Please don't ask me why `ancestors` is this value. I figured it
     # out with print-debugging, not by actually understanding what the
     # merge code is doing. :(
     ancs = [repo[b'.']]
     # Sniff-test to make sure we won't collide with untracked files in
     # the working directory. If we don't do this, we can get a
     # collision after we've started histedit and backing out gets ugly
     # for everyone, especially the user.
     for c in [ctxs[0].p1()] + ctxs:
         try:
             mergemod.calculateupdates(
                 repo,
                 wctx,
                 c,
                 ancs,
                 # These parameters were determined by print-debugging
                 # what happens later on inside histedit.
                 branchmerge=False,
                 force=False,
                 acceptremote=False,
                 followcopies=False,
             )
         except error.Abort:
             raise error.StateError(
                 _(
                     b"untracked files in working directory conflict with files in %s"
                 )
                 % c
             )
 
     if not rules:
         comment = geteditcomment(ui, short(root), short(topmost))
         actions = [pick(state, r) for r in revs]
         rules = ruleeditor(repo, ui, actions, comment)
     else:
         rules = _readfile(ui, rules)
     actions = parserules(rules, state)
     warnverifyactions(ui, repo, actions, state, ctxs)
 
     parentctxnode = repo[root].p1().node()
 
     state.parentctxnode = parentctxnode
     state.actions = actions
     state.topmost = topmost
     state.replacements = []
 
     ui.log(
         b"histedit",
         b"%d actions to histedit\n",
         len(actions),
         histedit_num_actions=len(actions),
     )
 
     # Create a backup so we can always abort completely.
     backupfile = None
     if not obsolete.isenabled(repo, obsolete.createmarkersopt):
         backupfile = repair.backupbundle(
             repo, [parentctxnode], [topmost], root, b'histedit'
         )
     state.backupfile = backupfile
 
 
 def _getsummary(ctx):
     # a common pattern is to extract the summary but default to the empty
     # string
     summary = ctx.description() or b''
     if summary:
         summary = summary.splitlines()[0]
     return summary
 
 
 def bootstrapcontinue(ui, state, opts):
     repo = state.repo
 
     ms = mergestatemod.mergestate.read(repo)
     mergeutil.checkunresolved(ms)
 
     if state.actions:
         actobj = state.actions.pop(0)
 
         if _isdirtywc(repo):
             actobj.continuedirty()
             if _isdirtywc(repo):
                 abortdirty()
 
         parentctx, replacements = actobj.continueclean()
 
         state.parentctxnode = parentctx.node()
         state.replacements.extend(replacements)
 
     return state
 
 
 def between(repo, old, new, keep):
     """select and validate the set of revision to edit
 
     When keep is false, the specified set can't have children."""
     revs = repo.revs(b'%n::%n', old, new)
     if revs and not keep:
         rewriteutil.precheck(repo, revs, b'edit')
         if repo.revs(b'(%ld) and merge()', revs):
             raise error.StateError(
                 _(b'cannot edit history that contains merges')
             )
     return pycompat.maplist(repo.changelog.node, revs)
 
 
 def ruleeditor(repo, ui, actions, editcomment=b""):
     """open an editor to edit rules
 
     rules are in the format [ [act, ctx], ...] like in state.rules
     """
     if repo.ui.configbool(b"experimental", b"histedit.autoverb"):
         newact = util.sortdict()
         for act in actions:
             ctx = repo[act.node]
             summary = _getsummary(ctx)
             fword = summary.split(b' ', 1)[0].lower()
             added = False
 
             # if it doesn't end with the special character '!' just skip this
             if fword.endswith(b'!'):
                 fword = fword[:-1]
                 if fword in primaryactions | secondaryactions | tertiaryactions:
                     act.verb = fword
                     # get the target summary
                     tsum = summary[len(fword) + 1 :].lstrip()
                     # safe but slow: reverse iterate over the actions so we
                     # don't clash on two commits having the same summary
                     for na, l in reversed(list(pycompat.iteritems(newact))):
                         actx = repo[na.node]
                         asum = _getsummary(actx)
                         if asum == tsum:
                             added = True
                             l.append(act)
                             break
 
             if not added:
                 newact[act] = []
 
         # copy over and flatten the new list
         actions = []
         for na, l in pycompat.iteritems(newact):
             actions.append(na)
             actions += l
 
     rules = b'\n'.join([act.torule() for act in actions])
     rules += b'\n\n'
     rules += editcomment
     rules = ui.edit(
         rules,
         ui.username(),
         {b'prefix': b'histedit'},
         repopath=repo.path,
         action=b'histedit',
     )
 
     # Save edit rules in .hg/histedit-last-edit.txt in case
     # the user needs to ask for help after something
     # surprising happens.
     with repo.vfs(b'histedit-last-edit.txt', b'wb') as f:
         f.write(rules)
 
     return rules
 
 
 def parserules(rules, state):
     """Read the histedit rules string and return list of action objects"""
     rules = [
         l
         for l in (r.strip() for r in rules.splitlines())
         if l and not l.startswith(b'#')
     ]
     actions = []
     for r in rules:
         if b' ' not in r:
             raise error.ParseError(_(b'malformed line "%s"') % r)
         verb, rest = r.split(b' ', 1)
 
         if verb not in actiontable:
             raise error.ParseError(_(b'unknown action "%s"') % verb)
 
         action = actiontable[verb].fromrule(state, rest)
         actions.append(action)
     return actions
 
 
 def warnverifyactions(ui, repo, actions, state, ctxs):
     try:
         verifyactions(actions, state, ctxs)
     except error.ParseError:
         if repo.vfs.exists(b'histedit-last-edit.txt'):
             ui.warn(
                 _(
                     b'warning: histedit rules saved '
                     b'to: .hg/histedit-last-edit.txt\n'
                 )
             )
         raise
 
 
 def verifyactions(actions, state, ctxs):
     """Verify that there exists exactly one action per given changeset and
     other constraints.
 
     Will abort if there are to many or too few rules, a malformed rule,
     or a rule on a changeset outside of the user-given range.
     """
     expected = {c.node() for c in ctxs}
     seen = set()
     prev = None
 
     if actions and actions[0].verb in [b'roll', b'fold']:
         raise error.ParseError(
             _(b'first changeset cannot use verb "%s"') % actions[0].verb
         )
 
     for action in actions:
         action.verify(prev, expected, seen)
         prev = action
         if action.node is not None:
             seen.add(action.node)
     missing = sorted(expected - seen)  # sort to stabilize output
 
     if state.repo.ui.configbool(b'histedit', b'dropmissing'):
         if len(actions) == 0:
             raise error.ParseError(
                 _(b'no rules provided'),
                 hint=_(b'use strip extension to remove commits'),
             )
 
         drops = [drop(state, n) for n in missing]
         # put the in the beginning so they execute immediately and
         # don't show in the edit-plan in the future
         actions[:0] = drops
     elif missing:
         raise error.ParseError(
             _(b'missing rules for changeset %s') % short(missing[0]),
             hint=_(
                 b'use "drop %s" to discard, see also: '
                 b"'hg help -e histedit.config'"
             )
             % short(missing[0]),
         )
 
 
 def adjustreplacementsfrommarkers(repo, oldreplacements):
     """Adjust replacements from obsolescence markers
 
     Replacements structure is originally generated based on
     histedit's state and does not account for changes that are
     not recorded there. This function fixes that by adding
     data read from obsolescence markers"""
     if not obsolete.isenabled(repo, obsolete.createmarkersopt):
         return oldreplacements
 
     unfi = repo.unfiltered()
     get_rev = unfi.changelog.index.get_rev
     obsstore = repo.obsstore
     newreplacements = list(oldreplacements)
     oldsuccs = [r[1] for r in oldreplacements]
     # successors that have already been added to succstocheck once
     seensuccs = set().union(
         *oldsuccs
     )  # create a set from an iterable of tuples
     succstocheck = list(seensuccs)
     while succstocheck:
         n = succstocheck.pop()
         missing = get_rev(n) is None
         markers = obsstore.successors.get(n, ())
         if missing and not markers:
             # dead end, mark it as such
             newreplacements.append((n, ()))
         for marker in markers:
             nsuccs = marker[1]
             newreplacements.append((n, nsuccs))
             for nsucc in nsuccs:
                 if nsucc not in seensuccs:
                     seensuccs.add(nsucc)
                     succstocheck.append(nsucc)
 
     return newreplacements
 
 
 def processreplacement(state):
     """process the list of replacements to return
 
     1) the final mapping between original and created nodes
     2) the list of temporary node created by histedit
     3) the list of new commit created by histedit"""
     replacements = adjustreplacementsfrommarkers(state.repo, state.replacements)
     allsuccs = set()
     replaced = set()
     fullmapping = {}
     # initialize basic set
     # fullmapping records all operations recorded in replacement
     for rep in replacements:
         allsuccs.update(rep[1])
         replaced.add(rep[0])
         fullmapping.setdefault(rep[0], set()).update(rep[1])
     new = allsuccs - replaced
     tmpnodes = allsuccs & replaced
     # Reduce content fullmapping into direct relation between original nodes
     # and final node created during history edition
     # Dropped changeset are replaced by an empty list
     toproceed = set(fullmapping)
     final = {}
     while toproceed:
         for x in list(toproceed):
             succs = fullmapping[x]
             for s in list(succs):
                 if s in toproceed:
                     # non final node with unknown closure
                     # We can't process this now
                     break
                 elif s in final:
                     # non final node, replace with closure
                     succs.remove(s)
                     succs.update(final[s])
             else:
                 final[x] = succs
                 toproceed.remove(x)
     # remove tmpnodes from final mapping
     for n in tmpnodes:
         del final[n]
     # we expect all changes involved in final to exist in the repo
     # turn `final` into list (topologically sorted)
     get_rev = state.repo.changelog.index.get_rev
     for prec, succs in final.items():
         final[prec] = sorted(succs, key=get_rev)
 
     # computed topmost element (necessary for bookmark)
     if new:
         newtopmost = sorted(new, key=state.repo.changelog.rev)[-1]
     elif not final:
         # Nothing rewritten at all. we won't need `newtopmost`
         # It is the same as `oldtopmost` and `processreplacement` know it
         newtopmost = None
     else:
         # every body died. The newtopmost is the parent of the root.
         r = state.repo.changelog.rev
         newtopmost = state.repo[sorted(final, key=r)[0]].p1().node()
 
     return final, tmpnodes, new, newtopmost
 
 
 def movetopmostbookmarks(repo, oldtopmost, newtopmost):
     """Move bookmark from oldtopmost to newly created topmost
 
     This is arguably a feature and we may only want that for the active
     bookmark. But the behavior is kept compatible with the old version for now.
     """
     if not oldtopmost or not newtopmost:
         return
     oldbmarks = repo.nodebookmarks(oldtopmost)
     if oldbmarks:
         with repo.lock(), repo.transaction(b'histedit') as tr:
             marks = repo._bookmarks
             changes = []
             for name in oldbmarks:
                 changes.append((name, newtopmost))
             marks.applychanges(repo, tr, changes)
 
 
 def cleanupnode(ui, repo, nodes, nobackup=False):
     """strip a group of nodes from the repository
 
     The set of node to strip may contains unknown nodes."""
     with repo.lock():
         # do not let filtering get in the way of the cleanse
         # we should probably get rid of obsolescence marker created during the
         # histedit, but we currently do not have such information.
         repo = repo.unfiltered()
         # Find all nodes that need to be stripped
         # (we use %lr instead of %ln to silently ignore unknown items)
         has_node = repo.changelog.index.has_node
         nodes = sorted(n for n in nodes if has_node(n))
         roots = [c.node() for c in repo.set(b"roots(%ln)", nodes)]
         if roots:
             backup = not nobackup
             repair.strip(ui, repo, roots, backup=backup)
 
 
 def stripwrapper(orig, ui, repo, nodelist, *args, **kwargs):
     if isinstance(nodelist, bytes):
         nodelist = [nodelist]
     state = histeditstate(repo)
     if state.inprogress():
         state.read()
         histedit_nodes = {
             action.node for action in state.actions if action.node
         }
         common_nodes = histedit_nodes & set(nodelist)
         if common_nodes:
             raise error.Abort(
                 _(b"histedit in progress, can't strip %s")
                 % b', '.join(short(x) for x in common_nodes)
             )
     return orig(ui, repo, nodelist, *args, **kwargs)
 
 
 extensions.wrapfunction(repair, b'strip', stripwrapper)
 
 
 def summaryhook(ui, repo):
     state = histeditstate(repo)
     if not state.inprogress():
         return
     state.read()
     if state.actions:
         # i18n: column positioning for "hg summary"
         ui.write(
             _(b'hist:   %s (histedit --continue)\n')
             % (
                 ui.label(_(b'%d remaining'), b'histedit.remaining')
                 % len(state.actions)
             )
         )
 
 
 def extsetup(ui):
     cmdutil.summaryhooks.add(b'histedit', summaryhook)
     statemod.addunfinished(
         b'histedit',
         fname=b'histedit-state',
         allowcommit=True,
         continueflag=True,
         abortfunc=hgaborthistedit,
     )

mercurial/formatter.py

@@ -1,873 +1,872 b''
 # formatter.py - generic output formatting for mercurial
 #
 # Copyright 2012 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Generic output formatting for Mercurial
 
 The formatter provides API to show data in various ways. The following
 functions should be used in place of ui.write():
 
 - fm.write() for unconditional output
 - fm.condwrite() to show some extra data conditionally in plain output
 - fm.context() to provide changectx to template output
 - fm.data() to provide extra data to JSON or template output
 - fm.plain() to show raw text that isn't provided to JSON or template output
 
 To show structured data (e.g. date tuples, dicts, lists), apply fm.format*()
 beforehand so the data is converted to the appropriate data type. Use
 fm.isplain() if you need to convert or format data conditionally which isn't
 supported by the formatter API.
 
 To build nested structure (i.e. a list of dicts), use fm.nested().
 
 See also https://www.mercurial-scm.org/wiki/GenericTemplatingPlan
 
 fm.condwrite() vs 'if cond:':
 
 In most cases, use fm.condwrite() so users can selectively show the data
 in template output. If it's costly to build data, use plain 'if cond:' with
 fm.write().
 
 fm.nested() vs fm.formatdict() (or fm.formatlist()):
 
 fm.nested() should be used to form a tree structure (a list of dicts of
 lists of dicts...) which can be accessed through template keywords, e.g.
 "{foo % "{bar % {...}} {baz % {...}}"}". On the other hand, fm.formatdict()
 exports a dict-type object to template, which can be accessed by e.g.
 "{get(foo, key)}" function.
 
 Doctest helper:
 
 >>> def show(fn, verbose=False, **opts):
 ...     import sys
 ...     from . import ui as uimod
 ...     ui = uimod.ui()
 ...     ui.verbose = verbose
 ...     ui.pushbuffer()
 ...     try:
 ...         return fn(ui, ui.formatter(pycompat.sysbytes(fn.__name__),
 ...                   pycompat.byteskwargs(opts)))
 ...     finally:
 ...         print(pycompat.sysstr(ui.popbuffer()), end='')
 
 Basic example:
 
 >>> def files(ui, fm):
 ...     files = [(b'foo', 123, (0, 0)), (b'bar', 456, (1, 0))]
 ...     for f in files:
 ...         fm.startitem()
 ...         fm.write(b'path', b'%s', f[0])
 ...         fm.condwrite(ui.verbose, b'date', b'  %s',
 ...                      fm.formatdate(f[2], b'%Y-%m-%d %H:%M:%S'))
 ...         fm.data(size=f[1])
 ...         fm.plain(b'\\n')
 ...     fm.end()
 >>> show(files)
 foo
 bar
 >>> show(files, verbose=True)
 foo  1970-01-01 00:00:00
 bar  1970-01-01 00:00:01
 >>> show(files, template=b'json')
 [
  {
   "date": [0, 0],
   "path": "foo",
   "size": 123
  },
  {
   "date": [1, 0],
   "path": "bar",
   "size": 456
  }
 ]
 >>> show(files, template=b'path: {path}\\ndate: {date|rfc3339date}\\n')
 path: foo
 date: 1970-01-01T00:00:00+00:00
 path: bar
 date: 1970-01-01T00:00:01+00:00
 
 Nested example:
 
 >>> def subrepos(ui, fm):
 ...     fm.startitem()
 ...     fm.write(b'reponame', b'[%s]\\n', b'baz')
 ...     files(ui, fm.nested(b'files', tmpl=b'{reponame}'))
 ...     fm.end()
 >>> show(subrepos)
 [baz]
 foo
 bar
 >>> show(subrepos, template=b'{reponame}: {join(files % "{path}", ", ")}\\n')
 baz: foo, bar
 """
 
 from __future__ import absolute_import, print_function
 
 import contextlib
 import itertools
 import os
+import pickle
 
 from .i18n import _
 from .node import (
     hex,
     short,
 )
 from .thirdparty import attr
 
 from . import (
     error,
     pycompat,
     templatefilters,
     templatekw,
     templater,
     templateutil,
     util,
 )
 from .utils import (
     cborutil,
     dateutil,
     stringutil,
 )
 
-pickle = util.pickle
-
 
 def isprintable(obj):
     """Check if the given object can be directly passed in to formatter's
     write() and data() functions
 
     Returns False if the object is unsupported or must be pre-processed by
     formatdate(), formatdict(), or formatlist().
     """
     return isinstance(obj, (type(None), bool, int, pycompat.long, float, bytes))
 
 
 class _nullconverter(object):
     '''convert non-primitive data types to be processed by formatter'''
 
     # set to True if context object should be stored as item
     storecontext = False
 
     @staticmethod
     def wrapnested(data, tmpl, sep):
         '''wrap nested data by appropriate type'''
         return data
 
     @staticmethod
     def formatdate(date, fmt):
         '''convert date tuple to appropriate format'''
         # timestamp can be float, but the canonical form should be int
         ts, tz = date
         return (int(ts), tz)
 
     @staticmethod
     def formatdict(data, key, value, fmt, sep):
         '''convert dict or key-value pairs to appropriate dict format'''
         # use plain dict instead of util.sortdict so that data can be
         # serialized as a builtin dict in pickle output
         return dict(data)
 
     @staticmethod
     def formatlist(data, name, fmt, sep):
         '''convert iterable to appropriate list format'''
         return list(data)
 
 
 class baseformatter(object):
 
     # set to True if the formater output a strict format that does not support
     # arbitrary output in the stream.
     strict_format = False
 
     def __init__(self, ui, topic, opts, converter):
         self._ui = ui
         self._topic = topic
         self._opts = opts
         self._converter = converter
         self._item = None
         # function to convert node to string suitable for this output
         self.hexfunc = hex
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, traceback):
         if exctype is None:
             self.end()
 
     def _showitem(self):
         '''show a formatted item once all data is collected'''
 
     def startitem(self):
         '''begin an item in the format list'''
         if self._item is not None:
             self._showitem()
         self._item = {}
 
     def formatdate(self, date, fmt=b'%a %b %d %H:%M:%S %Y %1%2'):
         '''convert date tuple to appropriate format'''
         return self._converter.formatdate(date, fmt)
 
     def formatdict(self, data, key=b'key', value=b'value', fmt=None, sep=b' '):
         '''convert dict or key-value pairs to appropriate dict format'''
         return self._converter.formatdict(data, key, value, fmt, sep)
 
     def formatlist(self, data, name, fmt=None, sep=b' '):
         '''convert iterable to appropriate list format'''
         # name is mandatory argument for now, but it could be optional if
         # we have default template keyword, e.g. {item}
         return self._converter.formatlist(data, name, fmt, sep)
 
     def context(self, **ctxs):
         '''insert context objects to be used to render template keywords'''
         ctxs = pycompat.byteskwargs(ctxs)
         assert all(k in {b'repo', b'ctx', b'fctx'} for k in ctxs)
         if self._converter.storecontext:
             # populate missing resources in fctx -> ctx -> repo order
             if b'fctx' in ctxs and b'ctx' not in ctxs:
                 ctxs[b'ctx'] = ctxs[b'fctx'].changectx()
             if b'ctx' in ctxs and b'repo' not in ctxs:
                 ctxs[b'repo'] = ctxs[b'ctx'].repo()
             self._item.update(ctxs)
 
     def datahint(self):
         '''set of field names to be referenced'''
         return set()
 
     def data(self, **data):
         '''insert data into item that's not shown in default output'''
         data = pycompat.byteskwargs(data)
         self._item.update(data)
 
     def write(self, fields, deftext, *fielddata, **opts):
         '''do default text output while assigning data to item'''
         fieldkeys = fields.split()
         assert len(fieldkeys) == len(fielddata), (fieldkeys, fielddata)
         self._item.update(zip(fieldkeys, fielddata))
 
     def condwrite(self, cond, fields, deftext, *fielddata, **opts):
         '''do conditional write (primarily for plain formatter)'''
         fieldkeys = fields.split()
         assert len(fieldkeys) == len(fielddata)
         self._item.update(zip(fieldkeys, fielddata))
 
     def plain(self, text, **opts):
         '''show raw text for non-templated mode'''
 
     def isplain(self):
         '''check for plain formatter usage'''
         return False
 
     def nested(self, field, tmpl=None, sep=b''):
         '''sub formatter to store nested data in the specified field'''
         data = []
         self._item[field] = self._converter.wrapnested(data, tmpl, sep)
         return _nestedformatter(self._ui, self._converter, data)
 
     def end(self):
         '''end output for the formatter'''
         if self._item is not None:
             self._showitem()
 
 
 def nullformatter(ui, topic, opts):
     '''formatter that prints nothing'''
     return baseformatter(ui, topic, opts, converter=_nullconverter)
 
 
 class _nestedformatter(baseformatter):
     '''build sub items and store them in the parent formatter'''
 
     def __init__(self, ui, converter, data):
         baseformatter.__init__(
             self, ui, topic=b'', opts={}, converter=converter
         )
         self._data = data
 
     def _showitem(self):
         self._data.append(self._item)
 
 
 def _iteritems(data):
     '''iterate key-value pairs in stable order'''
     if isinstance(data, dict):
         return sorted(pycompat.iteritems(data))
     return data
 
 
 class _plainconverter(object):
     '''convert non-primitive data types to text'''
 
     storecontext = False
 
     @staticmethod
     def wrapnested(data, tmpl, sep):
         raise error.ProgrammingError(b'plainformatter should never be nested')
 
     @staticmethod
     def formatdate(date, fmt):
         '''stringify date tuple in the given format'''
         return dateutil.datestr(date, fmt)
 
     @staticmethod
     def formatdict(data, key, value, fmt, sep):
         '''stringify key-value pairs separated by sep'''
         prefmt = pycompat.identity
         if fmt is None:
             fmt = b'%s=%s'
             prefmt = pycompat.bytestr
         return sep.join(
             fmt % (prefmt(k), prefmt(v)) for k, v in _iteritems(data)
         )
 
     @staticmethod
     def formatlist(data, name, fmt, sep):
         '''stringify iterable separated by sep'''
         prefmt = pycompat.identity
         if fmt is None:
             fmt = b'%s'
             prefmt = pycompat.bytestr
         return sep.join(fmt % prefmt(e) for e in data)
 
 
 class plainformatter(baseformatter):
     '''the default text output scheme'''
 
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _plainconverter)
         if ui.debugflag:
             self.hexfunc = hex
         else:
             self.hexfunc = short
         if ui is out:
             self._write = ui.write
         else:
             self._write = lambda s, **opts: out.write(s)
 
     def startitem(self):
         pass
 
     def data(self, **data):
         pass
 
     def write(self, fields, deftext, *fielddata, **opts):
         self._write(deftext % fielddata, **opts)
 
     def condwrite(self, cond, fields, deftext, *fielddata, **opts):
         '''do conditional write'''
         if cond:
             self._write(deftext % fielddata, **opts)
 
     def plain(self, text, **opts):
         self._write(text, **opts)
 
     def isplain(self):
         return True
 
     def nested(self, field, tmpl=None, sep=b''):
         # nested data will be directly written to ui
         return self
 
     def end(self):
         pass
 
 
 class debugformatter(baseformatter):
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._out.write(b"%s = [\n" % self._topic)
 
     def _showitem(self):
         self._out.write(
             b'    %s,\n' % stringutil.pprint(self._item, indent=4, level=1)
         )
 
     def end(self):
         baseformatter.end(self)
         self._out.write(b"]\n")
 
 
 class pickleformatter(baseformatter):
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._data = []
 
     def _showitem(self):
         self._data.append(self._item)
 
     def end(self):
         baseformatter.end(self)
         self._out.write(pickle.dumps(self._data))
 
 
 class cborformatter(baseformatter):
     '''serialize items as an indefinite-length CBOR array'''
 
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._out.write(cborutil.BEGIN_INDEFINITE_ARRAY)
 
     def _showitem(self):
         self._out.write(b''.join(cborutil.streamencode(self._item)))
 
     def end(self):
         baseformatter.end(self)
         self._out.write(cborutil.BREAK)
 
 
 class jsonformatter(baseformatter):
 
     strict_format = True
 
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._out.write(b"[")
         self._first = True
 
     def _showitem(self):
         if self._first:
             self._first = False
         else:
             self._out.write(b",")
 
         self._out.write(b"\n {\n")
         first = True
         for k, v in sorted(self._item.items()):
             if first:
                 first = False
             else:
                 self._out.write(b",\n")
             u = templatefilters.json(v, paranoid=False)
             self._out.write(b'  "%s": %s' % (k, u))
         self._out.write(b"\n }")
 
     def end(self):
         baseformatter.end(self)
         self._out.write(b"\n]\n")
 
 
 class _templateconverter(object):
     '''convert non-primitive data types to be processed by templater'''
 
     storecontext = True
 
     @staticmethod
     def wrapnested(data, tmpl, sep):
         '''wrap nested data by templatable type'''
         return templateutil.mappinglist(data, tmpl=tmpl, sep=sep)
 
     @staticmethod
     def formatdate(date, fmt):
         '''return date tuple'''
         return templateutil.date(date)
 
     @staticmethod
     def formatdict(data, key, value, fmt, sep):
         '''build object that can be evaluated as either plain string or dict'''
         data = util.sortdict(_iteritems(data))
 
         def f():
             yield _plainconverter.formatdict(data, key, value, fmt, sep)
 
         return templateutil.hybriddict(
             data, key=key, value=value, fmt=fmt, gen=f
         )
 
     @staticmethod
     def formatlist(data, name, fmt, sep):
         '''build object that can be evaluated as either plain string or list'''
         data = list(data)
 
         def f():
             yield _plainconverter.formatlist(data, name, fmt, sep)
 
         return templateutil.hybridlist(data, name=name, fmt=fmt, gen=f)
 
 
 class templateformatter(baseformatter):
     def __init__(self, ui, out, topic, opts, spec, overridetemplates=None):
         baseformatter.__init__(self, ui, topic, opts, _templateconverter)
         self._out = out
         self._tref = spec.ref
         self._t = loadtemplater(
             ui,
             spec,
             defaults=templatekw.keywords,
             resources=templateresources(ui),
             cache=templatekw.defaulttempl,
         )
         if overridetemplates:
             self._t.cache.update(overridetemplates)
         self._parts = templatepartsmap(
             spec, self._t, [b'docheader', b'docfooter', b'separator']
         )
         self._counter = itertools.count()
         self._renderitem(b'docheader', {})
 
     def _showitem(self):
         item = self._item.copy()
         item[b'index'] = index = next(self._counter)
         if index > 0:
             self._renderitem(b'separator', {})
         self._renderitem(self._tref, item)
 
     def _renderitem(self, part, item):
         if part not in self._parts:
             return
         ref = self._parts[part]
         # None can't be put in the mapping dict since it means <unset>
         for k, v in item.items():
             if v is None:
                 item[k] = templateutil.wrappedvalue(v)
         self._out.write(self._t.render(ref, item))
 
     @util.propertycache
     def _symbolsused(self):
         return self._t.symbolsused(self._tref)
 
     def datahint(self):
         '''set of field names to be referenced from the template'''
         return self._symbolsused[0]
 
     def end(self):
         baseformatter.end(self)
         self._renderitem(b'docfooter', {})
 
 
 @attr.s(frozen=True)
 class templatespec(object):
     ref = attr.ib()
     tmpl = attr.ib()
     mapfile = attr.ib()
     refargs = attr.ib(default=None)
     fp = attr.ib(default=None)
 
 
 def empty_templatespec():
     return templatespec(None, None, None)
 
 
 def reference_templatespec(ref, refargs=None):
     return templatespec(ref, None, None, refargs)
 
 
 def literal_templatespec(tmpl):
     if pycompat.ispy3:
         assert not isinstance(tmpl, str), b'tmpl must not be a str'
     return templatespec(b'', tmpl, None)
 
 
 def mapfile_templatespec(topic, mapfile, fp=None):
     return templatespec(topic, None, mapfile, fp=fp)
 
 
 def lookuptemplate(ui, topic, tmpl):
     """Find the template matching the given -T/--template spec 'tmpl'
 
     'tmpl' can be any of the following:
 
      - a literal template (e.g. '{rev}')
      - a reference to built-in template (i.e. formatter)
      - a map-file name or path (e.g. 'changelog')
      - a reference to [templates] in config file
      - a path to raw template file
 
     A map file defines a stand-alone template environment. If a map file
     selected, all templates defined in the file will be loaded, and the
     template matching the given topic will be rendered. Aliases won't be
     loaded from user config, but from the map file.
 
     If no map file selected, all templates in [templates] section will be
     available as well as aliases in [templatealias].
     """
 
     if not tmpl:
         return empty_templatespec()
 
     # looks like a literal template?
     if b'{' in tmpl:
         return literal_templatespec(tmpl)
 
     # a reference to built-in (formatter) template
     if tmpl in {b'cbor', b'json', b'pickle', b'debug'}:
         return reference_templatespec(tmpl)
 
     # a function-style reference to built-in template
     func, fsep, ftail = tmpl.partition(b'(')
     if func in {b'cbor', b'json'} and fsep and ftail.endswith(b')'):
         templater.parseexpr(tmpl)  # make sure syntax errors are confined
         return reference_templatespec(func, refargs=ftail[:-1])
 
     # perhaps a stock style?
     if not os.path.split(tmpl)[0]:
         (mapname, fp) = templater.try_open_template(
             b'map-cmdline.' + tmpl
         ) or templater.try_open_template(tmpl)
         if mapname:
             return mapfile_templatespec(topic, mapname, fp)
 
     # perhaps it's a reference to [templates]
     if ui.config(b'templates', tmpl):
         return reference_templatespec(tmpl)
 
     if tmpl == b'list':
         ui.write(_(b"available styles: %s\n") % templater.stylelist())
         raise error.Abort(_(b"specify a template"))
 
     # perhaps it's a path to a map or a template
     if (b'/' in tmpl or b'\\' in tmpl) and os.path.isfile(tmpl):
         # is it a mapfile for a style?
         if os.path.basename(tmpl).startswith(b"map-"):
             return mapfile_templatespec(topic, os.path.realpath(tmpl))
         with util.posixfile(tmpl, b'rb') as f:
             tmpl = f.read()
         return literal_templatespec(tmpl)
 
     # constant string?
     return literal_templatespec(tmpl)
 
 
 def templatepartsmap(spec, t, partnames):
     """Create a mapping of {part: ref}"""
     partsmap = {spec.ref: spec.ref}  # initial ref must exist in t
     if spec.mapfile:
         partsmap.update((p, p) for p in partnames if p in t)
     elif spec.ref:
         for part in partnames:
             ref = b'%s:%s' % (spec.ref, part)  # select config sub-section
             if ref in t:
                 partsmap[part] = ref
     return partsmap
 
 
 def loadtemplater(ui, spec, defaults=None, resources=None, cache=None):
     """Create a templater from either a literal template or loading from
     a map file"""
     assert not (spec.tmpl and spec.mapfile)
     if spec.mapfile:
         return templater.templater.frommapfile(
             spec.mapfile,
             spec.fp,
             defaults=defaults,
             resources=resources,
             cache=cache,
         )
     return maketemplater(
         ui, spec.tmpl, defaults=defaults, resources=resources, cache=cache
     )
 
 
 def maketemplater(ui, tmpl, defaults=None, resources=None, cache=None):
     """Create a templater from a string template 'tmpl'"""
     aliases = ui.configitems(b'templatealias')
     t = templater.templater(
         defaults=defaults, resources=resources, cache=cache, aliases=aliases
     )
     t.cache.update(
         (k, templater.unquotestring(v)) for k, v in ui.configitems(b'templates')
     )
     if tmpl:
         t.cache[b''] = tmpl
     return t
 
 
 # marker to denote a resource to be loaded on demand based on mapping values
 # (e.g. (ctx, path) -> fctx)
 _placeholder = object()
 
 
 class templateresources(templater.resourcemapper):
     """Resource mapper designed for the default templatekw and function"""
 
     def __init__(self, ui, repo=None):
         self._resmap = {
             b'cache': {},  # for templatekw/funcs to store reusable data
             b'repo': repo,
             b'ui': ui,
         }
 
     def availablekeys(self, mapping):
         return {
             k for k in self.knownkeys() if self._getsome(mapping, k) is not None
         }
 
     def knownkeys(self):
         return {b'cache', b'ctx', b'fctx', b'repo', b'revcache', b'ui'}
 
     def lookup(self, mapping, key):
         if key not in self.knownkeys():
             return None
         v = self._getsome(mapping, key)
         if v is _placeholder:
             v = mapping[key] = self._loadermap[key](self, mapping)
         return v
 
     def populatemap(self, context, origmapping, newmapping):
         mapping = {}
         if self._hasnodespec(newmapping):
             mapping[b'revcache'] = {}  # per-ctx cache
         if self._hasnodespec(origmapping) and self._hasnodespec(newmapping):
             orignode = templateutil.runsymbol(context, origmapping, b'node')
             mapping[b'originalnode'] = orignode
         # put marker to override 'ctx'/'fctx' in mapping if any, and flag
         # its existence to be reported by availablekeys()
         if b'ctx' not in newmapping and self._hasliteral(newmapping, b'node'):
             mapping[b'ctx'] = _placeholder
         if b'fctx' not in newmapping and self._hasliteral(newmapping, b'path'):
             mapping[b'fctx'] = _placeholder
         return mapping
 
     def _getsome(self, mapping, key):
         v = mapping.get(key)
         if v is not None:
             return v
         return self._resmap.get(key)
 
     def _hasliteral(self, mapping, key):
         """Test if a literal value is set or unset in the given mapping"""
         return key in mapping and not callable(mapping[key])
 
     def _getliteral(self, mapping, key):
         """Return value of the given name if it is a literal"""
         v = mapping.get(key)
         if callable(v):
             return None
         return v
 
     def _hasnodespec(self, mapping):
         """Test if context revision is set or unset in the given mapping"""
         return b'node' in mapping or b'ctx' in mapping
 
     def _loadctx(self, mapping):
         repo = self._getsome(mapping, b'repo')
         node = self._getliteral(mapping, b'node')
         if repo is None or node is None:
             return
         try:
             return repo[node]
         except error.RepoLookupError:
             return None  # maybe hidden/non-existent node
 
     def _loadfctx(self, mapping):
         ctx = self._getsome(mapping, b'ctx')
         path = self._getliteral(mapping, b'path')
         if ctx is None or path is None:
             return None
         try:
             return ctx[path]
         except error.LookupError:
             return None  # maybe removed file?
 
     _loadermap = {
         b'ctx': _loadctx,
         b'fctx': _loadfctx,
     }
 
 
 def _internaltemplateformatter(
     ui,
     out,
     topic,
     opts,
     spec,
     tmpl,
     docheader=b'',
     docfooter=b'',
     separator=b'',
 ):
     """Build template formatter that handles customizable built-in templates
     such as -Tjson(...)"""
     templates = {spec.ref: tmpl}
     if docheader:
         templates[b'%s:docheader' % spec.ref] = docheader
     if docfooter:
         templates[b'%s:docfooter' % spec.ref] = docfooter
     if separator:
         templates[b'%s:separator' % spec.ref] = separator
     return templateformatter(
         ui, out, topic, opts, spec, overridetemplates=templates
     )
 
 
 def formatter(ui, out, topic, opts):
     spec = lookuptemplate(ui, topic, opts.get(b'template', b''))
     if spec.ref == b"cbor" and spec.refargs is not None:
         return _internaltemplateformatter(
             ui,
             out,
             topic,
             opts,
             spec,
             tmpl=b'{dict(%s)|cbor}' % spec.refargs,
             docheader=cborutil.BEGIN_INDEFINITE_ARRAY,
             docfooter=cborutil.BREAK,
         )
     elif spec.ref == b"cbor":
         return cborformatter(ui, out, topic, opts)
     elif spec.ref == b"json" and spec.refargs is not None:
         return _internaltemplateformatter(
             ui,
             out,
             topic,
             opts,
             spec,
             tmpl=b'{dict(%s)|json}' % spec.refargs,
             docheader=b'[\n ',
             docfooter=b'\n]\n',
             separator=b',\n ',
         )
     elif spec.ref == b"json":
         return jsonformatter(ui, out, topic, opts)
     elif spec.ref == b"pickle":
         assert spec.refargs is None, r'function-style not supported'
         return pickleformatter(ui, out, topic, opts)
     elif spec.ref == b"debug":
         assert spec.refargs is None, r'function-style not supported'
         return debugformatter(ui, out, topic, opts)
     elif spec.ref or spec.tmpl or spec.mapfile:
         assert spec.refargs is None, r'function-style not supported'
         return templateformatter(ui, out, topic, opts, spec)
     # developer config: ui.formatdebug
     elif ui.configbool(b'ui', b'formatdebug'):
         return debugformatter(ui, out, topic, opts)
     # deprecated config: ui.formatjson
     elif ui.configbool(b'ui', b'formatjson'):
         return jsonformatter(ui, out, topic, opts)
     return plainformatter(ui, out, topic, opts)
 
 
 @contextlib.contextmanager
 def openformatter(ui, filename, topic, opts):
     """Create a formatter that writes outputs to the specified file
 
     Must be invoked using the 'with' statement.
     """
     with util.posixfile(filename, b'wb') as out:
         with formatter(ui, out, topic, opts) as fm:
             yield fm
 
 
 @contextlib.contextmanager
 def _neverending(fm):
     yield fm
 
 
 def maybereopen(fm, filename):
     """Create a formatter backed by file if filename specified, else return
     the given formatter
 
     Must be invoked using the 'with' statement. This will never call fm.end()
     of the given formatter.
     """
     if filename:
         return openformatter(fm._ui, filename, fm._topic, fm._opts)
     else:
         return _neverending(fm)

mercurial/util.py

@@ -1,3361 +1,3360 b''
 # util.py - Mercurial utility functions and platform specific implementations
 #
 #  Copyright 2005 K. Thananchayan <[e-mail]>
 #  Copyright 2005-2007 Olivia Mackall <[e-mail]>
 #  Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Mercurial utility functions and platform specific implementations.
 
 This contains helper routines that are independent of the SCM core and
 hide platform-specific details from the core.
 """
 
 from __future__ import absolute_import, print_function
 
 import abc
 import collections
 import contextlib
 import errno
 import gc
 import hashlib
 import itertools
 import locale
 import mmap
 import os
 import platform as pyplatform
 import re as remod
 import shutil
 import stat
 import sys
 import time
 import traceback
 import warnings
 
 from .node import hex
 from .thirdparty import attr
 from .pycompat import (
     delattr,
     getattr,
     open,
     setattr,
 )
 from .node import hex
 from hgdemandimport import tracing
 from . import (
     encoding,
     error,
     i18n,
     policy,
     pycompat,
     urllibcompat,
 )
 from .utils import (
     compression,
     hashutil,
     procutil,
     stringutil,
 )
 
 if pycompat.TYPE_CHECKING:
     from typing import (
         Iterator,
         List,
         Optional,
         Tuple,
     )
 
 
 base85 = policy.importmod('base85')
 osutil = policy.importmod('osutil')
 
 b85decode = base85.b85decode
 b85encode = base85.b85encode
 
 cookielib = pycompat.cookielib
 httplib = pycompat.httplib
-pickle = pycompat.pickle
 safehasattr = pycompat.safehasattr
 socketserver = pycompat.socketserver
 bytesio = pycompat.bytesio
 # TODO deprecate stringio name, as it is a lie on Python 3.
 stringio = bytesio
 xmlrpclib = pycompat.xmlrpclib
 
 httpserver = urllibcompat.httpserver
 urlerr = urllibcompat.urlerr
 urlreq = urllibcompat.urlreq
 
 # workaround for win32mbcs
 _filenamebytestr = pycompat.bytestr
 
 if pycompat.iswindows:
     from . import windows as platform
 else:
     from . import posix as platform
 
 _ = i18n._
 
 abspath = platform.abspath
 bindunixsocket = platform.bindunixsocket
 cachestat = platform.cachestat
 checkexec = platform.checkexec
 checklink = platform.checklink
 copymode = platform.copymode
 expandglobs = platform.expandglobs
 getfsmountpoint = platform.getfsmountpoint
 getfstype = platform.getfstype
 get_password = platform.get_password
 groupmembers = platform.groupmembers
 groupname = platform.groupname
 isexec = platform.isexec
 isowner = platform.isowner
 listdir = osutil.listdir
 localpath = platform.localpath
 lookupreg = platform.lookupreg
 makedir = platform.makedir
 nlinks = platform.nlinks
 normpath = platform.normpath
 normcase = platform.normcase
 normcasespec = platform.normcasespec
 normcasefallback = platform.normcasefallback
 openhardlinks = platform.openhardlinks
 oslink = platform.oslink
 parsepatchoutput = platform.parsepatchoutput
 pconvert = platform.pconvert
 poll = platform.poll
 posixfile = platform.posixfile
 readlink = platform.readlink
 rename = platform.rename
 removedirs = platform.removedirs
 samedevice = platform.samedevice
 samefile = platform.samefile
 samestat = platform.samestat
 setflags = platform.setflags
 split = platform.split
 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
 statisexec = platform.statisexec
 statislink = platform.statislink
 umask = platform.umask
 unlink = platform.unlink
 username = platform.username
 
 
 def setumask(val):
     # type: (int) -> None
     '''updates the umask. used by chg server'''
     if pycompat.iswindows:
         return
     os.umask(val)
     global umask
     platform.umask = umask = val & 0o777
 
 
 # small compat layer
 compengines = compression.compengines
 SERVERROLE = compression.SERVERROLE
 CLIENTROLE = compression.CLIENTROLE
 
 try:
     recvfds = osutil.recvfds
 except AttributeError:
     pass
 
 # Python compatibility
 
 _notset = object()
 
 
 def bitsfrom(container):
     bits = 0
     for bit in container:
         bits |= bit
     return bits
 
 
 # python 2.6 still have deprecation warning enabled by default. We do not want
 # to display anything to standard user so detect if we are running test and
 # only use python deprecation warning in this case.
 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
 if _dowarn:
     # explicitly unfilter our warning for python 2.7
     #
     # The option of setting PYTHONWARNINGS in the test runner was investigated.
     # However, module name set through PYTHONWARNINGS was exactly matched, so
     # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
     # makes the whole PYTHONWARNINGS thing useless for our usecase.
     warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
 if _dowarn and pycompat.ispy3:
     # silence warning emitted by passing user string to re.sub()
     warnings.filterwarnings(
         'ignore', 'bad escape', DeprecationWarning, 'mercurial'
     )
     warnings.filterwarnings(
         'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
     )
     # TODO: reinvent imp.is_frozen()
     warnings.filterwarnings(
         'ignore',
         'the imp module is deprecated',
         DeprecationWarning,
         'mercurial',
     )
 
 
 def nouideprecwarn(msg, version, stacklevel=1):
     """Issue an python native deprecation warning
 
     This is a noop outside of tests, use 'ui.deprecwarn' when possible.
     """
     if _dowarn:
         msg += (
             b"\n(compatibility will be dropped after Mercurial-%s,"
             b" update your code.)"
         ) % version
         warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
         # on python 3 with chg, we will need to explicitly flush the output
         sys.stderr.flush()
 
 
 DIGESTS = {
     b'md5': hashlib.md5,
     b'sha1': hashutil.sha1,
     b'sha512': hashlib.sha512,
 }
 # List of digest types from strongest to weakest
 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
 
 for k in DIGESTS_BY_STRENGTH:
     assert k in DIGESTS
 
 
 class digester(object):
     """helper to compute digests.
 
     This helper can be used to compute one or more digests given their name.
 
     >>> d = digester([b'md5', b'sha1'])
     >>> d.update(b'foo')
     >>> [k for k in sorted(d)]
     ['md5', 'sha1']
     >>> d[b'md5']
     'acbd18db4cc2f85cedef654fccc4a4d8'
     >>> d[b'sha1']
     '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
     >>> digester.preferred([b'md5', b'sha1'])
     'sha1'
     """
 
     def __init__(self, digests, s=b''):
         self._hashes = {}
         for k in digests:
             if k not in DIGESTS:
                 raise error.Abort(_(b'unknown digest type: %s') % k)
             self._hashes[k] = DIGESTS[k]()
         if s:
             self.update(s)
 
     def update(self, data):
         for h in self._hashes.values():
             h.update(data)
 
     def __getitem__(self, key):
         if key not in DIGESTS:
             raise error.Abort(_(b'unknown digest type: %s') % k)
         return hex(self._hashes[key].digest())
 
     def __iter__(self):
         return iter(self._hashes)
 
     @staticmethod
     def preferred(supported):
         """returns the strongest digest type in both supported and DIGESTS."""
 
         for k in DIGESTS_BY_STRENGTH:
             if k in supported:
                 return k
         return None
 
 
 class digestchecker(object):
     """file handle wrapper that additionally checks content against a given
     size and digests.
 
         d = digestchecker(fh, size, {'md5': '...'})
 
     When multiple digests are given, all of them are validated.
     """
 
     def __init__(self, fh, size, digests):
         self._fh = fh
         self._size = size
         self._got = 0
         self._digests = dict(digests)
         self._digester = digester(self._digests.keys())
 
     def read(self, length=-1):
         content = self._fh.read(length)
         self._digester.update(content)
         self._got += len(content)
         return content
 
     def validate(self):
         if self._size != self._got:
             raise error.Abort(
                 _(b'size mismatch: expected %d, got %d')
                 % (self._size, self._got)
             )
         for k, v in self._digests.items():
             if v != self._digester[k]:
                 # i18n: first parameter is a digest name
                 raise error.Abort(
                     _(b'%s mismatch: expected %s, got %s')
                     % (k, v, self._digester[k])
                 )
 
 
 try:
     buffer = buffer  # pytype: disable=name-error
 except NameError:
 
     def buffer(sliceable, offset=0, length=None):
         if length is not None:
             return memoryview(sliceable)[offset : offset + length]
         return memoryview(sliceable)[offset:]
 
 
 _chunksize = 4096
 
 
 class bufferedinputpipe(object):
     """a manually buffered input pipe
 
     Python will not let us use buffered IO and lazy reading with 'polling' at
     the same time. We cannot probe the buffer state and select will not detect
     that data are ready to read if they are already buffered.
 
     This class let us work around that by implementing its own buffering
     (allowing efficient readline) while offering a way to know if the buffer is
     empty from the output (allowing collaboration of the buffer with polling).
 
     This class lives in the 'util' module because it makes use of the 'os'
     module from the python stdlib.
     """
 
     def __new__(cls, fh):
         # If we receive a fileobjectproxy, we need to use a variation of this
         # class that notifies observers about activity.
         if isinstance(fh, fileobjectproxy):
             cls = observedbufferedinputpipe
 
         return super(bufferedinputpipe, cls).__new__(cls)
 
     def __init__(self, input):
         self._input = input
         self._buffer = []
         self._eof = False
         self._lenbuf = 0
 
     @property
     def hasbuffer(self):
         """True is any data is currently buffered
 
         This will be used externally a pre-step for polling IO. If there is
         already data then no polling should be set in place."""
         return bool(self._buffer)
 
     @property
     def closed(self):
         return self._input.closed
 
     def fileno(self):
         return self._input.fileno()
 
     def close(self):
         return self._input.close()
 
     def read(self, size):
         while (not self._eof) and (self._lenbuf < size):
             self._fillbuffer()
         return self._frombuffer(size)
 
     def unbufferedread(self, size):
         if not self._eof and self._lenbuf == 0:
             self._fillbuffer(max(size, _chunksize))
         return self._frombuffer(min(self._lenbuf, size))
 
     def readline(self, *args, **kwargs):
         if len(self._buffer) > 1:
             # this should not happen because both read and readline end with a
             # _frombuffer call that collapse it.
             self._buffer = [b''.join(self._buffer)]
             self._lenbuf = len(self._buffer[0])
         lfi = -1
         if self._buffer:
             lfi = self._buffer[-1].find(b'\n')
         while (not self._eof) and lfi < 0:
             self._fillbuffer()
             if self._buffer:
                 lfi = self._buffer[-1].find(b'\n')
         size = lfi + 1
         if lfi < 0:  # end of file
             size = self._lenbuf
         elif len(self._buffer) > 1:
             # we need to take previous chunks into account
             size += self._lenbuf - len(self._buffer[-1])
         return self._frombuffer(size)
 
     def _frombuffer(self, size):
         """return at most 'size' data from the buffer
 
         The data are removed from the buffer."""
         if size == 0 or not self._buffer:
             return b''
         buf = self._buffer[0]
         if len(self._buffer) > 1:
             buf = b''.join(self._buffer)
 
         data = buf[:size]
         buf = buf[len(data) :]
         if buf:
             self._buffer = [buf]
             self._lenbuf = len(buf)
         else:
             self._buffer = []
             self._lenbuf = 0
         return data
 
     def _fillbuffer(self, size=_chunksize):
         """read data to the buffer"""
         data = os.read(self._input.fileno(), size)
         if not data:
             self._eof = True
         else:
             self._lenbuf += len(data)
             self._buffer.append(data)
 
         return data
 
 
 def mmapread(fp, size=None):
     if size == 0:
         # size of 0 to mmap.mmap() means "all data"
         # rather than "zero bytes", so special case that.
         return b''
     elif size is None:
         size = 0
     fd = getattr(fp, 'fileno', lambda: fp)()
     try:
         return mmap.mmap(fd, size, access=mmap.ACCESS_READ)
     except ValueError:
         # Empty files cannot be mmapped, but mmapread should still work.  Check
         # if the file is empty, and if so, return an empty buffer.
         if os.fstat(fd).st_size == 0:
             return b''
         raise
 
 
 class fileobjectproxy(object):
     """A proxy around file objects that tells a watcher when events occur.
 
     This type is intended to only be used for testing purposes. Think hard
     before using it in important code.
     """
 
     __slots__ = (
         '_orig',
         '_observer',
     )
 
     def __init__(self, fh, observer):
         object.__setattr__(self, '_orig', fh)
         object.__setattr__(self, '_observer', observer)
 
     def __getattribute__(self, name):
         ours = {
             '_observer',
             # IOBase
             'close',
             # closed if a property
             'fileno',
             'flush',
             'isatty',
             'readable',
             'readline',
             'readlines',
             'seek',
             'seekable',
             'tell',
             'truncate',
             'writable',
             'writelines',
             # RawIOBase
             'read',
             'readall',
             'readinto',
             'write',
             # BufferedIOBase
             # raw is a property
             'detach',
             # read defined above
             'read1',
             # readinto defined above
             # write defined above
         }
 
         # We only observe some methods.
         if name in ours:
             return object.__getattribute__(self, name)
 
         return getattr(object.__getattribute__(self, '_orig'), name)
 
     def __nonzero__(self):
         return bool(object.__getattribute__(self, '_orig'))
 
     __bool__ = __nonzero__
 
     def __delattr__(self, name):
         return delattr(object.__getattribute__(self, '_orig'), name)
 
     def __setattr__(self, name, value):
         return setattr(object.__getattribute__(self, '_orig'), name, value)
 
     def __iter__(self):
         return object.__getattribute__(self, '_orig').__iter__()
 
     def _observedcall(self, name, *args, **kwargs):
         # Call the original object.
         orig = object.__getattribute__(self, '_orig')
         res = getattr(orig, name)(*args, **kwargs)
 
         # Call a method on the observer of the same name with arguments
         # so it can react, log, etc.
         observer = object.__getattribute__(self, '_observer')
         fn = getattr(observer, name, None)
         if fn:
             fn(res, *args, **kwargs)
 
         return res
 
     def close(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'close', *args, **kwargs
         )
 
     def fileno(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'fileno', *args, **kwargs
         )
 
     def flush(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'flush', *args, **kwargs
         )
 
     def isatty(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'isatty', *args, **kwargs
         )
 
     def readable(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readable', *args, **kwargs
         )
 
     def readline(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readline', *args, **kwargs
         )
 
     def readlines(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readlines', *args, **kwargs
         )
 
     def seek(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'seek', *args, **kwargs
         )
 
     def seekable(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'seekable', *args, **kwargs
         )
 
     def tell(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'tell', *args, **kwargs
         )
 
     def truncate(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'truncate', *args, **kwargs
         )
 
     def writable(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'writable', *args, **kwargs
         )
 
     def writelines(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'writelines', *args, **kwargs
         )
 
     def read(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'read', *args, **kwargs
         )
 
     def readall(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readall', *args, **kwargs
         )
 
     def readinto(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'readinto', *args, **kwargs
         )
 
     def write(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'write', *args, **kwargs
         )
 
     def detach(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'detach', *args, **kwargs
         )
 
     def read1(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'read1', *args, **kwargs
         )
 
 
 class observedbufferedinputpipe(bufferedinputpipe):
     """A variation of bufferedinputpipe that is aware of fileobjectproxy.
 
     ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
     bypass ``fileobjectproxy``. Because of this, we need to make
     ``bufferedinputpipe`` aware of these operations.
 
     This variation of ``bufferedinputpipe`` can notify observers about
     ``os.read()`` events. It also re-publishes other events, such as
     ``read()`` and ``readline()``.
     """
 
     def _fillbuffer(self):
         res = super(observedbufferedinputpipe, self)._fillbuffer()
 
         fn = getattr(self._input._observer, 'osread', None)
         if fn:
             fn(res, _chunksize)
 
         return res
 
     # We use different observer methods because the operation isn't
     # performed on the actual file object but on us.
     def read(self, size):
         res = super(observedbufferedinputpipe, self).read(size)
 
         fn = getattr(self._input._observer, 'bufferedread', None)
         if fn:
             fn(res, size)
 
         return res
 
     def readline(self, *args, **kwargs):
         res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
 
         fn = getattr(self._input._observer, 'bufferedreadline', None)
         if fn:
             fn(res)
 
         return res
 
 
 PROXIED_SOCKET_METHODS = {
     'makefile',
     'recv',
     'recvfrom',
     'recvfrom_into',
     'recv_into',
     'send',
     'sendall',
     'sendto',
     'setblocking',
     'settimeout',
     'gettimeout',
     'setsockopt',
 }
 
 
 class socketproxy(object):
     """A proxy around a socket that tells a watcher when events occur.
 
     This is like ``fileobjectproxy`` except for sockets.
 
     This type is intended to only be used for testing purposes. Think hard
     before using it in important code.
     """
 
     __slots__ = (
         '_orig',
         '_observer',
     )
 
     def __init__(self, sock, observer):
         object.__setattr__(self, '_orig', sock)
         object.__setattr__(self, '_observer', observer)
 
     def __getattribute__(self, name):
         if name in PROXIED_SOCKET_METHODS:
             return object.__getattribute__(self, name)
 
         return getattr(object.__getattribute__(self, '_orig'), name)
 
     def __delattr__(self, name):
         return delattr(object.__getattribute__(self, '_orig'), name)
 
     def __setattr__(self, name, value):
         return setattr(object.__getattribute__(self, '_orig'), name, value)
 
     def __nonzero__(self):
         return bool(object.__getattribute__(self, '_orig'))
 
     __bool__ = __nonzero__
 
     def _observedcall(self, name, *args, **kwargs):
         # Call the original object.
         orig = object.__getattribute__(self, '_orig')
         res = getattr(orig, name)(*args, **kwargs)
 
         # Call a method on the observer of the same name with arguments
         # so it can react, log, etc.
         observer = object.__getattribute__(self, '_observer')
         fn = getattr(observer, name, None)
         if fn:
             fn(res, *args, **kwargs)
 
         return res
 
     def makefile(self, *args, **kwargs):
         res = object.__getattribute__(self, '_observedcall')(
             'makefile', *args, **kwargs
         )
 
         # The file object may be used for I/O. So we turn it into a
         # proxy using our observer.
         observer = object.__getattribute__(self, '_observer')
         return makeloggingfileobject(
             observer.fh,
             res,
             observer.name,
             reads=observer.reads,
             writes=observer.writes,
             logdata=observer.logdata,
             logdataapis=observer.logdataapis,
         )
 
     def recv(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recv', *args, **kwargs
         )
 
     def recvfrom(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recvfrom', *args, **kwargs
         )
 
     def recvfrom_into(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recvfrom_into', *args, **kwargs
         )
 
     def recv_into(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'recv_info', *args, **kwargs
         )
 
     def send(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'send', *args, **kwargs
         )
 
     def sendall(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'sendall', *args, **kwargs
         )
 
     def sendto(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'sendto', *args, **kwargs
         )
 
     def setblocking(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'setblocking', *args, **kwargs
         )
 
     def settimeout(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'settimeout', *args, **kwargs
         )
 
     def gettimeout(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'gettimeout', *args, **kwargs
         )
 
     def setsockopt(self, *args, **kwargs):
         return object.__getattribute__(self, '_observedcall')(
             'setsockopt', *args, **kwargs
         )
 
 
 class baseproxyobserver(object):
     def __init__(self, fh, name, logdata, logdataapis):
         self.fh = fh
         self.name = name
         self.logdata = logdata
         self.logdataapis = logdataapis
 
     def _writedata(self, data):
         if not self.logdata:
             if self.logdataapis:
                 self.fh.write(b'\n')
                 self.fh.flush()
             return
 
         # Simple case writes all data on a single line.
         if b'\n' not in data:
             if self.logdataapis:
                 self.fh.write(b': %s\n' % stringutil.escapestr(data))
             else:
                 self.fh.write(
                     b'%s>     %s\n' % (self.name, stringutil.escapestr(data))
                 )
             self.fh.flush()
             return
 
         # Data with newlines is written to multiple lines.
         if self.logdataapis:
             self.fh.write(b':\n')
 
         lines = data.splitlines(True)
         for line in lines:
             self.fh.write(
                 b'%s>     %s\n' % (self.name, stringutil.escapestr(line))
             )
         self.fh.flush()
 
 
 class fileobjectobserver(baseproxyobserver):
     """Logs file object activity."""
 
     def __init__(
         self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
     ):
         super(fileobjectobserver, self).__init__(fh, name, logdata, logdataapis)
         self.reads = reads
         self.writes = writes
 
     def read(self, res, size=-1):
         if not self.reads:
             return
         # Python 3 can return None from reads at EOF instead of empty strings.
         if res is None:
             res = b''
 
         if size == -1 and res == b'':
             # Suppress pointless read(-1) calls that return
             # nothing. These happen _a lot_ on Python 3, and there
             # doesn't seem to be a better workaround to have matching
             # Python 2 and 3 behavior. :(
             return
 
         if self.logdataapis:
             self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
 
         self._writedata(res)
 
     def readline(self, res, limit=-1):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
 
         self._writedata(res)
 
     def readinto(self, res, dest):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
             )
 
         data = dest[0:res] if res is not None else b''
 
         # _writedata() uses "in" operator and is confused by memoryview because
         # characters are ints on Python 3.
         if isinstance(data, memoryview):
             data = data.tobytes()
 
         self._writedata(data)
 
     def write(self, res, data):
         if not self.writes:
             return
 
         # Python 2 returns None from some write() calls. Python 3 (reasonably)
         # returns the integer bytes written.
         if res is None and data:
             res = len(data)
 
         if self.logdataapis:
             self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
 
         self._writedata(data)
 
     def flush(self, res):
         if not self.writes:
             return
 
         self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
 
     # For observedbufferedinputpipe.
     def bufferedread(self, res, size):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
             )
 
         self._writedata(res)
 
     def bufferedreadline(self, res):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
             )
 
         self._writedata(res)
 
 
 def makeloggingfileobject(
     logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
 ):
     """Turn a file object into a logging file object."""
 
     observer = fileobjectobserver(
         logh,
         name,
         reads=reads,
         writes=writes,
         logdata=logdata,
         logdataapis=logdataapis,
     )
     return fileobjectproxy(fh, observer)
 
 
 class socketobserver(baseproxyobserver):
     """Logs socket activity."""
 
     def __init__(
         self,
         fh,
         name,
         reads=True,
         writes=True,
         states=True,
         logdata=False,
         logdataapis=True,
     ):
         super(socketobserver, self).__init__(fh, name, logdata, logdataapis)
         self.reads = reads
         self.writes = writes
         self.states = states
 
     def makefile(self, res, mode=None, bufsize=None):
         if not self.states:
             return
 
         self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
 
     def recv(self, res, size, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
             )
         self._writedata(res)
 
     def recvfrom(self, res, size, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recvfrom(%d, %d) -> %d'
                 % (self.name, size, flags, len(res[0]))
             )
 
         self._writedata(res[0])
 
     def recvfrom_into(self, res, buf, size, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recvfrom_into(%d, %d) -> %d'
                 % (self.name, size, flags, res[0])
             )
 
         self._writedata(buf[0 : res[0]])
 
     def recv_into(self, res, buf, size=0, flags=0):
         if not self.reads:
             return
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
             )
 
         self._writedata(buf[0:res])
 
     def send(self, res, data, flags=0):
         if not self.writes:
             return
 
         self.fh.write(
             b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
         )
         self._writedata(data)
 
     def sendall(self, res, data, flags=0):
         if not self.writes:
             return
 
         if self.logdataapis:
             # Returns None on success. So don't bother reporting return value.
             self.fh.write(
                 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
             )
 
         self._writedata(data)
 
     def sendto(self, res, data, flagsoraddress, address=None):
         if not self.writes:
             return
 
         if address:
             flags = flagsoraddress
         else:
             flags = 0
 
         if self.logdataapis:
             self.fh.write(
                 b'%s> sendto(%d, %d, %r) -> %d'
                 % (self.name, len(data), flags, address, res)
             )
 
         self._writedata(data)
 
     def setblocking(self, res, flag):
         if not self.states:
             return
 
         self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
 
     def settimeout(self, res, value):
         if not self.states:
             return
 
         self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
 
     def gettimeout(self, res):
         if not self.states:
             return
 
         self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
 
     def setsockopt(self, res, level, optname, value):
         if not self.states:
             return
 
         self.fh.write(
             b'%s> setsockopt(%r, %r, %r) -> %r\n'
             % (self.name, level, optname, value, res)
         )
 
 
 def makeloggingsocket(
     logh,
     fh,
     name,
     reads=True,
     writes=True,
     states=True,
     logdata=False,
     logdataapis=True,
 ):
     """Turn a socket into a logging socket."""
 
     observer = socketobserver(
         logh,
         name,
         reads=reads,
         writes=writes,
         states=states,
         logdata=logdata,
         logdataapis=logdataapis,
     )
     return socketproxy(fh, observer)
 
 
 def version():
     """Return version information if available."""
     try:
         from . import __version__
 
         return __version__.version
     except ImportError:
         return b'unknown'
 
 
 def versiontuple(v=None, n=4):
     """Parses a Mercurial version string into an N-tuple.
 
     The version string to be parsed is specified with the ``v`` argument.
     If it isn't defined, the current Mercurial version string will be parsed.
 
     ``n`` can be 2, 3, or 4. Here is how some version strings map to
     returned values:
 
     >>> v = b'3.6.1+190-df9b73d2d444'
     >>> versiontuple(v, 2)
     (3, 6)
     >>> versiontuple(v, 3)
     (3, 6, 1)
     >>> versiontuple(v, 4)
     (3, 6, 1, '190-df9b73d2d444')
 
     >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
     (3, 6, 1, '190-df9b73d2d444+20151118')
 
     >>> v = b'3.6'
     >>> versiontuple(v, 2)
     (3, 6)
     >>> versiontuple(v, 3)
     (3, 6, None)
     >>> versiontuple(v, 4)
     (3, 6, None, None)
 
     >>> v = b'3.9-rc'
     >>> versiontuple(v, 2)
     (3, 9)
     >>> versiontuple(v, 3)
     (3, 9, None)
     >>> versiontuple(v, 4)
     (3, 9, None, 'rc')
 
     >>> v = b'3.9-rc+2-02a8fea4289b'
     >>> versiontuple(v, 2)
     (3, 9)
     >>> versiontuple(v, 3)
     (3, 9, None)
     >>> versiontuple(v, 4)
     (3, 9, None, 'rc+2-02a8fea4289b')
 
     >>> versiontuple(b'4.6rc0')
     (4, 6, None, 'rc0')
     >>> versiontuple(b'4.6rc0+12-425d55e54f98')
     (4, 6, None, 'rc0+12-425d55e54f98')
     >>> versiontuple(b'.1.2.3')
     (None, None, None, '.1.2.3')
     >>> versiontuple(b'12.34..5')
     (12, 34, None, '..5')
     >>> versiontuple(b'1.2.3.4.5.6')
     (1, 2, 3, '.4.5.6')
     """
     if not v:
         v = version()
     m = remod.match(br'(\d+(?:\.\d+){,2})[+-]?(.*)', v)
     if not m:
         vparts, extra = b'', v
     elif m.group(2):
         vparts, extra = m.groups()
     else:
         vparts, extra = m.group(1), None
 
     assert vparts is not None  # help pytype
 
     vints = []
     for i in vparts.split(b'.'):
         try:
             vints.append(int(i))
         except ValueError:
             break
     # (3, 6) -> (3, 6, None)
     while len(vints) < 3:
         vints.append(None)
 
     if n == 2:
         return (vints[0], vints[1])
     if n == 3:
         return (vints[0], vints[1], vints[2])
     if n == 4:
         return (vints[0], vints[1], vints[2], extra)
 
     raise error.ProgrammingError(b"invalid version part request: %d" % n)
 
 
 def cachefunc(func):
     '''cache the result of function calls'''
     # XXX doesn't handle keywords args
     if func.__code__.co_argcount == 0:
         listcache = []
 
         def f():
             if len(listcache) == 0:
                 listcache.append(func())
             return listcache[0]
 
         return f
     cache = {}
     if func.__code__.co_argcount == 1:
         # we gain a small amount of time because
         # we don't need to pack/unpack the list
         def f(arg):
             if arg not in cache:
                 cache[arg] = func(arg)
             return cache[arg]
 
     else:
 
         def f(*args):
             if args not in cache:
                 cache[args] = func(*args)
             return cache[args]
 
     return f
 
 
 class cow(object):
     """helper class to make copy-on-write easier
 
     Call preparewrite before doing any writes.
     """
 
     def preparewrite(self):
         """call this before writes, return self or a copied new object"""
         if getattr(self, '_copied', 0):
             self._copied -= 1
             # Function cow.__init__ expects 1 arg(s), got 2 [wrong-arg-count]
             return self.__class__(self)  # pytype: disable=wrong-arg-count
         return self
 
     def copy(self):
         """always do a cheap copy"""
         self._copied = getattr(self, '_copied', 0) + 1
         return self
 
 
 class sortdict(collections.OrderedDict):
     """a simple sorted dictionary
 
     >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
     >>> d2 = d1.copy()
     >>> d2
     sortdict([('a', 0), ('b', 1)])
     >>> d2.update([(b'a', 2)])
     >>> list(d2.keys()) # should still be in last-set order
     ['b', 'a']
     >>> d1.insert(1, b'a.5', 0.5)
     >>> d1
     sortdict([('a', 0), ('a.5', 0.5), ('b', 1)])
     """
 
     def __setitem__(self, key, value):
         if key in self:
             del self[key]
         super(sortdict, self).__setitem__(key, value)
 
     if pycompat.ispypy:
         # __setitem__() isn't called as of PyPy 5.8.0
         def update(self, src, **f):
             if isinstance(src, dict):
                 src = pycompat.iteritems(src)
             for k, v in src:
                 self[k] = v
             for k in f:
                 self[k] = f[k]
 
     def insert(self, position, key, value):
         for (i, (k, v)) in enumerate(list(self.items())):
             if i == position:
                 self[key] = value
             if i >= position:
                 del self[k]
                 self[k] = v
 
 
 class cowdict(cow, dict):
     """copy-on-write dict
 
     Be sure to call d = d.preparewrite() before writing to d.
 
     >>> a = cowdict()
     >>> a is a.preparewrite()
     True
     >>> b = a.copy()
     >>> b is a
     True
     >>> c = b.copy()
     >>> c is a
     True
     >>> a = a.preparewrite()
     >>> b is a
     False
     >>> a is a.preparewrite()
     True
     >>> c = c.preparewrite()
     >>> b is c
     False
     >>> b is b.preparewrite()
     True
     """
 
 
 class cowsortdict(cow, sortdict):
     """copy-on-write sortdict
 
     Be sure to call d = d.preparewrite() before writing to d.
     """
 
 
 class transactional(object):  # pytype: disable=ignored-metaclass
     """Base class for making a transactional type into a context manager."""
 
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
     def close(self):
         """Successfully closes the transaction."""
 
     @abc.abstractmethod
     def release(self):
         """Marks the end of the transaction.
 
         If the transaction has not been closed, it will be aborted.
         """
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         try:
             if exc_type is None:
                 self.close()
         finally:
             self.release()
 
 
 @contextlib.contextmanager
 def acceptintervention(tr=None):
     """A context manager that closes the transaction on InterventionRequired
 
     If no transaction was provided, this simply runs the body and returns
     """
     if not tr:
         yield
         return
     try:
         yield
         tr.close()
     except error.InterventionRequired:
         tr.close()
         raise
     finally:
         tr.release()
 
 
 @contextlib.contextmanager
 def nullcontextmanager(enter_result=None):
     yield enter_result
 
 
 class _lrucachenode(object):
     """A node in a doubly linked list.
 
     Holds a reference to nodes on either side as well as a key-value
     pair for the dictionary entry.
     """
 
     __slots__ = ('next', 'prev', 'key', 'value', 'cost')
 
     def __init__(self):
         self.next = self
         self.prev = self
 
         self.key = _notset
         self.value = None
         self.cost = 0
 
     def markempty(self):
         """Mark the node as emptied."""
         self.key = _notset
         self.value = None
         self.cost = 0
 
 
 class lrucachedict(object):
     """Dict that caches most recent accesses and sets.
 
     The dict consists of an actual backing dict - indexed by original
     key - and a doubly linked circular list defining the order of entries in
     the cache.
 
     The head node is the newest entry in the cache. If the cache is full,
     we recycle head.prev and make it the new head. Cache accesses result in
     the node being moved to before the existing head and being marked as the
     new head node.
 
     Items in the cache can be inserted with an optional "cost" value. This is
     simply an integer that is specified by the caller. The cache can be queried
     for the total cost of all items presently in the cache.
 
     The cache can also define a maximum cost. If a cache insertion would
     cause the total cost of the cache to go beyond the maximum cost limit,
     nodes will be evicted to make room for the new code. This can be used
     to e.g. set a max memory limit and associate an estimated bytes size
     cost to each item in the cache. By default, no maximum cost is enforced.
     """
 
     def __init__(self, max, maxcost=0):
         self._cache = {}
 
         self._head = _lrucachenode()
         self._size = 1
         self.capacity = max
         self.totalcost = 0
         self.maxcost = maxcost
 
     def __len__(self):
         return len(self._cache)
 
     def __contains__(self, k):
         return k in self._cache
 
     def __iter__(self):
         # We don't have to iterate in cache order, but why not.
         n = self._head
         for i in range(len(self._cache)):
             yield n.key
             n = n.next
 
     def __getitem__(self, k):
         node = self._cache[k]
         self._movetohead(node)
         return node.value
 
     def insert(self, k, v, cost=0):
         """Insert a new item in the cache with optional cost value."""
         node = self._cache.get(k)
         # Replace existing value and mark as newest.
         if node is not None:
             self.totalcost -= node.cost
             node.value = v
             node.cost = cost
             self.totalcost += cost
             self._movetohead(node)
 
             if self.maxcost:
                 self._enforcecostlimit()
 
             return
 
         if self._size < self.capacity:
             node = self._addcapacity()
         else:
             # Grab the last/oldest item.
             node = self._head.prev
 
         # At capacity. Kill the old entry.
         if node.key is not _notset:
             self.totalcost -= node.cost
             del self._cache[node.key]
 
         node.key = k
         node.value = v
         node.cost = cost
         self.totalcost += cost
         self._cache[k] = node
         # And mark it as newest entry. No need to adjust order since it
         # is already self._head.prev.
         self._head = node
 
         if self.maxcost:
             self._enforcecostlimit()
 
     def __setitem__(self, k, v):
         self.insert(k, v)
 
     def __delitem__(self, k):
         self.pop(k)
 
     def pop(self, k, default=_notset):
         try:
             node = self._cache.pop(k)
         except KeyError:
             if default is _notset:
                 raise
             return default
 
         assert node is not None  # help pytype
         value = node.value
         self.totalcost -= node.cost
         node.markempty()
 
         # Temporarily mark as newest item before re-adjusting head to make
         # this node the oldest item.
         self._movetohead(node)
         self._head = node.next
 
         return value
 
     # Additional dict methods.
 
     def get(self, k, default=None):
         try:
             return self.__getitem__(k)
         except KeyError:
             return default
 
     def peek(self, k, default=_notset):
         """Get the specified item without moving it to the head
 
         Unlike get(), this doesn't mutate the internal state. But be aware
         that it doesn't mean peek() is thread safe.
         """
         try:
             node = self._cache[k]
             assert node is not None  # help pytype
             return node.value
         except KeyError:
             if default is _notset:
                 raise
             return default
 
     def clear(self):
         n = self._head
         while n.key is not _notset:
             self.totalcost -= n.cost
             n.markempty()
             n = n.next
 
         self._cache.clear()
 
     def copy(self, capacity=None, maxcost=0):
         """Create a new cache as a copy of the current one.
 
         By default, the new cache has the same capacity as the existing one.
         But, the cache capacity can be changed as part of performing the
         copy.
 
         Items in the copy have an insertion/access order matching this
         instance.
         """
 
         capacity = capacity or self.capacity
         maxcost = maxcost or self.maxcost
         result = lrucachedict(capacity, maxcost=maxcost)
 
         # We copy entries by iterating in oldest-to-newest order so the copy
         # has the correct ordering.
 
         # Find the first non-empty entry.
         n = self._head.prev
         while n.key is _notset and n is not self._head:
             n = n.prev
 
         # We could potentially skip the first N items when decreasing capacity.
         # But let's keep it simple unless it is a performance problem.
         for i in range(len(self._cache)):
             result.insert(n.key, n.value, cost=n.cost)
             n = n.prev
 
         return result
 
     def popoldest(self):
         """Remove the oldest item from the cache.
 
         Returns the (key, value) describing the removed cache entry.
         """
         if not self._cache:
             return
 
         # Walk the linked list backwards starting at tail node until we hit
         # a non-empty node.
         n = self._head.prev
 
         assert n is not None  # help pytype
 
         while n.key is _notset:
             n = n.prev
 
         assert n is not None  # help pytype
 
         key, value = n.key, n.value
 
         # And remove it from the cache and mark it as empty.
         del self._cache[n.key]
         self.totalcost -= n.cost
         n.markempty()
 
         return key, value
 
     def _movetohead(self, node):
         """Mark a node as the newest, making it the new head.
 
         When a node is accessed, it becomes the freshest entry in the LRU
         list, which is denoted by self._head.
 
         Visually, let's make ``N`` the new head node (* denotes head):
 
             previous/oldest <-> head <-> next/next newest
 
             ----<->--- A* ---<->-----
             |                       |
             E <-> D <-> N <-> C <-> B
 
         To:
 
             ----<->--- N* ---<->-----
             |                       |
             E <-> D <-> C <-> B <-> A
 
         This requires the following moves:
 
            C.next = D  (node.prev.next = node.next)
            D.prev = C  (node.next.prev = node.prev)
            E.next = N  (head.prev.next = node)
            N.prev = E  (node.prev = head.prev)
            N.next = A  (node.next = head)
            A.prev = N  (head.prev = node)
         """
         head = self._head
         # C.next = D
         node.prev.next = node.next
         # D.prev = C
         node.next.prev = node.prev
         # N.prev = E
         node.prev = head.prev
         # N.next = A
         # It is tempting to do just "head" here, however if node is
         # adjacent to head, this will do bad things.
         node.next = head.prev.next
         # E.next = N
         node.next.prev = node
         # A.prev = N
         node.prev.next = node
 
         self._head = node
 
     def _addcapacity(self):
         """Add a node to the circular linked list.
 
         The new node is inserted before the head node.
         """
         head = self._head
         node = _lrucachenode()
         head.prev.next = node
         node.prev = head.prev
         node.next = head
         head.prev = node
         self._size += 1
         return node
 
     def _enforcecostlimit(self):
         # This should run after an insertion. It should only be called if total
         # cost limits are being enforced.
         # The most recently inserted node is never evicted.
         if len(self) <= 1 or self.totalcost <= self.maxcost:
             return
 
         # This is logically equivalent to calling popoldest() until we
         # free up enough cost. We don't do that since popoldest() needs
         # to walk the linked list and doing this in a loop would be
         # quadratic. So we find the first non-empty node and then
         # walk nodes until we free up enough capacity.
         #
         # If we only removed the minimum number of nodes to free enough
         # cost at insert time, chances are high that the next insert would
         # also require pruning. This would effectively constitute quadratic
         # behavior for insert-heavy workloads. To mitigate this, we set a
         # target cost that is a percentage of the max cost. This will tend
         # to free more nodes when the high water mark is reached, which
         # lowers the chances of needing to prune on the subsequent insert.
         targetcost = int(self.maxcost * 0.75)
 
         n = self._head.prev
         while n.key is _notset:
             n = n.prev
 
         while len(self) > 1 and self.totalcost > targetcost:
             del self._cache[n.key]
             self.totalcost -= n.cost
             n.markempty()
             n = n.prev
 
 
 def lrucachefunc(func):
     '''cache most recent results of function calls'''
     cache = {}
     order = collections.deque()
     if func.__code__.co_argcount == 1:
 
         def f(arg):
             if arg not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[arg] = func(arg)
             else:
                 order.remove(arg)
             order.append(arg)
             return cache[arg]
 
     else:
 
         def f(*args):
             if args not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[args] = func(*args)
             else:
                 order.remove(args)
             order.append(args)
             return cache[args]
 
     return f
 
 
 class propertycache(object):
     def __init__(self, func):
         self.func = func
         self.name = func.__name__
 
     def __get__(self, obj, type=None):
         result = self.func(obj)
         self.cachevalue(obj, result)
         return result
 
     def cachevalue(self, obj, value):
         # __dict__ assignment required to bypass __setattr__ (eg: repoview)
         obj.__dict__[self.name] = value
 
 
 def clearcachedproperty(obj, prop):
     '''clear a cached property value, if one has been set'''
     prop = pycompat.sysstr(prop)
     if prop in obj.__dict__:
         del obj.__dict__[prop]
 
 
 def increasingchunks(source, min=1024, max=65536):
     """return no less than min bytes per chunk while data remains,
     doubling min after each chunk until it reaches max"""
 
     def log2(x):
         if not x:
             return 0
         i = 0
         while x:
             x >>= 1
             i += 1
         return i - 1
 
     buf = []
     blen = 0
     for chunk in source:
         buf.append(chunk)
         blen += len(chunk)
         if blen >= min:
             if min < max:
                 min = min << 1
                 nmin = 1 << log2(blen)
                 if nmin > min:
                     min = nmin
                 if min > max:
                     min = max
             yield b''.join(buf)
             blen = 0
             buf = []
     if buf:
         yield b''.join(buf)
 
 
 def always(fn):
     return True
 
 
 def never(fn):
     return False
 
 
 def nogc(func):
     """disable garbage collector
 
     Python's garbage collector triggers a GC each time a certain number of
     container objects (the number being defined by gc.get_threshold()) are
     allocated even when marked not to be tracked by the collector. Tracking has
     no effect on when GCs are triggered, only on what objects the GC looks
     into. As a workaround, disable GC while building complex (huge)
     containers.
 
     This garbage collector issue have been fixed in 2.7. But it still affect
     CPython's performance.
     """
 
     def wrapper(*args, **kwargs):
         gcenabled = gc.isenabled()
         gc.disable()
         try:
             return func(*args, **kwargs)
         finally:
             if gcenabled:
                 gc.enable()
 
     return wrapper
 
 
 if pycompat.ispypy:
     # PyPy runs slower with gc disabled
     nogc = lambda x: x
 
 
 def pathto(root, n1, n2):
     # type: (bytes, bytes, bytes) -> bytes
     """return the relative path from one place to another.
     root should use os.sep to separate directories
     n1 should use os.sep to separate directories
     n2 should use "/" to separate directories
     returns an os.sep-separated path.
 
     If n1 is a relative path, it's assumed it's
     relative to root.
     n2 should always be relative to root.
     """
     if not n1:
         return localpath(n2)
     if os.path.isabs(n1):
         if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
             return os.path.join(root, localpath(n2))
         n2 = b'/'.join((pconvert(root), n2))
     a, b = splitpath(n1), n2.split(b'/')
     a.reverse()
     b.reverse()
     while a and b and a[-1] == b[-1]:
         a.pop()
         b.pop()
     b.reverse()
     return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
 
 
 def checksignature(func, depth=1):
     '''wrap a function with code to check for calling errors'''
 
     def check(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except TypeError:
             if len(traceback.extract_tb(sys.exc_info()[2])) == depth:
                 raise error.SignatureError
             raise
 
     return check
 
 
 # a whilelist of known filesystems where hardlink works reliably
 _hardlinkfswhitelist = {
     b'apfs',
     b'btrfs',
     b'ext2',
     b'ext3',
     b'ext4',
     b'hfs',
     b'jfs',
     b'NTFS',
     b'reiserfs',
     b'tmpfs',
     b'ufs',
     b'xfs',
     b'zfs',
 }
 
 
 def copyfile(
     src,
     dest,
     hardlink=False,
     copystat=False,
     checkambig=False,
     nb_bytes=None,
     no_hardlink_cb=None,
     check_fs_hardlink=True,
 ):
     """copy a file, preserving mode and optionally other stat info like
     atime/mtime
 
     checkambig argument is used with filestat, and is useful only if
     destination file is guarded by any lock (e.g. repo.lock or
     repo.wlock).
 
     copystat and checkambig should be exclusive.
 
     nb_bytes: if set only copy the first `nb_bytes` of the source file.
     """
     assert not (copystat and checkambig)
     oldstat = None
     if os.path.lexists(dest):
         if checkambig:
             oldstat = checkambig and filestat.frompath(dest)
         unlink(dest)
     if hardlink and check_fs_hardlink:
         # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
         # unless we are confident that dest is on a whitelisted filesystem.
         try:
             fstype = getfstype(os.path.dirname(dest))
         except OSError:
             fstype = None
         if fstype not in _hardlinkfswhitelist:
             if no_hardlink_cb is not None:
                 no_hardlink_cb()
             hardlink = False
     if hardlink:
         try:
             oslink(src, dest)
             if nb_bytes is not None:
                 m = "the `nb_bytes` argument is incompatible with `hardlink`"
                 raise error.ProgrammingError(m)
             return
         except (IOError, OSError) as exc:
             if exc.errno != errno.EEXIST and no_hardlink_cb is not None:
                 no_hardlink_cb()
             # fall back to normal copy
     if os.path.islink(src):
         os.symlink(os.readlink(src), dest)
         # copytime is ignored for symlinks, but in general copytime isn't needed
         # for them anyway
         if nb_bytes is not None:
             m = "cannot use `nb_bytes` on a symlink"
             raise error.ProgrammingError(m)
     else:
         try:
             shutil.copyfile(src, dest)
             if copystat:
                 # copystat also copies mode
                 shutil.copystat(src, dest)
             else:
                 shutil.copymode(src, dest)
                 if oldstat and oldstat.stat:
                     newstat = filestat.frompath(dest)
                     if newstat.isambig(oldstat):
                         # stat of copied file is ambiguous to original one
                         advanced = (
                             oldstat.stat[stat.ST_MTIME] + 1
                         ) & 0x7FFFFFFF
                         os.utime(dest, (advanced, advanced))
             # We could do something smarter using `copy_file_range` call or similar
             if nb_bytes is not None:
                 with open(dest, mode='r+') as f:
                     f.truncate(nb_bytes)
         except shutil.Error as inst:
             raise error.Abort(stringutil.forcebytestr(inst))
 
 
 def copyfiles(src, dst, hardlink=None, progress=None):
     """Copy a directory tree using hardlinks if possible."""
     num = 0
 
     def settopic():
         if progress:
             progress.topic = _(b'linking') if hardlink else _(b'copying')
 
     if os.path.isdir(src):
         if hardlink is None:
             hardlink = (
                 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
             )
         settopic()
         os.mkdir(dst)
         for name, kind in listdir(src):
             srcname = os.path.join(src, name)
             dstname = os.path.join(dst, name)
             hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
             num += n
     else:
         if hardlink is None:
             hardlink = (
                 os.stat(os.path.dirname(src)).st_dev
                 == os.stat(os.path.dirname(dst)).st_dev
             )
         settopic()
 
         if hardlink:
             try:
                 oslink(src, dst)
             except (IOError, OSError) as exc:
                 if exc.errno != errno.EEXIST:
                     hardlink = False
                 # XXX maybe try to relink if the file exist ?
                 shutil.copy(src, dst)
         else:
             shutil.copy(src, dst)
         num += 1
         if progress:
             progress.increment()
 
     return hardlink, num
 
 
 _winreservednames = {
     b'con',
     b'prn',
     b'aux',
     b'nul',
     b'com1',
     b'com2',
     b'com3',
     b'com4',
     b'com5',
     b'com6',
     b'com7',
     b'com8',
     b'com9',
     b'lpt1',
     b'lpt2',
     b'lpt3',
     b'lpt4',
     b'lpt5',
     b'lpt6',
     b'lpt7',
     b'lpt8',
     b'lpt9',
 }
 _winreservedchars = b':*?"<>|'
 
 
 def checkwinfilename(path):
     # type: (bytes) -> Optional[bytes]
     r"""Check that the base-relative path is a valid filename on Windows.
     Returns None if the path is ok, or a UI string describing the problem.
 
     >>> checkwinfilename(b"just/a/normal/path")
     >>> checkwinfilename(b"foo/bar/con.xml")
     "filename contains 'con', which is reserved on Windows"
     >>> checkwinfilename(b"foo/con.xml/bar")
     "filename contains 'con', which is reserved on Windows"
     >>> checkwinfilename(b"foo/bar/xml.con")
     >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
     "filename contains 'AUX', which is reserved on Windows"
     >>> checkwinfilename(b"foo/bar/bla:.txt")
     "filename contains ':', which is reserved on Windows"
     >>> checkwinfilename(b"foo/bar/b\07la.txt")
     "filename contains '\\x07', which is invalid on Windows"
     >>> checkwinfilename(b"foo/bar/bla ")
     "filename ends with ' ', which is not allowed on Windows"
     >>> checkwinfilename(b"../bar")
     >>> checkwinfilename(b"foo\\")
     "filename ends with '\\', which is invalid on Windows"
     >>> checkwinfilename(b"foo\\/bar")
     "directory name ends with '\\', which is invalid on Windows"
     """
     if path.endswith(b'\\'):
         return _(b"filename ends with '\\', which is invalid on Windows")
     if b'\\/' in path:
         return _(b"directory name ends with '\\', which is invalid on Windows")
     for n in path.replace(b'\\', b'/').split(b'/'):
         if not n:
             continue
         for c in _filenamebytestr(n):
             if c in _winreservedchars:
                 return (
                     _(
                         b"filename contains '%s', which is reserved "
                         b"on Windows"
                     )
                     % c
                 )
             if ord(c) <= 31:
                 return _(
                     b"filename contains '%s', which is invalid on Windows"
                 ) % stringutil.escapestr(c)
         base = n.split(b'.')[0]
         if base and base.lower() in _winreservednames:
             return (
                 _(b"filename contains '%s', which is reserved on Windows")
                 % base
             )
         t = n[-1:]
         if t in b'. ' and n not in b'..':
             return (
                 _(
                     b"filename ends with '%s', which is not allowed "
                     b"on Windows"
                 )
                 % t
             )
 
 
 timer = getattr(time, "perf_counter", None)
 
 if pycompat.iswindows:
     checkosfilename = checkwinfilename
     if not timer:
         timer = time.clock
 else:
     # mercurial.windows doesn't have platform.checkosfilename
     checkosfilename = platform.checkosfilename  # pytype: disable=module-attr
     if not timer:
         timer = time.time
 
 
 def makelock(info, pathname):
     """Create a lock file atomically if possible
 
     This may leave a stale lock file if symlink isn't supported and signal
     interrupt is enabled.
     """
     try:
         return os.symlink(info, pathname)
     except OSError as why:
         if why.errno == errno.EEXIST:
             raise
     except AttributeError:  # no symlink in os
         pass
 
     flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
     ld = os.open(pathname, flags)
     os.write(ld, info)
     os.close(ld)
 
 
 def readlock(pathname):
     # type: (bytes) -> bytes
     try:
         return readlink(pathname)
     except OSError as why:
         if why.errno not in (errno.EINVAL, errno.ENOSYS):
             raise
     except AttributeError:  # no symlink in os
         pass
     with posixfile(pathname, b'rb') as fp:
         return fp.read()
 
 
 def fstat(fp):
     '''stat file object that may not have fileno method.'''
     try:
         return os.fstat(fp.fileno())
     except AttributeError:
         return os.stat(fp.name)
 
 
 # File system features
 
 
 def fscasesensitive(path):
     # type: (bytes) -> bool
     """
     Return true if the given path is on a case-sensitive filesystem
 
     Requires a path (like /foo/.hg) ending with a foldable final
     directory component.
     """
     s1 = os.lstat(path)
     d, b = os.path.split(path)
     b2 = b.upper()
     if b == b2:
         b2 = b.lower()
         if b == b2:
             return True  # no evidence against case sensitivity
     p2 = os.path.join(d, b2)
     try:
         s2 = os.lstat(p2)
         if s2 == s1:
             return False
         return True
     except OSError:
         return True
 
 
 _re2_input = lambda x: x
 try:
     import re2  # pytype: disable=import-error
 
     _re2 = None
 except ImportError:
     _re2 = False
 
 
 class _re(object):
     def _checkre2(self):
         global _re2
         global _re2_input
 
         check_pattern = br'\[([^\[]+)\]'
         check_input = b'[ui]'
         try:
             # check if match works, see issue3964
             _re2 = bool(re2.match(check_pattern, check_input))
         except ImportError:
             _re2 = False
         except TypeError:
             # the `pyre-2` project provides a re2 module that accept bytes
             # the `fb-re2` project provides a re2 module that acccept sysstr
             check_pattern = pycompat.sysstr(check_pattern)
             check_input = pycompat.sysstr(check_input)
             _re2 = bool(re2.match(check_pattern, check_input))
             _re2_input = pycompat.sysstr
 
     def compile(self, pat, flags=0):
         """Compile a regular expression, using re2 if possible
 
         For best performance, use only re2-compatible regexp features. The
         only flags from the re module that are re2-compatible are
         IGNORECASE and MULTILINE."""
         if _re2 is None:
             self._checkre2()
         if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
             if flags & remod.IGNORECASE:
                 pat = b'(?i)' + pat
             if flags & remod.MULTILINE:
                 pat = b'(?m)' + pat
             try:
                 return re2.compile(_re2_input(pat))
             except re2.error:
                 pass
         return remod.compile(pat, flags)
 
     @propertycache
     def escape(self):
         """Return the version of escape corresponding to self.compile.
 
         This is imperfect because whether re2 or re is used for a particular
         function depends on the flags, etc, but it's the best we can do.
         """
         global _re2
         if _re2 is None:
             self._checkre2()
         if _re2:
             return re2.escape
         else:
             return remod.escape
 
 
 re = _re()
 
 _fspathcache = {}
 
 
 def fspath(name, root):
     # type: (bytes, bytes) -> bytes
     """Get name in the case stored in the filesystem
 
     The name should be relative to root, and be normcase-ed for efficiency.
 
     Note that this function is unnecessary, and should not be
     called, for case-sensitive filesystems (simply because it's expensive).
 
     The root should be normcase-ed, too.
     """
 
     def _makefspathcacheentry(dir):
         return {normcase(n): n for n in os.listdir(dir)}
 
     seps = pycompat.ossep
     if pycompat.osaltsep:
         seps = seps + pycompat.osaltsep
     # Protect backslashes. This gets silly very quickly.
     seps.replace(b'\\', b'\\\\')
     pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
     dir = os.path.normpath(root)
     result = []
     for part, sep in pattern.findall(name):
         if sep:
             result.append(sep)
             continue
 
         if dir not in _fspathcache:
             _fspathcache[dir] = _makefspathcacheentry(dir)
         contents = _fspathcache[dir]
 
         found = contents.get(part)
         if not found:
             # retry "once per directory" per "dirstate.walk" which
             # may take place for each patches of "hg qpush", for example
             _fspathcache[dir] = contents = _makefspathcacheentry(dir)
             found = contents.get(part)
 
         result.append(found or part)
         dir = os.path.join(dir, part)
 
     return b''.join(result)
 
 
 def checknlink(testfile):
     # type: (bytes) -> bool
     '''check whether hardlink count reporting works properly'''
 
     # testfile may be open, so we need a separate file for checking to
     # work around issue2543 (or testfile may get lost on Samba shares)
     f1, f2, fp = None, None, None
     try:
         fd, f1 = pycompat.mkstemp(
             prefix=b'.%s-' % os.path.basename(testfile),
             suffix=b'1~',
             dir=os.path.dirname(testfile),
         )
         os.close(fd)
         f2 = b'%s2~' % f1[:-2]
 
         oslink(f1, f2)
         # nlinks() may behave differently for files on Windows shares if
         # the file is open.
         fp = posixfile(f2)
         return nlinks(f2) > 1
     except OSError:
         return False
     finally:
         if fp is not None:
             fp.close()
         for f in (f1, f2):
             try:
                 if f is not None:
                     os.unlink(f)
             except OSError:
                 pass
 
 
 def endswithsep(path):
     # type: (bytes) -> bool
     '''Check path ends with os.sep or os.altsep.'''
     return bool(  # help pytype
         path.endswith(pycompat.ossep)
         or pycompat.osaltsep
         and path.endswith(pycompat.osaltsep)
     )
 
 
 def splitpath(path):
     # type: (bytes) -> List[bytes]
     """Split path by os.sep.
     Note that this function does not use os.altsep because this is
     an alternative of simple "xxx.split(os.sep)".
     It is recommended to use os.path.normpath() before using this
     function if need."""
     return path.split(pycompat.ossep)
 
 
 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
     """Create a temporary file with the same contents from name
 
     The permission bits are copied from the original file.
 
     If the temporary file is going to be truncated immediately, you
     can use emptyok=True as an optimization.
 
     Returns the name of the temporary file.
     """
     d, fn = os.path.split(name)
     fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
     os.close(fd)
     # Temporary files are created with mode 0600, which is usually not
     # what we want.  If the original file already exists, just copy
     # its mode.  Otherwise, manually obey umask.
     copymode(name, temp, createmode, enforcewritable)
 
     if emptyok:
         return temp
     try:
         try:
             ifp = posixfile(name, b"rb")
         except IOError as inst:
             if inst.errno == errno.ENOENT:
                 return temp
             if not getattr(inst, 'filename', None):
                 inst.filename = name
             raise
         ofp = posixfile(temp, b"wb")
         for chunk in filechunkiter(ifp):
             ofp.write(chunk)
         ifp.close()
         ofp.close()
     except:  # re-raises
         try:
             os.unlink(temp)
         except OSError:
             pass
         raise
     return temp
 
 
 class filestat(object):
     """help to exactly detect change of a file
 
     'stat' attribute is result of 'os.stat()' if specified 'path'
     exists. Otherwise, it is None. This can avoid preparative
     'exists()' examination on client side of this class.
     """
 
     def __init__(self, stat):
         self.stat = stat
 
     @classmethod
     def frompath(cls, path):
         try:
             stat = os.stat(path)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             stat = None
         return cls(stat)
 
     @classmethod
     def fromfp(cls, fp):
         stat = os.fstat(fp.fileno())
         return cls(stat)
 
     __hash__ = object.__hash__
 
     def __eq__(self, old):
         try:
             # if ambiguity between stat of new and old file is
             # avoided, comparison of size, ctime and mtime is enough
             # to exactly detect change of a file regardless of platform
             return (
                 self.stat.st_size == old.stat.st_size
                 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
                 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
             )
         except AttributeError:
             pass
         try:
             return self.stat is None and old.stat is None
         except AttributeError:
             return False
 
     def isambig(self, old):
         """Examine whether new (= self) stat is ambiguous against old one
 
         "S[N]" below means stat of a file at N-th change:
 
         - S[n-1].ctime  < S[n].ctime: can detect change of a file
         - S[n-1].ctime == S[n].ctime
           - S[n-1].ctime  < S[n].mtime: means natural advancing (*1)
           - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
           - S[n-1].ctime  > S[n].mtime: never occurs naturally (don't care)
         - S[n-1].ctime  > S[n].ctime: never occurs naturally (don't care)
 
         Case (*2) above means that a file was changed twice or more at
         same time in sec (= S[n-1].ctime), and comparison of timestamp
         is ambiguous.
 
         Base idea to avoid such ambiguity is "advance mtime 1 sec, if
         timestamp is ambiguous".
 
         But advancing mtime only in case (*2) doesn't work as
         expected, because naturally advanced S[n].mtime in case (*1)
         might be equal to manually advanced S[n-1 or earlier].mtime.
 
         Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
         treated as ambiguous regardless of mtime, to avoid overlooking
         by confliction between such mtime.
 
         Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
         S[n].mtime", even if size of a file isn't changed.
         """
         try:
             return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
         except AttributeError:
             return False
 
     def avoidambig(self, path, old):
         """Change file stat of specified path to avoid ambiguity
 
         'old' should be previous filestat of 'path'.
 
         This skips avoiding ambiguity, if a process doesn't have
         appropriate privileges for 'path'. This returns False in this
         case.
 
         Otherwise, this returns True, as "ambiguity is avoided".
         """
         advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
         try:
             os.utime(path, (advanced, advanced))
         except OSError as inst:
             if inst.errno == errno.EPERM:
                 # utime() on the file created by another user causes EPERM,
                 # if a process doesn't have appropriate privileges
                 return False
             raise
         return True
 
     def __ne__(self, other):
         return not self == other
 
 
 class atomictempfile(object):
     """writable file object that atomically updates a file
 
     All writes will go to a temporary copy of the original file. Call
     close() when you are done writing, and atomictempfile will rename
     the temporary copy to the original name, making the changes
     visible. If the object is destroyed without being closed, all your
     writes are discarded.
 
     checkambig argument of constructor is used with filestat, and is
     useful only if target file is guarded by any lock (e.g. repo.lock
     or repo.wlock).
     """
 
     def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
         self.__name = name  # permanent name
         self._tempname = mktempcopy(
             name,
             emptyok=(b'w' in mode),
             createmode=createmode,
             enforcewritable=(b'w' in mode),
         )
 
         self._fp = posixfile(self._tempname, mode)
         self._checkambig = checkambig
 
         # delegated methods
         self.read = self._fp.read
         self.write = self._fp.write
         self.seek = self._fp.seek
         self.tell = self._fp.tell
         self.fileno = self._fp.fileno
 
     def close(self):
         if not self._fp.closed:
             self._fp.close()
             filename = localpath(self.__name)
             oldstat = self._checkambig and filestat.frompath(filename)
             if oldstat and oldstat.stat:
                 rename(self._tempname, filename)
                 newstat = filestat.frompath(filename)
                 if newstat.isambig(oldstat):
                     # stat of changed file is ambiguous to original one
                     advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
                     os.utime(filename, (advanced, advanced))
             else:
                 rename(self._tempname, filename)
 
     def discard(self):
         if not self._fp.closed:
             try:
                 os.unlink(self._tempname)
             except OSError:
                 pass
             self._fp.close()
 
     def __del__(self):
         if safehasattr(self, '_fp'):  # constructor actually did something
             self.discard()
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, traceback):
         if exctype is not None:
             self.discard()
         else:
             self.close()
 
 
 def unlinkpath(f, ignoremissing=False, rmdir=True):
     # type: (bytes, bool, bool) -> None
     """unlink and remove the directory if it is empty"""
     if ignoremissing:
         tryunlink(f)
     else:
         unlink(f)
     if rmdir:
         # try removing directories that might now be empty
         try:
             removedirs(os.path.dirname(f))
         except OSError:
             pass
 
 
 def tryunlink(f):
     # type: (bytes) -> None
     """Attempt to remove a file, ignoring ENOENT errors."""
     try:
         unlink(f)
     except OSError as e:
         if e.errno != errno.ENOENT:
             raise
 
 
 def makedirs(name, mode=None, notindexed=False):
     # type: (bytes, Optional[int], bool) -> None
     """recursive directory creation with parent mode inheritance
 
     Newly created directories are marked as "not to be indexed by
     the content indexing service", if ``notindexed`` is specified
     for "write" mode access.
     """
     try:
         makedir(name, notindexed)
     except OSError as err:
         if err.errno == errno.EEXIST:
             return
         if err.errno != errno.ENOENT or not name:
             raise
         parent = os.path.dirname(abspath(name))
         if parent == name:
             raise
         makedirs(parent, mode, notindexed)
         try:
             makedir(name, notindexed)
         except OSError as err:
             # Catch EEXIST to handle races
             if err.errno == errno.EEXIST:
                 return
             raise
     if mode is not None:
         os.chmod(name, mode)
 
 
 def readfile(path):
     # type: (bytes) -> bytes
     with open(path, b'rb') as fp:
         return fp.read()
 
 
 def writefile(path, text):
     # type: (bytes, bytes) -> None
     with open(path, b'wb') as fp:
         fp.write(text)
 
 
 def appendfile(path, text):
     # type: (bytes, bytes) -> None
     with open(path, b'ab') as fp:
         fp.write(text)
 
 
 class chunkbuffer(object):
     """Allow arbitrary sized chunks of data to be efficiently read from an
     iterator over chunks of arbitrary size."""
 
     def __init__(self, in_iter):
         """in_iter is the iterator that's iterating over the input chunks."""
 
         def splitbig(chunks):
             for chunk in chunks:
                 if len(chunk) > 2 ** 20:
                     pos = 0
                     while pos < len(chunk):
                         end = pos + 2 ** 18
                         yield chunk[pos:end]
                         pos = end
                 else:
                     yield chunk
 
         self.iter = splitbig(in_iter)
         self._queue = collections.deque()
         self._chunkoffset = 0
 
     def read(self, l=None):
         """Read L bytes of data from the iterator of chunks of data.
         Returns less than L bytes if the iterator runs dry.
 
         If size parameter is omitted, read everything"""
         if l is None:
             return b''.join(self.iter)
 
         left = l
         buf = []
         queue = self._queue
         while left > 0:
             # refill the queue
             if not queue:
                 target = 2 ** 18
                 for chunk in self.iter:
                     queue.append(chunk)
                     target -= len(chunk)
                     if target <= 0:
                         break
                 if not queue:
                     break
 
             # The easy way to do this would be to queue.popleft(), modify the
             # chunk (if necessary), then queue.appendleft(). However, for cases
             # where we read partial chunk content, this incurs 2 dequeue
             # mutations and creates a new str for the remaining chunk in the
             # queue. Our code below avoids this overhead.
 
             chunk = queue[0]
             chunkl = len(chunk)
             offset = self._chunkoffset
 
             # Use full chunk.
             if offset == 0 and left >= chunkl:
                 left -= chunkl
                 queue.popleft()
                 buf.append(chunk)
                 # self._chunkoffset remains at 0.
                 continue
 
             chunkremaining = chunkl - offset
 
             # Use all of unconsumed part of chunk.
             if left >= chunkremaining:
                 left -= chunkremaining
                 queue.popleft()
                 # offset == 0 is enabled by block above, so this won't merely
                 # copy via ``chunk[0:]``.
                 buf.append(chunk[offset:])
                 self._chunkoffset = 0
 
             # Partial chunk needed.
             else:
                 buf.append(chunk[offset : offset + left])
                 self._chunkoffset += left
                 left -= chunkremaining
 
         return b''.join(buf)
 
 
 def filechunkiter(f, size=131072, limit=None):
     """Create a generator that produces the data in the file size
     (default 131072) bytes at a time, up to optional limit (default is
     to read all data).  Chunks may be less than size bytes if the
     chunk is the last chunk in the file, or the file is a socket or
     some other type of file that sometimes reads less data than is
     requested."""
     assert size >= 0
     assert limit is None or limit >= 0
     while True:
         if limit is None:
             nbytes = size
         else:
             nbytes = min(limit, size)
         s = nbytes and f.read(nbytes)
         if not s:
             break
         if limit:
             limit -= len(s)
         yield s
 
 
 class cappedreader(object):
     """A file object proxy that allows reading up to N bytes.
 
     Given a source file object, instances of this type allow reading up to
     N bytes from that source file object. Attempts to read past the allowed
     limit are treated as EOF.
 
     It is assumed that I/O is not performed on the original file object
     in addition to I/O that is performed by this instance. If there is,
     state tracking will get out of sync and unexpected results will ensue.
     """
 
     def __init__(self, fh, limit):
         """Allow reading up to <limit> bytes from <fh>."""
         self._fh = fh
         self._left = limit
 
     def read(self, n=-1):
         if not self._left:
             return b''
 
         if n < 0:
             n = self._left
 
         data = self._fh.read(min(n, self._left))
         self._left -= len(data)
         assert self._left >= 0
 
         return data
 
     def readinto(self, b):
         res = self.read(len(b))
         if res is None:
             return None
 
         b[0 : len(res)] = res
         return len(res)
 
 
 def unitcountfn(*unittable):
     '''return a function that renders a readable count of some quantity'''
 
     def go(count):
         for multiplier, divisor, format in unittable:
             if abs(count) >= divisor * multiplier:
                 return format % (count / float(divisor))
         return unittable[-1][2] % count
 
     return go
 
 
 def processlinerange(fromline, toline):
     # type: (int, int) -> Tuple[int, int]
     """Check that linerange <fromline>:<toline> makes sense and return a
     0-based range.
 
     >>> processlinerange(10, 20)
     (9, 20)
     >>> processlinerange(2, 1)
     Traceback (most recent call last):
         ...
     ParseError: line range must be positive
     >>> processlinerange(0, 5)
     Traceback (most recent call last):
         ...
     ParseError: fromline must be strictly positive
     """
     if toline - fromline < 0:
         raise error.ParseError(_(b"line range must be positive"))
     if fromline < 1:
         raise error.ParseError(_(b"fromline must be strictly positive"))
     return fromline - 1, toline
 
 
 bytecount = unitcountfn(
     (100, 1 << 30, _(b'%.0f GB')),
     (10, 1 << 30, _(b'%.1f GB')),
     (1, 1 << 30, _(b'%.2f GB')),
     (100, 1 << 20, _(b'%.0f MB')),
     (10, 1 << 20, _(b'%.1f MB')),
     (1, 1 << 20, _(b'%.2f MB')),
     (100, 1 << 10, _(b'%.0f KB')),
     (10, 1 << 10, _(b'%.1f KB')),
     (1, 1 << 10, _(b'%.2f KB')),
     (1, 1, _(b'%.0f bytes')),
 )
 
 
 class transformingwriter(object):
     """Writable file wrapper to transform data by function"""
 
     def __init__(self, fp, encode):
         self._fp = fp
         self._encode = encode
 
     def close(self):
         self._fp.close()
 
     def flush(self):
         self._fp.flush()
 
     def write(self, data):
         return self._fp.write(self._encode(data))
 
 
 # Matches a single EOL which can either be a CRLF where repeated CR
 # are removed or a LF. We do not care about old Macintosh files, so a
 # stray CR is an error.
 _eolre = remod.compile(br'\r*\n')
 
 
 def tolf(s):
     # type: (bytes) -> bytes
     return _eolre.sub(b'\n', s)
 
 
 def tocrlf(s):
     # type: (bytes) -> bytes
     return _eolre.sub(b'\r\n', s)
 
 
 def _crlfwriter(fp):
     return transformingwriter(fp, tocrlf)
 
 
 if pycompat.oslinesep == b'\r\n':
     tonativeeol = tocrlf
     fromnativeeol = tolf
     nativeeolwriter = _crlfwriter
 else:
     tonativeeol = pycompat.identity
     fromnativeeol = pycompat.identity
     nativeeolwriter = pycompat.identity
 
 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
     3,
     0,
 ):
     # There is an issue in CPython that some IO methods do not handle EINTR
     # correctly. The following table shows what CPython version (and functions)
     # are affected (buggy: has the EINTR bug, okay: otherwise):
     #
     #                | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
     #   --------------------------------------------------
     #    fp.__iter__ | buggy   | buggy           | okay
     #    fp.read*    | buggy   | okay [1]        | okay
     #
     # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
     #
     # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
     # like "read*" work fine, as we do not support Python < 2.7.4.
     #
     # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
     # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
     # CPython 2, because CPython 2 maintains an internal readahead buffer for
     # fp.__iter__ but not other fp.read* methods.
     #
     # On modern systems like Linux, the "read" syscall cannot be interrupted
     # when reading "fast" files like on-disk files. So the EINTR issue only
     # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
     # files approximately as "fast" files and use the fast (unsafe) code path,
     # to minimize the performance impact.
 
     def iterfile(fp):
         fastpath = True
         if type(fp) is file:
             fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
         if fastpath:
             return fp
         else:
             # fp.readline deals with EINTR correctly, use it as a workaround.
             return iter(fp.readline, b'')
 
 
 else:
     # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
     def iterfile(fp):
         return fp
 
 
 def iterlines(iterator):
     # type: (Iterator[bytes]) -> Iterator[bytes]
     for chunk in iterator:
         for line in chunk.splitlines():
             yield line
 
 
 def expandpath(path):
     # type: (bytes) -> bytes
     return os.path.expanduser(os.path.expandvars(path))
 
 
 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
     """Return the result of interpolating items in the mapping into string s.
 
     prefix is a single character string, or a two character string with
     a backslash as the first character if the prefix needs to be escaped in
     a regular expression.
 
     fn is an optional function that will be applied to the replacement text
     just before replacement.
 
     escape_prefix is an optional flag that allows using doubled prefix for
     its escaping.
     """
     fn = fn or (lambda s: s)
     patterns = b'|'.join(mapping.keys())
     if escape_prefix:
         patterns += b'|' + prefix
         if len(prefix) > 1:
             prefix_char = prefix[1:]
         else:
             prefix_char = prefix
         mapping[prefix_char] = prefix_char
     r = remod.compile(br'%s(%s)' % (prefix, patterns))
     return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
 
 
 timecount = unitcountfn(
     (1, 1e3, _(b'%.0f s')),
     (100, 1, _(b'%.1f s')),
     (10, 1, _(b'%.2f s')),
     (1, 1, _(b'%.3f s')),
     (100, 0.001, _(b'%.1f ms')),
     (10, 0.001, _(b'%.2f ms')),
     (1, 0.001, _(b'%.3f ms')),
     (100, 0.000001, _(b'%.1f us')),
     (10, 0.000001, _(b'%.2f us')),
     (1, 0.000001, _(b'%.3f us')),
     (100, 0.000000001, _(b'%.1f ns')),
     (10, 0.000000001, _(b'%.2f ns')),
     (1, 0.000000001, _(b'%.3f ns')),
 )
 
 
 @attr.s
 class timedcmstats(object):
     """Stats information produced by the timedcm context manager on entering."""
 
     # the starting value of the timer as a float (meaning and resulution is
     # platform dependent, see util.timer)
     start = attr.ib(default=attr.Factory(lambda: timer()))
     # the number of seconds as a floating point value; starts at 0, updated when
     # the context is exited.
     elapsed = attr.ib(default=0)
     # the number of nested timedcm context managers.
     level = attr.ib(default=1)
 
     def __bytes__(self):
         return timecount(self.elapsed) if self.elapsed else b'<unknown>'
 
     __str__ = encoding.strmethod(__bytes__)
 
 
 @contextlib.contextmanager
 def timedcm(whencefmt, *whenceargs):
     """A context manager that produces timing information for a given context.
 
     On entering a timedcmstats instance is produced.
 
     This context manager is reentrant.
 
     """
     # track nested context managers
     timedcm._nested += 1
     timing_stats = timedcmstats(level=timedcm._nested)
     try:
         with tracing.log(whencefmt, *whenceargs):
             yield timing_stats
     finally:
         timing_stats.elapsed = timer() - timing_stats.start
         timedcm._nested -= 1
 
 
 timedcm._nested = 0
 
 
 def timed(func):
     """Report the execution time of a function call to stderr.
 
     During development, use as a decorator when you need to measure
     the cost of a function, e.g. as follows:
 
     @util.timed
     def foo(a, b, c):
         pass
     """
 
     def wrapper(*args, **kwargs):
         with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
             result = func(*args, **kwargs)
         stderr = procutil.stderr
         stderr.write(
             b'%s%s: %s\n'
             % (
                 b' ' * time_stats.level * 2,
                 pycompat.bytestr(func.__name__),
                 time_stats,
             )
         )
         return result
 
     return wrapper
 
 
 _sizeunits = (
     (b'm', 2 ** 20),
     (b'k', 2 ** 10),
     (b'g', 2 ** 30),
     (b'kb', 2 ** 10),
     (b'mb', 2 ** 20),
     (b'gb', 2 ** 30),
     (b'b', 1),
 )
 
 
 def sizetoint(s):
     # type: (bytes) -> int
     """Convert a space specifier to a byte count.
 
     >>> sizetoint(b'30')
     30
     >>> sizetoint(b'2.2kb')
     2252
     >>> sizetoint(b'6M')
     6291456
     """
     t = s.strip().lower()
     try:
         for k, u in _sizeunits:
             if t.endswith(k):
                 return int(float(t[: -len(k)]) * u)
         return int(t)
     except ValueError:
         raise error.ParseError(_(b"couldn't parse size: %s") % s)
 
 
 class hooks(object):
     """A collection of hook functions that can be used to extend a
     function's behavior. Hooks are called in lexicographic order,
     based on the names of their sources."""
 
     def __init__(self):
         self._hooks = []
 
     def add(self, source, hook):
         self._hooks.append((source, hook))
 
     def __call__(self, *args):
         self._hooks.sort(key=lambda x: x[0])
         results = []
         for source, hook in self._hooks:
             results.append(hook(*args))
         return results
 
 
 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
     """Yields lines for a nicely formatted stacktrace.
     Skips the 'skip' last entries, then return the last 'depth' entries.
     Each file+linenumber is formatted according to fileline.
     Each line is formatted according to line.
     If line is None, it yields:
       length of longest filepath+line number,
       filepath+linenumber,
       function
 
     Not be used in production code but very convenient while developing.
     """
     entries = [
         (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
         for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
     ][-depth:]
     if entries:
         fnmax = max(len(entry[0]) for entry in entries)
         for fnln, func in entries:
             if line is None:
                 yield (fnmax, fnln, func)
             else:
                 yield line % (fnmax, fnln, func)
 
 
 def debugstacktrace(
     msg=b'stacktrace',
     skip=0,
     f=procutil.stderr,
     otherf=procutil.stdout,
     depth=0,
     prefix=b'',
 ):
     """Writes a message to f (stderr) with a nicely formatted stacktrace.
     Skips the 'skip' entries closest to the call, then show 'depth' entries.
     By default it will flush stdout first.
     It can be used everywhere and intentionally does not require an ui object.
     Not be used in production code but very convenient while developing.
     """
     if otherf:
         otherf.flush()
     f.write(b'%s%s at:\n' % (prefix, msg.rstrip()))
     for line in getstackframes(skip + 1, depth=depth):
         f.write(prefix + line)
     f.flush()
 
 
 # convenient shortcut
 dst = debugstacktrace
 
 
 def safename(f, tag, ctx, others=None):
     """
     Generate a name that it is safe to rename f to in the given context.
 
     f:      filename to rename
     tag:    a string tag that will be included in the new name
     ctx:    a context, in which the new name must not exist
     others: a set of other filenames that the new name must not be in
 
     Returns a file name of the form oldname~tag[~number] which does not exist
     in the provided context and is not in the set of other names.
     """
     if others is None:
         others = set()
 
     fn = b'%s~%s' % (f, tag)
     if fn not in ctx and fn not in others:
         return fn
     for n in itertools.count(1):
         fn = b'%s~%s~%s' % (f, tag, n)
         if fn not in ctx and fn not in others:
             return fn
 
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
     s = stream.read(n)
     if len(s) < n:
         raise error.Abort(
             _(b"stream ended unexpectedly (got %d bytes, expected %d)")
             % (len(s), n)
         )
     return s
 
 
 def uvarintencode(value):
     """Encode an unsigned integer value to a varint.
 
     A varint is a variable length integer of 1 or more bytes. Each byte
     except the last has the most significant bit set. The lower 7 bits of
     each byte store the 2's complement representation, least significant group
     first.
 
     >>> uvarintencode(0)
     '\\x00'
     >>> uvarintencode(1)
     '\\x01'
     >>> uvarintencode(127)
     '\\x7f'
     >>> uvarintencode(1337)
     '\\xb9\\n'
     >>> uvarintencode(65536)
     '\\x80\\x80\\x04'
     >>> uvarintencode(-1)
     Traceback (most recent call last):
         ...
     ProgrammingError: negative value for uvarint: -1
     """
     if value < 0:
         raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
     bits = value & 0x7F
     value >>= 7
     bytes = []
     while value:
         bytes.append(pycompat.bytechr(0x80 | bits))
         bits = value & 0x7F
         value >>= 7
     bytes.append(pycompat.bytechr(bits))
 
     return b''.join(bytes)
 
 
 def uvarintdecodestream(fh):
     """Decode an unsigned variable length integer from a stream.
 
     The passed argument is anything that has a ``.read(N)`` method.
 
     >>> try:
     ...     from StringIO import StringIO as BytesIO
     ... except ImportError:
     ...     from io import BytesIO
     >>> uvarintdecodestream(BytesIO(b'\\x00'))
     0
     >>> uvarintdecodestream(BytesIO(b'\\x01'))
     1
     >>> uvarintdecodestream(BytesIO(b'\\x7f'))
     127
     >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
     1337
     >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
     65536
     >>> uvarintdecodestream(BytesIO(b'\\x80'))
     Traceback (most recent call last):
         ...
     Abort: stream ended unexpectedly (got 0 bytes, expected 1)
     """
     result = 0
     shift = 0
     while True:
         byte = ord(readexactly(fh, 1))
         result |= (byte & 0x7F) << shift
         if not (byte & 0x80):
             return result
         shift += 7
 
 
 # Passing the '' locale means that the locale should be set according to the
 # user settings (environment variables).
 # Python sometimes avoids setting the global locale settings. When interfacing
 # with C code (e.g. the curses module or the Subversion bindings), the global
 # locale settings must be initialized correctly. Python 2 does not initialize
 # the global locale settings on interpreter startup. Python 3 sometimes
 # initializes LC_CTYPE, but not consistently at least on Windows. Therefore we
 # explicitly initialize it to get consistent behavior if it's not already
 # initialized. Since CPython commit 177d921c8c03d30daa32994362023f777624b10d,
 # LC_CTYPE is always initialized. If we require Python 3.8+, we should re-check
 # if we can remove this code.
 @contextlib.contextmanager
 def with_lc_ctype():
     oldloc = locale.setlocale(locale.LC_CTYPE, None)
     if oldloc == 'C':
         try:
             try:
                 locale.setlocale(locale.LC_CTYPE, '')
             except locale.Error:
                 # The likely case is that the locale from the environment
                 # variables is unknown.
                 pass
             yield
         finally:
             locale.setlocale(locale.LC_CTYPE, oldloc)
     else:
         yield
 
 
 def _estimatememory():
     # type: () -> Optional[int]
     """Provide an estimate for the available system memory in Bytes.
 
     If no estimate can be provided on the platform, returns None.
     """
     if pycompat.sysplatform.startswith(b'win'):
         # On Windows, use the GlobalMemoryStatusEx kernel function directly.
         from ctypes import c_long as DWORD, c_ulonglong as DWORDLONG
         from ctypes.wintypes import (  # pytype: disable=import-error
             Structure,
             byref,
             sizeof,
             windll,
         )
 
         class MEMORYSTATUSEX(Structure):
             _fields_ = [
                 ('dwLength', DWORD),
                 ('dwMemoryLoad', DWORD),
                 ('ullTotalPhys', DWORDLONG),
                 ('ullAvailPhys', DWORDLONG),
                 ('ullTotalPageFile', DWORDLONG),
                 ('ullAvailPageFile', DWORDLONG),
                 ('ullTotalVirtual', DWORDLONG),
                 ('ullAvailVirtual', DWORDLONG),
                 ('ullExtendedVirtual', DWORDLONG),
             ]
 
         x = MEMORYSTATUSEX()
         x.dwLength = sizeof(x)
         windll.kernel32.GlobalMemoryStatusEx(byref(x))
         return x.ullAvailPhys
 
     # On newer Unix-like systems and Mac OSX, the sysconf interface
     # can be used. _SC_PAGE_SIZE is part of POSIX; _SC_PHYS_PAGES
     # seems to be implemented on most systems.
     try:
         pagesize = os.sysconf(os.sysconf_names['SC_PAGE_SIZE'])
         pages = os.sysconf(os.sysconf_names['SC_PHYS_PAGES'])
         return pagesize * pages
     except OSError:  # sysconf can fail
         pass
     except KeyError:  # unknown parameter
         pass

mercurial/worker.py

@@ -1,468 +1,468 b''
 # worker.py - master-slave parallelism support
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import os
+import pickle
 import signal
 import sys
 import threading
 import time
 
 try:
     import selectors
 
     selectors.BaseSelector
 except ImportError:
     from .thirdparty import selectors2 as selectors
 
 from .i18n import _
 from . import (
     encoding,
     error,
     pycompat,
     scmutil,
-    util,
 )
 
 
 def countcpus():
     '''try to count the number of CPUs on the system'''
 
     # posix
     try:
         n = int(os.sysconf('SC_NPROCESSORS_ONLN'))
         if n > 0:
             return n
     except (AttributeError, ValueError):
         pass
 
     # windows
     try:
         n = int(encoding.environ[b'NUMBER_OF_PROCESSORS'])
         if n > 0:
             return n
     except (KeyError, ValueError):
         pass
 
     return 1
 
 
 def _numworkers(ui):
     s = ui.config(b'worker', b'numcpus')
     if s:
         try:
             n = int(s)
             if n >= 1:
                 return n
         except ValueError:
             raise error.Abort(_(b'number of cpus must be an integer'))
     return min(max(countcpus(), 4), 32)
 
 
 if pycompat.ispy3:
 
     def ismainthread():
         return threading.current_thread() == threading.main_thread()
 
     class _blockingreader(object):
         def __init__(self, wrapped):
             self._wrapped = wrapped
 
         # Do NOT implement readinto() by making it delegate to
         # _wrapped.readinto(), since that is unbuffered. The unpickler is fine
         # with just read() and readline(), so we don't need to implement it.
 
         def readline(self):
             return self._wrapped.readline()
 
         # issue multiple reads until size is fulfilled
         def read(self, size=-1):
             if size < 0:
                 return self._wrapped.readall()
 
             buf = bytearray(size)
             view = memoryview(buf)
             pos = 0
 
             while pos < size:
                 ret = self._wrapped.readinto(view[pos:])
                 if not ret:
                     break
                 pos += ret
 
             del view
             del buf[pos:]
             return bytes(buf)
 
 
 else:
 
     def ismainthread():
         # pytype: disable=module-attr
         return isinstance(threading.current_thread(), threading._MainThread)
         # pytype: enable=module-attr
 
     def _blockingreader(wrapped):
         return wrapped
 
 
 if pycompat.isposix or pycompat.iswindows:
     _STARTUP_COST = 0.01
     # The Windows worker is thread based. If tasks are CPU bound, threads
     # in the presence of the GIL result in excessive context switching and
     # this overhead can slow down execution.
     _DISALLOW_THREAD_UNSAFE = pycompat.iswindows
 else:
     _STARTUP_COST = 1e30
     _DISALLOW_THREAD_UNSAFE = False
 
 
 def worthwhile(ui, costperop, nops, threadsafe=True):
     """try to determine whether the benefit of multiple processes can
     outweigh the cost of starting them"""
 
     if not threadsafe and _DISALLOW_THREAD_UNSAFE:
         return False
 
     linear = costperop * nops
     workers = _numworkers(ui)
     benefit = linear - (_STARTUP_COST * workers + linear / workers)
     return benefit >= 0.15
 
 
 def worker(
     ui, costperarg, func, staticargs, args, hasretval=False, threadsafe=True
 ):
     """run a function, possibly in parallel in multiple worker
     processes.
 
     returns a progress iterator
 
     costperarg - cost of a single task
 
     func - function to run. It is expected to return a progress iterator.
 
     staticargs - arguments to pass to every invocation of the function
 
     args - arguments to split into chunks, to pass to individual
     workers
 
     hasretval - when True, func and the current function return an progress
     iterator then a dict (encoded as an iterator that yield many (False, ..)
     then a (True, dict)). The dicts are joined in some arbitrary order, so
     overlapping keys are a bad idea.
 
     threadsafe - whether work items are thread safe and can be executed using
     a thread-based worker. Should be disabled for CPU heavy tasks that don't
     release the GIL.
     """
     enabled = ui.configbool(b'worker', b'enabled')
     if enabled and _platformworker is _posixworker and not ismainthread():
         # The POSIX worker has to install a handler for SIGCHLD.
         # Python up to 3.9 only allows this in the main thread.
         enabled = False
 
     if enabled and worthwhile(ui, costperarg, len(args), threadsafe=threadsafe):
         return _platformworker(ui, func, staticargs, args, hasretval)
     return func(*staticargs + (args,))
 
 
 def _posixworker(ui, func, staticargs, args, hasretval):
     workers = _numworkers(ui)
     oldhandler = signal.getsignal(signal.SIGINT)
     signal.signal(signal.SIGINT, signal.SIG_IGN)
     pids, problem = set(), [0]
 
     def killworkers():
         # unregister SIGCHLD handler as all children will be killed. This
         # function shouldn't be interrupted by another SIGCHLD; otherwise pids
         # could be updated while iterating, which would cause inconsistency.
         signal.signal(signal.SIGCHLD, oldchldhandler)
         # if one worker bails, there's no good reason to wait for the rest
         for p in pids:
             try:
                 os.kill(p, signal.SIGTERM)
             except OSError as err:
                 if err.errno != errno.ESRCH:
                     raise
 
     def waitforworkers(blocking=True):
         for pid in pids.copy():
             p = st = 0
             while True:
                 try:
                     p, st = os.waitpid(pid, (0 if blocking else os.WNOHANG))
                     break
                 except OSError as e:
                     if e.errno == errno.EINTR:
                         continue
                     elif e.errno == errno.ECHILD:
                         # child would already be reaped, but pids yet been
                         # updated (maybe interrupted just after waitpid)
                         pids.discard(pid)
                         break
                     else:
                         raise
             if not p:
                 # skip subsequent steps, because child process should
                 # be still running in this case
                 continue
             pids.discard(p)
             st = _exitstatus(st)
             if st and not problem[0]:
                 problem[0] = st
 
     def sigchldhandler(signum, frame):
         waitforworkers(blocking=False)
         if problem[0]:
             killworkers()
 
     oldchldhandler = signal.signal(signal.SIGCHLD, sigchldhandler)
     ui.flush()
     parentpid = os.getpid()
     pipes = []
     retval = {}
     for pargs in partition(args, min(workers, len(args))):
         # Every worker gets its own pipe to send results on, so we don't have to
         # implement atomic writes larger than PIPE_BUF. Each forked process has
         # its own pipe's descriptors in the local variables, and the parent
         # process has the full list of pipe descriptors (and it doesn't really
         # care what order they're in).
         rfd, wfd = os.pipe()
         pipes.append((rfd, wfd))
         # make sure we use os._exit in all worker code paths. otherwise the
         # worker may do some clean-ups which could cause surprises like
         # deadlock. see sshpeer.cleanup for example.
         # override error handling *before* fork. this is necessary because
         # exception (signal) may arrive after fork, before "pid =" assignment
         # completes, and other exception handler (dispatch.py) can lead to
         # unexpected code path without os._exit.
         ret = -1
         try:
             pid = os.fork()
             if pid == 0:
                 signal.signal(signal.SIGINT, oldhandler)
                 signal.signal(signal.SIGCHLD, oldchldhandler)
 
                 def workerfunc():
                     for r, w in pipes[:-1]:
                         os.close(r)
                         os.close(w)
                     os.close(rfd)
                     for result in func(*(staticargs + (pargs,))):
-                        os.write(wfd, util.pickle.dumps(result))
+                        os.write(wfd, pickle.dumps(result))
                     return 0
 
                 ret = scmutil.callcatch(ui, workerfunc)
         except:  # parent re-raises, child never returns
             if os.getpid() == parentpid:
                 raise
             exctype = sys.exc_info()[0]
             force = not issubclass(exctype, KeyboardInterrupt)
             ui.traceback(force=force)
         finally:
             if os.getpid() != parentpid:
                 try:
                     ui.flush()
                 except:  # never returns, no re-raises
                     pass
                 finally:
                     os._exit(ret & 255)
         pids.add(pid)
     selector = selectors.DefaultSelector()
     for rfd, wfd in pipes:
         os.close(wfd)
         selector.register(os.fdopen(rfd, 'rb', 0), selectors.EVENT_READ)
 
     def cleanup():
         signal.signal(signal.SIGINT, oldhandler)
         waitforworkers()
         signal.signal(signal.SIGCHLD, oldchldhandler)
         selector.close()
         return problem[0]
 
     try:
         openpipes = len(pipes)
         while openpipes > 0:
             for key, events in selector.select():
                 try:
-                    res = util.pickle.load(_blockingreader(key.fileobj))
+                    res = pickle.load(_blockingreader(key.fileobj))
                     if hasretval and res[0]:
                         retval.update(res[1])
                     else:
                         yield res
                 except EOFError:
                     selector.unregister(key.fileobj)
                     key.fileobj.close()
                     openpipes -= 1
                 except IOError as e:
                     if e.errno == errno.EINTR:
                         continue
                     raise
     except:  # re-raises
         killworkers()
         cleanup()
         raise
     status = cleanup()
     if status:
         if status < 0:
             os.kill(os.getpid(), -status)
         raise error.WorkerError(status)
     if hasretval:
         yield True, retval
 
 
 def _posixexitstatus(code):
     """convert a posix exit status into the same form returned by
     os.spawnv
 
     returns None if the process was stopped instead of exiting"""
     if os.WIFEXITED(code):
         return os.WEXITSTATUS(code)
     elif os.WIFSIGNALED(code):
         return -(os.WTERMSIG(code))
 
 
 def _windowsworker(ui, func, staticargs, args, hasretval):
     class Worker(threading.Thread):
         def __init__(
             self, taskqueue, resultqueue, func, staticargs, *args, **kwargs
         ):
             threading.Thread.__init__(self, *args, **kwargs)
             self._taskqueue = taskqueue
             self._resultqueue = resultqueue
             self._func = func
             self._staticargs = staticargs
             self._interrupted = False
             self.daemon = True
             self.exception = None
 
         def interrupt(self):
             self._interrupted = True
 
         def run(self):
             try:
                 while not self._taskqueue.empty():
                     try:
                         args = self._taskqueue.get_nowait()
                         for res in self._func(*self._staticargs + (args,)):
                             self._resultqueue.put(res)
                             # threading doesn't provide a native way to
                             # interrupt execution. handle it manually at every
                             # iteration.
                             if self._interrupted:
                                 return
                     except pycompat.queue.Empty:
                         break
             except Exception as e:
                 # store the exception such that the main thread can resurface
                 # it as if the func was running without workers.
                 self.exception = e
                 raise
 
     threads = []
 
     def trykillworkers():
         # Allow up to 1 second to clean worker threads nicely
         cleanupend = time.time() + 1
         for t in threads:
             t.interrupt()
         for t in threads:
             remainingtime = cleanupend - time.time()
             t.join(remainingtime)
             if t.is_alive():
                 # pass over the workers joining failure. it is more
                 # important to surface the inital exception than the
                 # fact that one of workers may be processing a large
                 # task and does not get to handle the interruption.
                 ui.warn(
                     _(
                         b"failed to kill worker threads while "
                         b"handling an exception\n"
                     )
                 )
                 return
 
     workers = _numworkers(ui)
     resultqueue = pycompat.queue.Queue()
     taskqueue = pycompat.queue.Queue()
     retval = {}
     # partition work to more pieces than workers to minimize the chance
     # of uneven distribution of large tasks between the workers
     for pargs in partition(args, workers * 20):
         taskqueue.put(pargs)
     for _i in range(workers):
         t = Worker(taskqueue, resultqueue, func, staticargs)
         threads.append(t)
         t.start()
     try:
         while len(threads) > 0:
             while not resultqueue.empty():
                 res = resultqueue.get()
                 if hasretval and res[0]:
                     retval.update(res[1])
                 else:
                     yield res
             threads[0].join(0.05)
             finishedthreads = [_t for _t in threads if not _t.is_alive()]
             for t in finishedthreads:
                 if t.exception is not None:
                     raise t.exception
                 threads.remove(t)
     except (Exception, KeyboardInterrupt):  # re-raises
         trykillworkers()
         raise
     while not resultqueue.empty():
         res = resultqueue.get()
         if hasretval and res[0]:
             retval.update(res[1])
         else:
             yield res
     if hasretval:
         yield True, retval
 
 
 if pycompat.iswindows:
     _platformworker = _windowsworker
 else:
     _platformworker = _posixworker
     _exitstatus = _posixexitstatus
 
 
 def partition(lst, nslices):
     """partition a list into N slices of roughly equal size
 
     The current strategy takes every Nth element from the input. If
     we ever write workers that need to preserve grouping in input
     we should consider allowing callers to specify a partition strategy.
 
     olivia is not a fan of this partitioning strategy when files are involved.
     In his words:
 
         Single-threaded Mercurial makes a point of creating and visiting
         files in a fixed order (alphabetical). When creating files in order,
         a typical filesystem is likely to allocate them on nearby regions on
         disk. Thus, when revisiting in the same order, locality is maximized
         and various forms of OS and disk-level caching and read-ahead get a
         chance to work.
 
         This effect can be quite significant on spinning disks. I discovered it
         circa Mercurial v0.4 when revlogs were named by hashes of filenames.
         Tarring a repo and copying it to another disk effectively randomized
         the revlog ordering on disk by sorting the revlogs by hash and suddenly
         performance of my kernel checkout benchmark dropped by ~10x because the
         "working set" of sectors visited no longer fit in the drive's cache and
         the workload switched from streaming to random I/O.
 
         What we should really be doing is have workers read filenames from a
         ordered queue. This preserves locality and also keeps any worker from
         getting more than one file out of balance.
     """
     for i in range(nslices):
         yield lst[i::nslices]

tests/test-status.t

@@ -1,973 +1,973 b''
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
   $ cat >> $HGRCPATH << EOF
   > [format]
   > use-dirstate-v2=1
   > [storage]
   > dirstate-v2.slow-path=allow
   > EOF
 #endif
 
   $ hg init repo1
   $ cd repo1
   $ mkdir a b a/1 b/1 b/2
   $ touch in_root a/in_a b/in_b a/1/in_a_1 b/1/in_b_1 b/2/in_b_2
 
 hg status in repo root:
 
   $ hg status
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
 
 hg status . in repo root:
 
   $ hg status .
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
 
   $ hg status --cwd a
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd a .
   ? 1/in_a_1
   ? in_a
   $ hg status --cwd a ..
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
 
   $ hg status --cwd b
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd b .
   ? 1/in_b_1
   ? 2/in_b_2
   ? in_b
   $ hg status --cwd b ..
   ? ../a/1/in_a_1
   ? ../a/in_a
   ? 1/in_b_1
   ? 2/in_b_2
   ? in_b
   ? ../in_root
 
   $ hg status --cwd a/1
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd a/1 .
   ? in_a_1
   $ hg status --cwd a/1 ..
   ? in_a_1
   ? ../in_a
 
   $ hg status --cwd b/1
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd b/1 .
   ? in_b_1
   $ hg status --cwd b/1 ..
   ? in_b_1
   ? ../2/in_b_2
   ? ../in_b
 
   $ hg status --cwd b/2
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
   $ hg status --cwd b/2 .
   ? in_b_2
   $ hg status --cwd b/2 ..
   ? ../1/in_b_1
   ? in_b_2
   ? ../in_b
 
 combining patterns with root and patterns without a root works
 
   $ hg st a/in_a re:.*b$
   ? a/in_a
   ? b/in_b
 
 tweaking defaults works
   $ hg status --cwd a --config ui.tweakdefaults=yes
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
   $ HGPLAIN=1 hg status --cwd a --config ui.tweakdefaults=yes
   ? a/1/in_a_1 (glob)
   ? a/in_a (glob)
   ? b/1/in_b_1 (glob)
   ? b/2/in_b_2 (glob)
   ? b/in_b (glob)
   ? in_root
   $ HGPLAINEXCEPT=tweakdefaults hg status --cwd a --config ui.tweakdefaults=yes
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root (glob)
 
 relative paths can be requested
 
   $ hg status --cwd a --config ui.relative-paths=yes
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
 
   $ hg status --cwd a . --config ui.relative-paths=legacy
   ? 1/in_a_1
   ? in_a
   $ hg status --cwd a . --config ui.relative-paths=no
   ? a/1/in_a_1
   ? a/in_a
 
 commands.status.relative overrides ui.relative-paths
 
   $ cat >> $HGRCPATH <<EOF
   > [ui]
   > relative-paths = False
   > [commands]
   > status.relative = True
   > EOF
   $ hg status --cwd a
   ? 1/in_a_1
   ? in_a
   ? ../b/1/in_b_1
   ? ../b/2/in_b_2
   ? ../b/in_b
   ? ../in_root
   $ HGPLAIN=1 hg status --cwd a
   ? a/1/in_a_1 (glob)
   ? a/in_a (glob)
   ? b/1/in_b_1 (glob)
   ? b/2/in_b_2 (glob)
   ? b/in_b (glob)
   ? in_root
 
 if relative paths are explicitly off, tweakdefaults doesn't change it
   $ cat >> $HGRCPATH <<EOF
   > [commands]
   > status.relative = False
   > EOF
   $ hg status --cwd a --config ui.tweakdefaults=yes
   ? a/1/in_a_1
   ? a/in_a
   ? b/1/in_b_1
   ? b/2/in_b_2
   ? b/in_b
   ? in_root
 
   $ cd ..
 
   $ hg init repo2
   $ cd repo2
   $ touch modified removed deleted ignored
   $ echo "^ignored$" > .hgignore
   $ hg ci -A -m 'initial checkin'
   adding .hgignore
   adding deleted
   adding modified
   adding removed
   $ touch modified added unknown ignored
   $ hg add added
   $ hg remove removed
   $ rm deleted
 
 hg status:
 
   $ hg status
   A added
   R removed
   ! deleted
   ? unknown
 
 hg status -n:
   $ env RHG_ON_UNSUPPORTED=abort hg status -n
   added
   removed
   deleted
   unknown
 
 hg status modified added removed deleted unknown never-existed ignored:
 
   $ hg status modified added removed deleted unknown never-existed ignored
   never-existed: * (glob)
   A added
   R removed
   ! deleted
   ? unknown
 
   $ hg copy modified copied
 
 hg status -C:
 
   $ hg status -C
   A added
   A copied
     modified
   R removed
   ! deleted
   ? unknown
 
 hg status -A:
 
   $ hg status -A
   A added
   A copied
     modified
   R removed
   ! deleted
   ? unknown
   I ignored
   C .hgignore
   C modified
 
   $ hg status -A -T '{status} {path} {node|shortest}\n'
   A added ffff
   A copied ffff
   R removed ffff
   ! deleted ffff
   ? unknown ffff
   I ignored ffff
   C .hgignore ffff
   C modified ffff
 
   $ hg status -A -Tjson
   [
    {
     "itemtype": "file",
     "path": "added",
     "status": "A"
    },
    {
     "itemtype": "file",
     "path": "copied",
     "source": "modified",
     "status": "A"
    },
    {
     "itemtype": "file",
     "path": "removed",
     "status": "R"
    },
    {
     "itemtype": "file",
     "path": "deleted",
     "status": "!"
    },
    {
     "itemtype": "file",
     "path": "unknown",
     "status": "?"
    },
    {
     "itemtype": "file",
     "path": "ignored",
     "status": "I"
    },
    {
     "itemtype": "file",
     "path": ".hgignore",
     "status": "C"
    },
    {
     "itemtype": "file",
     "path": "modified",
     "status": "C"
    }
   ]
 
   $ hg status -A -Tpickle > pickle
   >>> from __future__ import print_function
+  >>> import pickle
   >>> from mercurial import util
-  >>> pickle = util.pickle
   >>> data = sorted((x[b'status'].decode(), x[b'path'].decode()) for x in pickle.load(open("pickle", r"rb")))
   >>> for s, p in data: print("%s %s" % (s, p))
   ! deleted
   ? pickle
   ? unknown
   A added
   A copied
   C .hgignore
   C modified
   I ignored
   R removed
   $ rm pickle
 
   $ echo "^ignoreddir$" > .hgignore
   $ mkdir ignoreddir
   $ touch ignoreddir/file
 
 Test templater support:
 
   $ hg status -AT "[{status}]\t{if(source, '{source} -> ')}{path}\n"
   [M]	.hgignore
   [A]	added
   [A]	modified -> copied
   [R]	removed
   [!]	deleted
   [?]	ignored
   [?]	unknown
   [I]	ignoreddir/file
   [C]	modified
   $ hg status -AT default
   M .hgignore
   A added
   A copied
     modified
   R removed
   ! deleted
   ? ignored
   ? unknown
   I ignoreddir/file
   C modified
   $ hg status -T compact
   abort: "status" not in template map
   [255]
 
 hg status ignoreddir/file:
 
   $ hg status ignoreddir/file
 
 hg status -i ignoreddir/file:
 
   $ hg status -i ignoreddir/file
   I ignoreddir/file
   $ cd ..
 
 Check 'status -q' and some combinations
 
   $ hg init repo3
   $ cd repo3
   $ touch modified removed deleted ignored
   $ echo "^ignored$" > .hgignore
   $ hg commit -A -m 'initial checkin'
   adding .hgignore
   adding deleted
   adding modified
   adding removed
   $ touch added unknown ignored
   $ hg add added
   $ echo "test" >> modified
   $ hg remove removed
   $ rm deleted
   $ hg copy modified copied
 
 Specify working directory revision explicitly, that should be the same as
 "hg status"
 
   $ hg status --change "wdir()"
   M modified
   A added
   A copied
   R removed
   ! deleted
   ? unknown
 
 Run status with 2 different flags.
 Check if result is the same or different.
 If result is not as expected, raise error
 
   $ assert() {
   >     hg status $1 > ../a
   >     hg status $2 > ../b
   >     if diff ../a ../b > /dev/null; then
   >         out=0
   >     else
   >         out=1
   >     fi
   >     if [ $3 -eq 0 ]; then
   >         df="same"
   >     else
   >         df="different"
   >     fi
   >     if [ $out -ne $3 ]; then
   >         echo "Error on $1 and $2, should be $df."
   >     fi
   > }
 
 Assert flag1 flag2 [0-same | 1-different]
 
   $ assert "-q" "-mard"      0
   $ assert "-A" "-marduicC"  0
   $ assert "-qA" "-mardcC"   0
   $ assert "-qAui" "-A"      0
   $ assert "-qAu" "-marducC" 0
   $ assert "-qAi" "-mardicC" 0
   $ assert "-qu" "-u"        0
   $ assert "-q" "-u"         1
   $ assert "-m" "-a"         1
   $ assert "-r" "-d"         1
   $ cd ..
 
   $ hg init repo4
   $ cd repo4
   $ touch modified removed deleted
   $ hg ci -q -A -m 'initial checkin'
   $ touch added unknown
   $ hg add added
   $ hg remove removed
   $ rm deleted
   $ echo x > modified
   $ hg copy modified copied
   $ hg ci -m 'test checkin' -d "1000001 0"
   $ rm *
   $ touch unrelated
   $ hg ci -q -A -m 'unrelated checkin' -d "1000002 0"
 
 hg status --change 1:
 
   $ hg status --change 1
   M modified
   A added
   A copied
   R removed
 
 hg status --change 1 unrelated:
 
   $ hg status --change 1 unrelated
 
 hg status -C --change 1 added modified copied removed deleted:
 
   $ hg status -C --change 1 added modified copied removed deleted
   M modified
   A added
   A copied
     modified
   R removed
 
 hg status -A --change 1 and revset:
 
   $ hg status -A --change '1|1'
   M modified
   A added
   A copied
     modified
   R removed
   C deleted
 
   $ cd ..
 
 hg status with --rev and reverted changes:
 
   $ hg init reverted-changes-repo
   $ cd reverted-changes-repo
   $ echo a > file
   $ hg add file
   $ hg ci -m a
   $ echo b > file
   $ hg ci -m b
 
 reverted file should appear clean
 
   $ hg revert -r 0 .
   reverting file
   $ hg status -A --rev 0
   C file
 
 #if execbit
 reverted file with changed flag should appear modified
 
   $ chmod +x file
   $ hg status -A --rev 0
   M file
 
   $ hg revert -r 0 .
   reverting file
 
 reverted and committed file with changed flag should appear modified
 
   $ hg co -C .
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ chmod +x file
   $ hg ci -m 'change flag'
   $ hg status -A --rev 1 --rev 2
   M file
   $ hg diff -r 1 -r 2
 
 #endif
 
   $ cd ..
 
 hg status of binary file starting with '\1\n', a separator for metadata:
 
   $ hg init repo5
   $ cd repo5
   >>> open("010a", r"wb").write(b"\1\nfoo") and None
   $ hg ci -q -A -m 'initial checkin'
   $ hg status -A
   C 010a
 
   >>> open("010a", r"wb").write(b"\1\nbar") and None
   $ hg status -A
   M 010a
   $ hg ci -q -m 'modify 010a'
   $ hg status -A --rev 0:1
   M 010a
 
   $ touch empty
   $ hg ci -q -A -m 'add another file'
   $ hg status -A --rev 1:2 010a
   C 010a
 
   $ cd ..
 
 test "hg status" with "directory pattern" which matches against files
 only known on target revision.
 
   $ hg init repo6
   $ cd repo6
 
   $ echo a > a.txt
   $ hg add a.txt
   $ hg commit -m '#0'
   $ mkdir -p 1/2/3/4/5
   $ echo b > 1/2/3/4/5/b.txt
   $ hg add 1/2/3/4/5/b.txt
   $ hg commit -m '#1'
 
   $ hg update -C 0 > /dev/null
   $ hg status -A
   C a.txt
 
 the directory matching against specified pattern should be removed,
 because directory existence prevents 'dirstate.walk()' from showing
 warning message about such pattern.
 
   $ test ! -d 1
   $ hg status -A --rev 1 1/2/3/4/5/b.txt
   R 1/2/3/4/5/b.txt
   $ hg status -A --rev 1 1/2/3/4/5
   R 1/2/3/4/5/b.txt
   $ hg status -A --rev 1 1/2/3
   R 1/2/3/4/5/b.txt
   $ hg status -A --rev 1 1
   R 1/2/3/4/5/b.txt
 
   $ hg status --config ui.formatdebug=True --rev 1 1
   status = [
       {
           'itemtype': 'file',
           'path': '1/2/3/4/5/b.txt',
           'status': 'R'
       },
   ]
 
 #if windows
   $ hg --config ui.slash=false status -A --rev 1 1
   R 1\2\3\4\5\b.txt
 #endif
 
   $ cd ..
 
 Status after move overwriting a file (issue4458)
 =================================================
 
 
   $ hg init issue4458
   $ cd issue4458
   $ echo a > a
   $ echo b > b
   $ hg commit -Am base
   adding a
   adding b
 
 
 with --force
 
   $ hg mv b --force a
   $ hg st --copies
   M a
     b
   R b
   $ hg revert --all
   reverting a
   undeleting b
   $ rm *.orig
 
 without force
 
   $ hg rm a
   $ hg st --copies
   R a
   $ hg mv b a
   $ hg st --copies
   M a
     b
   R b
 
 using ui.statuscopies setting
   $ hg st --config ui.statuscopies=true
   M a
     b
   R b
   $ hg st --config ui.statuscopies=false
   M a
   R b
   $ hg st --config ui.tweakdefaults=yes
   M a
     b
   R b
 
 using log status template (issue5155)
   $ hg log -Tstatus -r 'wdir()' -C
   changeset:   2147483647:ffffffffffff
   parent:      0:8c55c58b4c0e
   user:        test
   date:        * (glob)
   files:
   M a
     b
   R b
   
   $ hg log -GTstatus -r 'wdir()' -C
   o  changeset:   2147483647:ffffffffffff
   |  parent:      0:8c55c58b4c0e
   ~  user:        test
      date:        * (glob)
      files:
      M a
        b
      R b
   
 
 Other "bug" highlight, the revision status does not report the copy information.
 This is buggy behavior.
 
   $ hg commit -m 'blah'
   $ hg st --copies --change .
   M a
   R b
 
 using log status template, the copy information is displayed correctly.
   $ hg log -Tstatus -r. -C
   changeset:   1:6685fde43d21
   tag:         tip
   user:        test
   date:        * (glob)
   summary:     blah
   files:
   M a
     b
   R b
   
 
   $ cd ..
 
 Make sure .hg doesn't show up even as a symlink
 
   $ hg init repo0
   $ mkdir symlink-repo0
   $ cd symlink-repo0
   $ ln -s ../repo0/.hg
   $ hg status
 
 If the size hasn’t changed but mtime has, status needs to read the contents
 of the file to check whether it has changed
 
   $ echo 1 > a
   $ echo 1 > b
   $ touch -t 200102030000 a b
   $ hg commit -Aqm '#0'
   $ echo 2 > a
   $ touch -t 200102040000 a b
   $ hg status
   M a
 
 Asking specifically for the status of a deleted/removed file
 
   $ rm a
   $ rm b
   $ hg status a
   ! a
   $ hg rm a
   $ hg rm b
   $ hg status a
   R a
   $ hg commit -qm '#1'
   $ hg status a
   a: $ENOENT$
 
 Check using include flag with pattern when status does not need to traverse
 the working directory (issue6483)
 
   $ cd ..
   $ hg init issue6483
   $ cd issue6483
   $ touch a.py b.rs
   $ hg add a.py b.rs
   $ hg st -aI "*.py"
   A a.py
 
 Also check exclude pattern
 
   $ hg st -aX "*.rs"
   A a.py
 
 issue6335
 When a directory containing a tracked file gets symlinked, as of 5.8
 `hg st` only gives the correct answer about clean (or deleted) files
 if also listing unknowns.
 The tree-based dirstate and status algorithm fix this:
 
 #if symlink no-dirstate-v1 rust
 
   $ cd ..
   $ hg init issue6335
   $ cd issue6335
   $ mkdir foo
   $ touch foo/a
   $ hg ci -Ama
   adding foo/a
   $ mv foo bar
   $ ln -s bar foo
   $ hg status
   ! foo/a
   ? bar/a
   ? foo
 
   $ hg status -c  # incorrect output without the Rust implementation
   $ hg status -cu
   ? bar/a
   ? foo
   $ hg status -d  # incorrect output without the Rust implementation
   ! foo/a
   $ hg status -du
   ! foo/a
   ? bar/a
   ? foo
 
 #endif
 
 
 Create a repo with files in each possible status
 
   $ cd ..
   $ hg init repo7
   $ cd repo7
   $ mkdir subdir
   $ touch clean modified deleted removed
   $ touch subdir/clean subdir/modified subdir/deleted subdir/removed
   $ echo ignored > .hgignore
   $ hg ci -Aqm '#0'
   $ echo 1 > modified
   $ echo 1 > subdir/modified
   $ rm deleted
   $ rm subdir/deleted
   $ hg rm removed
   $ hg rm subdir/removed
   $ touch unknown ignored
   $ touch subdir/unknown subdir/ignored
 
 Check the output
 
   $ hg status
   M modified
   M subdir/modified
   R removed
   R subdir/removed
   ! deleted
   ! subdir/deleted
   ? subdir/unknown
   ? unknown
 
   $ hg status -mard
   M modified
   M subdir/modified
   R removed
   R subdir/removed
   ! deleted
   ! subdir/deleted
 
   $ hg status -A
   M modified
   M subdir/modified
   R removed
   R subdir/removed
   ! deleted
   ! subdir/deleted
   ? subdir/unknown
   ? unknown
   I ignored
   I subdir/ignored
   C .hgignore
   C clean
   C subdir/clean
 
 Note: `hg status some-name` creates a patternmatcher which is not supported
 yet by the Rust implementation of status, but includematcher is supported.
 --include is used below for that reason
 
 #if unix-permissions
 
 Not having permission to read a directory that contains tracked files makes
 status emit a warning then behave as if the directory was empty or removed
 entirely:
 
   $ chmod 0 subdir
   $ hg status --include subdir
   subdir: Permission denied
   R subdir/removed
   ! subdir/clean
   ! subdir/deleted
   ! subdir/modified
   $ chmod 755 subdir
 
 #endif
 
 Remove a directory that contains tracked files
 
   $ rm -r subdir
   $ hg status --include subdir
   R subdir/removed
   ! subdir/clean
   ! subdir/deleted
   ! subdir/modified
 
 … and replace it by a file
 
   $ touch subdir
   $ hg status --include subdir
   R subdir/removed
   ! subdir/clean
   ! subdir/deleted
   ! subdir/modified
   ? subdir
 
 Replaced a deleted or removed file with a directory
 
   $ mkdir deleted removed
   $ touch deleted/1 removed/1
   $ hg status --include deleted --include removed
   R removed
   ! deleted
   ? deleted/1
   ? removed/1
   $ hg add removed/1
   $ hg status --include deleted --include removed
   A removed/1
   R removed
   ! deleted
   ? deleted/1
 
 Deeply nested files in an ignored directory are still listed on request
 
   $ echo ignored-dir >> .hgignore
   $ mkdir ignored-dir
   $ mkdir ignored-dir/subdir
   $ touch ignored-dir/subdir/1
   $ hg status --ignored
   I ignored
   I ignored-dir/subdir/1
 
 Check using include flag while listing ignored composes correctly (issue6514)
 
   $ cd ..
   $ hg init issue6514
   $ cd issue6514
   $ mkdir ignored-folder
   $ touch A.hs B.hs C.hs ignored-folder/other.txt ignored-folder/ctest.hs
   $ cat >.hgignore <<EOF
   > A.hs
   > B.hs
   > ignored-folder/
   > EOF
   $ hg st -i -I 're:.*\.hs$'
   I A.hs
   I B.hs
   I ignored-folder/ctest.hs
 
 #if rust dirstate-v2
 
 Check read_dir caching
 
   $ cd ..
   $ hg init repo8
   $ cd repo8
   $ mkdir subdir
   $ touch subdir/a subdir/b
   $ hg ci -Aqm '#0'
 
 The cached mtime is initially unset
 
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 unset               subdir
 
 It is still not set when there are unknown files
 
   $ touch subdir/unknown
   $ hg status
   ? subdir/unknown
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 unset               subdir
 
 Now the directory is eligible for caching, so its mtime is save in the dirstate
 
   $ rm subdir/unknown
   $ sleep 0.1 # ensure the kernel’s internal clock for mtimes has ticked
   $ hg status
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 set                 subdir
 
 This time the command should be ever so slightly faster since it does not need `read_dir("subdir")`
 
   $ hg status
 
 Creating a new file changes the directory’s mtime, invalidating the cache
 
   $ touch subdir/unknown
   $ hg status
   ? subdir/unknown
 
   $ rm subdir/unknown
   $ hg status
 
 Removing a node from the dirstate resets the cache for its parent directory
 
   $ hg forget subdir/a
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 set                 subdir
   $ hg ci -qm '#1'
   $ hg debugdirstate --all --no-dates | grep '^ '
       0         -1 unset               subdir
   $ hg status
   ? subdir/a
 
 #endif

tests/test-stdio.py

@@ -1,365 +1,366 b''
 #!/usr/bin/env python
 """
 Tests the buffering behavior of stdio streams in `mercurial.utils.procutil`.
 """
 from __future__ import absolute_import
 
 import contextlib
 import errno
 import os
+import pickle
 import signal
 import subprocess
 import sys
 import tempfile
 import unittest
 
 from mercurial import pycompat, util
 
 
 if pycompat.ispy3:
 
     def set_noninheritable(fd):
         # On Python 3, file descriptors are non-inheritable by default.
         pass
 
 
 else:
     if pycompat.iswindows:
         # unused
         set_noninheritable = None
     else:
         import fcntl
 
         def set_noninheritable(fd):
             old = fcntl.fcntl(fd, fcntl.F_GETFD)
             fcntl.fcntl(fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
 
 
 TEST_BUFFERING_CHILD_SCRIPT = r'''
 import os
 
 from mercurial import dispatch
 from mercurial.utils import procutil
 
 dispatch.initstdio()
 procutil.{stream}.write(b'aaa')
 os.write(procutil.{stream}.fileno(), b'[written aaa]')
 procutil.{stream}.write(b'bbb\n')
 os.write(procutil.{stream}.fileno(), b'[written bbb\\n]')
 '''
 UNBUFFERED = b'aaa[written aaa]bbb\n[written bbb\\n]'
 LINE_BUFFERED = b'[written aaa]aaabbb\n[written bbb\\n]'
 FULLY_BUFFERED = b'[written aaa][written bbb\\n]aaabbb\n'
 
 
 TEST_LARGE_WRITE_CHILD_SCRIPT = r'''
 import os
 import signal
 import sys
 
 from mercurial import dispatch
 from mercurial.utils import procutil
 
 signal.signal(signal.SIGINT, lambda *x: None)
 dispatch.initstdio()
 write_result = procutil.{stream}.write(b'x' * 1048576)
 with os.fdopen(
     os.open({write_result_fn!r}, os.O_WRONLY | getattr(os, 'O_TEMPORARY', 0)),
     'w',
 ) as write_result_f:
     write_result_f.write(str(write_result))
 '''
 
 
 TEST_BROKEN_PIPE_CHILD_SCRIPT = r'''
 import os
 import pickle
 
 from mercurial import dispatch
 from mercurial.utils import procutil
 
 dispatch.initstdio()
 procutil.stdin.read(1)  # wait until parent process closed pipe
 try:
     procutil.{stream}.write(b'test')
     procutil.{stream}.flush()
 except EnvironmentError as e:
     with os.fdopen(
         os.open(
             {err_fn!r},
             os.O_WRONLY
             | getattr(os, 'O_BINARY', 0)
             | getattr(os, 'O_TEMPORARY', 0),
         ),
         'wb',
     ) as err_f:
         pickle.dump(e, err_f)
 # Exit early to suppress further broken pipe errors at interpreter shutdown.
 os._exit(0)
 '''
 
 
 @contextlib.contextmanager
 def _closing(fds):
     try:
         yield
     finally:
         for fd in fds:
             try:
                 os.close(fd)
             except EnvironmentError:
                 pass
 
 
 # In the following, we set the FDs non-inheritable mainly to make it possible
 # for tests to close the receiving end of the pipe / PTYs.
 
 
 @contextlib.contextmanager
 def _devnull():
     devnull = os.open(os.devnull, os.O_WRONLY)
     # We don't have a receiving end, so it's not worth the effort on Python 2
     # on Windows to make the FD non-inheritable.
     with _closing([devnull]):
         yield (None, devnull)
 
 
 @contextlib.contextmanager
 def _pipes():
     rwpair = os.pipe()
     # Pipes are already non-inheritable on Windows.
     if not pycompat.iswindows:
         set_noninheritable(rwpair[0])
         set_noninheritable(rwpair[1])
     with _closing(rwpair):
         yield rwpair
 
 
 @contextlib.contextmanager
 def _ptys():
     if pycompat.iswindows:
         raise unittest.SkipTest("PTYs are not supported on Windows")
     import pty
     import tty
 
     rwpair = pty.openpty()
     set_noninheritable(rwpair[0])
     set_noninheritable(rwpair[1])
     with _closing(rwpair):
         tty.setraw(rwpair[0])
         yield rwpair
 
 
 def _readall(fd, buffer_size, initial_buf=None):
     buf = initial_buf or []
     while True:
         try:
             s = os.read(fd, buffer_size)
         except OSError as e:
             if e.errno == errno.EIO:
                 # If the child-facing PTY got closed, reading from the
                 # parent-facing PTY raises EIO.
                 break
             raise
         if not s:
             break
         buf.append(s)
     return b''.join(buf)
 
 
 class TestStdio(unittest.TestCase):
     def _test(
         self,
         child_script,
         stream,
         rwpair_generator,
         check_output,
         python_args=[],
         post_child_check=None,
         stdin_generator=None,
     ):
         assert stream in ('stdout', 'stderr')
         if stdin_generator is None:
             stdin_generator = open(os.devnull, 'rb')
         with rwpair_generator() as (
             stream_receiver,
             child_stream,
         ), stdin_generator as child_stdin:
             proc = subprocess.Popen(
                 [sys.executable] + python_args + ['-c', child_script],
                 stdin=child_stdin,
                 stdout=child_stream if stream == 'stdout' else None,
                 stderr=child_stream if stream == 'stderr' else None,
             )
             try:
                 os.close(child_stream)
                 if stream_receiver is not None:
                     check_output(stream_receiver, proc)
             except:  # re-raises
                 proc.terminate()
                 raise
             finally:
                 retcode = proc.wait()
             self.assertEqual(retcode, 0)
             if post_child_check is not None:
                 post_child_check()
 
     def _test_buffering(
         self, stream, rwpair_generator, expected_output, python_args=[]
     ):
         def check_output(stream_receiver, proc):
             self.assertEqual(_readall(stream_receiver, 1024), expected_output)
 
         self._test(
             TEST_BUFFERING_CHILD_SCRIPT.format(stream=stream),
             stream,
             rwpair_generator,
             check_output,
             python_args,
         )
 
     def test_buffering_stdout_devnull(self):
         self._test_buffering('stdout', _devnull, None)
 
     def test_buffering_stdout_pipes(self):
         self._test_buffering('stdout', _pipes, FULLY_BUFFERED)
 
     def test_buffering_stdout_ptys(self):
         self._test_buffering('stdout', _ptys, LINE_BUFFERED)
 
     def test_buffering_stdout_devnull_unbuffered(self):
         self._test_buffering('stdout', _devnull, None, python_args=['-u'])
 
     def test_buffering_stdout_pipes_unbuffered(self):
         self._test_buffering('stdout', _pipes, UNBUFFERED, python_args=['-u'])
 
     def test_buffering_stdout_ptys_unbuffered(self):
         self._test_buffering('stdout', _ptys, UNBUFFERED, python_args=['-u'])
 
     if not pycompat.ispy3 and not pycompat.iswindows:
         # On Python 2 on non-Windows, we manually open stdout in line-buffered
         # mode if connected to a TTY. We should check if Python was configured
         # to use unbuffered stdout, but it's hard to do that.
         test_buffering_stdout_ptys_unbuffered = unittest.expectedFailure(
             test_buffering_stdout_ptys_unbuffered
         )
 
     def _test_large_write(self, stream, rwpair_generator, python_args=[]):
         if not pycompat.ispy3 and pycompat.isdarwin:
             # Python 2 doesn't always retry on EINTR, but the libc might retry.
             # So far, it was observed only on macOS that EINTR is raised at the
             # Python level. As Python 2 support will be dropped soon-ish, we
             # won't attempt to fix it.
             raise unittest.SkipTest("raises EINTR on macOS")
 
         def check_output(stream_receiver, proc):
             if not pycompat.iswindows:
                 # On Unix, we can provoke a partial write() by interrupting it
                 # by a signal handler as soon as a bit of data was written.
                 # We test that write() is called until all data is written.
                 buf = [os.read(stream_receiver, 1)]
                 proc.send_signal(signal.SIGINT)
             else:
                 # On Windows, there doesn't seem to be a way to cause partial
                 # writes.
                 buf = []
             self.assertEqual(
                 _readall(stream_receiver, 131072, buf), b'x' * 1048576
             )
 
         def post_child_check():
             write_result_str = write_result_f.read()
             if pycompat.ispy3:
                 # On Python 3, we test that the correct number of bytes is
                 # claimed to have been written.
                 expected_write_result_str = '1048576'
             else:
                 # On Python 2, we only check that the large write does not
                 # crash.
                 expected_write_result_str = 'None'
             self.assertEqual(write_result_str, expected_write_result_str)
 
         with tempfile.NamedTemporaryFile('r') as write_result_f:
             self._test(
                 TEST_LARGE_WRITE_CHILD_SCRIPT.format(
                     stream=stream, write_result_fn=write_result_f.name
                 ),
                 stream,
                 rwpair_generator,
                 check_output,
                 python_args,
                 post_child_check=post_child_check,
             )
 
     def test_large_write_stdout_devnull(self):
         self._test_large_write('stdout', _devnull)
 
     def test_large_write_stdout_pipes(self):
         self._test_large_write('stdout', _pipes)
 
     def test_large_write_stdout_ptys(self):
         self._test_large_write('stdout', _ptys)
 
     def test_large_write_stdout_devnull_unbuffered(self):
         self._test_large_write('stdout', _devnull, python_args=['-u'])
 
     def test_large_write_stdout_pipes_unbuffered(self):
         self._test_large_write('stdout', _pipes, python_args=['-u'])
 
     def test_large_write_stdout_ptys_unbuffered(self):
         self._test_large_write('stdout', _ptys, python_args=['-u'])
 
     def test_large_write_stderr_devnull(self):
         self._test_large_write('stderr', _devnull)
 
     def test_large_write_stderr_pipes(self):
         self._test_large_write('stderr', _pipes)
 
     def test_large_write_stderr_ptys(self):
         self._test_large_write('stderr', _ptys)
 
     def test_large_write_stderr_devnull_unbuffered(self):
         self._test_large_write('stderr', _devnull, python_args=['-u'])
 
     def test_large_write_stderr_pipes_unbuffered(self):
         self._test_large_write('stderr', _pipes, python_args=['-u'])
 
     def test_large_write_stderr_ptys_unbuffered(self):
         self._test_large_write('stderr', _ptys, python_args=['-u'])
 
     def _test_broken_pipe(self, stream):
         assert stream in ('stdout', 'stderr')
 
         def check_output(stream_receiver, proc):
             os.close(stream_receiver)
             proc.stdin.write(b'x')
             proc.stdin.close()
 
         def post_child_check():
-            err = util.pickle.load(err_f)
+            err = pickle.load(err_f)
             self.assertEqual(err.errno, errno.EPIPE)
             self.assertEqual(err.strerror, "Broken pipe")
 
         with tempfile.NamedTemporaryFile('rb') as err_f:
             self._test(
                 TEST_BROKEN_PIPE_CHILD_SCRIPT.format(
                     stream=stream, err_fn=err_f.name
                 ),
                 stream,
                 _pipes,
                 check_output,
                 post_child_check=post_child_check,
                 stdin_generator=util.nullcontextmanager(subprocess.PIPE),
             )
 
     def test_broken_pipe_stdout(self):
         self._test_broken_pipe('stdout')
 
     def test_broken_pipe_stderr(self):
         self._test_broken_pipe('stderr')
 
 
 if __name__ == '__main__':
     import silenttestrunner
 
     silenttestrunner.main(__name__)