py2: remove simple from __future__ statements

These were needed for Python 2 support. Now that our linter no longer
mandates these, we can start deleting them.

Differential Revision: https://phab.mercurial-scm.org/D12254

contrib/bdiff-torture.py

@@ -1,108 +1,107 b''
 # Randomized torture test generation for bdiff
 
-from __future__ import absolute_import, print_function
 import random
 import sys
 
 from mercurial import (
     mdiff,
     pycompat,
 )
 
 
 def reducetest(a, b):
     tries = 0
     reductions = 0
     print("reducing...")
     while tries < 1000:
         a2 = (
             "\n".join(l for l in a.splitlines() if random.randint(0, 100) > 0)
             + "\n"
         )
         b2 = (
             "\n".join(l for l in b.splitlines() if random.randint(0, 100) > 0)
             + "\n"
         )
         if a2 == a and b2 == b:
             continue
         if a2 == b2:
             continue
         tries += 1
 
         try:
             test1(a, b)
         except Exception:
             reductions += 1
             tries = 0
             a = a2
             b = b2
 
     print("reduced:", reductions, len(a) + len(b), repr(a), repr(b))
     try:
         test1(a, b)
     except Exception as inst:
         print("failed:", inst)
 
     sys.exit(0)
 
 
 def test1(a, b):
     d = mdiff.textdiff(a, b)
     if not d:
         raise ValueError("empty")
     c = mdiff.patches(a, [d])
     if c != b:
         raise ValueError("bad")
 
 
 def testwrap(a, b):
     try:
         test1(a, b)
         return
     except Exception as inst:
         print("exception:", inst)
     reducetest(a, b)
 
 
 def test(a, b):
     testwrap(a, b)
     testwrap(b, a)
 
 
 def rndtest(size, noise):
     a = []
     src = "                aaaaaaaabbbbccd"
     for x in pycompat.xrange(size):
         a.append(src[random.randint(0, len(src) - 1)])
 
     while True:
         b = [c for c in a if random.randint(0, 99) > noise]
         b2 = []
         for c in b:
             b2.append(c)
             while random.randint(0, 99) < noise:
                 b2.append(src[random.randint(0, len(src) - 1)])
         if b2 != a:
             break
 
     a = "\n".join(a) + "\n"
     b = "\n".join(b2) + "\n"
 
     test(a, b)
 
 
 maxvol = 10000
 startsize = 2
 while True:
     size = startsize
     count = 0
     while size < maxvol:
         print(size)
         volume = 0
         while volume < maxvol:
             rndtest(size, 2)
             volume += size
             count += 2
         size *= 2
     maxvol *= 4
     startsize *= 4

contrib/benchmarks/__init__.py

@@ -1,126 +1,125 b''
 # __init__.py - asv benchmark suite
 #
 # Copyright 2016 Logilab SA <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # "historical portability" policy of contrib/benchmarks:
 #
 # We have to make this code work correctly with current mercurial stable branch
 # and if possible with reasonable cost with early Mercurial versions.
 
 '''ASV (https://asv.readthedocs.io) benchmark suite
 
 Benchmark are parameterized against reference repositories found in the
 directory pointed by the REPOS_DIR environment variable.
 
 Invocation example:
 
     $ export REPOS_DIR=~/hgperf/repos
     # run suite on given revision
     $ asv --config contrib/asv.conf.json run REV
     # run suite on new changesets found in stable and default branch
     $ asv --config contrib/asv.conf.json run NEW
     # display a comparative result table of benchmark results between two given
     # revisions
     $ asv --config contrib/asv.conf.json compare REV1 REV2
     # compute regression detection and generate ASV static website
     $ asv --config contrib/asv.conf.json publish
     # serve the static website
     $ asv --config contrib/asv.conf.json preview
 '''
 
-from __future__ import absolute_import
 
 import functools
 import os
 import re
 
 from mercurial import (
     extensions,
     hg,
     ui as uimod,
     util,
 )
 
 basedir = os.path.abspath(
     os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
 )
 reposdir = os.environ['REPOS_DIR']
 reposnames = [
     name
     for name in os.listdir(reposdir)
     if os.path.isdir(os.path.join(reposdir, name, ".hg"))
 ]
 if not reposnames:
     raise ValueError("No repositories found in $REPO_DIR")
 outputre = re.compile(
     (
         r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
         r'\d+.\d+ \(best of \d+\)'
     )
 )
 
 
 def runperfcommand(reponame, command, *args, **kwargs):
     os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "")
     # for "historical portability"
     # ui.load() has been available since d83ca85
     if util.safehasattr(uimod.ui, "load"):
         ui = uimod.ui.load()
     else:
         ui = uimod.ui()
     repo = hg.repository(ui, os.path.join(reposdir, reponame))
     perfext = extensions.load(
         ui, 'perfext', os.path.join(basedir, 'contrib', 'perf.py')
     )
     cmd = getattr(perfext, command)
     ui.pushbuffer()
     cmd(ui, repo, *args, **kwargs)
     output = ui.popbuffer()
     match = outputre.search(output)
     if not match:
         raise ValueError("Invalid output {}".format(output))
     return float(match.group(1))
 
 
 def perfbench(repos=reposnames, name=None, params=None):
     """decorator to declare ASV benchmark based on contrib/perf.py extension
 
     An ASV benchmark is a python function with the given attributes:
 
     __name__: should start with track_, time_ or mem_ to be collected by ASV
     params and param_name: parameter matrix to display multiple graphs on the
     same page.
     pretty_name: If defined it's displayed in web-ui instead of __name__
     (useful for revsets)
     the module name is prepended to the benchmark name and displayed as
     "category" in webui.
 
     Benchmarks are automatically parameterized with repositories found in the
     REPOS_DIR environment variable.
 
     `params` is the param matrix in the form of a list of tuple
     (param_name, [value0, value1])
 
     For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
     (a, c), (a, d), (b, c) and (b, d).
     """
     params = list(params or [])
     params.insert(0, ("repo", repos))
 
     def decorator(func):
         @functools.wraps(func)
         def wrapped(repo, *args):
             def perf(command, *a, **kw):
                 return runperfcommand(repo, command, *a, **kw)
 
             return func(perf, *args)
 
         wrapped.params = [p[1] for p in params]
         wrapped.param_names = [p[0] for p in params]
         wrapped.pretty_name = name
         return wrapped
 
     return decorator

contrib/benchmarks/perf.py

@@ -1,30 +1,29 b''
 # perf.py - asv benchmarks using contrib/perf.py extension
 #
 # Copyright 2016 Logilab SA <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import
 
 from . import perfbench
 
 
 @perfbench()
 def track_tags(perf):
     return perf("perftags")
 
 
 @perfbench()
 def track_status(perf):
     return perf("perfstatus", unknown=False)
 
 
 @perfbench(params=[('rev', ['1000', '10000', 'tip'])])
 def track_manifest(perf, rev):
     return perf("perfmanifest", rev)
 
 
 @perfbench()
 def track_heads(perf):
     return perf("perfheads")

contrib/benchmarks/revset.py

@@ -1,57 +1,56 b''
 # revset.py - asv revset benchmarks
 #
 # Copyright 2016 Logilab SA <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''ASV revset benchmarks generated from contrib/base-revsets.txt
 
 Each revset benchmark is parameterized with variants (first, last, sort, ...)
 '''
 
-from __future__ import absolute_import
 
 import os
 import string
 import sys
 
 from . import basedir, perfbench
 
 
 def createrevsetbenchmark(baseset, variants=None):
     if variants is None:
         # Default variants
         variants = ["plain", "first", "last", "sort", "sort+first", "sort+last"]
     fname = "track_" + "_".join(
         "".join(
             [c if c in string.digits + string.letters else " " for c in baseset]
         ).split()
     )
 
     def wrap(fname, baseset):
         @perfbench(name=baseset, params=[("variant", variants)])
         def f(perf, variant):
             revset = baseset
             if variant != "plain":
                 for var in variant.split("+"):
                     revset = "%s(%s)" % (var, revset)
             return perf("perfrevset", revset)
 
         f.__name__ = fname
         return f
 
     return wrap(fname, baseset)
 
 
 def initializerevsetbenchmarks():
     mod = sys.modules[__name__]
     with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'), 'rb') as fh:
         for line in fh:
             baseset = line.strip()
             if baseset and not baseset.startswith('#'):
                 func = createrevsetbenchmark(baseset)
                 setattr(mod, func.__name__, func)
 
 
 initializerevsetbenchmarks()

contrib/byteify-strings.py

@@ -1,350 +1,349 b''
 #!/usr/bin/env python3
 #
 # byteify-strings.py - transform string literals to be Python 3 safe
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import, print_function
 
 import argparse
 import contextlib
 import errno
 import os
 import sys
 import tempfile
 import token
 import tokenize
 
 
 def adjusttokenpos(t, ofs):
     """Adjust start/end column of the given token"""
     return t._replace(
         start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs)
     )
 
 
 def replacetokens(tokens, opts):
     """Transform a stream of tokens from raw to Python 3.
 
     Returns a generator of possibly rewritten tokens.
 
     The input token list may be mutated as part of processing. However,
     its changes do not necessarily match the output token stream.
     """
     sysstrtokens = set()
 
     # The following utility functions access the tokens list and i index of
     # the for i, t enumerate(tokens) loop below
     def _isop(j, *o):
         """Assert that tokens[j] is an OP with one of the given values"""
         try:
             return tokens[j].type == token.OP and tokens[j].string in o
         except IndexError:
             return False
 
     def _findargnofcall(n):
         """Find arg n of a call expression (start at 0)
 
         Returns index of the first token of that argument, or None if
         there is not that many arguments.
 
         Assumes that token[i + 1] is '('.
 
         """
         nested = 0
         for j in range(i + 2, len(tokens)):
             if _isop(j, ')', ']', '}'):
                 # end of call, tuple, subscription or dict / set
                 nested -= 1
                 if nested < 0:
                     return None
             elif n == 0:
                 # this is the starting position of arg
                 return j
             elif _isop(j, '(', '[', '{'):
                 nested += 1
             elif _isop(j, ',') and nested == 0:
                 n -= 1
 
         return None
 
     def _ensuresysstr(j):
         """Make sure the token at j is a system string
 
         Remember the given token so the string transformer won't add
         the byte prefix.
 
         Ignores tokens that are not strings. Assumes bounds checking has
         already been done.
 
         """
         k = j
         currtoken = tokens[k]
         while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL):
             k += 1
             if currtoken.type == token.STRING and currtoken.string.startswith(
                 ("'", '"')
             ):
                 sysstrtokens.add(currtoken)
             try:
                 currtoken = tokens[k]
             except IndexError:
                 break
 
     def _isitemaccess(j):
         """Assert the next tokens form an item access on `tokens[j]` and that
         `tokens[j]` is a name.
         """
         try:
             return (
                 tokens[j].type == token.NAME
                 and _isop(j + 1, '[')
                 and tokens[j + 2].type == token.STRING
                 and _isop(j + 3, ']')
             )
         except IndexError:
             return False
 
     def _ismethodcall(j, *methodnames):
         """Assert the next tokens form a call to `methodname` with a string
         as first argument on `tokens[j]` and that `tokens[j]` is a name.
         """
         try:
             return (
                 tokens[j].type == token.NAME
                 and _isop(j + 1, '.')
                 and tokens[j + 2].type == token.NAME
                 and tokens[j + 2].string in methodnames
                 and _isop(j + 3, '(')
                 and tokens[j + 4].type == token.STRING
             )
         except IndexError:
             return False
 
     coldelta = 0  # column increment for new opening parens
     coloffset = -1  # column offset for the current line (-1: TBD)
     parens = [(0, 0, 0, -1)]  # stack of (line, end-column, column-offset, type)
     ignorenextline = False  # don't transform the next line
     insideignoreblock = False  # don't transform until turned off
     for i, t in enumerate(tokens):
         # Compute the column offset for the current line, such that
         # the current line will be aligned to the last opening paren
         # as before.
         if coloffset < 0:
             lastparen = parens[-1]
             if t.start[1] == lastparen[1]:
                 coloffset = lastparen[2]
             elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in (
                 token.NEWLINE,
                 tokenize.NL,
             ):
                 # fix misaligned indent of s/util.Abort/error.Abort/
                 coloffset = lastparen[2] + (lastparen[1] - t.start[1])
             else:
                 coloffset = 0
 
         # Reset per-line attributes at EOL.
         if t.type in (token.NEWLINE, tokenize.NL):
             yield adjusttokenpos(t, coloffset)
             coldelta = 0
             coloffset = -1
             if not insideignoreblock:
                 ignorenextline = (
                     tokens[i - 1].type == token.COMMENT
                     and tokens[i - 1].string == "# no-py3-transform"
                 )
             continue
 
         if t.type == token.COMMENT:
             if t.string == "# py3-transform: off":
                 insideignoreblock = True
             if t.string == "# py3-transform: on":
                 insideignoreblock = False
 
         if ignorenextline or insideignoreblock:
             yield adjusttokenpos(t, coloffset)
             continue
 
         # Remember the last paren position.
         if _isop(i, '(', '[', '{'):
             parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type))
         elif _isop(i, ')', ']', '}'):
             parens.pop()
 
         # Convert most string literals to byte literals. String literals
         # in Python 2 are bytes. String literals in Python 3 are unicode.
         # Most strings in Mercurial are bytes and unicode strings are rare.
         # Rather than rewrite all string literals to use ``b''`` to indicate
         # byte strings, we apply this token transformer to insert the ``b``
         # prefix nearly everywhere.
         if t.type == token.STRING and t not in sysstrtokens:
             s = t.string
 
             # Preserve docstrings as string literals. This is inconsistent
             # with regular unprefixed strings. However, the
             # "from __future__" parsing (which allows a module docstring to
             # exist before it) doesn't properly handle the docstring if it
             # is b''' prefixed, leading to a SyntaxError. We leave all
             # docstrings as unprefixed to avoid this. This means Mercurial
             # components touching docstrings need to handle unicode,
             # unfortunately.
             if s[0:3] in ("'''", '"""'):
                 # If it's assigned to something, it's not a docstring
                 if not _isop(i - 1, '='):
                     yield adjusttokenpos(t, coloffset)
                     continue
 
             # If the first character isn't a quote, it is likely a string
             # prefixing character (such as 'b', 'u', or 'r'. Ignore.
             if s[0] not in ("'", '"'):
                 yield adjusttokenpos(t, coloffset)
                 continue
 
             # String literal. Prefix to make a b'' string.
             yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset)
             coldelta += 1
             continue
 
         # This looks like a function call.
         if t.type == token.NAME and _isop(i + 1, '('):
             fn = t.string
 
             # *attr() builtins don't accept byte strings to 2nd argument.
             if (
                 fn
                 in (
                     'getattr',
                     'setattr',
                     'hasattr',
                     'safehasattr',
                     'wrapfunction',
                     'wrapclass',
                     'addattr',
                 )
                 and (opts['allow-attr-methods'] or not _isop(i - 1, '.'))
             ):
                 arg1idx = _findargnofcall(1)
                 if arg1idx is not None:
                     _ensuresysstr(arg1idx)
 
             # .encode() and .decode() on str/bytes/unicode don't accept
             # byte strings on Python 3.
             elif fn in ('encode', 'decode') and _isop(i - 1, '.'):
                 for argn in range(2):
                     argidx = _findargnofcall(argn)
                     if argidx is not None:
                         _ensuresysstr(argidx)
 
             # It changes iteritems/values to items/values as they are not
             # present in Python 3 world.
             elif opts['dictiter'] and fn in ('iteritems', 'itervalues'):
                 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset)
                 continue
 
         if t.type == token.NAME and t.string in opts['treat-as-kwargs']:
             if _isitemaccess(i):
                 _ensuresysstr(i + 2)
             if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'):
                 _ensuresysstr(i + 4)
 
         # Looks like "if __name__ == '__main__'".
         if (
             t.type == token.NAME
             and t.string == '__name__'
             and _isop(i + 1, '==')
         ):
             _ensuresysstr(i + 2)
 
         # Emit unmodified token.
         yield adjusttokenpos(t, coloffset)
 
 
 def process(fin, fout, opts):
     tokens = tokenize.tokenize(fin.readline)
     tokens = replacetokens(list(tokens), opts)
     fout.write(tokenize.untokenize(tokens))
 
 
 def tryunlink(fname):
     try:
         os.unlink(fname)
     except OSError as err:
         if err.errno != errno.ENOENT:
             raise
 
 
 @contextlib.contextmanager
 def editinplace(fname):
     n = os.path.basename(fname)
     d = os.path.dirname(fname)
     fp = tempfile.NamedTemporaryFile(
         prefix='.%s-' % n, suffix='~', dir=d, delete=False
     )
     try:
         yield fp
         fp.close()
         if os.name == 'nt':
             tryunlink(fname)
         os.rename(fp.name, fname)
     finally:
         fp.close()
         tryunlink(fp.name)
 
 
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument(
         '--version', action='version', version='Byteify strings 1.0'
     )
     ap.add_argument(
         '-i',
         '--inplace',
         action='store_true',
         default=False,
         help='edit files in place',
     )
     ap.add_argument(
         '--dictiter',
         action='store_true',
         default=False,
         help='rewrite iteritems() and itervalues()',
     ),
     ap.add_argument(
         '--allow-attr-methods',
         action='store_true',
         default=False,
         help='also handle attr*() when they are methods',
     ),
     ap.add_argument(
         '--treat-as-kwargs',
         nargs="+",
         default=[],
         help="ignore kwargs-like objects",
     ),
     ap.add_argument('files', metavar='FILE', nargs='+', help='source file')
     args = ap.parse_args()
     opts = {
         'dictiter': args.dictiter,
         'treat-as-kwargs': set(args.treat_as_kwargs),
         'allow-attr-methods': args.allow_attr_methods,
     }
     for fname in args.files:
         fname = os.path.realpath(fname)
         if args.inplace:
             with editinplace(fname) as fout:
                 with open(fname, 'rb') as fin:
                     process(fin, fout, opts)
         else:
             with open(fname, 'rb') as fin:
                 fout = sys.stdout.buffer
                 process(fin, fout, opts)
 
 
 if __name__ == '__main__':
     if sys.version_info[0:2] < (3, 7):
         print('This script must be run under Python 3.7+')
         sys.exit(3)
     main()

contrib/casesmash.py

@@ -1,41 +1,40 b''
-from __future__ import absolute_import
 import __builtin__
 import os
 from mercurial import util
 
 
 def lowerwrap(scope, funcname):
     f = getattr(scope, funcname)
 
     def wrap(fname, *args, **kwargs):
         d, base = os.path.split(fname)
         try:
             files = os.listdir(d or '.')
         except OSError:
             files = []
         if base in files:
             return f(fname, *args, **kwargs)
         for fn in files:
             if fn.lower() == base.lower():
                 return f(os.path.join(d, fn), *args, **kwargs)
         return f(fname, *args, **kwargs)
 
     scope.__dict__[funcname] = wrap
 
 
 def normcase(path):
     return path.lower()
 
 
 os.path.normcase = normcase
 
 for f in 'file open'.split():
     lowerwrap(__builtin__, f)
 
 for f in "chmod chown open lstat stat remove unlink".split():
     lowerwrap(os, f)
 
 for f in "exists lexists".split():
     lowerwrap(os.path, f)
 
 lowerwrap(util, 'posixfile')

contrib/catapipe.py

@@ -1,121 +1,120 b''
 #!/usr/bin/env python3
 #
 # Copyright 2018 Google LLC.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """Tool read primitive events from a pipe to produce a catapult trace.
 
 Usage:
     Terminal 1: $ catapipe.py /tmp/mypipe /tmp/trace.json
     Terminal 2: $ HGCATAPULTSERVERPIPE=/tmp/mypipe hg root
     <ctrl-c catapipe.py in Terminal 1>
     $ catapult/tracing/bin/trace2html /tmp/trace.json  # produce /tmp/trace.html
     <open trace.html in your browser of choice; the WASD keys are very useful>
     (catapult is located at https://github.com/catapult-project/catapult)
 
 For now the event stream supports
 
   START $SESSIONID ...
 
 and
 
   END $SESSIONID ...
 
 events. Everything after the SESSIONID (which must not contain spaces)
 is used as a label for the event. Events are timestamped as of when
 they arrive in this process and are then used to produce catapult
 traces that can be loaded in Chrome's about:tracing utility. It's
 important that the event stream *into* this process stay simple,
 because we have to emit it from the shell scripts produced by
 run-tests.py.
 
 Typically you'll want to place the path to the named pipe in the
 HGCATAPULTSERVERPIPE environment variable, which both run-tests and hg
 understand. To trace *only* run-tests, use HGTESTCATAPULTSERVERPIPE instead.
 """
-from __future__ import absolute_import, print_function
 
 import argparse
 import json
 import os
 import timeit
 
 _TYPEMAP = {
     'START': 'B',
     'END': 'E',
     'COUNTER': 'C',
 }
 
 _threadmap = {}
 
 # Timeit already contains the whole logic about which timer to use based on
 # Python version and OS
 timer = timeit.default_timer
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'pipe',
         type=str,
         nargs=1,
         help='Path of named pipe to create and listen on.',
     )
     parser.add_argument(
         'output',
         default='trace.json',
         type=str,
         nargs='?',
         help='Path of json file to create where the traces ' 'will be stored.',
     )
     parser.add_argument(
         '--debug',
         default=False,
         action='store_true',
         help='Print useful debug messages',
     )
     args = parser.parse_args()
     fn = args.pipe[0]
     os.mkfifo(fn)
     try:
         with open(fn) as f, open(args.output, 'w') as out:
             out.write('[\n')
             start = timer()
             while True:
                 ev = f.readline().strip()
                 if not ev:
                     continue
                 now = timer()
                 if args.debug:
                     print(ev)
                 verb, session, label = ev.split(' ', 2)
                 if session not in _threadmap:
                     _threadmap[session] = len(_threadmap)
                 if verb == 'COUNTER':
                     amount, label = label.split(' ', 1)
                     payload_args = {'value': int(amount)}
                 else:
                     payload_args = {}
                 pid = _threadmap[session]
                 ts_micros = (now - start) * 1000000
                 out.write(
                     json.dumps(
                         {
                             "name": label,
                             "cat": "misc",
                             "ph": _TYPEMAP[verb],
                             "ts": ts_micros,
                             "pid": pid,
                             "tid": 1,
                             "args": payload_args,
                         }
                     )
                 )
                 out.write(',\n')
     finally:
         os.unlink(fn)
 
 
 if __name__ == '__main__':
     main()

contrib/check-code.py

@@ -1,1124 +1,1123 b''
 #!/usr/bin/env python3
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """style and portability checker for Mercurial
 
 when a rule triggers wrong, do one of the following (prefer one from top):
  * do the work-around the rule suggests
  * doublecheck that it is a false match
  * improve the rule pattern
  * add an ignore pattern to the rule (3rd arg) which matches your good line
    (you can append a short comment and match this, like: #re-raises)
  * change the pattern to a warning and list the exception in test-check-code-hg
  * ONLY use no--check-code for skipping entire files from external sources
 """
 
-from __future__ import absolute_import, print_function
 import glob
 import keyword
 import optparse
 import os
 import re
 import sys
 
 if sys.version_info[0] < 3:
     opentext = open
 else:
 
     def opentext(f):
         return open(f, encoding='latin1')
 
 
 try:
     xrange
 except NameError:
     xrange = range
 try:
     import re2
 except ImportError:
     re2 = None
 
 import testparseutil
 
 
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
     if re2:
         try:
             return re2.compile(pat)
         except re2.error:
             pass
     return re.compile(pat)
 
 
 # check "rules depending on implementation of repquote()" in each
 # patterns (especially pypats), before changing around repquote()
 _repquotefixedmap = {
     ' ': ' ',
     '\n': '\n',
     '.': 'p',
     ':': 'q',
     '%': '%',
     '\\': 'b',
     '*': 'A',
     '+': 'P',
     '-': 'M',
 }
 
 
 def _repquoteencodechr(i):
     if i > 255:
         return 'u'
     c = chr(i)
     if c in _repquotefixedmap:
         return _repquotefixedmap[c]
     if c.isalpha():
         return 'x'
     if c.isdigit():
         return 'n'
     return 'o'
 
 
 _repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256))
 
 
 def repquote(m):
     t = m.group('text')
     t = t.translate(_repquotett)
     return m.group('quote') + t + m.group('quote')
 
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         l = len(comment.rstrip())
         return "#" * l + comment[l:]
     return repquote(m)
 
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
     [
         (r'\b(push|pop)d\b', "don't use 'pushd' or 'popd', use 'cd'"),
         (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
         (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
         (r'(?<!hg )grep.* -a', "don't use 'grep -a', use in-line python"),
         (r'sed.*-i', "don't use 'sed -i', use a temporary file"),
         (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"),
         (r'echo -n', "don't use 'echo -n', use printf"),
         (r'(^|\|\s*)\bwc\b[^|]*$\n(?!.*\(re\))', "filter wc output"),
         (r'head -c', "don't use 'head -c', use 'dd'"),
         (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
         (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
         (r'\bls\b.*-\w*R', "don't use 'ls -R', use 'find'"),
         (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"),
         (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
         (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
         (
             r'\[[^\]]+==',
             '[ foo == bar ] is a bashism, use [ foo = bar ] instead',
         ),
         (
             r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
             "use egrep for extended grep syntax",
         ),
         (r'(^|\|\s*)e?grep .*\\S', "don't use \\S in regular expression"),
         (r'(?<!!)/bin/', "don't use explicit paths for tools"),
         (r'#!.*/bash', "don't use bash in shebang, use sh"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'export .*=', "don't export and assign at once"),
         (r'^source\b', "don't use 'source', use '.'"),
         (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
         (r'\bls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
         (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
         (r'^stop\(\)', "don't use 'stop' as a shell function name"),
         (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
         (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"),
         (r'^alias\b.*=', "don't use alias, use a function"),
         (r'if\s*!', "don't use '!' to negate exit status"),
         (r'/dev/u?random', "don't use entropy, use /dev/zero"),
         (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"),
         (
             r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)',
             "put a backslash-escaped newline after sed 'i' command",
         ),
         (r'^diff *-\w*[uU].*$\n(^  \$ |^$)', "prefix diff -u/-U with cmp"),
         (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"),
         (r'[\s="`\']python\s(?!bindings)', "don't use 'python', use '$PYTHON'"),
         (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (r'\|&', "don't use |&, use 2>&1"),
         (r'\w =  +\w', "only one space after = allowed"),
         (
             r'\bsed\b.*[^\\]\\n',
             "don't use 'sed ... \\n', use a \\ and a newline",
         ),
         (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"),
         (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"),
         (r'grep.* -[ABC]', "don't use grep's context flags"),
         (
             r'find.*-printf',
             "don't use 'find -printf', it doesn't exist on BSD find(1)",
         ),
         (r'\$RANDOM ', "don't use bash-only $RANDOM to generate random values"),
     ],
     # warnings
     [
         (r'^function', "don't use 'function', use old style"),
         (r'^diff.*-\w*N', "don't use 'diff -N'"),
         (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`", "no-pwd-check"),
         (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"),
         (r'kill (`|\$\()', "don't use kill, use killdaemons.py"),
     ],
 ]
 
 testfilters = [
     (r"( *)(#([^!][^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 uprefix = r"^  \$ "
 utestpats = [
     [
         (r'^(\S.*||  [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"),
         (
             uprefix + r'.*\|\s*sed[^|>\n]*\n',
             "use regex test output patterns instead of sed",
         ),
         (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
         (
             uprefix + r'.*\|\| echo.*(fail|error)',
             "explicit exit code checks unnecessary",
         ),
         (uprefix + r'set -e', "don't use set -e"),
         (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"),
         (
             uprefix + r'.*:\.\S*/',
             "x:.y in a path does not work on msys, rewrite "
             "as x://.y, or see `hg log -k msys` for alternatives",
             r'-\S+:\.|' '# no-msys',  # -Rxxx
         ),  # in test-pull.t which is skipped on windows
         (
             r'^  [^$>].*27\.0\.0\.1',
             'use $LOCALIP not an explicit loopback address',
         ),
         (
             r'^  (?![>$] ).*\$LOCALIP.*[^)]$',
             'mark $LOCALIP output lines with (glob) to help tests in BSD jails',
         ),
         (
             r'^  (cat|find): .*: \$ENOENT\$',
             'use test -f to test for file existence',
         ),
         (
             r'^  diff -[^ -]*p',
             "don't use (external) diff with -p for portability",
         ),
         (r' readlink ', 'use readlink.py instead of readlink'),
         (
             r'^  [-+][-+][-+] .* [-+]0000 \(glob\)',
             "glob timezone field in diff output for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+,[0-9]+ @@',
             "use '@@ -N* +N,n @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+,[0-9]+ [+][0-9]+ @@',
             "use '@@ -N,n +N* @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+ @@',
             "use '@@ -N* +N* @@ (glob)' style chunk header for portability",
         ),
         (
             uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff'
             r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$',
             "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)",
         ),
     ],
     # warnings
     [
         (
             r'^  (?!.*\$LOCALIP)[^*?/\n]* \(glob\)$',
             "glob match with no glob string (?, *, /, and $LOCALIP)",
         ),
     ],
 ]
 
 # transform plain test rules to unified test's
 for i in [0, 1]:
     for tp in testpats[i]:
         p = tp[0]
         m = tp[1]
         if p.startswith('^'):
             p = "^  [$>] (%s)" % p[1:]
         else:
             p = "^  [$>] .*(%s)" % p
         utestpats[i].append((p, m) + tp[2:])
 
 # don't transform the following rules:
 # "  > \t" and "  \t" should be allowed in unified tests
 testpats[0].append((r'^( *)\t', "don't use tabs to indent"))
 utestpats[0].append((r'^( ?)\t', "don't use tabs to indent"))
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
     (r"( +)(#([^!][^\n]*\S)?)", repcomment),
 ]
 
 # common patterns to check *.py
 commonpypats = [
     [
         (r'\\$', 'Use () to wrap long lines in Python, not \\'),
         (
             r'^\s*def\s*\w+\s*\(.*,\s*\(',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (
             r'lambda\s*\(.*,.*\)',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
         (r'(?<!\.)\breduce\s*\(.*', "reduce is not available in Python 3+"),
         (
             r'\bdict\(.*=',
             'dict() is different in Py2 and 3 and is slower than {}',
             'dict-from-generator',
         ),
         (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
         (r'\s<>\s', '<> operator is not available in Python 3+, use !='),
         (r'^\s*\t', "don't use tabs"),
         (r'\S;\s*\n', "semicolon"),
         (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"),
         (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"),
         (r'(\w|\)),\w', "missing whitespace after ,"),
         (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (
             (
                 # a line ending with a colon, potentially with trailing comments
                 r':([ \t]*#[^\n]*)?\n'
                 # one that is not a pass and not only a comment
                 r'(?P<indent>[ \t]+)[^#][^\n]+\n'
                 # more lines at the same indent level
                 r'((?P=indent)[^\n]+\n)*'
                 # a pass at the same indent level, which is bogus
                 r'(?P=indent)pass[ \t\n#]'
             ),
             'omit superfluous pass',
         ),
         (r'[^\n]\Z', "no trailing newline"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (
             r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
             "linebreak after :",
         ),
         (
             r'class\s[^( \n]+:',
             "old-style class, use class foo(object)",
             r'#.*old-style',
         ),
         (
             r'class\s[^( \n]+\(\):',
             "class foo() creates old style object, use class foo(object)",
             r'#.*old-style',
         ),
         (
             r'\b(%s)\('
             % '|'.join(k for k in keyword.kwlist if k not in ('print', 'exec')),
             "Python keyword is not a function",
         ),
         #    (r'class\s[A-Z][^\(]*\((?!Exception)',
         #     "don't capitalize non-exception classes"),
         #    (r'in range\(', "use xrange"),
         #    (r'^\s*print\s+', "avoid using print in core and extensions"),
         (r'[\x80-\xff]', "non-ASCII character literal"),
         (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"),
         (
             r'([\(\[][ \t]\S)|(\S[ \t][\)\]])',
             "gratuitous whitespace in () or []",
         ),
         #    (r'\s\s=', "gratuitous whitespace before ="),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s',
             "missing whitespace around operator",
         ),
         (
             r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', "wrong whitespace around ="),
         (
             r'\([^()]*( =[^=]|[^<>!=]= )',
             "no whitespace around = for named parameters",
         ),
         (
             r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
             "don't use old-style two-argument raise, use Exception(message)",
         ),
         (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
         (
             r' [=!]=\s+(True|False|None)',
             "comparison with singleton, use 'is' or 'is not' instead",
         ),
         (
             r'^\s*(while|if) [01]:',
             "use True/False for constant Boolean expression",
         ),
         (r'^\s*if False(:| +and)', 'Remove code instead of using `if False`'),
         (
             r'(?:(?<!def)\s+|\()hasattr\(',
             'hasattr(foo, bar) is broken on py2, use util.safehasattr(foo, bar) '
             'instead',
             r'#.*hasattr-py3-only',
         ),
         (r'opener\([^)]*\).read\(', "use opener.read() instead"),
         (r'opener\([^)]*\).write\(', "use opener.write() instead"),
         (r'(?i)descend[e]nt', "the proper spelling is descendAnt"),
         (r'\.debug\(\_', "don't mark debug messages for translation"),
         (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
         (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
         (
             r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
             'legacy exception syntax; use "as" instead of ","',
         ),
         (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
         (r'\bdef\s+__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
         (
             r'os\.path\.join\(.*, *(""|\'\')\)',
             "use pathutil.normasprefix(path) instead of os.path.join(path, '')",
         ),
         (r'\s0[0-7]+\b', 'legacy octal syntax; use "0o" prefix instead of "0"'),
         # XXX only catch mutable arguments on the first line of the definition
         (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (
             r'^@(\w*\.)?cachefunc',
             "module-level @cachefunc is risky, please avoid",
         ),
         (
             r'^import Queue',
             "don't use Queue, use pycompat.queue.Queue + "
             "pycompat.queue.Empty",
         ),
         (
             r'^import cStringIO',
             "don't use cStringIO.StringIO, use util.stringio",
         ),
         (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"),
         (
             r'^import SocketServer',
             "don't use SockerServer, use util.socketserver",
         ),
         (r'^import urlparse', "don't use urlparse, use util.urlreq"),
         (r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"),
         (r'^import httplib', "don't use httplib, use util.httplib"),
         (r'^import BaseHTTPServer', "use util.httpserver instead"),
         (
             r'^(from|import) mercurial\.(cext|pure|cffi)',
             "use mercurial.policy.importmod instead",
         ),
         (r'\.next\(\)', "don't use .next(), use next(...)"),
         (
             r'([a-z]*).revision\(\1\.node\(',
             "don't convert rev to node before passing to revision(nodeorrev)",
         ),
         (r'platform\.system\(\)', "don't use platform.system(), use pycompat"),
     ],
     # warnings
     [],
 ]
 
 # patterns to check normal *.py files
 pypats = [
     [
         # Ideally, these should be placed in "commonpypats" for
         # consistency of coding rules in Mercurial source tree.
         # But on the other hand, these are not so seriously required for
         # python code fragments embedded in test scripts. Fixing test
         # scripts for these patterns requires many changes, and has less
         # profit than effort.
         (r'raise Exception', "don't raise generic exceptions"),
         (r'[\s\(](open|file)\([^)]*\)\.read\(', "use util.readfile() instead"),
         (
             r'[\s\(](open|file)\([^)]*\)\.write\(',
             "use util.writefile() instead",
         ),
         (
             r'^[\s\(]*(open(er)?|file)\([^)]*\)(?!\.close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (
             r'[\s\(](open|file)\([^)]*\)\.(?!close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
         (r'^import atexit', "don't use atexit, use ui.atexit"),
         # rules depending on implementation of repquote()
         (
             r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
             'string join across lines with no space',
         ),
         (
             r'''(?x)ui\.(status|progress|write|note|warn)\(
          [ \t\n#]*
          (?# any strings/comments might precede a string, which
            # contains translatable message)
          b?((['"]|\'\'\'|""")[ \npq%bAPMxno]*(['"]|\'\'\'|""")[ \t\n#]+)*
          (?# sequence consisting of below might precede translatable message
            # - formatting string: "% 10s", "%05d", "% -3.2f", "%*s", "%%" ...
            # - escaped character: "\\", "\n", "\0" ...
            # - character other than '%', 'b' as '\', and 'x' as alphabet)
          (['"]|\'\'\'|""")
          ((%([ n]?[PM]?([np]+|A))?x)|%%|b[bnx]|[ \nnpqAPMo])*x
          (?# this regexp can't use [^...] style,
            # because _preparepats forcibly adds "\n" into [^...],
            # even though this regexp wants match it against "\n")''',
             "missing _() in ui message (use () to hide false-positives)",
         ),
     ]
     + commonpypats[0],
     # warnings
     [
         # rules depending on implementation of repquote()
         (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
     ]
     + commonpypats[1],
 ]
 
 # patterns to check *.py for embedded ones in test script
 embeddedpypats = [
     [] + commonpypats[0],
     # warnings
     [] + commonpypats[1],
 ]
 
 # common filters to convert *.py
 commonpyfilters = [
     (
         r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""",
         reppython,
     ),
 ]
 
 # pattern only for mercurial and extensions
 core_py_pats = [
     [
         # Windows tend to get confused about capitalization of the drive letter
         #
         # see mercurial.windows.abspath for details
         (
             r'os\.path\.abspath',
             "use util.abspath instead (windows)",
             r'#.*re-exports',
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert normal *.py files
 pyfilters = [] + commonpyfilters
 
 # non-filter patterns
 pynfpats = [
     [
         (r'pycompat\.osname\s*[=!]=\s*[\'"]nt[\'"]', "use pycompat.iswindows"),
         (r'pycompat\.osname\s*[=!]=\s*[\'"]posix[\'"]', "use pycompat.isposix"),
         (
             r'pycompat\.sysplatform\s*[!=]=\s*[\'"]darwin[\'"]',
             "use pycompat.isdarwin",
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert *.py for embedded ones in test script
 embeddedpyfilters = [] + commonpyfilters
 
 # extension non-filter patterns
 pyextnfpats = [
     [(r'^"""\n?[A-Z]', "don't capitalize docstring title")],
     # warnings
     [],
 ]
 
 txtfilters = []
 
 txtpats = [
     [
         (r'\s$', 'trailing whitespace'),
         ('.. note::[ \n][^\n]', 'add two newlines after note::'),
     ],
     [],
 ]
 
 cpats = [
     [
         (r'//', "don't use //-style comments"),
         (r'\S\t', "don't use tabs except for indent"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (r'(while|if|do|for)\(', "use space after while/if/do/for"),
         (r'return\(', "return is not a function"),
         (r' ;', "no space before ;"),
         (r'[^;] \)', "no space before )"),
         (r'[)][{]', "space between ) and {"),
         (r'\w+\* \w+', "use int *foo, not int* foo"),
         (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
         (r'\w+ (\+\+|--)', "use foo++, not foo ++"),
         (r'\w,\w', "missing whitespace after ,"),
         (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (r'^#\s+\w', "use #foo, not # foo"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'^\s*#import\b', "use only #include in standard C code"),
         (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"),
         (r'strcat\(', "don't use strcat"),
         # rules depending on implementation of repquote()
     ],
     # warnings
     [
         # rules depending on implementation of repquote()
     ],
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
     [
         (r'\bui\.', "don't use ui in util"),
     ],
     # warnings
     [],
 ]
 
 inrevlogpats = [
     [
         (r'\brepo\.', "don't use repo in revlog"),
     ],
     # warnings
     [],
 ]
 
 webtemplatefilters = []
 
 webtemplatepats = [
     [],
     [
         (
             r'{desc(\|(?!websub|firstline)[^\|]*)+}',
             'follow desc keyword with either firstline or websub',
         ),
     ],
 ]
 
 allfilesfilters = []
 
 allfilespats = [
     [
         (
             r'(http|https)://[a-zA-Z0-9./]*selenic.com/',
             'use mercurial-scm.org domain URL',
         ),
         (
             r'mercurial@selenic\.com',
             'use mercurial-scm.org domain for mercurial ML address',
         ),
         (
             r'mercurial-devel@selenic\.com',
             'use mercurial-scm.org domain for mercurial-devel ML address',
         ),
     ],
     # warnings
     [],
 ]
 
 py3pats = [
     [
         (
             r'os\.environ',
             "use encoding.environ instead (py3)",
             r'#.*re-exports',
         ),
         (r'os\.name', "use pycompat.osname instead (py3)"),
         (r'os\.getcwd', "use encoding.getcwd instead (py3)", r'#.*re-exports'),
         (r'os\.sep', "use pycompat.ossep instead (py3)"),
         (r'os\.pathsep', "use pycompat.ospathsep instead (py3)"),
         (r'os\.altsep', "use pycompat.osaltsep instead (py3)"),
         (r'sys\.platform', "use pycompat.sysplatform instead (py3)"),
         (r'getopt\.getopt', "use pycompat.getoptb instead (py3)"),
         (r'os\.getenv', "use encoding.environ.get instead"),
         (r'os\.setenv', "modifying the environ dict is not preferred"),
         (r'(?<!pycompat\.)xrange', "use pycompat.xrange instead (py3)"),
     ],
     # warnings
     [],
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', pyfilters, pypats),
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', [], pynfpats),
     ('python', r'.*hgext.*\.py$', '', [], pyextnfpats),
     (
         'python 3',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         py3pats,
     ),
     (
         'core files',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         core_py_pats,
     ),
     ('test script', r'(.*/)?test-[^.~]*$', '', testfilters, testpats),
     ('c', r'.*\.[ch]$', '', cfilters, cpats),
     ('unified test', r'.*\.t$', '', utestfilters, utestpats),
     (
         'layering violation repo in revlog',
         r'mercurial/revlog\.py',
         '',
         pyfilters,
         inrevlogpats,
     ),
     (
         'layering violation ui in util',
         r'mercurial/util\.py',
         '',
         pyfilters,
         inutilpats,
     ),
     ('txt', r'.*\.txt$', '', txtfilters, txtpats),
     (
         'web template',
         r'mercurial/templates/.*\.tmpl',
         '',
         webtemplatefilters,
         webtemplatepats,
     ),
     ('all except for .po', r'.*(?<!\.po)$', '', allfilesfilters, allfilespats),
 ]
 
 # (desc,
 #  func to pick up embedded code fragments,
 #  list of patterns to convert target files
 #  list of patterns to detect errors/warnings)
 embeddedchecks = [
     (
         'embedded python',
         testparseutil.pyembedded,
         embeddedpyfilters,
         embeddedpypats,
     )
 ]
 
 
 def _preparepats():
     def preparefailandwarn(failandwarn):
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
                 p = pseq[0]
                 # \s doesn't match \n (done in two steps)
                 # first, we replace \s that appears in a set already
                 p = re.sub(r'\[\\s', r'[ \\t', p)
                 # now we replace other \s instances.
                 p = re.sub(r'(?<!(\\|\[))\\s', r'[ \\t]', p)
                 # [^...] doesn't match newline
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
 
     def preparefilters(filters):
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
 
     for cs in (checks, embeddedchecks):
         for c in cs:
             failandwarn = c[-1]
             preparefailandwarn(failandwarn)
 
             filters = c[-2]
             preparefilters(filters)
 
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print("%s:%d (%s):" % (fname, lineno, blame))
             else:
                 print("%s:%d:" % (fname, lineno))
             print(" > %s" % line)
             self._lastseen = msgid
         print(" " + msg)
 
 
 _defaultlogger = norepeatlogger()
 
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 
 def checkfile(
     f,
     logfunc=_defaultlogger.log,
     maxerr=None,
     warnings=False,
     blame=False,
     debug=False,
     lineno=True,
 ):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting.
              Set to false (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     result = True
 
     try:
         with opentext(f) as fp:
             try:
                 pre = fp.read()
             except UnicodeDecodeError as e:
                 print("%s while reading %s" % (e, f))
                 return result
     except IOError as e:
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
 
     # context information shared while single checkfile() invocation
     context = {'blamecache': None}
 
     for name, match, magic, filters, pats in checks:
         if debug:
             print(name, f)
         if not (re.match(match, f) or (magic and re.search(magic, pre))):
             if debug:
                 print(
                     "Skipping %s for %s it doesn't match %s" % (name, match, f)
                 )
             continue
         if "no-" "check-code" in pre:
             # If you're looking at this line, it's because a file has:
             # no- check- code
             # but the reason to output skipping is to make life for
             # tests easier. So, instead of writing it with a normal
             # spelling, we write it with the expected spelling from
             # tests/test-check-code.t
             print("Skipping %s it has no-che?k-code (glob)" % f)
             return "Skip"  # skip checking this file
 
         fc = _checkfiledata(
             name,
             f,
             pre,
             filters,
             pats,
             context,
             logfunc,
             maxerr,
             warnings,
             blame,
             debug,
             lineno,
         )
         if fc:
             result = False
 
     if f.endswith('.t') and "no-" "check-code" not in pre:
         if debug:
             print("Checking embedded code in %s" % f)
 
         prelines = pre.splitlines()
         embeddederros = []
         for name, embedded, filters, pats in embeddedchecks:
             # "reset curmax at each repetition" treats maxerr as "max
             # nubmer of errors in an actual file per entry of
             # (embedded)checks"
             curmaxerr = maxerr
 
             for found in embedded(f, prelines, embeddederros):
                 filename, starts, ends, code = found
                 fc = _checkfiledata(
                     name,
                     f,
                     code,
                     filters,
                     pats,
                     context,
                     logfunc,
                     curmaxerr,
                     warnings,
                     blame,
                     debug,
                     lineno,
                     offset=starts - 1,
                 )
                 if fc:
                     result = False
                     if curmaxerr:
                         if fc >= curmaxerr:
                             break
                         curmaxerr -= fc
 
     return result
 
 
 def _checkfiledata(
     name,
     f,
     filedata,
     filters,
     pats,
     context,
     logfunc,
     maxerr,
     warnings,
     blame,
     debug,
     lineno,
     offset=None,
 ):
     """Execute actual error check for file data
 
     :name: of the checking category
     :f: filepath
     :filedata: content of a file
     :filters: to be applied before checking
     :pats: to detect errors
     :context: a dict of information shared while single checkfile() invocation
               Valid keys: 'blamecache'.
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting, or False to
              report all errors
     :warnings: whether warning level checks should be applied
     :blame: whether blame information should be displayed at error reporting
     :debug: whether debug information should be displayed
     :lineno: whether lineno should be displayed at error reporting
     :offset: line number offset of 'filedata' in 'f' for checking
              an embedded code fragment, or None (offset=0 is different
              from offset=None)
 
     returns number of detected errors.
     """
     blamecache = context['blamecache']
     if offset is None:
         lineoffset = 0
     else:
         lineoffset = offset
 
     fc = 0
     pre = post = filedata
 
     if True:  # TODO: get rid of this redundant 'if' block
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0])  # nerr elements are errors
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
 
         if debug:
             print("Checking %s for %s" % (name, f))
 
         prelines = None
         errors = []
         for i, pat in enumerate(pats):
             if len(pat) == 3:
                 p, msg, ignore = pat
             else:
                 p, msg = pat
                 ignore = None
             if i >= nerrs:
                 msg = "warning: " + msg
 
             pos = 0
             n = 0
             for m in p.finditer(post):
                 if prelines is None:
                     prelines = pre.splitlines()
                     postlines = post.splitlines(True)
 
                 start = m.start()
                 while n < len(postlines):
                     step = len(postlines[n])
                     if pos + step > start:
                         break
                     pos += step
                     n += 1
                 l = prelines[n]
 
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print(
                             "Skipping %s for %s:%s (ignore pattern)"
                             % (name, f, (n + lineoffset))
                         )
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
                     if blamecache is None:
                         blamecache = getblame(f)
                         context['blamecache'] = blamecache
                     if (n + lineoffset) < len(blamecache):
                         bl, bu, br = blamecache[(n + lineoffset)]
                         if offset is None and bl == l:
                             bd = '%s@%s' % (bu, br)
                         elif offset is not None and bl.endswith(l):
                             # "offset is not None" means "checking
                             # embedded code fragment". In this case,
                             # "l" does not have information about the
                             # beginning of an *original* line in the
                             # file (e.g. '  > ').
                             # Therefore, use "str.endswith()", and
                             # show "maybe" for a little loose
                             # examination.
                             bd = '%s@%s, maybe' % (bu, br)
 
                 errors.append((f, lineno and (n + lineoffset + 1), l, msg, bd))
 
         errors.sort()
         for e in errors:
             logfunc(*e)
             fc += 1
             if maxerr and fc >= maxerr:
                 print(" (too many errors, giving up)")
                 break
 
     return fc
 
 
 def main():
     parser = optparse.OptionParser("%prog [options] [files | -]")
     parser.add_option(
         "-w",
         "--warnings",
         action="store_true",
         help="include warning-level checks",
     )
     parser.add_option(
         "-p", "--per-file", type="int", help="max warnings per file"
     )
     parser.add_option(
         "-b",
         "--blame",
         action="store_true",
         help="use annotate to generate blame info",
     )
     parser.add_option(
         "", "--debug", action="store_true", help="show debug information"
     )
     parser.add_option(
         "",
         "--nolineno",
         action="store_false",
         dest='lineno',
         help="don't show line numbers",
     )
 
     parser.set_defaults(
         per_file=15, warnings=False, blame=False, debug=False, lineno=True
     )
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     elif args == ['-']:
         # read file list from stdin
         check = sys.stdin.read().splitlines()
     else:
         check = args
 
     _preparepats()
 
     ret = 0
     for f in check:
         if not checkfile(
             f,
             maxerr=options.per_file,
             warnings=options.warnings,
             blame=options.blame,
             debug=options.debug,
             lineno=options.lineno,
         ):
             ret = 1
     return ret
 
 
 if __name__ == "__main__":
     sys.exit(main())

contrib/check-commit

@@ -1,115 +1,114 b''
 #!/usr/bin/env python3
 #
 # Copyright 2014 Olivia Mackall <[e-mail]>
 #
 # A tool/hook to run basic sanity checks on commits/patches for
 # submission to Mercurial. Install by adding the following to your
 # .hg/hgrc:
 #
 # [hooks]
 # pretxncommit = contrib/check-commit
 #
 # The hook can be temporarily bypassed with:
 #
 # $ BYPASS= hg commit
 #
 # See also: https://mercurial-scm.org/wiki/ContributingChanges
 
-from __future__ import absolute_import, print_function
 
 import os
 import re
 import sys
 
 commitheader = r"^(?:# [^\n]*\n)*"
 afterheader = commitheader + r"(?!#)"
 beforepatch = afterheader + r"(?!\n(?!@@))"
 
 errors = [
     (beforepatch + r".*[(]bc[)]", "(BC) needs to be uppercase"),
     (
         beforepatch + r".*[(]issue \d\d\d",
         "no space allowed between issue and number",
     ),
     (beforepatch + r".*[(]bug(\d|\s)", "use (issueDDDD) instead of bug"),
     (commitheader + r"# User [^@\n]+\n", "username is not an email address"),
     (
         commitheader + r"(?!merge with )[^#]\S+[^:] ",
         "summary line doesn't start with 'topic: '",
     ),
     (afterheader + r"[A-Z][a-z]\S+", "don't capitalize summary lines"),
     (afterheader + r"^\S+: *[A-Z][a-z]\S+", "don't capitalize summary lines"),
     (
         afterheader + r"\S*[^A-Za-z0-9-_]\S*: ",
         "summary keyword should be most user-relevant one-word command or topic",
     ),
     (afterheader + r".*\.\s*\n", "don't add trailing period on summary line"),
     (afterheader + r".{79,}", "summary line too long (limit is 78)"),
 ]
 
 word = re.compile(r'\S')
 
 
 def nonempty(first, second):
     if word.search(first):
         return first
     return second
 
 
 def checkcommit(commit, node=None):
     exitcode = 0
     printed = node is None
     hits = []
     signtag = (
         afterheader + r'Added (tag [^ ]+|signature) for changeset [a-f0-9]{12}'
     )
     if re.search(signtag, commit):
         return 0
     for exp, msg in errors:
         for m in re.finditer(exp, commit):
             end = m.end()
             trailing = re.search(r'(\\n)+$', exp)
             if trailing:
                 end -= len(trailing.group()) / 2
             hits.append((end, exp, msg))
     if hits:
         hits.sort()
         pos = 0
         last = ''
         for n, l in enumerate(commit.splitlines(True)):
             pos += len(l)
             while len(hits):
                 end, exp, msg = hits[0]
                 if pos < end:
                     break
                 if not printed:
                     printed = True
                     print("node: %s" % node)
                 print("%d: %s" % (n, msg))
                 print(" %s" % nonempty(l, last)[:-1])
                 if "BYPASS" not in os.environ:
                     exitcode = 1
                 del hits[0]
             last = nonempty(l, last)
 
     return exitcode
 
 
 def readcommit(node):
     return os.popen("hg export %s" % node).read()
 
 
 if __name__ == "__main__":
     exitcode = 0
     node = os.environ.get("HG_NODE")
 
     if node:
         commit = readcommit(node)
         exitcode = checkcommit(commit)
     elif sys.argv[1:]:
         for node in sys.argv[1:]:
             exitcode |= checkcommit(readcommit(node), node)
     else:
         commit = sys.stdin.read()
         exitcode = checkcommit(commit)
     sys.exit(exitcode)

contrib/check-config.py

@@ -1,191 +1,190 b''
 #!/usr/bin/env python3
 #
 # check-config - a config flag documentation checker for Mercurial
 #
 # Copyright 2015 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import, print_function
 import re
 import sys
 
 foundopts = {}
 documented = {}
 allowinconsistent = set()
 
 configre = re.compile(
     br'''
     # Function call
     ui\.config(?P<ctype>|int|bool|list)\(
         # First argument.
         ['"](?P<section>\S+)['"],\s*
         # Second argument
         ['"](?P<option>\S+)['"](,\s+
         (?:default=)?(?P<default>\S+?))?
     \)''',
     re.VERBOSE | re.MULTILINE,
 )
 
 configwithre = re.compile(
     br'''
     ui\.config(?P<ctype>with)\(
         # First argument is callback function. This doesn't parse robustly
         # if it is e.g. a function call.
         [^,]+,\s*
         ['"](?P<section>\S+)['"],\s*
         ['"](?P<option>\S+)['"](,\s+
         (?:default=)?(?P<default>\S+?))?
     \)''',
     re.VERBOSE | re.MULTILINE,
 )
 
 configpartialre = br"""ui\.config"""
 
 ignorere = re.compile(
     br'''
     \#\s(?P<reason>internal|experimental|deprecated|developer|inconsistent)\s
     config:\s(?P<config>\S+\.\S+)$
     ''',
     re.VERBOSE | re.MULTILINE,
 )
 
 if sys.version_info[0] > 2:
 
     def mkstr(b):
         if isinstance(b, str):
             return b
         return b.decode('utf8')
 
 
 else:
     mkstr = lambda x: x
 
 
 def main(args):
     for f in args:
         sect = b''
         prevname = b''
         confsect = b''
         carryover = b''
         linenum = 0
         for l in open(f, 'rb'):
             linenum += 1
 
             # check topic-like bits
             m = re.match(br'\s*``(\S+)``', l)
             if m:
                 prevname = m.group(1)
             if re.match(br'^\s*-+$', l):
                 sect = prevname
                 prevname = b''
 
             if sect and prevname:
                 name = sect + b'.' + prevname
                 documented[name] = 1
 
             # check docstring bits
             m = re.match(br'^\s+\[(\S+)\]', l)
             if m:
                 confsect = m.group(1)
                 continue
             m = re.match(br'^\s+(?:#\s*)?(\S+) = ', l)
             if m:
                 name = confsect + b'.' + m.group(1)
                 documented[name] = 1
 
             # like the bugzilla extension
             m = re.match(br'^\s*(\S+\.\S+)$', l)
             if m:
                 documented[m.group(1)] = 1
 
             # like convert
             m = re.match(br'^\s*:(\S+\.\S+):\s+', l)
             if m:
                 documented[m.group(1)] = 1
 
             # quoted in help or docstrings
             m = re.match(br'.*?``(\S+\.\S+)``', l)
             if m:
                 documented[m.group(1)] = 1
 
             # look for ignore markers
             m = ignorere.search(l)
             if m:
                 if m.group('reason') == b'inconsistent':
                     allowinconsistent.add(m.group('config'))
                 else:
                     documented[m.group('config')] = 1
 
             # look for code-like bits
             line = carryover + l
             m = configre.search(line) or configwithre.search(line)
             if m:
                 ctype = m.group('ctype')
                 if not ctype:
                     ctype = 'str'
                 name = m.group('section') + b"." + m.group('option')
                 default = m.group('default')
                 if default in (
                     None,
                     b'False',
                     b'None',
                     b'0',
                     b'[]',
                     b'""',
                     b"''",
                 ):
                     default = b''
                 if re.match(b'[a-z.]+$', default):
                     default = b'<variable>'
                 if (
                     name in foundopts
                     and (ctype, default) != foundopts[name]
                     and name not in allowinconsistent
                 ):
                     print(mkstr(l.rstrip()))
                     fctype, fdefault = foundopts[name]
                     print(
                         "conflict on %s: %r != %r"
                         % (
                             mkstr(name),
                             (mkstr(ctype), mkstr(default)),
                             (mkstr(fctype), mkstr(fdefault)),
                         )
                     )
                     print("at %s:%d:" % (mkstr(f), linenum))
                 foundopts[name] = (ctype, default)
                 carryover = b''
             else:
                 m = re.search(configpartialre, line)
                 if m:
                     carryover = line
                 else:
                     carryover = b''
 
     for name in sorted(foundopts):
         if name not in documented:
             if not (
                 name.startswith(b"devel.")
                 or name.startswith(b"experimental.")
                 or name.startswith(b"debug.")
             ):
                 ctype, default = foundopts[name]
                 if default:
                     if isinstance(default, bytes):
                         default = mkstr(default)
                     default = ' [%s]' % default
                 elif isinstance(default, bytes):
                     default = mkstr(default)
                 print(
                     "undocumented: %s (%s)%s"
                     % (mkstr(name), mkstr(ctype), default)
                 )
 
 
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         sys.exit(main(sys.argv[1:]))
     else:
         sys.exit(main([l.rstrip() for l in sys.stdin]))

contrib/check-py3-compat.py

@@ -1,93 +1,92 b''
 #!/usr/bin/env python3
 #
 # check-py3-compat - check Python 3 compatibility of Mercurial files
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import, print_function
 
 import ast
 import importlib
 import os
 import sys
 import traceback
 import warnings
 
 
 def check_compat_py3(f):
     """Check Python 3 compatibility of a file with Python 3."""
     with open(f, 'rb') as fh:
         content = fh.read()
 
     try:
         ast.parse(content, filename=f)
     except SyntaxError as e:
         print('%s: invalid syntax: %s' % (f, e))
         return
 
     # Try to import the module.
     # For now we only support modules in packages because figuring out module
     # paths for things not in a package can be confusing.
     if f.startswith(
         ('hgdemandimport/', 'hgext/', 'mercurial/')
     ) and not f.endswith('__init__.py'):
         assert f.endswith('.py')
         name = f.replace('/', '.')[:-3]
         try:
             importlib.import_module(name)
         except Exception as e:
             exc_type, exc_value, tb = sys.exc_info()
             # We walk the stack and ignore frames from our custom importer,
             # import mechanisms, and stdlib modules. This kinda/sorta
             # emulates CPython behavior in import.c while also attempting
             # to pin blame on a Mercurial file.
             for frame in reversed(traceback.extract_tb(tb)):
                 if frame.name == '_call_with_frames_removed':
                     continue
                 if 'importlib' in frame.filename:
                     continue
                 if 'mercurial/__init__.py' in frame.filename:
                     continue
                 if frame.filename.startswith(sys.prefix):
                     continue
                 break
 
             if frame.filename:
                 filename = os.path.basename(frame.filename)
                 print(
                     '%s: error importing: <%s> %s (error at %s:%d)'
                     % (f, type(e).__name__, e, filename, frame.lineno)
                 )
             else:
                 print(
                     '%s: error importing module: <%s> %s (line %d)'
                     % (f, type(e).__name__, e, frame.lineno)
                 )
 
 
 if __name__ == '__main__':
     # check_compat_py3 will import every filename we specify as long as it
     # starts with one of a few prefixes. It does this by converting
     # specified filenames like 'mercurial/foo.py' to 'mercurial.foo' and
     # importing that. When running standalone (not as part of a test), this
     # means we actually import the installed versions, not the files we just
     # specified. When running as test-check-py3-compat.t, we technically
     # would import the correct paths, but it's cleaner to have both cases
     # use the same import logic.
     sys.path.insert(0, '.')
 
     for f in sys.argv[1:]:
         with warnings.catch_warnings(record=True) as warns:
             check_compat_py3(f)
 
         for w in warns:
             print(
                 warnings.formatwarning(
                     w.message, w.category, w.filename, w.lineno
                 ).rstrip()
             )
 
     sys.exit(0)

contrib/debugcmdserver.py

@@ -1,51 +1,50 b''
 #!/usr/bin/env python3
 #
 # Dumps output generated by Mercurial's command server in a formatted style to a
 # given file or stderr if '-' is specified. Output is also written in its raw
 # format to stdout.
 #
 # $ ./hg serve --cmds pipe | ./contrib/debugcmdserver.py -
 # o, 52   -> 'capabilities: getencoding runcommand\nencoding: UTF-8'
 
-from __future__ import absolute_import, print_function
 import struct
 import sys
 
 if len(sys.argv) != 2:
     print('usage: debugcmdserver.py FILE')
     sys.exit(1)
 
 outputfmt = '>cI'
 outputfmtsize = struct.calcsize(outputfmt)
 
 if sys.argv[1] == '-':
     log = sys.stderr
 else:
     log = open(sys.argv[1], 'a')
 
 
 def read(size):
     data = sys.stdin.read(size)
     if not data:
         raise EOFError
     sys.stdout.write(data)
     sys.stdout.flush()
     return data
 
 
 try:
     while True:
         header = read(outputfmtsize)
         channel, length = struct.unpack(outputfmt, header)
         log.write('%s, %-4d' % (channel, length))
         if channel in 'IL':
             log.write(' -> waiting for input\n')
         else:
             data = read(length)
             log.write(' -> %r\n' % data)
         log.flush()
 except EOFError:
     pass
 finally:
     if log != sys.stderr:
         log.close()

contrib/debugshell.py

@@ -1,65 +1,64 b''
 # debugshell extension
 """a python shell with repo, changelog & manifest objects"""
 
-from __future__ import absolute_import
 import code
 import mercurial
 import sys
 from mercurial import (
     demandimport,
     pycompat,
     registrar,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 
 def pdb(ui, repo, msg, **opts):
     objects = {
         'mercurial': mercurial,
         'repo': repo,
         'cl': repo.changelog,
         'mf': repo.manifestlog,
     }
 
     code.interact(msg, local=objects)
 
 
 def ipdb(ui, repo, msg, **opts):
     import IPython
 
     cl = repo.changelog
     mf = repo.manifestlog
     cl, mf  # use variables to appease pyflakes
 
     IPython.embed()
 
 
 @command(b'debugshell|dbsh', [])
 def debugshell(ui, repo, **opts):
     bannermsg = "loaded repo : %s\n" "using source: %s" % (
         pycompat.sysstr(repo.root),
         mercurial.__path__[0],
     )
 
     pdbmap = {'pdb': 'code', 'ipdb': 'IPython'}
 
     debugger = ui.config(b"ui", b"debugger")
     if not debugger:
         debugger = 'pdb'
     else:
         debugger = pycompat.sysstr(debugger)
 
     # if IPython doesn't exist, fallback to code.interact
     try:
         with demandimport.deactivated():
             __import__(pdbmap[debugger])
     except ImportError:
         ui.warnnoi18n(
             b"%s debugger specified but %s module was not found\n"
             % (debugger, pdbmap[debugger])
         )
         debugger = b'pdb'
 
     getattr(sys.modules[__name__], debugger)(ui, repo, bannermsg, **opts)

contrib/dumprevlog

@@ -1,60 +1,59 b''
 #!/usr/bin/env python3
 # Dump revlogs as raw data stream
 # $ find .hg/store/ -name "*.i" | xargs dumprevlog > repo.dump
 
-from __future__ import absolute_import, print_function
 
 import sys
 from mercurial.node import hex
 from mercurial import (
     encoding,
     pycompat,
     revlog,
 )
 from mercurial.utils import procutil
 
 from mercurial.revlogutils import (
     constants as revlog_constants,
 )
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     procutil.setbinary(fp)
 
 
 def binopen(path, mode=b'rb'):
     if b'b' not in mode:
         mode = mode + b'b'
     return open(path, pycompat.sysstr(mode))
 
 
 binopen.options = {}
 
 
 def printb(data, end=b'\n'):
     sys.stdout.flush()
     procutil.stdout.write(data + end)
 
 
 for f in sys.argv[1:]:
     localf = encoding.strtolocal(f)
     if not localf.endswith(b'.i'):
         print("file:", f, file=sys.stderr)
         print("  invalid filename", file=sys.stderr)
 
     r = revlog.revlog(
         binopen,
         target=(revlog_constants.KIND_OTHER, b'dump-revlog'),
         radix=localf[:-2],
     )
     print("file:", f)
     for i in r:
         n = r.node(i)
         p = r.parents(n)
         d = r.revision(n)
         printb(b"node: %s" % hex(n))
         printb(b"linkrev: %d" % r.linkrev(i))
         printb(b"parents: %s %s" % (hex(p[0]), hex(p[1])))
         printb(b"length: %d" % len(d))
         printb(b"-start-")
         printb(d)
         printb(b"-end-")

contrib/fuzz/dirs_corpus.py

@@ -1,29 +1,27 b''
-from __future__ import absolute_import, print_function
-
 import argparse
 import zipfile
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     zf.writestr(
         "greek-tree",
         "\n".join(
             [
                 "iota",
                 "A/mu",
                 "A/B/lambda",
                 "A/B/E/alpha",
                 "A/B/E/beta",
                 "A/D/gamma",
                 "A/D/G/pi",
                 "A/D/G/rho",
                 "A/D/G/tau",
                 "A/D/H/chi",
                 "A/D/H/omega",
                 "A/D/H/psi",
             ]
         ),
     )

contrib/fuzz/dirstate_corpus.py

@@ -1,17 +1,15 b''
-from __future__ import absolute_import, print_function
-
 import argparse
 import os
 import zipfile
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 reporoot = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
 dirstate = os.path.join(reporoot, '.hg', 'dirstate')
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     if os.path.exists(dirstate):
         with open(dirstate, 'rb') as f:
             zf.writestr("dirstate", f.read())

contrib/fuzz/fm1readmarkers_corpus.py

@@ -1,38 +1,36 b''
-from __future__ import absolute_import, print_function
-
 import argparse
 import zipfile
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     zf.writestr(
         'smallish_obsstore',
         (
             # header: fm1readmarkers should start at offset 1, and
             # read until byte 597.
             '1\x00597\x00'
             # body of obsstore file
             '\x01\x00\x00\x00vA\xd7\x02+C\x1a<)\x01,\x00\x00\x01\x03\x03\xe6'
             '\x92\xde)x\x16\xd1Xph\xc7\xa7[\xe5\xe2\x1a\xab\x1e6e\xaf\xc2\xae'
             '\xe7\xbc\x83\xe1\x88\xa5\xda\xce>O\xbd\x04\xe9\x03\xc4o\xeb\x03'
             '\x01\t\x05\x04\x1fef18operationamenduserAugie Fackler <raf@duri'
             'n42.com>\x00\x00\x00vA\xd7\x02-\x8aD\xaf-\x01,\x00\x00\x01\x03\x03'
             '\x17*\xca\x8f\x9e}i\xe0i\xbb\xdf\x9fb\x03\xd2XG?\xd3h\x98\x89\x1a'
             '=2\xeb\xc3\xc5<\xb3\x9e\xcc\x0e;#\xee\xc3\x10ux\x03\x01\t\x05\x04'
             '\x1fef18operationamenduserAugie Fackler <[e-mail]>\x00\x00'
             '\x00vA\xd7\x02Mn\xd9%\xea\x01,\x00\x00\x01\x03\x03\x98\x89\x1a='
             '2\xeb\xc3\xc5<\xb3\x9e\xcc\x0e;#\xee\xc3\x10ux\xe0*\xcaT\x86Z8J'
             '\x85)\x97\xff7\xcc)\xc1\x7f\x19\x0c\x01\x03\x01\t\x05\x04\x1fef'
             '18operationamenduserAugie Fackler <[e-mail]>\x00\x00\x00'
             'yA\xd7\x02MtA\xbfj\x01,\x00\x00\x01\x03\x03\xe0*\xcaT\x86Z8J\x85'
             ')\x97\xff7\xcc)\xc1\x7f\x19\x0c\x01\x00\x94\x01\xa9\n\xf80\x92\xa3'
             'j\xc5X\xb1\xc9:\xd51\xb8*\xa9\x03\x01\t\x08\x04\x1fef11operatio'
             'nhistedituserAugie Fackler <[e-mail]>\x00\x00\x00yA\xd7\x02'
             'MtA\xd4\xe1\x01,\x00\x00\x01\x03\x03"\xa5\xcb\x86\xb6\xf4\xbaO\xa0'
             'sH\xe7?\xcb\x9b\xc2n\xcfI\x9e\x14\xf0D\xf0!\x18DN\xcd\x97\x016\xa5'
             '\xef\xa06\xcb\x884\x8a\x03\x01\t\x08\x04\x1fef14operationhisted'
         ),
     )

contrib/fuzz/manifest_corpus.py

@@ -1,38 +1,36 b''
-from __future__ import absolute_import, print_function
-
 import argparse
 import zipfile
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     zf.writestr(
         "manifest_zero",
         '''\0PKG-INFO\09b3ed8f2b81095a13064402e930565f083346e9a
 README\080b6e76643dcb44d4bc729e932fc464b3e36dbe3
 hg\0b6444347c629cc058d478023905cfb83b7f5bb9d
 mercurial/__init__.py\0b80de5d138758541c5f05265ad144ab9fa86d1db
 mercurial/byterange.py\017f5a9fbd99622f31a392c33ac1e903925dc80ed
 mercurial/fancyopts.py\0b6f52e23e356748c5039313d8b639cda16bf67ba
 mercurial/hg.py\023cc12f225f1b42f32dc0d897a4f95a38ddc8f4a
 mercurial/mdiff.py\0a05f65c44bfbeec6a42336cd2ff0b30217899ca3
 mercurial/revlog.py\0217bc3fde6d82c0210cf56aeae11d05a03f35b2b
 mercurial/transaction.py\09d180df101dc14ce3dd582fd998b36c98b3e39aa
 notes.txt\0703afcec5edb749cf5cec67831f554d6da13f2fb
 setup.py\0ccf3f6daf0f13101ca73631f7a1769e328b472c9
 tkmerge\03c922edb43a9c143682f7bc7b00f98b3c756ebe7
 ''',
     )
     zf.writestr("badmanifest_shorthashes", "\0narf\0aa\nnarf2\0aaa\n")
     zf.writestr(
         "badmanifest_nonull",
         "\0narf\0cccccccccccccccccccccccccccccccccccccccc\n"
         "narf2aaaaaaaaaaaaaaaaaaaa\n",
     )
 
     zf.writestr(
         "manifest_long_nodes",
         "\1a\0ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\n",
     )

contrib/fuzz/mpatch_corpus.py

@@ -1,371 +1,369 b''
-from __future__ import absolute_import, print_function
-
 import argparse
 import os
 import struct
 import sys
 import zipfile
 
 # Add ../.. to sys.path as an absolute path so we can import hg modules
 hgloc = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
 sys.path[0:0] = [hgloc]
 
 from mercurial import (
     hg,
     ui as uimod,
 )
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 
 if sys.version_info[0] < 3:
 
     class py2reprhack(object):
         def __repr__(self):
             """Py2 calls __repr__ for `bytes(foo)`, forward to __bytes__"""
             return self.__bytes__()
 
 
 else:
 
     class py2reprhack(object):
         """Not needed on py3."""
 
 
 class deltafrag(py2reprhack):
     def __init__(self, start, end, data):
         self.start = start
         self.end = end
         self.data = data
 
     def __bytes__(self):
         return (
             struct.pack(">lll", self.start, self.end, len(self.data))
             + self.data
         )
 
 
 class delta(py2reprhack):
     def __init__(self, frags):
         self.frags = frags
 
     def __bytes__(self):
         return b''.join(bytes(f) for f in self.frags)
 
 
 class corpus(py2reprhack):
     def __init__(self, base, deltas):
         self.base = base
         self.deltas = deltas
 
     def __bytes__(self):
         deltas = [bytes(d) for d in self.deltas]
         parts = (
             [
                 struct.pack(">B", len(deltas) + 1),
                 struct.pack(">H", len(self.base)),
             ]
             + [struct.pack(">H", len(d)) for d in deltas]
             + [self.base]
             + deltas
         )
         return b''.join(parts)
 
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     # Manually constructed entries
     zf.writestr(
         "one_delta_applies",
         bytes(corpus(b'a', [delta([deltafrag(0, 1, b'b')])])),
     )
     zf.writestr(
         "one_delta_starts_late",
         bytes(corpus(b'a', [delta([deltafrag(3, 1, b'b')])])),
     )
     zf.writestr(
         "one_delta_ends_late",
         bytes(corpus(b'a', [delta([deltafrag(0, 20, b'b')])])),
     )
 
     try:
         # Generated from repo data
         r = hg.repository(uimod.ui(), b'../..')
         fl = r.file(b'mercurial/manifest.py')
         rl = getattr(fl, '_revlog', fl)
         bins = rl._chunks(rl._deltachain(10)[0])
         zf.writestr('manifest_py_rev_10', bytes(corpus(bins[0], bins[1:])))
     except:  # skip this, so no re-raises
         print('skipping seed file from repo data')
     # Automatically discovered by running the fuzzer
     zf.writestr(
         "mpatch_decode_old_overread", b"\x02\x00\x00\x00\x02\x00\x00\x00"
     )
     # https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=8876
     zf.writestr(
         "mpatch_ossfuzz_getbe32_ubsan",
         b"\x02\x00\x00\x00\x0c    \xff\xff\xff\xff    ",
     )
     zf.writestr(
         "mpatch_apply_over_memcpy",
         b'\x13\x01\x00\x05\xd0\x00\x00\x00\x00\x00\x00\x00\x00\n \x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8c\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00)\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00A\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x94\x18'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfa\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x94\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfa\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00se\x00\x00'
         b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
         b'\x00\x00\x00\x00',
     )

contrib/fuzz/revlog_corpus.py

@@ -1,25 +1,23 b''
-from __future__ import absolute_import
-
 import argparse
 import os
 import zipfile
 
 ap = argparse.ArgumentParser()
 ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
 args = ap.parse_args()
 
 reporoot = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
 # typically a standalone index
 changelog = os.path.join(reporoot, '.hg', 'store', '00changelog.i')
 # an inline revlog with only a few revisions
 contributing = os.path.join(
     reporoot, '.hg', 'store', 'data', 'contrib', 'fuzz', 'mpatch.cc.i'
 )
 
 with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
     if os.path.exists(changelog):
         with open(changelog, 'rb') as f:
             zf.writestr("00changelog.i", f.read())
     if os.path.exists(contributing):
         with open(contributing, 'rb') as f:
             zf.writestr("contributing.i", f.read())

contrib/genosxversion.py

@@ -1,141 +1,140 b''
 #!/usr/bin/env python2
-from __future__ import absolute_import, print_function
 
 import argparse
 import os
 import subprocess
 import sys
 
 try:
     # Always load hg libraries from the hg we can find on $PATH.
     hglib = subprocess.check_output(['hg', 'debuginstall', '-T', '{hgmodules}'])
     sys.path.insert(0, os.path.dirname(hglib))
 except subprocess.CalledProcessError:
     # We're probably running with a PyOxidized Mercurial, so just
     # proceed and hope it works out okay.
     pass
 
 from mercurial import util
 
 ap = argparse.ArgumentParser()
 ap.add_argument(
     '--paranoid',
     action='store_true',
     help=(
         "Be paranoid about how version numbers compare and "
         "produce something that's more likely to sort "
         "reasonably."
     ),
 )
 ap.add_argument('--selftest', action='store_true', help='Run self-tests.')
 ap.add_argument('versionfile', help='Path to a valid mercurial __version__.py')
 
 
 def paranoidver(ver):
     """Given an hg version produce something that distutils can sort.
 
     Some Mac package management systems use distutils code in order to
     figure out upgrades, which makes life difficult. The test case is
     a reduced version of code in the Munki tool used by some large
     organizations to centrally manage OS X packages, which is what
     inspired this kludge.
 
     >>> paranoidver('3.4')
     '3.4.0'
     >>> paranoidver('3.4.2')
     '3.4.2'
     >>> paranoidver('3.0-rc+10')
     '2.9.9999-rc+10'
     >>> paranoidver('4.2+483-5d44d7d4076e')
     '4.2.0+483-5d44d7d4076e'
     >>> paranoidver('4.2.1+598-48d1e1214d8c')
     '4.2.1+598-48d1e1214d8c'
     >>> paranoidver('4.3-rc')
     '4.2.9999-rc'
     >>> paranoidver('4.3')
     '4.3.0'
     >>> from distutils import version
     >>> class LossyPaddedVersion(version.LooseVersion):
     ...     '''Subclass version.LooseVersion to compare things like
     ...     "10.6" and "10.6.0" as equal'''
     ...     def __init__(self, s):
     ...             self.parse(s)
     ...
     ...     def _pad(self, version_list, max_length):
     ...         'Pad a version list by adding extra 0 components to the end'
     ...         # copy the version_list so we don't modify it
     ...         cmp_list = list(version_list)
     ...         while len(cmp_list) < max_length:
     ...             cmp_list.append(0)
     ...         return cmp_list
     ...
     ...     def __cmp__(self, other):
     ...         if isinstance(other, str):
     ...             other = MunkiLooseVersion(other)
     ...         max_length = max(len(self.version), len(other.version))
     ...         self_cmp_version = self._pad(self.version, max_length)
     ...         other_cmp_version = self._pad(other.version, max_length)
     ...         return cmp(self_cmp_version, other_cmp_version)
     >>> def testver(older, newer):
     ...   o = LossyPaddedVersion(paranoidver(older))
     ...   n = LossyPaddedVersion(paranoidver(newer))
     ...   return o < n
     >>> testver('3.4', '3.5')
     True
     >>> testver('3.4.0', '3.5-rc')
     True
     >>> testver('3.4-rc', '3.5')
     True
     >>> testver('3.4-rc+10-deadbeef', '3.5')
     True
     >>> testver('3.4.2', '3.5-rc')
     True
     >>> testver('3.4.2', '3.5-rc+10-deadbeef')
     True
     >>> testver('4.2+483-5d44d7d4076e', '4.2.1+598-48d1e1214d8c')
     True
     >>> testver('4.3-rc', '4.3')
     True
     >>> testver('4.3', '4.3-rc')
     False
     """
     major, minor, micro, extra = util.versiontuple(ver, n=4)
     if micro is None:
         micro = 0
     if extra:
         if extra.startswith('rc'):
             if minor == 0:
                 major -= 1
                 minor = 9
             else:
                 minor -= 1
             micro = 9999
             extra = '-' + extra
         else:
             extra = '+' + extra
     else:
         extra = ''
     return '%d.%d.%d%s' % (major, minor, micro, extra)
 
 
 def main(argv):
     opts = ap.parse_args(argv[1:])
     if opts.selftest:
         import doctest
 
         doctest.testmod()
         return
     with open(opts.versionfile) as f:
         for l in f:
             if l.startswith('version = b'):
                 # version number is entire line minus the quotes
                 ver = l[len('version = b') + 1 : -2]
                 break
     if opts.paranoid:
         print(paranoidver(ver))
     else:
         print(ver)
 
 
 if __name__ == '__main__':
     main(sys.argv)

contrib/hg-ssh

@@ -1,118 +1,117 b''
 #!/usr/bin/env python3
 #
 # Copyright 2005-2007 by Intevation GmbH <[e-mail]>
 #
 # Author(s):
 # Thomas Arendsen Hein <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """
 hg-ssh - a wrapper for ssh access to a limited set of mercurial repos
 
 To be used in ~/.ssh/authorized_keys with the "command" option, see sshd(8):
 command="hg-ssh path/to/repo1 /path/to/repo2 ~/repo3 ~user/repo4" ssh-dss ...
 (probably together with these other useful options:
  no-port-forwarding,no-X11-forwarding,no-agent-forwarding)
 
 This allows pull/push over ssh from/to the repositories given as arguments.
 
 If all your repositories are subdirectories of a common directory, you can
 allow shorter paths with:
 command="cd path/to/my/repositories && hg-ssh repo1 subdir/repo2"
 
 You can use pattern matching of your normal shell, e.g.:
 command="cd repos && hg-ssh user/thomas/* projects/{mercurial,foo}"
 
 You can also add a --read-only flag to allow read-only access to a key, e.g.:
 command="hg-ssh --read-only repos/*"
 """
-from __future__ import absolute_import
 
 import os
 import re
 import shlex
 import sys
 
 # enable importing on demand to reduce startup time
 import hgdemandimport
 
 hgdemandimport.enable()
 
 from mercurial import (
     dispatch,
     pycompat,
     ui as uimod,
 )
 
 
 def main():
     # Prevent insertion/deletion of CRs
     dispatch.initstdio()
 
     cwd = os.getcwd()
     if os.name == 'nt':
         # os.getcwd() is inconsistent on the capitalization of the drive
         # letter, so adjust it. see https://bugs.python.org/issue40368
         if re.match('^[a-z]:', cwd):
             cwd = cwd[0:1].upper() + cwd[1:]
 
     readonly = False
     args = sys.argv[1:]
     while len(args):
         if args[0] == '--read-only':
             readonly = True
             args.pop(0)
         else:
             break
     allowed_paths = [
         os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
         for path in args
     ]
     orig_cmd = os.getenv('SSH_ORIGINAL_COMMAND', '?')
     try:
         cmdargv = shlex.split(orig_cmd)
     except ValueError as e:
         sys.stderr.write('Illegal command "%s": %s\n' % (orig_cmd, e))
         sys.exit(255)
 
     if cmdargv[:2] == ['hg', '-R'] and cmdargv[3:] == ['serve', '--stdio']:
         path = cmdargv[2]
         repo = os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
         if repo in allowed_paths:
             cmd = [b'-R', pycompat.fsencode(repo), b'serve', b'--stdio']
             req = dispatch.request(cmd)
             if readonly:
                 if not req.ui:
                     req.ui = uimod.ui.load()
                 req.ui.setconfig(
                     b'hooks',
                     b'pretxnopen.hg-ssh',
                     b'python:__main__.rejectpush',
                     b'hg-ssh',
                 )
                 req.ui.setconfig(
                     b'hooks',
                     b'prepushkey.hg-ssh',
                     b'python:__main__.rejectpush',
                     b'hg-ssh',
                 )
             dispatch.dispatch(req)
         else:
             sys.stderr.write('Illegal repository "%s"\n' % repo)
             sys.exit(255)
     else:
         sys.stderr.write('Illegal command "%s"\n' % orig_cmd)
         sys.exit(255)
 
 
 def rejectpush(ui, **kwargs):
     ui.warn((b"Permission denied\n"))
     # mercurial hooks use unix process conventions for hook return values
     # so a truthy return means failure
     return True
 
 
 if __name__ == '__main__':
     main()

contrib/hgclient.py

@@ -1,163 +1,162 b''
 # A minimal client for Mercurial's command server
 
-from __future__ import absolute_import, print_function
 
 import io
 import os
 import re
 import signal
 import socket
 import struct
 import subprocess
 import sys
 import time
 
 if sys.version_info[0] >= 3:
     stdout = sys.stdout.buffer
     stderr = sys.stderr.buffer
     stringio = io.BytesIO
 
     def bprint(*args):
         # remove b'' as well for ease of test migration
         pargs = [re.sub(br'''\bb(['"])''', br'\1', b'%s' % a) for a in args]
         stdout.write(b' '.join(pargs) + b'\n')
 
 
 else:
     import cStringIO
 
     stdout = sys.stdout
     stderr = sys.stderr
     stringio = cStringIO.StringIO
     bprint = print
 
 
 def connectpipe(path=None, extraargs=()):
     cmdline = [b'hg', b'serve', b'--cmdserver', b'pipe']
     if path:
         cmdline += [b'-R', path]
     cmdline.extend(extraargs)
 
     def tonative(cmdline):
         if os.name != 'nt':
             return cmdline
         return [arg.decode("utf-8") for arg in cmdline]
 
     server = subprocess.Popen(
         tonative(cmdline), stdin=subprocess.PIPE, stdout=subprocess.PIPE
     )
 
     return server
 
 
 class unixconnection(object):
     def __init__(self, sockpath):
         self.sock = sock = socket.socket(socket.AF_UNIX)
         sock.connect(sockpath)
         self.stdin = sock.makefile('wb')
         self.stdout = sock.makefile('rb')
 
     def wait(self):
         self.stdin.close()
         self.stdout.close()
         self.sock.close()
 
 
 class unixserver(object):
     def __init__(self, sockpath, logpath=None, repopath=None):
         self.sockpath = sockpath
         cmdline = [b'hg', b'serve', b'--cmdserver', b'unix', b'-a', sockpath]
         if repopath:
             cmdline += [b'-R', repopath]
         if logpath:
             stdout = open(logpath, 'a')
             stderr = subprocess.STDOUT
         else:
             stdout = stderr = None
         self.server = subprocess.Popen(cmdline, stdout=stdout, stderr=stderr)
         # wait for listen()
         while self.server.poll() is None:
             if os.path.exists(sockpath):
                 break
             time.sleep(0.1)
 
     def connect(self):
         return unixconnection(self.sockpath)
 
     def shutdown(self):
         os.kill(self.server.pid, signal.SIGTERM)
         self.server.wait()
 
 
 def writeblock(server, data):
     server.stdin.write(struct.pack(b'>I', len(data)))
     server.stdin.write(data)
     server.stdin.flush()
 
 
 def readchannel(server):
     data = server.stdout.read(5)
     if not data:
         raise EOFError
     channel, length = struct.unpack('>cI', data)
     if channel in b'IL':
         return channel, length
     else:
         return channel, server.stdout.read(length)
 
 
 def sep(text):
     return text.replace(b'\\', b'/')
 
 
 def runcommand(
     server, args, output=stdout, error=stderr, input=None, outfilter=lambda x: x
 ):
     bprint(b'*** runcommand', b' '.join(args))
     stdout.flush()
     server.stdin.write(b'runcommand\n')
     writeblock(server, b'\0'.join(args))
 
     if not input:
         input = stringio()
 
     while True:
         ch, data = readchannel(server)
         if ch == b'o':
             output.write(outfilter(data))
             output.flush()
         elif ch == b'e':
             error.write(data)
             error.flush()
         elif ch == b'I':
             writeblock(server, input.read(data))
         elif ch == b'L':
             writeblock(server, input.readline(data))
         elif ch == b'm':
             bprint(b"message: %r" % data)
         elif ch == b'r':
             (ret,) = struct.unpack('>i', data)
             if ret != 0:
                 bprint(b' [%d]' % ret)
             return ret
         else:
             bprint(b"unexpected channel %c: %r" % (ch, data))
             if ch.isupper():
                 return
 
 
 def check(func, connect=connectpipe):
     stdout.flush()
     server = connect()
     try:
         return func(server)
     finally:
         server.stdin.close()
         server.wait()
 
 
 def checkwith(connect=connectpipe, **kwargs):
     def wrap(func):
         return check(func, lambda: connect(**kwargs))
 
     return wrap

contrib/import-checker.py

@@ -1,771 +1,770 b''
 #!/usr/bin/env python3
 
-from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import io
 import os
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
 if True:  # disable lexical sorting checks
     try:
         import BaseHTTPServer as basehttpserver
     except ImportError:
         basehttpserver = None
     import zlib
 
 import testparseutil
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
     'breezy',
     'concurrent',
     'hgclient',
     'mercurial',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
     'mercurial.interfaces',
     'mercurial.node',
     'mercurial.pycompat',
     # for revlog to re-export constant to extensions
     'mercurial.revlogutils.constants',
     'mercurial.revlogutils.flagutil',
     # for cffi modules to re-export pure functions
     'mercurial.pure.base85',
     'mercurial.pure.bdiff',
     'mercurial.pure.mpatch',
     'mercurial.pure.osutil',
     'mercurial.pure.parsers',
     # third-party imports should be directly imported
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
 )
 
 # Whitelist of symbols that can be directly imported.
 directsymbols = ('demandimport',)
 
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 
 def dotted_name_of_path(path):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0]  # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     return '.'.join(parts)
 
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
     paths of locally defined (= Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__', 'foo.foo1',
     ...              'foo.bar.__init__', 'foo.bar.bar1',
     ...              'baz.__init__', 'baz.baz1'}
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
     >>> fromlocal('foo1', 1)
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
     >>> fromlocal2('bar2', 1)
     False
     >>> fromlocal2('bar', 2)
     ('foo.bar', 'foo.bar.__init__', True)
     """
     if not isinstance(modulename, str):
         modulename = modulename.decode('ascii')
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
 
     def fromlocal(name, level=0):
         # name is false value when relative imports are used.
         if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
             if not level:
                 # Check relative name first.
                 candidates = [prefix + name, name]
             else:
                 candidates = [
                     '.'.join(modulename.split('.')[:-level]) + '.' + name
                 ]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
 
     return fromlocal
 
 
 def populateextmods(localmods):
     """Populate C extension modules based on pure modules"""
     newlocalmods = set(localmods)
     for n in localmods:
         if n.startswith('mercurial.pure.'):
             m = n[len('mercurial.pure.') :]
             newlocalmods.add('mercurial.cext.' + m)
             newlocalmods.add('mercurial.cffi._' + m)
     return newlocalmods
 
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> py3 = sys.version_info[0] >= 3
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods or py3
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods or py3
     True
 
     >>> 'cffi' in mods
     True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield '__builtin__'
     yield 'builtins'  # python3 only
     yield 'importlib.abc'  # python3 only
     yield 'importlib.machinery'  # python3 only
     yield 'importlib.util'  # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime':  # in Python (not C) on PyPy
         yield m
     for m in ['cffi']:
         yield m
     stdlib_prefixes = {sys.prefix, sys.exec_prefix}
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (basehttpserver, zlib):
         if mod is None:
             continue
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
     sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
     for libpath in sys.path:
         # We want to walk everything in sys.path that starts with something in
         # stdlib_prefixes, but not directories from the hg sources.
         if os.path.abspath(libpath).startswith(sourceroot) or not any(
             libpath.startswith(p) for p in stdlib_prefixes
         ):
             continue
         for top, dirs, files in os.walk(libpath):
             if 'dist-packages' in top.split(os.path.sep):
                 continue
             for i, d in reversed(list(enumerate(dirs))):
                 if (
                     not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath
                     and d in ('hgdemandimport', 'hgext', 'mercurial')
                 ):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1 :]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 
 stdlib_modules = set(list_stdlib_modules())
 
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: set of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
             if modnotfound and dottedpath != modulename:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
                 # specifically allow "from . import foo" from __init__.py
                 yield dottedpath
 
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention."""
     root = ast.parse(source)
 
     return verify_modern_convention(module, root, localmods)
 
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
       must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed, unless that
       library is also a local module.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     if not isinstance(module, str):
         module = module.decode('ascii')
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
     # Whether a local/non-stdlib, non-symbol import has been seen.
     seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     laststdlib = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
 
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
 
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(
                 module, node, localmods, node.col_offset + 4
             ):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg(
                     'multiple imported names: %s',
                     ', '.join(n.name for n in node.names),
                 )
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             stdlib = name in stdlib_modules
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname and laststdlib == stdlib:
                     yield msg(
                         'imports not lexically sorted: %s < %s', name, lastname
                     )
 
             lastname = name
             laststdlib = stdlib
 
             # stdlib imports should be before local imports.
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg(
                     'stdlib import "%s" follows local import: %s',
                     name,
                     seenlocal,
                 )
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg(
                     '%s module must be "as" aliased to %s',
                     name,
                     requirealias[name],
                 )
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[: -node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or (
                     fullname in stdlib_modules
                     # allow standard 'from typing import ...' style
                     and fullname.startswith('.')
                     and fullname not in localmods
                     and fullname + '.__init__' not in localmods
                 ):
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
             found = fromlocal(node.module, node.level)
             if found and found[2]:  # node.module is a package
                 prefix = found[0] + '.'
                 symbols = (
                     n.name for n in node.names if not fromlocal(prefix + n.name)
                 )
             else:
                 symbols = (n.name for n in node.names)
             symbols = [sym for sym in symbols if sym not in directsymbols]
             if node.module and node.col_offset == root_col_offset:
                 if symbols and fullname not in allowsymbolimports:
                     yield msg(
                         'direct symbol import %s from %s',
                         ', '.join(symbols),
                         fullname,
                     )
 
                 if symbols and seennonsymbollocal:
                     yield msg(
                         'symbol import follows non-symbol import: %s', fullname
                     )
             if not symbols and fullname not in stdlib_modules:
                 seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
 
                 # Only allow 1 group per level.
                 if (
                     node.level in seenlevels
                     and node.col_offset == root_col_offset
                 ):
                     yield msg(
                         'multiple "from %s import" statements', '.' * node.level
                     )
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg(
                         'higher-level import should come first: %s', fullname
                     )
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg(
                         'imports from %s not lexically sorted: %s < %s',
                         fullname,
                         n.name,
                         lastentryname,
                     )
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg(
                         '%s from %s must be "as" aliased to %s',
                         n.name,
                         fullname,
                         requirealias[n.name],
                     )
 
 
 class CircularImport(Exception):
     pass
 
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 
 def _cycle_sortkey(c):
     return len(c), c
 
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def _forcestr(thing):
     ...     if not isinstance(thing, str):
     ...         return thing.decode('ascii')
     ...     return thing
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, b"example", lines):
     ...         print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
     ...         print(repr(_forcestr(s)))
     >>> lines = [
     ...   'comment',
     ...   '  >>> from __future__ import print_function',
     ...   "  >>> ' multiline",
     ...   "  ... string'",
     ...   '  ',
     ...   'comment',
     ...   '  $ cat > foo.py <<EOF',
     ...   '  > from __future__ import print_function',
     ...   '  > EOF',
     ... ]
     >>> test(b"example.t", lines)
     example[2] doctest.py 1
     "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
     example[8] foo.py 7
     'from __future__ import print_function\\n'
     """
     errors = []
     for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
         if not name:
             # use 'doctest.py', in order to make already existing
             # doctest above pass instantly
             name = 'doctest.py'
         # "starts" is "line number" (1-origin), but embedded() is
         # expected to return "line offset" (0-origin). Therefore, this
         # yields "starts - 1".
         if not isinstance(modname, str):
             modname = modname.decode('utf8')
         yield code, "%s[%d]" % (modname, starts), name, starts - 1
 
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
     if not f.endswith('.t'):
         with open(f, 'rb') as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         # Strictly speaking we should sniff for the magic header that denotes
         # Python source file encoding. But in reality we don't use anything
         # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
         # simplicity is fine.
         with io.open(f, 'r', encoding='utf-8') as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname.encode('utf8'), t, line
 
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmodpaths = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
         modname = dotted_name_of_path(source_path)
         localmodpaths[modname] = source_path
     localmods = populateextmods(localmodpaths)
     for localmodname, source_path in sorted(localmodpaths.items()):
         if not isinstance(localmodname, bytes):
             # This is only safe because all hg's files are ascii
             localmodname = localmodname.encode('ascii')
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(
                         src, modname, name, localmods, ignore_nested=True
                     )
                 )
                 for error, lineno in verify_import_convention(
                     modname, src, localmods
                 ):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print(
                     '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
                 )
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

contrib/memory.py

@@ -1,37 +1,35 b''
 # memory.py - track memory usage
 #
 # Copyright 2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''helper extension to measure memory usage
 
 Reads current and peak memory usage from ``/proc/self/status`` and
 prints it to ``stderr`` on exit.
 '''
 
-from __future__ import absolute_import
-
 
 def memusage(ui):
     """Report memory usage of the current process."""
     result = {'peak': 0, 'rss': 0}
     with open('/proc/self/status', 'r') as status:
         # This will only work on systems with a /proc file system
         # (like Linux).
         for line in status:
             parts = line.split()
             key = parts[0][2:-1].lower()
             if key in result:
                 result[key] = int(parts[1])
     ui.write_err(
         ", ".join(
             ["%s: %.1f MiB" % (k, v / 1024.0) for k, v in result.iteritems()]
         )
         + "\n"
     )
 
 
 def extsetup(ui):
     ui.atexit(memusage, ui)

contrib/perf-utils/perf-revlog-write-plot.py

@@ -1,126 +1,125 b''
 #!/usr/bin/env python3
 #
 #  Copyright 2018 Paul Morelle <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # This script use the output of `hg perfrevlogwrite -T json --details` to draw
 # various plot related to write performance in a revlog
 #
 # usage: perf-revlog-write-plot.py details.json
-from __future__ import absolute_import, print_function
 import json
 import re
 
 import numpy as np
 import scipy.signal
 
 from matplotlib import (
     pyplot as plt,
     ticker as mticker,
 )
 
 
 def plot(data, title=None):
     items = {}
     re_title = re.compile(r'^revisions #\d+ of \d+, rev (\d+)$')
     for item in data:
         m = re_title.match(item['title'])
         if m is None:
             continue
 
         rev = int(m.group(1))
         items[rev] = item
 
     min_rev = min(items.keys())
     max_rev = max(items.keys())
     ary = np.empty((2, max_rev - min_rev + 1))
     for rev, item in items.items():
         ary[0][rev - min_rev] = rev
         ary[1][rev - min_rev] = item['wall']
 
     fig = plt.figure()
     comb_plt = fig.add_subplot(211)
     other_plt = fig.add_subplot(212)
 
     comb_plt.plot(
         ary[0], np.cumsum(ary[1]), color='red', linewidth=1, label='comb'
     )
 
     plots = []
     p = other_plt.plot(ary[0], ary[1], color='red', linewidth=1, label='wall')
     plots.append(p)
 
     colors = {
         10: ('green', 'xkcd:grass green'),
         100: ('blue', 'xkcd:bright blue'),
         1000: ('purple', 'xkcd:dark pink'),
     }
     for n, color in colors.items():
         avg_n = np.convolve(ary[1], np.full(n, 1.0 / n), 'valid')
         p = other_plt.plot(
             ary[0][n - 1 :],
             avg_n,
             color=color[0],
             linewidth=1,
             label='avg time last %d' % n,
         )
         plots.append(p)
 
         med_n = scipy.signal.medfilt(ary[1], n + 1)
         p = other_plt.plot(
             ary[0],
             med_n,
             color=color[1],
             linewidth=1,
             label='median time last %d' % n,
         )
         plots.append(p)
 
     formatter = mticker.ScalarFormatter()
     formatter.set_scientific(False)
     formatter.set_useOffset(False)
 
     comb_plt.grid()
     comb_plt.xaxis.set_major_formatter(formatter)
     comb_plt.legend()
 
     other_plt.grid()
     other_plt.xaxis.set_major_formatter(formatter)
     leg = other_plt.legend()
     leg2plot = {}
     for legline, plot in zip(leg.get_lines(), plots):
         legline.set_picker(5)
         leg2plot[legline] = plot
 
     def onpick(event):
         legline = event.artist
         plot = leg2plot[legline]
         visible = not plot[0].get_visible()
         for l in plot:
             l.set_visible(visible)
 
         if visible:
             legline.set_alpha(1.0)
         else:
             legline.set_alpha(0.2)
         fig.canvas.draw()
 
     if title is not None:
         fig.canvas.set_window_title(title)
     fig.canvas.mpl_connect('pick_event', onpick)
 
     plt.show()
 
 
 if __name__ == '__main__':
     import sys
 
     if len(sys.argv) > 1:
         print('reading from %r' % sys.argv[1])
         with open(sys.argv[1], 'r') as fp:
             plot(json.load(fp), title=sys.argv[1])
     else:
         print('reading from stdin')
         plot(json.load(sys.stdin))

contrib/perf-utils/search-discovery-case

@@ -1,198 +1,197 b''
 #!/usr/bin/env python3
 # Search for interesting discovery instance
 #
 #  search-discovery-case REPO [REPO]…
 #
 # This use a subsetmaker extension (next to this script) to generate a steam of
 # random discovery instance. When interesting case are discovered, information
 # about them are print on the stdout.
-from __future__ import print_function
 
 import json
 import os
 import queue
 import random
 import signal
 import subprocess
 import sys
 import threading
 
 this_script = os.path.abspath(sys.argv[0])
 this_dir = os.path.dirname(this_script)
 hg_dir = os.path.join(this_dir, '..', '..')
 HG_REPO = os.path.normpath(hg_dir)
 HG_BIN = os.path.join(HG_REPO, 'hg')
 
 JOB = int(os.environ.get('NUMBER_OF_PROCESSORS', 8))
 
 
 SLICING = ('scratch', 'randomantichain', 'rev')
 
 
 def nb_revs(repo_path):
     cmd = [
         HG_BIN,
         '--repository',
         repo_path,
         'log',
         '--template',
         '{rev}',
         '--rev',
         'tip',
     ]
     s = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     out, err = s.communicate()
     return int(out)
 
 
 repos = []
 for repo in sys.argv[1:]:
     size = nb_revs(repo)
     repos.append((repo, size))
 
 
 def pick_one(repo):
     pick = random.choice(SLICING)
     seed = random.randint(0, 100000)
     if pick == 'scratch':
         start = int(repo[1] * 0.3)
         end = int(repo[1] * 0.7)
         nb = random.randint(start, end)
         return ('scratch', nb, seed)
     elif pick == 'randomantichain':
         return ('randomantichain', seed)
     elif pick == 'rev':
         start = int(repo[1] * 0.3)
         end = int(repo[1])
         rev = random.randint(start, end)
         return ('rev', rev)
     else:
         assert False
 
 
 done = threading.Event()
 cases = queue.Queue(maxsize=10 * JOB)
 results = queue.Queue()
 
 
 def worker():
     while not done.is_set():
         c = cases.get()
         if c is None:
             return
         try:
             res = process(c)
             results.put((c, res))
         except Exception as exc:
             print('processing-failed: %s %s' % (c, exc), file=sys.stderr)
         c = (c[0], c[2], c[1])
         try:
             res = process(c)
             results.put((c, res))
         except Exception as exc:
             print('processing-failed: %s %s' % (c, exc), file=sys.stderr)
 
 
 SUBSET_PATH = os.path.join(HG_REPO, 'contrib', 'perf-utils', 'subsetmaker.py')
 
 
 CMD_BASE = (
     HG_BIN,
     'debugdiscovery',
     '--template',
     'json',
     '--config',
     'extensions.subset=%s' % SUBSET_PATH,
 )
 #    '--local-as-revs "$left" --local-as-revs "$right"'
 #    > /data/discovery-references/results/disco-mozilla-unified-$1-$2.txt
 #        )
 
 
 def to_revsets(case):
     t = case[0]
     if t == 'scratch':
         return 'not scratch(all(), %d, "%d")' % (case[1], case[2])
     elif t == 'randomantichain':
         return '::randomantichain(all(), "%d")' % case[1]
     elif t == 'rev':
         return '::%d' % case[1]
     else:
         assert False
 
 
 def process(case):
     (repo, left, right) = case
     cmd = list(CMD_BASE)
     cmd.append('-R')
     cmd.append(repo[0])
     cmd.append('--local-as-revs')
     cmd.append(to_revsets(left))
     cmd.append('--remote-as-revs')
     cmd.append(to_revsets(right))
     s = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     out, err = s.communicate()
     return json.loads(out)[0]
 
 
 def interesting_boundary(res):
     """check if a case is interesting or not
 
     For now we are mostly interrested in case were we do multiple roundstrip
     and where the boundary is somewhere in the middle of the undecided set.
 
     Ideally, we would make this configurable, but this is not a focus for now
 
     return None or (round-trip, undecided-common, undecided-missing)
     """
     roundtrips = res["total-roundtrips"]
     if roundtrips <= 1:
         return None
     undecided_common = res["nb-ini_und-common"]
     undecided_missing = res["nb-ini_und-missing"]
     if undecided_common == 0:
         return None
     if undecided_missing == 0:
         return None
     return (roundtrips, undecided_common, undecided_missing)
 
 
 def end(*args, **kwargs):
     done.set()
 
 
 def format_case(case):
     return '-'.join(str(s) for s in case)
 
 
 signal.signal(signal.SIGINT, end)
 
 for i in range(JOB):
     threading.Thread(target=worker).start()
 
 nb_cases = 0
 while not done.is_set():
     repo = random.choice(repos)
     left = pick_one(repo)
     right = pick_one(repo)
     cases.put((repo, left, right))
     while not results.empty():
         # results has a single reader so this is fine
         c, res = results.get_nowait()
         boundary = interesting_boundary(res)
         if boundary is not None:
             print(c[0][0], format_case(c[1]), format_case(c[2]), *boundary)
             sys.stdout.flush()
 
     nb_cases += 1
     if not nb_cases % 100:
         print('[%d cases generated]' % nb_cases, file=sys.stderr)
 
 for i in range(JOB):
     try:
         cases.put_nowait(None)
     except queue.Full:
         pass
 
 print('[%d cases generated]' % nb_cases, file=sys.stderr)
 print('[ouput generation is over]' % nb_cases, file=sys.stderr)

contrib/perf.py

@@ -1,3980 +1,3979 b''
 # perf.py - performance test routines
 '''helper extension to measure performance
 
 Configurations
 ==============
 
 ``perf``
 --------
 
 ``all-timing``
     When set, additional statistics will be reported for each benchmark: best,
     worst, median average. If not set only the best timing is reported
     (default: off).
 
 ``presleep``
   number of second to wait before any group of runs (default: 1)
 
 ``pre-run``
   number of run to perform before starting measurement.
 
 ``profile-benchmark``
   Enable profiling for the benchmarked section.
   (The first iteration is benchmarked)
 
 ``run-limits``
   Control the number of runs each benchmark will perform. The option value
   should be a list of `<time>-<numberofrun>` pairs. After each run the
   conditions are considered in order with the following logic:
 
       If benchmark has been running for <time> seconds, and we have performed
       <numberofrun> iterations, stop the benchmark,
 
   The default value is: `3.0-100, 10.0-3`
 
 ``stub``
     When set, benchmarks will only be run once, useful for testing
     (default: off)
 '''
 
 # "historical portability" policy of perf.py:
 #
 # We have to do:
 # - make perf.py "loadable" with as wide Mercurial version as possible
 #   This doesn't mean that perf commands work correctly with that Mercurial.
 #   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
 # - make historical perf command work correctly with as wide Mercurial
 #   version as possible
 #
 # We have to do, if possible with reasonable cost:
 # - make recent perf command for historical feature work correctly
 #   with early Mercurial
 #
 # We don't have to do:
 # - make perf command for recent feature work correctly with early
 #   Mercurial
 
-from __future__ import absolute_import
 import contextlib
 import functools
 import gc
 import os
 import random
 import shutil
 import struct
 import sys
 import tempfile
 import threading
 import time
 
 import mercurial.revlog
 from mercurial import (
     changegroup,
     cmdutil,
     commands,
     copies,
     error,
     extensions,
     hg,
     mdiff,
     merge,
     util,
 )
 
 # for "historical portability":
 # try to import modules separately (in dict order), and ignore
 # failure, because these aren't available with early Mercurial
 try:
     from mercurial import branchmap  # since 2.5 (or bcee63733aad)
 except ImportError:
     pass
 try:
     from mercurial import obsolete  # since 2.3 (or ad0d6c2b3279)
 except ImportError:
     pass
 try:
     from mercurial import registrar  # since 3.7 (or 37d50250b696)
 
     dir(registrar)  # forcibly load it
 except ImportError:
     registrar = None
 try:
     from mercurial import repoview  # since 2.5 (or 3a6ddacb7198)
 except ImportError:
     pass
 try:
     from mercurial.utils import repoviewutil  # since 5.0
 except ImportError:
     repoviewutil = None
 try:
     from mercurial import scmutil  # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
 try:
     from mercurial import setdiscovery  # since 1.9 (or cb98fed52495)
 except ImportError:
     pass
 
 try:
     from mercurial import profiling
 except ImportError:
     profiling = None
 
 try:
     from mercurial.revlogutils import constants as revlog_constants
 
     perf_rl_kind = (revlog_constants.KIND_OTHER, b'created-by-perf')
 
     def revlog(opener, *args, **kwargs):
         return mercurial.revlog.revlog(opener, perf_rl_kind, *args, **kwargs)
 
 
 except (ImportError, AttributeError):
     perf_rl_kind = None
 
     def revlog(opener, *args, **kwargs):
         return mercurial.revlog.revlog(opener, *args, **kwargs)
 
 
 def identity(a):
     return a
 
 
 try:
     from mercurial import pycompat
 
     getargspec = pycompat.getargspec  # added to module after 4.5
     _byteskwargs = pycompat.byteskwargs  # since 4.1 (or fbc3f73dc802)
     _sysstr = pycompat.sysstr  # since 4.0 (or 2219f4f82ede)
     _bytestr = pycompat.bytestr  # since 4.2 (or b70407bd84d5)
     _xrange = pycompat.xrange  # since 4.8 (or 7eba8f83129b)
     fsencode = pycompat.fsencode  # since 3.9 (or f4a5e0e86a7e)
     if pycompat.ispy3:
         _maxint = sys.maxsize  # per py3 docs for replacing maxint
     else:
         _maxint = sys.maxint
 except (NameError, ImportError, AttributeError):
     import inspect
 
     getargspec = inspect.getargspec
     _byteskwargs = identity
     _bytestr = str
     fsencode = identity  # no py3 support
     _maxint = sys.maxint  # no py3 support
     _sysstr = lambda x: x  # no py3 support
     _xrange = xrange
 
 try:
     # 4.7+
     queue = pycompat.queue.Queue
 except (NameError, AttributeError, ImportError):
     # <4.7.
     try:
         queue = pycompat.queue
     except (NameError, AttributeError, ImportError):
         import Queue as queue
 
 try:
     from mercurial import logcmdutil
 
     makelogtemplater = logcmdutil.maketemplater
 except (AttributeError, ImportError):
     try:
         makelogtemplater = cmdutil.makelogtemplater
     except (AttributeError, ImportError):
         makelogtemplater = None
 
 # for "historical portability":
 # define util.safehasattr forcibly, because util.safehasattr has been
 # available since 1.9.3 (or 94b200a11cf7)
 _undefined = object()
 
 
 def safehasattr(thing, attr):
     return getattr(thing, _sysstr(attr), _undefined) is not _undefined
 
 
 setattr(util, 'safehasattr', safehasattr)
 
 # for "historical portability":
 # define util.timer forcibly, because util.timer has been available
 # since ae5d60bb70c9
 if safehasattr(time, 'perf_counter'):
     util.timer = time.perf_counter
 elif os.name == b'nt':
     util.timer = time.clock
 else:
     util.timer = time.time
 
 # for "historical portability":
 # use locally defined empty option list, if formatteropts isn't
 # available, because commands.formatteropts has been available since
 # 3.2 (or 7a7eed5176a4), even though formatting itself has been
 # available since 2.2 (or ae5f92e154d3)
 formatteropts = getattr(
     cmdutil, "formatteropts", getattr(commands, "formatteropts", [])
 )
 
 # for "historical portability":
 # use locally defined option list, if debugrevlogopts isn't available,
 # because commands.debugrevlogopts has been available since 3.7 (or
 # 5606f7d0d063), even though cmdutil.openrevlog() has been available
 # since 1.9 (or a79fea6b3e77).
 revlogopts = getattr(
     cmdutil,
     "debugrevlogopts",
     getattr(
         commands,
         "debugrevlogopts",
         [
             (b'c', b'changelog', False, b'open changelog'),
             (b'm', b'manifest', False, b'open manifest'),
             (b'', b'dir', False, b'open directory manifest'),
         ],
     ),
 )
 
 cmdtable = {}
 
 # for "historical portability":
 # define parsealiases locally, because cmdutil.parsealiases has been
 # available since 1.5 (or 6252852b4332)
 def parsealiases(cmd):
     return cmd.split(b"|")
 
 
 if safehasattr(registrar, 'command'):
     command = registrar.command(cmdtable)
 elif safehasattr(cmdutil, 'command'):
     command = cmdutil.command(cmdtable)
     if 'norepo' not in getargspec(command).args:
         # for "historical portability":
         # wrap original cmdutil.command, because "norepo" option has
         # been available since 3.1 (or 75a96326cecb)
         _command = command
 
         def command(name, options=(), synopsis=None, norepo=False):
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return _command(name, list(options), synopsis)
 
 
 else:
     # for "historical portability":
     # define "@command" annotation locally, because cmdutil.command
     # has been available since 1.9 (or 2daa5179e73f)
     def command(name, options=(), synopsis=None, norepo=False):
         def decorator(func):
             if synopsis:
                 cmdtable[name] = func, list(options), synopsis
             else:
                 cmdtable[name] = func, list(options)
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return func
 
         return decorator
 
 
 try:
     import mercurial.registrar
     import mercurial.configitems
 
     configtable = {}
     configitem = mercurial.registrar.configitem(configtable)
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'pre-run',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
 except (ImportError, AttributeError):
     pass
 except TypeError:
     # compatibility fix for a11fd395e83f
     # hg version: 5.2
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'pre-run',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
     )
 
 
 def getlen(ui):
     if ui.configbool(b"perf", b"stub", False):
         return lambda x: 1
     return len
 
 
 class noop(object):
     """dummy context manager"""
 
     def __enter__(self):
         pass
 
     def __exit__(self, *args):
         pass
 
 
 NOOPCTX = noop()
 
 
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
     This function exists to gather the creation of formatter in a single
     place instead of duplicating it in all performance commands."""
 
     # enforce an idle period before execution to counteract power management
     # experimental config: perf.presleep
     time.sleep(getint(ui, b"perf", b"presleep", 1))
 
     if opts is None:
         opts = {}
     # redirect all to stderr unless buffer api is in use
     if not ui._buffers:
         ui = ui.copy()
         uifout = safeattrsetter(ui, b'fout', ignoremissing=True)
         if uifout:
             # for "historical portability":
             # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d)
             uifout.set(ui.ferr)
 
     # get a formatter
     uiformatter = getattr(ui, 'formatter', None)
     if uiformatter:
         fm = uiformatter(b'perf', opts)
     else:
         # for "historical portability":
         # define formatter locally, because ui.formatter has been
         # available since 2.2 (or ae5f92e154d3)
         from mercurial import node
 
         class defaultformatter(object):
             """Minimized composition of baseformatter and plainformatter"""
 
             def __init__(self, ui, topic, opts):
                 self._ui = ui
                 if ui.debugflag:
                     self.hexfunc = node.hex
                 else:
                     self.hexfunc = node.short
 
             def __nonzero__(self):
                 return False
 
             __bool__ = __nonzero__
 
             def startitem(self):
                 pass
 
             def data(self, **data):
                 pass
 
             def write(self, fields, deftext, *fielddata, **opts):
                 self._ui.write(deftext % fielddata, **opts)
 
             def condwrite(self, cond, fields, deftext, *fielddata, **opts):
                 if cond:
                     self._ui.write(deftext % fielddata, **opts)
 
             def plain(self, text, **opts):
                 self._ui.write(text, **opts)
 
             def end(self):
                 pass
 
         fm = defaultformatter(ui, b'perf', opts)
 
     # stub function, runs code only once instead of in a loop
     # experimental config: perf.stub
     if ui.configbool(b"perf", b"stub", False):
         return functools.partial(stub_timer, fm), fm
 
     # experimental config: perf.all-timing
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # experimental config: perf.run-limits
     limitspec = ui.configlist(b"perf", b"run-limits", [])
     limits = []
     for item in limitspec:
         parts = item.split(b'-', 1)
         if len(parts) < 2:
             ui.warn((b'malformatted run limit entry, missing "-": %s\n' % item))
             continue
         try:
             time_limit = float(_sysstr(parts[0]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         try:
             run_limit = int(_sysstr(parts[1]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         limits.append((time_limit, run_limit))
     if not limits:
         limits = DEFAULTLIMITS
 
     profiler = None
     if profiling is not None:
         if ui.configbool(b"perf", b"profile-benchmark", False):
             profiler = profiling.profile(ui)
 
     prerun = getint(ui, b"perf", b"pre-run", 0)
     t = functools.partial(
         _timer,
         fm,
         displayall=displayall,
         limits=limits,
         prerun=prerun,
         profiler=profiler,
     )
     return t, fm
 
 
 def stub_timer(fm, func, setup=None, title=None):
     if setup is not None:
         setup()
     func()
 
 
 @contextlib.contextmanager
 def timeone():
     r = []
     ostart = os.times()
     cstart = util.timer()
     yield r
     cstop = util.timer()
     ostop = os.times()
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1] - a[1]))
 
 
 # list of stop condition (elapsed time, minimal run count)
 DEFAULTLIMITS = (
     (3.0, 100),
     (10.0, 3),
 )
 
 
 def _timer(
     fm,
     func,
     setup=None,
     title=None,
     displayall=False,
     limits=DEFAULTLIMITS,
     prerun=0,
     profiler=None,
 ):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
     if profiler is None:
         profiler = NOOPCTX
     for i in range(prerun):
         if setup is not None:
             setup()
         func()
     keepgoing = True
     while keepgoing:
         if setup is not None:
             setup()
         with profiler:
             with timeone() as item:
                 r = func()
         profiler = NOOPCTX
         count += 1
         results.append(item[0])
         cstop = util.timer()
         # Look for a stop condition.
         elapsed = cstop - begin
         for t, mincount in limits:
             if elapsed >= t and count >= mincount:
                 keepgoing = False
                 break
 
     formatone(fm, results, title=title, result=r, displayall=displayall)
 
 
 def formatone(fm, timings, title=None, result=None, displayall=False):
 
     count = len(timings)
 
     fm.startitem()
 
     if title:
         fm.write(b'title', b'! %s\n', title)
     if result:
         fm.write(b'result', b'! result: %s\n', result)
 
     def display(role, entry):
         prefix = b''
         if role != b'best':
             prefix = b'%s.' % role
         fm.plain(b'!')
         fm.write(prefix + b'wall', b' wall %f', entry[0])
         fm.write(prefix + b'comb', b' comb %f', entry[1] + entry[2])
         fm.write(prefix + b'user', b' user %f', entry[1])
         fm.write(prefix + b'sys', b' sys %f', entry[2])
         fm.write(prefix + b'count', b' (%s of %%d)' % role, count)
         fm.plain(b'\n')
 
     timings.sort()
     min_val = timings[0]
     display(b'best', min_val)
     if displayall:
         max_val = timings[-1]
         display(b'max', max_val)
         avg = tuple([sum(x) / count for x in zip(*timings)])
         display(b'avg', avg)
         median = timings[len(timings) // 2]
         display(b'median', median)
 
 
 # utilities for historical portability
 
 
 def getint(ui, section, name, default):
     # for "historical portability":
     # ui.configint has been available since 1.9 (or fa2b596db182)
     v = ui.config(section, name, None)
     if v is None:
         return default
     try:
         return int(v)
     except ValueError:
         raise error.ConfigError(
             b"%s.%s is not an integer ('%s')" % (section, name, v)
         )
 
 
 def safeattrsetter(obj, name, ignoremissing=False):
     """Ensure that 'obj' has 'name' attribute before subsequent setattr
 
     This function is aborted, if 'obj' doesn't have 'name' attribute
     at runtime. This avoids overlooking removal of an attribute, which
     breaks assumption of performance measurement, in the future.
 
     This function returns the object to (1) assign a new value, and
     (2) restore an original value to the attribute.
 
     If 'ignoremissing' is true, missing 'name' attribute doesn't cause
     abortion, and this function returns None. This is useful to
     examine an attribute, which isn't ensured in all Mercurial
     versions.
     """
     if not util.safehasattr(obj, name):
         if ignoremissing:
             return None
         raise error.Abort(
             (
                 b"missing attribute %s of %s might break assumption"
                 b" of performance measurement"
             )
             % (name, obj)
         )
 
     origvalue = getattr(obj, _sysstr(name))
 
     class attrutil(object):
         def set(self, newvalue):
             setattr(obj, _sysstr(name), newvalue)
 
         def restore(self):
             setattr(obj, _sysstr(name), origvalue)
 
     return attrutil()
 
 
 # utilities to examine each internal API changes
 
 
 def getbranchmapsubsettable():
     # for "historical portability":
     # subsettable is defined in:
     # - branchmap since 2.9 (or 175c6fd8cacc)
     # - repoview since 2.5 (or 59a9f18d4587)
     # - repoviewutil since 5.0
     for mod in (branchmap, repoview, repoviewutil):
         subsettable = getattr(mod, 'subsettable', None)
         if subsettable:
             return subsettable
 
     # bisecting in bcee63733aad::59a9f18d4587 can reach here (both
     # branchmap and repoview modules exist, but subsettable attribute
     # doesn't)
     raise error.Abort(
         b"perfbranchmap not available with this Mercurial",
         hint=b"use 2.5 or later",
     )
 
 
 def getsvfs(repo):
     """Return appropriate object to access files under .hg/store"""
     # for "historical portability":
     # repo.svfs has been available since 2.3 (or 7034365089bf)
     svfs = getattr(repo, 'svfs', None)
     if svfs:
         return svfs
     else:
         return getattr(repo, 'sopener')
 
 
 def getvfs(repo):
     """Return appropriate object to access files under .hg"""
     # for "historical portability":
     # repo.vfs has been available since 2.3 (or 7034365089bf)
     vfs = getattr(repo, 'vfs', None)
     if vfs:
         return vfs
     else:
         return getattr(repo, 'opener')
 
 
 def repocleartagscachefunc(repo):
     """Return the function to clear tags cache according to repo internal API"""
     if util.safehasattr(repo, b'_tagscache'):  # since 2.0 (or 9dca7653b525)
         # in this case, setattr(repo, '_tagscache', None) or so isn't
         # correct way to clear tags cache, because existing code paths
         # expect _tagscache to be a structured object.
         def clearcache():
             # _tagscache has been filteredpropertycache since 2.5 (or
             # 98c867ac1330), and delattr() can't work in such case
             if '_tagscache' in vars(repo):
                 del repo.__dict__['_tagscache']
 
         return clearcache
 
     repotags = safeattrsetter(repo, b'_tags', ignoremissing=True)
     if repotags:  # since 1.4 (or 5614a628d173)
         return lambda: repotags.set(None)
 
     repotagscache = safeattrsetter(repo, b'tagscache', ignoremissing=True)
     if repotagscache:  # since 0.6 (or d7df759d0e97)
         return lambda: repotagscache.set(None)
 
     # Mercurial earlier than 0.6 (or d7df759d0e97) logically reaches
     # this point, but it isn't so problematic, because:
     # - repo.tags of such Mercurial isn't "callable", and repo.tags()
     #   in perftags() causes failure soon
     # - perf.py itself has been available since 1.1 (or eb240755386d)
     raise error.Abort(b"tags API of this hg command is unknown")
 
 
 # utilities to clear cache
 
 
 def clearfilecache(obj, attrname):
     unfiltered = getattr(obj, 'unfiltered', None)
     if unfiltered is not None:
         obj = obj.unfiltered()
     if attrname in vars(obj):
         delattr(obj, attrname)
     obj._filecache.pop(attrname, None)
 
 
 def clearchangelog(repo):
     if repo is not repo.unfiltered():
         object.__setattr__(repo, '_clcachekey', None)
         object.__setattr__(repo, '_clcache', None)
     clearfilecache(repo.unfiltered(), 'changelog')
 
 
 # perf commands
 
 
 @command(b'perf::walk|perfwalk', formatteropts)
 def perfwalk(ui, repo, *pats, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     m = scmutil.match(repo[None], pats, {})
     timer(
         lambda: len(
             list(
                 repo.dirstate.walk(m, subrepos=[], unknown=True, ignored=False)
             )
         )
     )
     fm.end()
 
 
 @command(b'perf::annotate|perfannotate', formatteropts)
 def perfannotate(ui, repo, f, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     fc = repo[b'.'][f]
     timer(lambda: len(fc.annotate(True)))
     fm.end()
 
 
 @command(
     b'perf::status|perfstatus',
     [
         (b'u', b'unknown', False, b'ask status to look for unknown files'),
         (b'', b'dirstate', False, b'benchmark the internal dirstate call'),
     ]
     + formatteropts,
 )
 def perfstatus(ui, repo, **opts):
     """benchmark the performance of a single status call
 
     The repository data are preserved between each call.
 
     By default, only the status of the tracked file are requested. If
     `--unknown` is passed, the "unknown" files are also tracked.
     """
     opts = _byteskwargs(opts)
     # m = match.always(repo.root, repo.getcwd())
     # timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer, fm = gettimer(ui, opts)
     if opts[b'dirstate']:
         dirstate = repo.dirstate
         m = scmutil.matchall(repo)
         unknown = opts[b'unknown']
 
         def status_dirstate():
             s = dirstate.status(
                 m, subrepos=[], ignored=False, clean=False, unknown=unknown
             )
             sum(map(bool, s))
 
         timer(status_dirstate)
     else:
         timer(lambda: sum(map(len, repo.status(unknown=opts[b'unknown']))))
     fm.end()
 
 
 @command(b'perf::addremove|perfaddremove', formatteropts)
 def perfaddremove(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     try:
         oldquiet = repo.ui.quiet
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         opts[b'dry_run'] = True
         if 'uipathfn' in getargspec(scmutil.addremove).args:
             uipathfn = scmutil.getuipathfn(repo)
             timer(lambda: scmutil.addremove(repo, matcher, b"", uipathfn, opts))
         else:
             timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
 
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, b'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, b'_nodecache'):
         # <= hg-5.2
         from mercurial.node import nullid, nullrev
 
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 
 @command(b'perf::heads|perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
     """benchmark the computation of a changelog heads"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def s():
         clearcaches(cl)
 
     def d():
         len(cl.headrevs())
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::tags|perftags',
     formatteropts
     + [
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ],
 )
 def perftags(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repocleartagscache = repocleartagscachefunc(repo)
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
             clearfilecache(repo.unfiltered(), 'manifest')
         repocleartagscache()
 
     def t():
         return len(repo.tags())
 
     timer(t, setup=s)
     fm.end()
 
 
 @command(b'perf::ancestors|perfancestors', formatteropts)
 def perfancestors(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     heads = repo.changelog.headrevs()
 
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ancestorset|perfancestorset', formatteropts)
 def perfancestorset(ui, repo, revset, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
 
     def d():
         s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::discovery|perfdiscovery', formatteropts, b'PATH')
 def perfdiscovery(ui, repo, path, **opts):
     """benchmark discovery between local repo and the peer at given path"""
     repos = [repo, None]
     timer, fm = gettimer(ui, opts)
 
     try:
         from mercurial.utils.urlutil import get_unique_pull_path
 
         path = get_unique_pull_path(b'perfdiscovery', repo, ui, path)[0]
     except ImportError:
         path = ui.expandpath(path)
 
     def s():
         repos[1] = hg.peer(ui, opts, path)
 
     def d():
         setdiscovery.findcommonheads(ui, *repos)
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::bookmarks|perfbookmarks',
     formatteropts
     + [
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ],
 )
 def perfbookmarks(ui, repo, **opts):
     """benchmark parsing bookmarks from disk to memory"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
         clearfilecache(repo, b'_bookmarks')
 
     def d():
         repo._bookmarks
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(b'perf::bundleread|perfbundleread', formatteropts, b'BUNDLE')
 def perfbundleread(ui, repo, bundlepath, **opts):
     """Benchmark reading of bundle files.
 
     This command is meant to isolate the I/O part of bundle reading as
     much as possible.
     """
     from mercurial import (
         bundle2,
         exchange,
         streamclone,
     )
 
     opts = _byteskwargs(opts)
 
     def makebench(fn):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 fn(bundle)
 
         return run
 
     def makereadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 while bundle.read(size):
                     pass
 
         return run
 
     def makestdioread(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 while fh.read(size):
                     pass
 
         return run
 
     # bundle1
 
     def deltaiter(bundle):
         for delta in bundle.deltaiter():
             pass
 
     def iterchunks(bundle):
         for chunk in bundle.getchunks():
             pass
 
     # bundle2
 
     def forwardchunks(bundle):
         for chunk in bundle._forwardchunks():
             pass
 
     def iterparts(bundle):
         for part in bundle.iterparts():
             pass
 
     def iterpartsseekable(bundle):
         for part in bundle.iterparts(seekable=True):
             pass
 
     def seek(bundle):
         for part in bundle.iterparts(seekable=True):
             part.seek(0, os.SEEK_END)
 
     def makepartreadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 for part in bundle.iterparts():
                     while part.read(size):
                         pass
 
         return run
 
     benches = [
         (makestdioread(8192), b'read(8k)'),
         (makestdioread(16384), b'read(16k)'),
         (makestdioread(32768), b'read(32k)'),
         (makestdioread(131072), b'read(128k)'),
     ]
 
     with open(bundlepath, b'rb') as fh:
         bundle = exchange.readbundle(ui, fh, bundlepath)
 
         if isinstance(bundle, changegroup.cg1unpacker):
             benches.extend(
                 [
                     (makebench(deltaiter), b'cg1 deltaiter()'),
                     (makebench(iterchunks), b'cg1 getchunks()'),
                     (makereadnbytes(8192), b'cg1 read(8k)'),
                     (makereadnbytes(16384), b'cg1 read(16k)'),
                     (makereadnbytes(32768), b'cg1 read(32k)'),
                     (makereadnbytes(131072), b'cg1 read(128k)'),
                 ]
             )
         elif isinstance(bundle, bundle2.unbundle20):
             benches.extend(
                 [
                     (makebench(forwardchunks), b'bundle2 forwardchunks()'),
                     (makebench(iterparts), b'bundle2 iterparts()'),
                     (
                         makebench(iterpartsseekable),
                         b'bundle2 iterparts() seekable',
                     ),
                     (makebench(seek), b'bundle2 part seek()'),
                     (makepartreadnbytes(8192), b'bundle2 part read(8k)'),
                     (makepartreadnbytes(16384), b'bundle2 part read(16k)'),
                     (makepartreadnbytes(32768), b'bundle2 part read(32k)'),
                     (makepartreadnbytes(131072), b'bundle2 part read(128k)'),
                 ]
             )
         elif isinstance(bundle, streamclone.streamcloneapplier):
             raise error.Abort(b'stream clone bundles not supported')
         else:
             raise error.Abort(b'unhandled bundle type: %s' % type(bundle))
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::changegroupchangelog|perfchangegroupchangelog',
     formatteropts
     + [
         (b'', b'cgversion', b'02', b'changegroup version'),
         (b'r', b'rev', b'', b'revisions to add to changegroup'),
     ],
 )
 def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
     """Benchmark producing a changelog group for a changegroup.
 
     This measures the time spent processing the changelog during a
     bundle operation. This occurs during `hg bundle` and on a server
     processing a `getbundle` wire protocol request (handles clones
     and pull requests).
 
     By default, all revisions are added to the changegroup.
     """
     opts = _byteskwargs(opts)
     cl = repo.changelog
     nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
     bundler = changegroup.getbundler(cgversion, repo)
 
     def d():
         state, chunks = bundler._generatechangelog(cl, nodes)
         for chunk in chunks:
             pass
 
     timer, fm = gettimer(ui, opts)
 
     # Terminal printing can interfere with timing. So disable it.
     with ui.configoverride({(b'progress', b'disable'): True}):
         timer(d)
 
     fm.end()
 
 
 @command(b'perf::dirs|perfdirs', formatteropts)
 def perfdirs(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
 
     def d():
         dirstate.hasdir(b'a')
         try:
             del dirstate._map._dirs
         except AttributeError:
             pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::dirstate|perfdirstate',
     [
         (
             b'',
             b'iteration',
             None,
             b'benchmark a full iteration for the dirstate',
         ),
         (
             b'',
             b'contains',
             None,
             b'benchmark a large amount of `nf in dirstate` calls',
         ),
     ]
     + formatteropts,
 )
 def perfdirstate(ui, repo, **opts):
     """benchmap the time of various distate operations
 
     By default benchmark the time necessary to load a dirstate from scratch.
     The dirstate is loaded to the point were a "contains" request can be
     answered.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     b"a" in repo.dirstate
 
     if opts[b'iteration'] and opts[b'contains']:
         msg = b'only specify one of --iteration or --contains'
         raise error.Abort(msg)
 
     if opts[b'iteration']:
         setup = None
         dirstate = repo.dirstate
 
         def d():
             for f in dirstate:
                 pass
 
     elif opts[b'contains']:
         setup = None
         dirstate = repo.dirstate
         allfiles = list(dirstate)
         # also add file path that will be "missing" from the dirstate
         allfiles.extend([f[::-1] for f in allfiles])
 
         def d():
             for f in allfiles:
                 f in dirstate
 
     else:
 
         def setup():
             repo.dirstate.invalidate()
 
         def d():
             b"a" in repo.dirstate
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatedirs|perfdirstatedirs', formatteropts)
 def perfdirstatedirs(ui, repo, **opts):
     """benchmap a 'dirstate.hasdir' call from an empty `dirs` cache"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo.dirstate.hasdir(b"a")
 
     def setup():
         try:
             del repo.dirstate._map._dirs
         except AttributeError:
             pass
 
     def d():
         repo.dirstate.hasdir(b"a")
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatefoldmap|perfdirstatefoldmap', formatteropts)
 def perfdirstatefoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.filefoldmap.get()` request
 
     The dirstate filefoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.filefoldmap.get(b'a')
 
     def setup():
         del dirstate._map.filefoldmap
 
     def d():
         dirstate._map.filefoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirfoldmap|perfdirfoldmap', formatteropts)
 def perfdirfoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.dirfoldmap.get()` request
 
     The dirstate dirfoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.dirfoldmap.get(b'a')
 
     def setup():
         del dirstate._map.dirfoldmap
         try:
             del dirstate._map._dirs
         except AttributeError:
             pass
 
     def d():
         dirstate._map.dirfoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatewrite|perfdirstatewrite', formatteropts)
 def perfdirstatewrite(ui, repo, **opts):
     """benchmap the time it take to write a dirstate on disk"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ds = repo.dirstate
     b"a" in ds
 
     def setup():
         ds._dirty = True
 
     def d():
         ds.write(repo.currenttransaction())
 
     timer(d, setup=setup)
     fm.end()
 
 
 def _getmergerevs(repo, opts):
     """parse command argument to return rev involved in merge
 
     input: options dictionnary with `rev`, `from` and `bse`
     output: (localctx, otherctx, basectx)
     """
     if opts[b'from']:
         fromrev = scmutil.revsingle(repo, opts[b'from'])
         wctx = repo[fromrev]
     else:
         wctx = repo[None]
         # we don't want working dir files to be stat'd in the benchmark, so
         # prime that cache
         wctx.dirty()
     rctx = scmutil.revsingle(repo, opts[b'rev'], opts[b'rev'])
     if opts[b'base']:
         fromrev = scmutil.revsingle(repo, opts[b'base'])
         ancestor = repo[fromrev]
     else:
         ancestor = wctx.ancestor(rctx)
     return (wctx, rctx, ancestor)
 
 
 @command(
     b'perf::mergecalculate|perfmergecalculate',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecalculate(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         merge.calculateupdates(
             repo,
             wctx,
             rctx,
             [ancestor],
             branchmerge=False,
             force=False,
             acceptremote=True,
             followcopies=True,
         )
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::mergecopies|perfmergecopies',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecopies(ui, repo, **opts):
     """measure runtime of `copies.mergecopies`"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         copies.mergecopies(repo, wctx, rctx, ancestor)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::pathcopies|perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
     """benchmark the copy tracing logic"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
     ctx2 = scmutil.revsingle(repo, rev2, rev2)
 
     def d():
         copies.pathcopies(ctx1, ctx2)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::phases|perfphases',
     [
         (b'', b'full', False, b'include file reading time too'),
     ],
     b"",
 )
 def perfphases(ui, repo, **opts):
     """benchmark phasesets computation"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     _phases = repo._phasecache
     full = opts.get(b'full')
 
     def d():
         phases = _phases
         if full:
             clearfilecache(repo, b'_phasecache')
             phases = repo._phasecache
         phases.invalidate()
         phases.loadphaserevs(repo)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::phasesremote|perfphasesremote', [], b"[DEST]")
 def perfphasesremote(ui, repo, dest=None, **opts):
     """benchmark time needed to analyse phases of the remote server"""
     from mercurial.node import bin
     from mercurial import (
         exchange,
         hg,
         phases,
     )
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     path = ui.getpath(dest, default=(b'default-push', b'default'))
     if not path:
         raise error.Abort(
             b'default repository not configured!',
             hint=b"see 'hg help config.paths'",
         )
     dest = path.pushloc or path.loc
     ui.statusnoi18n(b'analysing phase of %s\n' % util.hidepassword(dest))
     other = hg.peer(repo, opts, dest)
 
     # easier to perform discovery through the operation
     op = exchange.pushoperation(repo, other)
     exchange._pushdiscoverychangeset(op)
 
     remotesubset = op.fallbackheads
 
     with other.commandexecutor() as e:
         remotephases = e.callcommand(
             b'listkeys', {b'namespace': b'phases'}
         ).result()
     del other
     publishing = remotephases.get(b'publishing', False)
     if publishing:
         ui.statusnoi18n(b'publishing: yes\n')
     else:
         ui.statusnoi18n(b'publishing: no\n')
 
     has_node = getattr(repo.changelog.index, 'has_node', None)
     if has_node is None:
         has_node = repo.changelog.nodemap.__contains__
     nonpublishroots = 0
     for nhex, phase in remotephases.iteritems():
         if nhex == b'publishing':  # ignore data related to publish option
             continue
         node = bin(nhex)
         if has_node(node) and int(phase):
             nonpublishroots += 1
     ui.statusnoi18n(b'number of roots: %d\n' % len(remotephases))
     ui.statusnoi18n(b'number of known non public roots: %d\n' % nonpublishroots)
 
     def d():
         phases.remotephasessummary(repo, remotesubset, remotephases)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::manifest|perfmanifest',
     [
         (b'm', b'manifest-rev', False, b'Look up a manifest node revision'),
         (b'', b'clear-disk', False, b'clear on-disk caches too'),
     ]
     + formatteropts,
     b'REV|NODE',
 )
 def perfmanifest(ui, repo, rev, manifest_rev=False, clear_disk=False, **opts):
     """benchmark the time to read a manifest from disk and return a usable
     dict-like object
 
     Manifest caches are cleared before retrieval."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     if not manifest_rev:
         ctx = scmutil.revsingle(repo, rev, rev)
         t = ctx.manifestnode()
     else:
         from mercurial.node import bin
 
         if len(rev) == 40:
             t = bin(rev)
         else:
             try:
                 rev = int(rev)
 
                 if util.safehasattr(repo.manifestlog, b'getstorage'):
                     t = repo.manifestlog.getstorage(b'').node(rev)
                 else:
                     t = repo.manifestlog._revlog.lookup(rev)
             except ValueError:
                 raise error.Abort(
                     b'manifest revision must be integer or full node'
                 )
 
     def d():
         repo.manifestlog.clearcaches(clear_persisted_data=clear_disk)
         repo.manifestlog[t].read()
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::changeset|perfchangeset', formatteropts)
 def perfchangeset(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     n = scmutil.revsingle(repo, rev).node()
 
     def d():
         repo.changelog.read(n)
         # repo.changelog._cache = None
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ignore|perfignore', formatteropts)
 def perfignore(ui, repo, **opts):
     """benchmark operation related to computing ignore"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
 
     def setupone():
         dirstate.invalidate()
         clearfilecache(dirstate, b'_ignore')
 
     def runone():
         dirstate._ignore
 
     timer(runone, setup=setupone, title=b"load")
     fm.end()
 
 
 @command(
     b'perf::index|perfindex',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'no-lookup', None, b'do not revision lookup post creation'),
     ]
     + formatteropts,
 )
 def perfindex(ui, repo, **opts):
     """benchmark index creation time followed by a lookup
 
     The default is to look `tip` up. Depending on the index implementation,
     the revision looked up can matters. For example, an implementation
     scanning the index will have a faster lookup time for `--rev tip` than for
     `--rev 0`. The number of looked up revisions and their order can also
     matters.
 
     Example of useful set to test:
 
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     It is not currently possible to check for lookup of a missing node. For
     deeper lookup benchmarking, checkout the `perfnodemap` command."""
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     if opts[b'no_lookup']:
         if opts['rev']:
             raise error.Abort('--no-lookup and --rev are mutually exclusive')
         nodes = []
     elif not opts[b'rev']:
         nodes = [repo[b"tip"].node()]
     else:
         revs = scmutil.revrange(repo, opts[b'rev'])
         cl = repo.changelog
         nodes = [cl.node(r) for r in revs]
 
     unfi = repo.unfiltered()
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
 
     def setup():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
 
     def d():
         cl = makecl(unfi)
         for n in nodes:
             cl.rev(n)
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(
     b'perf::nodemap|perfnodemap',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ]
     + formatteropts,
 )
 def perfnodemap(ui, repo, **opts):
     """benchmark the time necessary to look up revision from a cold nodemap
 
     Depending on the implementation, the amount and order of revision we look
     up can varies. Example of useful set to test:
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     The command currently focus on valid binary lookup. Benchmarking for
     hexlookup, prefix lookup and missing lookup would also be valuable.
     """
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
 
     unfi = repo.unfiltered()
     clearcaches = opts[b'clear_caches']
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
     if not opts[b'rev']:
         raise error.Abort(b'use --rev to specify revisions to look up')
     revs = scmutil.revrange(repo, opts[b'rev'])
     cl = repo.changelog
     nodes = [cl.node(r) for r in revs]
 
     # use a list to pass reference to a nodemap from one closure to the next
     nodeget = [None]
 
     def setnodeget():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
         cl = makecl(unfi)
         if util.safehasattr(cl.index, 'get_rev'):
             nodeget[0] = cl.index.get_rev
         else:
             nodeget[0] = cl.nodemap.get
 
     def d():
         get = nodeget[0]
         for n in nodes:
             get(n)
 
     setup = None
     if clearcaches:
 
         def setup():
             setnodeget()
 
     else:
         setnodeget()
         d()  # prewarm the data structure
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::startup|perfstartup', formatteropts)
 def perfstartup(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def d():
         if os.name != 'nt':
             os.system(
                 b"HGRCPATH= %s version -q > /dev/null" % fsencode(sys.argv[0])
             )
         else:
             os.environ['HGRCPATH'] = r' '
             os.system("%s version -q > NUL" % sys.argv[0])
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::parents|perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
     """benchmark the time necessary to fetch one changeset's parents.
 
     The fetch is done using the `node identifier`, traversing all object layers
     from the repository object. The first N revisions will be used for this
     benchmark. N is controlled by the ``perf.parentscount`` config option
     (default: 1000).
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
     # experimental config: perf.parentscount
     count = getint(ui, b"perf", b"parentscount", 1000)
     if len(repo.changelog) < count:
         raise error.Abort(b"repo needs %d commits for this test" % count)
     repo = repo.unfiltered()
     nl = [repo.changelog.node(i) for i in _xrange(count)]
 
     def d():
         for n in nl:
             repo.changelog.parents(n)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ctxfiles|perfctxfiles', formatteropts)
 def perfctxfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
 
     def d():
         len(repo[x].files())
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::rawfiles|perfrawfiles', formatteropts)
 def perfrawfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def d():
         len(cl.read(x)[3])
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::lookup|perflookup', formatteropts)
 def perflookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: len(repo.lookup(rev)))
     fm.end()
 
 
 @command(
     b'perf::linelogedits|perflinelogedits',
     [
         (b'n', b'edits', 10000, b'number of edits'),
         (b'', b'max-hunk-lines', 10, b'max lines in a hunk'),
     ],
     norepo=True,
 )
 def perflinelogedits(ui, **opts):
     from mercurial import linelog
 
     opts = _byteskwargs(opts)
 
     edits = opts[b'edits']
     maxhunklines = opts[b'max_hunk_lines']
 
     maxb1 = 100000
     random.seed(0)
     randint = random.randint
     currentlines = 0
     arglist = []
     for rev in _xrange(edits):
         a1 = randint(0, currentlines)
         a2 = randint(a1, min(currentlines, a1 + maxhunklines))
         b1 = randint(0, maxb1)
         b2 = randint(b1, b1 + maxhunklines)
         currentlines += (b2 - b1) - (a2 - a1)
         arglist.append((rev, a1, a2, b1, b2))
 
     def d():
         ll = linelog.linelog()
         for args in arglist:
             ll.replacelines(*args)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perf::revrange|perfrevrange', formatteropts)
 def perfrevrange(ui, repo, *specs, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
     fm.end()
 
 
 @command(b'perf::nodelookup|perfnodelookup', formatteropts)
 def perfnodelookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     import mercurial.revlog
 
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     n = scmutil.revsingle(repo, rev).node()
 
     try:
         cl = revlog(getsvfs(repo), radix=b"00changelog")
     except TypeError:
         cl = revlog(getsvfs(repo), indexfile=b"00changelog.i")
 
     def d():
         cl.rev(n)
         clearcaches(cl)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::log|perflog',
     [(b'', b'rename', False, b'ask log to follow renames')] + formatteropts,
 )
 def perflog(ui, repo, rev=None, **opts):
     opts = _byteskwargs(opts)
     if rev is None:
         rev = []
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(
         lambda: commands.log(
             ui, repo, rev=rev, date=b'', user=b'', copies=opts.get(b'rename')
         )
     )
     ui.popbuffer()
     fm.end()
 
 
 @command(b'perf::moonwalk|perfmoonwalk', formatteropts)
 def perfmoonwalk(ui, repo, **opts):
     """benchmark walking the changelog backwards
 
     This also loads the changelog data for each revision in the changelog.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def moonwalk():
         for i in repo.changelog.revs(start=(len(repo) - 1), stop=-1):
             ctx = repo[i]
             ctx.branch()  # read changelog data (in addition to the index)
 
     timer(moonwalk)
     fm.end()
 
 
 @command(
     b'perf::templating|perftemplating',
     [
         (b'r', b'rev', [], b'revisions to run the template on'),
     ]
     + formatteropts,
 )
 def perftemplating(ui, repo, testedtemplate=None, **opts):
     """test the rendering time of a given template"""
     if makelogtemplater is None:
         raise error.Abort(
             b"perftemplating not available with this Mercurial",
             hint=b"use 4.3 or later",
         )
 
     opts = _byteskwargs(opts)
 
     nullui = ui.copy()
     nullui.fout = open(os.devnull, 'wb')
     nullui.disablepager()
     revs = opts.get(b'rev')
     if not revs:
         revs = [b'all()']
     revs = list(scmutil.revrange(repo, revs))
 
     defaulttemplate = (
         b'{date|shortdate} [{rev}:{node|short}]'
         b' {author|person}: {desc|firstline}\n'
     )
     if testedtemplate is None:
         testedtemplate = defaulttemplate
     displayer = makelogtemplater(nullui, repo, testedtemplate)
 
     def format():
         for r in revs:
             ctx = repo[r]
             displayer.show(ctx)
             displayer.flush(ctx)
 
     timer, fm = gettimer(ui, opts)
     timer(format)
     fm.end()
 
 
 def _displaystats(ui, opts, entries, data):
     # use a second formatter because the data are quite different, not sure
     # how it flies with the templater.
     fm = ui.formatter(b'perf-stats', opts)
     for key, title in entries:
         values = data[key]
         nbvalues = len(data)
         values.sort()
         stats = {
             'key': key,
             'title': title,
             'nbitems': len(values),
             'min': values[0][0],
             '10%': values[(nbvalues * 10) // 100][0],
             '25%': values[(nbvalues * 25) // 100][0],
             '50%': values[(nbvalues * 50) // 100][0],
             '75%': values[(nbvalues * 75) // 100][0],
             '80%': values[(nbvalues * 80) // 100][0],
             '85%': values[(nbvalues * 85) // 100][0],
             '90%': values[(nbvalues * 90) // 100][0],
             '95%': values[(nbvalues * 95) // 100][0],
             '99%': values[(nbvalues * 99) // 100][0],
             'max': values[-1][0],
         }
         fm.startitem()
         fm.data(**stats)
         # make node pretty for the human output
         fm.plain('### %s (%d items)\n' % (title, len(values)))
         lines = [
             'min',
             '10%',
             '25%',
             '50%',
             '75%',
             '80%',
             '85%',
             '90%',
             '95%',
             '99%',
             'max',
         ]
         for l in lines:
             fm.plain('%s: %s\n' % (l, stats[l]))
     fm.end()
 
 
 @command(
     b'perf::helper-mergecopies|perfhelper-mergecopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelpermergecopies(ui, repo, revs=[], **opts):
     """find statistics about potential parameters for `perfmergecopies`
 
     This command find (base, p1, p2) triplet relevant for copytracing
     benchmarking in the context of a merge.  It reports values for some of the
     parameters that impact merge copy tracing time during merge.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details come at the cost of slower command
     execution.
 
     Since rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision triplets are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     output_template = [
         ("base", "%(base)12s"),
         ("p1", "%(p1.node)12s"),
         ("p2", "%(p2.node)12s"),
         ("p1.nb-revs", "%(p1.nbrevs)12d"),
         ("p1.nb-files", "%(p1.nbmissingfiles)12d"),
         ("p1.renames", "%(p1.renamedfiles)12d"),
         ("p1.time", "%(p1.time)12.3f"),
         ("p2.nb-revs", "%(p2.nbrevs)12d"),
         ("p2.nb-files", "%(p2.nbmissingfiles)12d"),
         ("p2.renames", "%(p2.renamedfiles)12d"),
         ("p2.time", "%(p2.time)12.3f"),
         ("renames", "%(nbrenamedfiles)12d"),
         ("total.time", "%(time)12.3f"),
     ]
     if not dotiming:
         output_template = [
             i
             for i in output_template
             if not ('time' in i[0] or 'renames' in i[0])
         ]
     header_names = [h for (h, v) in output_template]
     output = ' '.join([v for (h, v) in output_template]) + '\n'
     header = ' '.join(['%12s'] * len(header_names)) + '\n'
     fm.plain(header % tuple(header_names))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['parentnbrenames'] = []
             alldata['totalnbrenames'] = []
             alldata['parenttime'] = []
             alldata['totaltime'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1()
         p2 = ctx.p2()
         bases = repo.changelog._commonancestorsheads(p1.rev(), p2.rev())
         for b in bases:
             b = repo[b]
             p1missing = copies._computeforwardmissing(b, p1)
             p2missing = copies._computeforwardmissing(b, p2)
             data = {
                 b'base': b.hex(),
                 b'p1.node': p1.hex(),
                 b'p1.nbrevs': len(repo.revs('only(%d, %d)', p1.rev(), b.rev())),
                 b'p1.nbmissingfiles': len(p1missing),
                 b'p2.node': p2.hex(),
                 b'p2.nbrevs': len(repo.revs('only(%d, %d)', p2.rev(), b.rev())),
                 b'p2.nbmissingfiles': len(p2missing),
             }
             if dostats:
                 if p1missing:
                     alldata['nbrevs'].append(
                         (data['p1.nbrevs'], b.hex(), p1.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p1.nbmissingfiles'], b.hex(), p1.hex())
                     )
                 if p2missing:
                     alldata['nbrevs'].append(
                         (data['p2.nbrevs'], b.hex(), p2.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p2.nbmissingfiles'], b.hex(), p2.hex())
                     )
             if dotiming:
                 begin = util.timer()
                 mergedata = copies.mergecopies(repo, p1, p2, b)
                 end = util.timer()
                 # not very stable timing since we did only one run
                 data['time'] = end - begin
                 # mergedata contains five dicts: "copy", "movewithdir",
                 # "diverge", "renamedelete" and "dirmove".
                 # The first 4 are about renamed file so lets count that.
                 renames = len(mergedata[0])
                 renames += len(mergedata[1])
                 renames += len(mergedata[2])
                 renames += len(mergedata[3])
                 data['nbrenamedfiles'] = renames
                 begin = util.timer()
                 p1renames = copies.pathcopies(b, p1)
                 end = util.timer()
                 data['p1.time'] = end - begin
                 begin = util.timer()
                 p2renames = copies.pathcopies(b, p2)
                 end = util.timer()
                 data['p2.time'] = end - begin
                 data['p1.renamedfiles'] = len(p1renames)
                 data['p2.renamedfiles'] = len(p2renames)
 
                 if dostats:
                     if p1missing:
                         alldata['parentnbrenames'].append(
                             (data['p1.renamedfiles'], b.hex(), p1.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p1.time'], b.hex(), p1.hex())
                         )
                     if p2missing:
                         alldata['parentnbrenames'].append(
                             (data['p2.renamedfiles'], b.hex(), p2.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p2.time'], b.hex(), p2.hex())
                         )
                     if p1missing or p2missing:
                         alldata['totalnbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 b.hex(),
                                 p1.hex(),
                                 p2.hex(),
                             )
                         )
                         alldata['totaltime'].append(
                             (data['time'], b.hex(), p1.hex(), p2.hex())
                         )
             fm.startitem()
             fm.data(**data)
             # make node pretty for the human output
             out = data.copy()
             out['base'] = fm.hexfunc(b.node())
             out['p1.node'] = fm.hexfunc(p1.node())
             out['p2.node'] = fm.hexfunc(p2.node())
             fm.plain(output % out)
 
     fm.end()
     if dostats:
         # use a second formatter because the data are quite different, not sure
         # how it flies with the templater.
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(
                 ('parentnbrenames', 'rename from one parent to base')
             )
             entries.append(('totalnbrenames', 'total number of renames'))
             entries.append(('parenttime', 'time for one parent'))
             entries.append(('totaltime', 'time for both parents'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(
     b'perf::helper-pathcopies|perfhelper-pathcopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelperpathcopies(ui, repo, revs=[], **opts):
     """find statistic about potential parameters for the `perftracecopies`
 
     This command find source-destination pair relevant for copytracing testing.
     It report value for some of the parameters that impact copy tracing time.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details comes at the cost of a slower command
     execution.
 
     Since the rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision pairs are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     if dotiming:
         header = '%12s %12s %12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d "
             "%(nbrenamedfiles)12d %(time)18.5f\n"
         )
         header_names = (
             "source",
             "destination",
             "nb-revs",
             "nb-files",
             "nb-renames",
             "time",
         )
         fm.plain(header % header_names)
     else:
         header = '%12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d\n"
         )
         fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['nbrenames'] = []
             alldata['time'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1().rev()
         p2 = ctx.p2().rev()
         bases = repo.changelog._commonancestorsheads(p1, p2)
         for p in (p1, p2):
             for b in bases:
                 base = repo[b]
                 parent = repo[p]
                 missing = copies._computeforwardmissing(base, parent)
                 if not missing:
                     continue
                 data = {
                     b'source': base.hex(),
                     b'destination': parent.hex(),
                     b'nbrevs': len(repo.revs('only(%d, %d)', p, b)),
                     b'nbmissingfiles': len(missing),
                 }
                 if dostats:
                     alldata['nbrevs'].append(
                         (
                             data['nbrevs'],
                             base.hex(),
                             parent.hex(),
                         )
                     )
                     alldata['nbmissingfiles'].append(
                         (
                             data['nbmissingfiles'],
                             base.hex(),
                             parent.hex(),
                         )
                     )
                 if dotiming:
                     begin = util.timer()
                     renames = copies.pathcopies(base, parent)
                     end = util.timer()
                     # not very stable timing since we did only one run
                     data['time'] = end - begin
                     data['nbrenamedfiles'] = len(renames)
                     if dostats:
                         alldata['time'].append(
                             (
                                 data['time'],
                                 base.hex(),
                                 parent.hex(),
                             )
                         )
                         alldata['nbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 base.hex(),
                                 parent.hex(),
                             )
                         )
                 fm.startitem()
                 fm.data(**data)
                 out = data.copy()
                 out['source'] = fm.hexfunc(base.node())
                 out['destination'] = fm.hexfunc(parent.node())
                 fm.plain(output % out)
 
     fm.end()
     if dostats:
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(('nbrenames', 'renamed files'))
             entries.append(('time', 'time'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(b'perf::cca|perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
     fm.end()
 
 
 @command(b'perf::fncacheload|perffncacheload', formatteropts)
 def perffncacheload(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
 
     def d():
         s.fncache._load()
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::fncachewrite|perffncachewrite', formatteropts)
 def perffncachewrite(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     lock = repo.lock()
     s.fncache._load()
     tr = repo.transaction(b'perffncachewrite')
     tr.addbackup(b'fncache')
 
     def d():
         s.fncache._dirty = True
         s.fncache.write(tr)
 
     timer(d)
     tr.close()
     lock.release()
     fm.end()
 
 
 @command(b'perf::fncacheencode|perffncacheencode', formatteropts)
 def perffncacheencode(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
 
     def d():
         for p in s.fncache.entries:
             s.encode(p)
 
     timer(d)
     fm.end()
 
 
 def _bdiffworker(q, blocks, xdiff, ready, done):
     while not done.is_set():
         pair = q.get()
         while pair is not None:
             if xdiff:
                 mdiff.bdiff.xdiffblocks(*pair)
             elif blocks:
                 mdiff.bdiff.blocks(*pair)
             else:
                 mdiff.textdiff(*pair)
             q.task_done()
             pair = q.get()
         q.task_done()  # for the None one
         with ready:
             ready.wait()
 
 
 def _manifestrevision(repo, mnode):
     ml = repo.manifestlog
 
     if util.safehasattr(ml, b'getstorage'):
         store = ml.getstorage(b'')
     else:
         store = ml._revlog
 
     return store.revision(mnode)
 
 
 @command(
     b'perf::bdiff|perfbdiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test bdiffs for all associated revisions'),
         (b'', b'threads', 0, b'number of thread to use (disable with 0)'),
         (b'', b'blocks', False, b'test computing diffs into blocks'),
         (b'', b'xdiff', False, b'use xdiff algorithm'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfbdiff(ui, repo, file_, rev=None, count=None, threads=0, **opts):
     """benchmark a bdiff between revisions
 
     By default, benchmark a bdiff between its delta parent and itself.
 
     With ``--count``, benchmark bdiffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure bdiffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
 
     if opts[b'xdiff'] and not opts[b'blocks']:
         raise error.CommandError(b'perfbdiff', b'--xdiff requires --blocks')
 
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfbdiff', b'invalid arguments')
 
     blocks = opts[b'blocks']
     xdiff = opts[b'xdiff']
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfbdiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     withthreads = threads > 0
     if not withthreads:
 
         def d():
             for pair in textpairs:
                 if xdiff:
                     mdiff.bdiff.xdiffblocks(*pair)
                 elif blocks:
                     mdiff.bdiff.blocks(*pair)
                 else:
                     mdiff.textdiff(*pair)
 
     else:
         q = queue()
         for i in _xrange(threads):
             q.put(None)
         ready = threading.Condition()
         done = threading.Event()
         for i in _xrange(threads):
             threading.Thread(
                 target=_bdiffworker, args=(q, blocks, xdiff, ready, done)
             ).start()
         q.join()
 
         def d():
             for pair in textpairs:
                 q.put(pair)
             for i in _xrange(threads):
                 q.put(None)
             with ready:
                 ready.notify_all()
             q.join()
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
     if withthreads:
         done.set()
         for i in _xrange(threads):
             q.put(None)
         with ready:
             ready.notify_all()
 
 
 @command(
     b'perf::unidiff|perfunidiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test unidiffs for all associated revisions'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfunidiff(ui, repo, file_, rev=None, count=None, **opts):
     """benchmark a unified diff between revisions
 
     This doesn't include any copy tracing - it's just a unified diff
     of the texts.
 
     By default, benchmark a diff between its delta parent and itself.
 
     With ``--count``, benchmark diffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure diffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfunidiff', b'invalid arguments')
 
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfunidiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     def d():
         for left, right in textpairs:
             # The date strings don't matter, so we pass empty strings.
             headerlines, hunks = mdiff.unidiff(
                 left, b'', right, b'', b'left', b'right', binary=False
             )
             # consume iterators in roughly the way patch.py does
             b'\n'.join(headerlines)
             b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perf::diffwd|perfdiffwd', formatteropts)
 def perfdiffwd(ui, repo, **opts):
     """Profile diff of working directory changes"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
     }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
         opts = {options[c]: b'1' for c in diffopt}
 
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
 
         diffopt = diffopt.encode('ascii')
         title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
         timer(d, title=title)
     fm.end()
 
 
 @command(
     b'perf::revlogindex|perfrevlogindex',
     revlogopts + formatteropts,
     b'-c|-m|FILE',
 )
 def perfrevlogindex(ui, repo, file_=None, **opts):
     """Benchmark operations against a revlog index.
 
     This tests constructing a revlog instance, reading index data,
     parsing index data, and performing various operations related to
     index data.
     """
 
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
 
     opener = getattr(rl, 'opener')  # trick linter
     # compat with hg <= 5.8
     radix = getattr(rl, 'radix', None)
     indexfile = getattr(rl, '_indexfile', None)
     if indexfile is None:
         # compatibility with <= hg-5.8
         indexfile = getattr(rl, 'indexfile')
     data = opener.read(indexfile)
 
     header = struct.unpack(b'>I', data[0:4])[0]
     version = header & 0xFFFF
     if version == 1:
         inline = header & (1 << 16)
     else:
         raise error.Abort(b'unsupported revlog version: %d' % version)
 
     parse_index_v1 = getattr(mercurial.revlog, 'parse_index_v1', None)
     if parse_index_v1 is None:
         parse_index_v1 = mercurial.revlog.revlogio().parseindex
 
     rllen = len(rl)
 
     node0 = rl.node(0)
     node25 = rl.node(rllen // 4)
     node50 = rl.node(rllen // 2)
     node75 = rl.node(rllen // 4 * 3)
     node100 = rl.node(rllen - 1)
 
     allrevs = range(rllen)
     allrevsrev = list(reversed(allrevs))
     allnodes = [rl.node(rev) for rev in range(rllen)]
     allnodesrev = list(reversed(allnodes))
 
     def constructor():
         if radix is not None:
             revlog(opener, radix=radix)
         else:
             # hg <= 5.8
             revlog(opener, indexfile=indexfile)
 
     def read():
         with opener(indexfile) as fh:
             fh.read()
 
     def parseindex():
         parse_index_v1(data, inline)
 
     def getentry(revornode):
         index = parse_index_v1(data, inline)[0]
         index[revornode]
 
     def getentries(revs, count=1):
         index = parse_index_v1(data, inline)[0]
 
         for i in range(count):
             for rev in revs:
                 index[rev]
 
     def resolvenode(node):
         index = parse_index_v1(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
             nodemap = getattr(parse_index_v1(data, inline)[0], 'nodemap', None)
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         try:
             rev(node)
         except error.RevlogError:
             pass
 
     def resolvenodes(nodes, count=1):
         index = parse_index_v1(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
             nodemap = getattr(parse_index_v1(data, inline)[0], 'nodemap', None)
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         for i in range(count):
             for node in nodes:
                 try:
                     rev(node)
                 except error.RevlogError:
                     pass
 
     benches = [
         (constructor, b'revlog constructor'),
         (read, b'read'),
         (parseindex, b'create index object'),
         (lambda: getentry(0), b'retrieve index entry for rev 0'),
         (lambda: resolvenode(b'a' * 20), b'look up missing node'),
         (lambda: resolvenode(node0), b'look up node at rev 0'),
         (lambda: resolvenode(node25), b'look up node at 1/4 len'),
         (lambda: resolvenode(node50), b'look up node at 1/2 len'),
         (lambda: resolvenode(node75), b'look up node at 3/4 len'),
         (lambda: resolvenode(node100), b'look up node at tip'),
         # 2x variation is to measure caching impact.
         (lambda: resolvenodes(allnodes), b'look up all nodes (forward)'),
         (lambda: resolvenodes(allnodes, 2), b'look up all nodes 2x (forward)'),
         (lambda: resolvenodes(allnodesrev), b'look up all nodes (reverse)'),
         (
             lambda: resolvenodes(allnodesrev, 2),
             b'look up all nodes 2x (reverse)',
         ),
         (lambda: getentries(allrevs), b'retrieve all index entries (forward)'),
         (
             lambda: getentries(allrevs, 2),
             b'retrieve all index entries 2x (forward)',
         ),
         (
             lambda: getentries(allrevsrev),
             b'retrieve all index entries (reverse)',
         ),
         (
             lambda: getentries(allrevsrev, 2),
             b'retrieve all index entries 2x (reverse)',
         ),
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::revlogrevisions|perfrevlogrevisions',
     revlogopts
     + formatteropts
     + [
         (b'd', b'dist', 100, b'distance between the revisions'),
         (b's', b'startrev', 0, b'revision to start reading at'),
         (b'', b'reverse', False, b'read in reverse'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogrevisions(
     ui, repo, file_=None, startrev=0, reverse=False, **opts
 ):
     """Benchmark reading a series of revisions from a revlog.
 
     By default, we read every ``-d/--dist`` revision from 0 to tip of
     the specified revlog.
 
     The start revision can be defined via ``-s/--startrev``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogrevisions', file_, opts)
     rllen = getlen(ui)(rl)
 
     if startrev < 0:
         startrev = rllen + startrev
 
     def d():
         rl.clearcaches()
 
         beginrev = startrev
         endrev = rllen
         dist = opts[b'dist']
 
         if reverse:
             beginrev, endrev = endrev - 1, beginrev - 1
             dist = -1 * dist
 
         for x in _xrange(beginrev, endrev, dist):
             # Old revisions don't support passing int.
             n = rl.node(x)
             rl.revision(n)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::revlogwrite|perfrevlogwrite',
     revlogopts
     + formatteropts
     + [
         (b's', b'startrev', 1000, b'revision to start writing at'),
         (b'', b'stoprev', -1, b'last revision to write'),
         (b'', b'count', 3, b'number of passes to perform'),
         (b'', b'details', False, b'print timing for every revisions tested'),
         (b'', b'source', b'full', b'the kind of data feed in the revlog'),
         (b'', b'lazydeltabase', True, b'try the provided delta first'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
     """Benchmark writing a series of revisions to a revlog.
 
     Possible source values are:
     * `full`: add from a full text (default).
     * `parent-1`: add from a delta to the first parent
     * `parent-2`: add from a delta to the second parent if it exists
                   (use a delta from the first parent otherwise)
     * `parent-smallest`: add from the smallest delta (either p1 or p2)
     * `storage`: add from the existing precomputed deltas
 
     Note: This performance command measures performance in a custom way. As a
     result some of the global configuration of the 'perf' command does not
     apply to it:
 
     * ``pre-run``: disabled
 
     * ``profile-benchmark``: disabled
 
     * ``run-limits``: disabled use --count instead
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
     rllen = getlen(ui)(rl)
     if startrev < 0:
         startrev = rllen + startrev
     if stoprev < 0:
         stoprev = rllen + stoprev
 
     lazydeltabase = opts['lazydeltabase']
     source = opts['source']
     clearcaches = opts['clear_caches']
     validsource = (
         b'full',
         b'parent-1',
         b'parent-2',
         b'parent-smallest',
         b'storage',
     )
     if source not in validsource:
         raise error.Abort('invalid source type: %s' % source)
 
     ### actually gather results
     count = opts['count']
     if count <= 0:
         raise error.Abort('invalide run count: %d' % count)
     allresults = []
     for c in range(count):
         timing = _timeonewrite(
             ui,
             rl,
             source,
             startrev,
             stoprev,
             c + 1,
             lazydeltabase=lazydeltabase,
             clearcaches=clearcaches,
         )
         allresults.append(timing)
 
     ### consolidate the results in a single list
     results = []
     for idx, (rev, t) in enumerate(allresults[0]):
         ts = [t]
         for other in allresults[1:]:
             orev, ot = other[idx]
             assert orev == rev
             ts.append(ot)
         results.append((rev, ts))
     resultcount = len(results)
 
     ### Compute and display relevant statistics
 
     # get a formatter
     fm = ui.formatter(b'perf', opts)
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # print individual details if requested
     if opts['details']:
         for idx, item in enumerate(results, 1):
             rev, data = item
             title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
             formatone(fm, data, title=title, displayall=displayall)
 
     # sorts results by median time
     results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
     # list of (name, index) to display)
     relevants = [
         ("min", 0),
         ("10%", resultcount * 10 // 100),
         ("25%", resultcount * 25 // 100),
         ("50%", resultcount * 70 // 100),
         ("75%", resultcount * 75 // 100),
         ("90%", resultcount * 90 // 100),
         ("95%", resultcount * 95 // 100),
         ("99%", resultcount * 99 // 100),
         ("99.9%", resultcount * 999 // 1000),
         ("99.99%", resultcount * 9999 // 10000),
         ("99.999%", resultcount * 99999 // 100000),
         ("max", -1),
     ]
     if not ui.quiet:
         for name, idx in relevants:
             data = results[idx]
             title = '%s of %d, rev %d' % (name, resultcount, data[0])
             formatone(fm, data[1], title=title, displayall=displayall)
 
     # XXX summing that many float will not be very precise, we ignore this fact
     # for now
     totaltime = []
     for item in allresults:
         totaltime.append(
             (
                 sum(x[1][0] for x in item),
                 sum(x[1][1] for x in item),
                 sum(x[1][2] for x in item),
             )
         )
     formatone(
         fm,
         totaltime,
         title="total time (%d revs)" % resultcount,
         displayall=displayall,
     )
     fm.end()
 
 
 class _faketr(object):
     def add(s, x, y, z=None):
         return None
 
 
 def _timeonewrite(
     ui,
     orig,
     source,
     startrev,
     stoprev,
     runidx=None,
     lazydeltabase=True,
     clearcaches=True,
 ):
     timings = []
     tr = _faketr()
     with _temprevlog(ui, orig, startrev) as dest:
         dest._lazydeltabase = lazydeltabase
         revs = list(orig.revs(startrev, stoprev))
         total = len(revs)
         topic = 'adding'
         if runidx is not None:
             topic += ' (run #%d)' % runidx
         # Support both old and new progress API
         if util.safehasattr(ui, 'makeprogress'):
             progress = ui.makeprogress(topic, unit='revs', total=total)
 
             def updateprogress(pos):
                 progress.update(pos)
 
             def completeprogress():
                 progress.complete()
 
         else:
 
             def updateprogress(pos):
                 ui.progress(topic, pos, unit='revs', total=total)
 
             def completeprogress():
                 ui.progress(topic, None, unit='revs', total=total)
 
         for idx, rev in enumerate(revs):
             updateprogress(idx)
             addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
             if clearcaches:
                 dest.index.clearcaches()
                 dest.clearcaches()
             with timeone() as r:
                 dest.addrawrevision(*addargs, **addkwargs)
             timings.append((rev, r[0]))
         updateprogress(total)
         completeprogress()
     return timings
 
 
 def _getrevisionseed(orig, rev, tr, source):
     from mercurial.node import nullid
 
     linkrev = orig.linkrev(rev)
     node = orig.node(rev)
     p1, p2 = orig.parents(node)
     flags = orig.flags(rev)
     cachedelta = None
     text = None
 
     if source == b'full':
         text = orig.revision(rev)
     elif source == b'parent-1':
         baserev = orig.rev(p1)
         cachedelta = (baserev, orig.revdiff(p1, rev))
     elif source == b'parent-2':
         parent = p2
         if p2 == nullid:
             parent = p1
         baserev = orig.rev(parent)
         cachedelta = (baserev, orig.revdiff(parent, rev))
     elif source == b'parent-smallest':
         p1diff = orig.revdiff(p1, rev)
         parent = p1
         diff = p1diff
         if p2 != nullid:
             p2diff = orig.revdiff(p2, rev)
             if len(p1diff) > len(p2diff):
                 parent = p2
                 diff = p2diff
         baserev = orig.rev(parent)
         cachedelta = (baserev, diff)
     elif source == b'storage':
         baserev = orig.deltaparent(rev)
         cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
 
     return (
         (text, tr, linkrev, p1, p2),
         {'node': node, 'flags': flags, 'cachedelta': cachedelta},
     )
 
 
 @contextlib.contextmanager
 def _temprevlog(ui, orig, truncaterev):
     from mercurial import vfs as vfsmod
 
     if orig._inline:
         raise error.Abort('not supporting inline revlog (yet)')
     revlogkwargs = {}
     k = 'upperboundcomp'
     if util.safehasattr(orig, k):
         revlogkwargs[k] = getattr(orig, k)
 
     indexfile = getattr(orig, '_indexfile', None)
     if indexfile is None:
         # compatibility with <= hg-5.8
         indexfile = getattr(orig, 'indexfile')
     origindexpath = orig.opener.join(indexfile)
 
     datafile = getattr(orig, '_datafile', getattr(orig, 'datafile'))
     origdatapath = orig.opener.join(datafile)
     radix = b'revlog'
     indexname = b'revlog.i'
     dataname = b'revlog.d'
 
     tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
     try:
         # copy the data file in a temporary directory
         ui.debug('copying data in %s\n' % tmpdir)
         destindexpath = os.path.join(tmpdir, 'revlog.i')
         destdatapath = os.path.join(tmpdir, 'revlog.d')
         shutil.copyfile(origindexpath, destindexpath)
         shutil.copyfile(origdatapath, destdatapath)
 
         # remove the data we want to add again
         ui.debug('truncating data to be rewritten\n')
         with open(destindexpath, 'ab') as index:
             index.seek(0)
             index.truncate(truncaterev * orig._io.size)
         with open(destdatapath, 'ab') as data:
             data.seek(0)
             data.truncate(orig.start(truncaterev))
 
         # instantiate a new revlog from the temporary copy
         ui.debug('truncating adding to be rewritten\n')
         vfs = vfsmod.vfs(tmpdir)
         vfs.options = getattr(orig.opener, 'options', None)
 
         try:
             dest = revlog(vfs, radix=radix, **revlogkwargs)
         except TypeError:
             dest = revlog(
                 vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
             )
         if dest._inline:
             raise error.Abort('not supporting inline revlog (yet)')
         # make sure internals are initialized
         dest.revision(len(dest) - 1)
         yield dest
         del dest, vfs
     finally:
         shutil.rmtree(tmpdir, True)
 
 
 @command(
     b'perf::revlogchunks|perfrevlogchunks',
     revlogopts
     + formatteropts
     + [
         (b'e', b'engines', b'', b'compression engines to use'),
         (b's', b'startrev', 0, b'revision to start at'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
     stored within each revlog are "chunks" of possibly compressed data. This
     data needs to be read and decompressed or compressed and written.
 
     This command measures the time it takes to read+decompress and recompress
     chunks in a revlog. It effectively isolates I/O and compression performance.
     For measurements of higher-level operations like resolving revisions,
     see ``perfrevlogrevisions`` and ``perfrevlogrevision``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogchunks', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = rl._getsegmentforrevs
     except AttributeError:
         segmentforrevs = rl._chunkraw
 
     # Verify engines argument.
     if engines:
         engines = {e.strip() for e in engines.split(b',')}
         for engine in engines:
             try:
                 util.compressionengines[engine]
             except KeyError:
                 raise error.Abort(b'unknown compression engine: %s' % engine)
     else:
         engines = []
         for e in util.compengines:
             engine = util.compengines[e]
             try:
                 if engine.available():
                     engine.revlogcompressor().compress(b'dummy')
                     engines.append(e)
             except NotImplementedError:
                 pass
 
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
         if rl._inline:
             indexfile = getattr(rl, '_indexfile', None)
             if indexfile is None:
                 # compatibility with <= hg-5.8
                 indexfile = getattr(rl, 'indexfile')
             return getsvfs(repo)(indexfile)
         else:
             datafile = getattr(rl, 'datafile', getattr(rl, 'datafile'))
             return getsvfs(repo)(datafile)
 
     def doread():
         rl.clearcaches()
         for rev in revs:
             segmentforrevs(rev, rev)
 
     def doreadcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             segmentforrevs(rev, rev, df=fh)
 
     def doreadbatch():
         rl.clearcaches()
         segmentforrevs(revs[0], revs[-1])
 
     def doreadbatchcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         segmentforrevs(revs[0], revs[-1], df=fh)
 
     def dochunk():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             rl._chunk(rev, df=fh)
 
     chunks = [None]
 
     def dochunkbatch():
         rl.clearcaches()
         fh = rlfh(rl)
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
     def docompress(compressor):
         rl.clearcaches()
 
         try:
             # Swap in the requested compression engine.
             oldcompressor = rl._compressor
             rl._compressor = compressor
             for chunk in chunks[0]:
                 rl.compress(chunk)
         finally:
             rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), b'read'),
         (lambda: doreadcachedfh(), b'read w/ reused fd'),
         (lambda: doreadbatch(), b'read batch'),
         (lambda: doreadbatchcachedfh(), b'read batch w/ reused fd'),
         (lambda: dochunk(), b'chunk'),
         (lambda: dochunkbatch(), b'chunk batch'),
     ]
 
     for engine in sorted(engines):
         compressor = util.compengines[engine].revlogcompressor()
         benches.append(
             (
                 functools.partial(docompress, compressor),
                 b'compress w/ %s' % engine,
             )
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::revlogrevision|perfrevlogrevision',
     revlogopts
     + formatteropts
     + [(b'', b'cache', False, b'use caches instead of clearing')],
     b'-c|-m|FILE REV',
 )
 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
     """Benchmark obtaining a revlog revision.
 
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
     2. Slice the delta chain if applicable
     3. Obtain the raw chunks for that delta chain
     4. Decompress each raw chunk
     5. Apply binary patches to obtain fulltext
     6. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
     opts = _byteskwargs(opts)
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfrevlogrevision', b'invalid arguments')
 
     r = cmdutil.openrevlog(repo, b'perfrevlogrevision', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = r._getsegmentforrevs
     except AttributeError:
         segmentforrevs = r._chunkraw
 
     node = r.lookup(rev)
     rev = r.rev(node)
 
     def getrawchunks(data, chain):
         start = r.start
         length = r.length
         inline = r._inline
         try:
             iosize = r.index.entry_size
         except AttributeError:
             iosize = r._io.size
         buffer = util.buffer
 
         chunks = []
         ladd = chunks.append
         for idx, item in enumerate(chain):
             offset = start(item[0])
             bits = data[idx]
             for rev in item:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(buffer(bits, chunkstart - offset, chunklength))
 
         return chunks
 
     def dodeltachain(rev):
         if not cache:
             r.clearcaches()
         r._deltachain(rev)
 
     def doread(chain):
         if not cache:
             r.clearcaches()
         for item in slicedchain:
             segmentforrevs(item[0], item[-1])
 
     def doslice(r, chain, size):
         for s in slicechunk(r, chain, targetsize=size):
             pass
 
     def dorawchunks(data, chain):
         if not cache:
             r.clearcaches()
         getrawchunks(data, chain)
 
     def dodecompress(chunks):
         decomp = r.decompress
         for chunk in chunks:
             decomp(chunk)
 
     def dopatch(text, bins):
         if not cache:
             r.clearcaches()
         mdiff.patches(text, bins)
 
     def dohash(text):
         if not cache:
             r.clearcaches()
         r.checkhash(text, node, rev=rev)
 
     def dorevision():
         if not cache:
             r.clearcaches()
         r.revision(node)
 
     try:
         from mercurial.revlogutils.deltas import slicechunk
     except ImportError:
         slicechunk = getattr(revlog, '_slicechunk', None)
 
     size = r.length(rev)
     chain = r._deltachain(rev)[0]
     if not getattr(r, '_withsparseread', False):
         slicedchain = (chain,)
     else:
         slicedchain = tuple(slicechunk(r, chain, targetsize=size))
     data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
     rawchunks = getrawchunks(data, slicedchain)
     bins = r._chunks(chain)
     text = bytes(bins[0])
     bins = bins[1:]
     text = mdiff.patches(text, bins)
 
     benches = [
         (lambda: dorevision(), b'full'),
         (lambda: dodeltachain(rev), b'deltachain'),
         (lambda: doread(chain), b'read'),
     ]
 
     if getattr(r, '_withsparseread', False):
         slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
         benches.append(slicing)
 
     benches.extend(
         [
             (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
             (lambda: dodecompress(rawchunks), b'decompress'),
             (lambda: dopatch(text, bins), b'patch'),
             (lambda: dohash(text), b'hash'),
         ]
     )
 
     timer, fm = gettimer(ui, opts)
     for fn, title in benches:
         timer(fn, title=title)
     fm.end()
 
 
 @command(
     b'perf::revset|perfrevset',
     [
         (b'C', b'clear', False, b'clear volatile cache between each call.'),
         (b'', b'contexts', False, b'obtain changectx for each revision'),
     ]
     + formatteropts,
     b"REVSET",
 )
 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
     """benchmark the execution time of a revset
 
     Use the --clean option if need to evaluate the impact of build volatile
     revisions set cache on the revset execution. Volatile cache hold filtered
     and obsolete related cache."""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def d():
         if clear:
             repo.invalidatevolatilesets()
         if contexts:
             for ctx in repo.set(expr):
                 pass
         else:
             for r in repo.revs(expr):
                 pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::volatilesets|perfvolatilesets',
     [
         (b'', b'clear-obsstore', False, b'drop obsstore between each call.'),
     ]
     + formatteropts,
 )
 def perfvolatilesets(ui, repo, *names, **opts):
     """benchmark the computation of various volatile set
 
     Volatile set computes element related to filtering and obsolescence."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo = repo.unfiltered()
 
     def getobs(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             obsolete.getrevs(repo, name)
 
         return d
 
     allobs = sorted(obsolete.cachefuncs)
     if names:
         allobs = [n for n in allobs if n in names]
 
     for name in allobs:
         timer(getobs(name), title=name)
 
     def getfiltered(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             repoview.filterrevs(repo, name)
 
         return d
 
     allfilter = sorted(repoview.filtertable)
     if names:
         allfilter = [n for n in allfilter if n in names]
 
     for name in allfilter:
         timer(getfiltered(name), title=name)
     fm.end()
 
 
 @command(
     b'perf::branchmap|perfbranchmap',
     [
         (b'f', b'full', False, b'Includes build time of subset'),
         (
             b'',
             b'clear-revbranch',
             False,
             b'purge the revbranch cache between computation',
         ),
     ]
     + formatteropts,
 )
 def perfbranchmap(ui, repo, *filternames, **opts):
     """benchmark the update of a branchmap
 
     This benchmarks the full repo.branchmap() call with read and write disabled
     """
     opts = _byteskwargs(opts)
     full = opts.get(b"full", False)
     clear_revbranch = opts.get(b"clear_revbranch", False)
     timer, fm = gettimer(ui, opts)
 
     def getbranchmap(filtername):
         """generate a benchmark function for the filtername"""
         if filtername is None:
             view = repo
         else:
             view = repo.filtered(filtername)
         if util.safehasattr(view._branchcaches, '_per_filter'):
             filtered = view._branchcaches._per_filter
         else:
             # older versions
             filtered = view._branchcaches
 
         def d():
             if clear_revbranch:
                 repo.revbranchcache()._clear()
             if full:
                 view._branchcaches.clear()
             else:
                 filtered.pop(filtername, None)
             view.branchmap()
 
         return d
 
     # add filter in smaller subset to bigger subset
     possiblefilters = set(repoview.filtertable)
     if filternames:
         possiblefilters &= set(filternames)
     subsettable = getbranchmapsubsettable()
     allfilters = []
     while possiblefilters:
         for name in possiblefilters:
             subset = subsettable.get(name)
             if subset not in possiblefilters:
                 break
         else:
             assert False, b'subset cycle %s!' % possiblefilters
         allfilters.append(name)
         possiblefilters.remove(name)
 
     # warm the cache
     if not full:
         for name in allfilters:
             repo.filtered(name).branchmap()
     if not filternames or b'unfiltered' in filternames:
         # add unfiltered
         allfilters.append(None)
 
     if util.safehasattr(branchmap.branchcache, 'fromfile'):
         branchcacheread = safeattrsetter(branchmap.branchcache, b'fromfile')
         branchcacheread.set(classmethod(lambda *args: None))
     else:
         # older versions
         branchcacheread = safeattrsetter(branchmap, b'read')
         branchcacheread.set(lambda *args: None)
     branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
     branchcachewrite.set(lambda *args: None)
     try:
         for name in allfilters:
             printname = name
             if name is None:
                 printname = b'unfiltered'
             timer(getbranchmap(name), title=printname)
     finally:
         branchcacheread.restore()
         branchcachewrite.restore()
     fm.end()
 
 
 @command(
     b'perf::branchmapupdate|perfbranchmapupdate',
     [
         (b'', b'base', [], b'subset of revision to start from'),
         (b'', b'target', [], b'subset of revision to end with'),
         (b'', b'clear-caches', False, b'clear cache between each runs'),
     ]
     + formatteropts,
 )
 def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
     """benchmark branchmap update from for <base> revs to <target> revs
 
     If `--clear-caches` is passed, the following items will be reset before
     each update:
         * the changelog instance and associated indexes
         * the rev-branch-cache instance
 
     Examples:
 
        # update for the one last revision
        $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
 
        $ update for change coming with a new branch
        $ hg perfbranchmapupdate --base 'stable' --target 'default'
     """
     from mercurial import branchmap
     from mercurial import repoview
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     clearcaches = opts[b'clear_caches']
     unfi = repo.unfiltered()
     x = [None]  # used to pass data between closure
 
     # we use a `list` here to avoid possible side effect from smartset
     baserevs = list(scmutil.revrange(repo, base))
     targetrevs = list(scmutil.revrange(repo, target))
     if not baserevs:
         raise error.Abort(b'no revisions selected for --base')
     if not targetrevs:
         raise error.Abort(b'no revisions selected for --target')
 
     # make sure the target branchmap also contains the one in the base
     targetrevs = list(set(baserevs) | set(targetrevs))
     targetrevs.sort()
 
     cl = repo.changelog
     allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
     allbaserevs.sort()
     alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
 
     newrevs = list(alltargetrevs.difference(allbaserevs))
     newrevs.sort()
 
     allrevs = frozenset(unfi.changelog.revs())
     basefilterrevs = frozenset(allrevs.difference(allbaserevs))
     targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
 
     def basefilter(repo, visibilityexceptions=None):
         return basefilterrevs
 
     def targetfilter(repo, visibilityexceptions=None):
         return targetfilterrevs
 
     msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
     ui.status(msg % (len(allbaserevs), len(newrevs)))
     if targetfilterrevs:
         msg = b'(%d revisions still filtered)\n'
         ui.status(msg % len(targetfilterrevs))
 
     try:
         repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
         repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
 
         baserepo = repo.filtered(b'__perf_branchmap_update_base')
         targetrepo = repo.filtered(b'__perf_branchmap_update_target')
 
         # try to find an existing branchmap to reuse
         subsettable = getbranchmapsubsettable()
         candidatefilter = subsettable.get(None)
         while candidatefilter is not None:
             candidatebm = repo.filtered(candidatefilter).branchmap()
             if candidatebm.validfor(baserepo):
                 filtered = repoview.filterrevs(repo, candidatefilter)
                 missing = [r for r in allbaserevs if r in filtered]
                 base = candidatebm.copy()
                 base.update(baserepo, missing)
                 break
             candidatefilter = subsettable.get(candidatefilter)
         else:
             # no suitable subset where found
             base = branchmap.branchcache()
             base.update(baserepo, allbaserevs)
 
         def setup():
             x[0] = base.copy()
             if clearcaches:
                 unfi._revbranchcache = None
                 clearchangelog(repo)
 
         def bench():
             x[0].update(targetrepo, newrevs)
 
         timer(bench, setup=setup)
         fm.end()
     finally:
         repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
         repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
 
 
 @command(
     b'perf::branchmapload|perfbranchmapload',
     [
         (b'f', b'filter', b'', b'Specify repoview filter'),
         (b'', b'list', False, b'List brachmap filter caches'),
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ]
     + formatteropts,
 )
 def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
     """benchmark reading the branchmap"""
     opts = _byteskwargs(opts)
     clearrevlogs = opts[b'clear_revlogs']
 
     if list:
         for name, kind, st in repo.cachevfs.readdir(stat=True):
             if name.startswith(b'branch2'):
                 filtername = name.partition(b'-')[2] or b'unfiltered'
                 ui.status(
                     b'%s - %s\n' % (filtername, util.bytecount(st.st_size))
                 )
         return
     if not filter:
         filter = None
     subsettable = getbranchmapsubsettable()
     if filter is None:
         repo = repo.unfiltered()
     else:
         repo = repoview.repoview(repo, filter)
 
     repo.branchmap()  # make sure we have a relevant, up to date branchmap
 
     try:
         fromfile = branchmap.branchcache.fromfile
     except AttributeError:
         # older versions
         fromfile = branchmap.read
 
     currentfilter = filter
     # try once without timer, the filter may not be cached
     while fromfile(repo) is None:
         currentfilter = subsettable.get(currentfilter)
         if currentfilter is None:
             raise error.Abort(
                 b'No branchmap cached for %s repo' % (filter or b'unfiltered')
             )
         repo = repo.filtered(currentfilter)
     timer, fm = gettimer(ui, opts)
 
     def setup():
         if clearrevlogs:
             clearchangelog(repo)
 
     def bench():
         fromfile(repo)
 
     timer(bench, setup=setup)
     fm.end()
 
 
 @command(b'perf::loadmarkers|perfloadmarkers')
 def perfloadmarkers(ui, repo):
     """benchmark the time to parse the on-disk markers for a repo
 
     Result is the number of markers in the repo."""
     timer, fm = gettimer(ui)
     svfs = getsvfs(repo)
     timer(lambda: len(obsolete.obsstore(repo, svfs)))
     fm.end()
 
 
 @command(
     b'perf::lrucachedict|perflrucachedict',
     formatteropts
     + [
         (b'', b'costlimit', 0, b'maximum total cost of items in cache'),
         (b'', b'mincost', 0, b'smallest cost of items in cache'),
         (b'', b'maxcost', 100, b'maximum cost of items in cache'),
         (b'', b'size', 4, b'size of cache'),
         (b'', b'gets', 10000, b'number of key lookups'),
         (b'', b'sets', 10000, b'number of key sets'),
         (b'', b'mixed', 10000, b'number of mixed mode operations'),
         (
             b'',
             b'mixedgetfreq',
             50,
             b'frequency of get vs set ops in mixed mode',
         ),
     ],
     norepo=True,
 )
 def perflrucache(
     ui,
     mincost=0,
     maxcost=100,
     costlimit=0,
     size=4,
     gets=10000,
     sets=10000,
     mixed=10000,
     mixedgetfreq=50,
     **opts
 ):
     opts = _byteskwargs(opts)
 
     def doinit():
         for i in _xrange(10000):
             util.lrucachedict(size)
 
     costrange = list(range(mincost, maxcost + 1))
 
     values = []
     for i in _xrange(size):
         values.append(random.randint(0, _maxint))
 
     # Get mode fills the cache and tests raw lookup performance with no
     # eviction.
     getseq = []
     for i in _xrange(gets):
         getseq.append(random.choice(values))
 
     def dogets():
         d = util.lrucachedict(size)
         for v in values:
             d[v] = v
         for key in getseq:
             value = d[key]
             value  # silence pyflakes warning
 
     def dogetscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(values):
             d.insert(v, v, cost=costs[i])
         for key in getseq:
             try:
                 value = d[key]
                 value  # silence pyflakes warning
             except KeyError:
                 pass
 
     # Set mode tests insertion speed with cache eviction.
     setseq = []
     costs = []
     for i in _xrange(sets):
         setseq.append(random.randint(0, _maxint))
         costs.append(random.choice(costrange))
 
     def doinserts():
         d = util.lrucachedict(size)
         for v in setseq:
             d.insert(v, v)
 
     def doinsertscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(setseq):
             d.insert(v, v, cost=costs[i])
 
     def dosets():
         d = util.lrucachedict(size)
         for v in setseq:
             d[v] = v
 
     # Mixed mode randomly performs gets and sets with eviction.
     mixedops = []
     for i in _xrange(mixed):
         r = random.randint(0, 100)
         if r < mixedgetfreq:
             op = 0
         else:
             op = 1
 
         mixedops.append(
             (op, random.randint(0, size * 2), random.choice(costrange))
         )
 
     def domixed():
         d = util.lrucachedict(size)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d[v] = v
 
     def domixedcost():
         d = util.lrucachedict(size, maxcost=costlimit)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d.insert(v, v, cost=cost)
 
     benches = [
         (doinit, b'init'),
     ]
 
     if costlimit:
         benches.extend(
             [
                 (dogetscost, b'gets w/ cost limit'),
                 (doinsertscost, b'inserts w/ cost limit'),
                 (domixedcost, b'mixed w/ cost limit'),
             ]
         )
     else:
         benches.extend(
             [
                 (dogets, b'gets'),
                 (doinserts, b'inserts'),
                 (dosets, b'sets'),
                 (domixed, b'mixed'),
             ]
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::write|perfwrite',
     formatteropts
     + [
         (b'', b'write-method', b'write', b'ui write method'),
         (b'', b'nlines', 100, b'number of lines'),
         (b'', b'nitems', 100, b'number of items (per line)'),
         (b'', b'item', b'x', b'item that is written'),
         (b'', b'batch-line', None, b'pass whole line to write method at once'),
         (b'', b'flush-line', None, b'flush after each line'),
     ],
 )
 def perfwrite(ui, repo, **opts):
     """microbenchmark ui.write (and others)"""
     opts = _byteskwargs(opts)
 
     write = getattr(ui, _sysstr(opts[b'write_method']))
     nlines = int(opts[b'nlines'])
     nitems = int(opts[b'nitems'])
     item = opts[b'item']
     batch_line = opts.get(b'batch_line')
     flush_line = opts.get(b'flush_line')
 
     if batch_line:
         line = item * nitems + b'\n'
 
     def benchmark():
         for i in pycompat.xrange(nlines):
             if batch_line:
                 write(line)
             else:
                 for i in pycompat.xrange(nitems):
                     write(item)
                 write(b'\n')
             if flush_line:
                 ui.flush()
         ui.flush()
 
     timer, fm = gettimer(ui, opts)
     timer(benchmark)
     fm.end()
 
 
 def uisetup(ui):
     if util.safehasattr(cmdutil, b'openrevlog') and not util.safehasattr(
         commands, b'debugrevlogopts'
     ):
         # for "historical portability":
         # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
         # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
         # openrevlog() should cause failure, because it has been
         # available since 3.5 (or 49c583ca48c4).
         def openrevlog(orig, repo, cmd, file_, opts):
             if opts.get(b'dir') and not util.safehasattr(repo, b'dirlog'):
                 raise error.Abort(
                     b"This version doesn't support --dir option",
                     hint=b"use 3.5 or later",
                 )
             return orig(repo, cmd, file_, opts)
 
         extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
 
 
 @command(
     b'perf::progress|perfprogress',
     formatteropts
     + [
         (b'', b'topic', b'topic', b'topic for progress messages'),
         (b'c', b'total', 1000000, b'total value we are progressing to'),
     ],
     norepo=True,
 )
 def perfprogress(ui, topic=None, total=None, **opts):
     """printing of progress bars"""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def doprogress():
         with ui.makeprogress(topic, total=total) as progress:
             for i in _xrange(total):
                 progress.increment()
 
     timer(doprogress)
     fm.end()

contrib/phab-clean.py

@@ -1,93 +1,92 b''
 #!/usr/bin/env python3
 #
 # A small script to automatically reject idle Diffs
 #
 # you need to set the PHABBOT_USER and PHABBOT_TOKEN environment variable for authentication
-from __future__ import absolute_import, print_function
 
 import datetime
 import os
 import sys
 
 import phabricator
 
 MESSAGE = """There seems to have been no activities on this Diff for the past 3 Months.
 
 By policy, we are automatically moving it out of the `need-review` state.
 
 Please, move it back to `need-review` without hesitation if this diff should still be discussed.
 
 :baymax:need-review-idle:
 """
 
 
 PHAB_URL = "https://phab.mercurial-scm.org/api/"
 USER = os.environ.get("PHABBOT_USER", "baymax")
 TOKEN = os.environ.get("PHABBOT_TOKEN")
 
 
 NOW = datetime.datetime.now()
 
 # 3 months in seconds
 DELAY = 60 * 60 * 24 * 30 * 3
 
 
 def get_all_diff(phab):
     """Fetch all the diff that the need review"""
     return phab.differential.query(
         status="status-needs-review",
         order="order-modified",
         paths=[('HG', None)],
     )
 
 
 def filter_diffs(diffs, older_than):
     """filter diffs to only keep the one unmodified sin <older_than> seconds"""
     olds = []
     for d in diffs:
         modified = int(d['dateModified'])
         modified = datetime.datetime.fromtimestamp(modified)
         d["idleFor"] = idle_for = NOW - modified
         if idle_for.total_seconds() > older_than:
             olds.append(d)
     return olds
 
 
 def nudge_diff(phab, diff):
     """Comment on the idle diff and reject it"""
     diff_id = int(d['id'])
     phab.differential.createcomment(
         revision_id=diff_id, message=MESSAGE, action="reject"
     )
 
 
 if not USER:
     print(
         "not user specified please set PHABBOT_USER and PHABBOT_TOKEN",
         file=sys.stderr,
     )
 elif not TOKEN:
     print(
         "not api-token specified please set PHABBOT_USER and PHABBOT_TOKEN",
         file=sys.stderr,
     )
     sys.exit(1)
 
 phab = phabricator.Phabricator(USER, host=PHAB_URL, token=TOKEN)
 phab.connect()
 phab.update_interfaces()
 print('Hello "%s".' % phab.user.whoami()['realName'])
 
 diffs = get_all_diff(phab)
 print("Found %d Diffs" % len(diffs))
 olds = filter_diffs(diffs, DELAY)
 print("Found %d old Diffs" % len(olds))
 for d in olds:
     diff_id = d['id']
     status = d['statusName']
     modified = int(d['dateModified'])
     idle_for = d["idleFor"]
     msg = 'nudging D%s in "%s" state for %s'
     print(msg % (diff_id, status, idle_for))
     # uncomment to actually affect phab
     nudge_diff(phab, d)

contrib/python-hook-examples.py

@@ -1,27 +1,26 b''
 '''
 Examples of useful python hooks for Mercurial.
 '''
-from __future__ import absolute_import
 from mercurial import (
     patch,
     util,
 )
 
 
 def diffstat(ui, repo, **kwargs):
     """Example usage:
 
     [hooks]
     commit.diffstat = python:/path/to/this/file.py:diffstat
     changegroup.diffstat = python:/path/to/this/file.py:diffstat
     """
     if kwargs.get('parent2'):
         return
     node = kwargs['node']
     first = repo[node].p1().node()
     if 'url' in kwargs:
         last = repo.changelog.tip()
     else:
         last = node
     diff = patch.diff(repo, first, last)
     ui.write(patch.diffstat(util.iterlines(diff)))

contrib/python-zstandard/make_cffi.py

@@ -1,239 +1,238 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
-from __future__ import absolute_import
 
 import cffi
 import distutils.ccompiler
 import os
 import re
 import subprocess
 import tempfile
 
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 SOURCES = [
     "zstd/%s" % p
     for p in (
         "common/debug.c",
         "common/entropy_common.c",
         "common/error_private.c",
         "common/fse_decompress.c",
         "common/pool.c",
         "common/threading.c",
         "common/xxhash.c",
         "common/zstd_common.c",
         "compress/fse_compress.c",
         "compress/hist.c",
         "compress/huf_compress.c",
         "compress/zstd_compress.c",
         "compress/zstd_compress_literals.c",
         "compress/zstd_compress_sequences.c",
         "compress/zstd_double_fast.c",
         "compress/zstd_fast.c",
         "compress/zstd_lazy.c",
         "compress/zstd_ldm.c",
         "compress/zstd_opt.c",
         "compress/zstdmt_compress.c",
         "decompress/huf_decompress.c",
         "decompress/zstd_ddict.c",
         "decompress/zstd_decompress.c",
         "decompress/zstd_decompress_block.c",
         "dictBuilder/cover.c",
         "dictBuilder/fastcover.c",
         "dictBuilder/divsufsort.c",
         "dictBuilder/zdict.c",
     )
 ]
 
 # Headers whose preprocessed output will be fed into cdef().
 HEADERS = [
     os.path.join(HERE, "zstd", *p)
     for p in (
         ("zstd.h",),
         ("dictBuilder", "zdict.h"),
     )
 ]
 
 INCLUDE_DIRS = [
     os.path.join(HERE, d)
     for d in (
         "zstd",
         "zstd/common",
         "zstd/compress",
         "zstd/decompress",
         "zstd/dictBuilder",
     )
 ]
 
 # cffi can't parse some of the primitives in zstd.h. So we invoke the
 # preprocessor and feed its output into cffi.
 compiler = distutils.ccompiler.new_compiler()
 
 # Needed for MSVC.
 if hasattr(compiler, "initialize"):
     compiler.initialize()
 
 # Distutils doesn't set compiler.preprocessor, so invoke the preprocessor
 # manually.
 if compiler.compiler_type == "unix":
     args = list(compiler.executables["compiler"])
     args.extend(
         [
             "-E",
             "-DZSTD_STATIC_LINKING_ONLY",
             "-DZDICT_STATIC_LINKING_ONLY",
         ]
     )
 elif compiler.compiler_type == "msvc":
     args = [compiler.cc]
     args.extend(
         [
             "/EP",
             "/DZSTD_STATIC_LINKING_ONLY",
             "/DZDICT_STATIC_LINKING_ONLY",
         ]
     )
 else:
     raise Exception("unsupported compiler type: %s" % compiler.compiler_type)
 
 
 def preprocess(path):
     with open(path, "rb") as fh:
         lines = []
         it = iter(fh)
 
         for l in it:
             # zstd.h includes <stddef.h>, which is also included by cffi's
             # boilerplate. This can lead to duplicate declarations. So we strip
             # this include from the preprocessor invocation.
             #
             # The same things happens for including zstd.h, so give it the same
             # treatment.
             #
             # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
             # #define in zstdmt_compress.h and results in a compiler warning. So drop
             # the inline #define.
             if l.startswith(
                 (
                     b"#include <stddef.h>",
                     b'#include "zstd.h"',
                     b"#define ZSTD_STATIC_LINKING_ONLY",
                 )
             ):
                 continue
 
             # The preprocessor environment on Windows doesn't define include
             # paths, so the #include of limits.h fails. We work around this
             # by removing that import and defining INT_MAX ourselves. This is
             # a bit hacky. But it gets the job done.
             # TODO make limits.h work on Windows so we ensure INT_MAX is
             # correct.
             if l.startswith(b"#include <limits.h>"):
                 l = b"#define INT_MAX 2147483647\n"
 
             # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
             # important so just filter it out.
             if l.startswith(b"ZSTDLIB_API"):
                 l = l[len(b"ZSTDLIB_API ") :]
 
             lines.append(l)
 
     fd, input_file = tempfile.mkstemp(suffix=".h")
     os.write(fd, b"".join(lines))
     os.close(fd)
 
     try:
         env = dict(os.environ)
         if getattr(compiler, "_paths", None):
             env["PATH"] = compiler._paths
         process = subprocess.Popen(
             args + [input_file], stdout=subprocess.PIPE, env=env
         )
         output = process.communicate()[0]
         ret = process.poll()
         if ret:
             raise Exception("preprocessor exited with error")
 
         return output
     finally:
         os.unlink(input_file)
 
 
 def normalize_output(output):
     lines = []
     for line in output.splitlines():
         # CFFI's parser doesn't like __attribute__ on UNIX compilers.
         if line.startswith(b'__attribute__ ((visibility ("default"))) '):
             line = line[len(b'__attribute__ ((visibility ("default"))) ') :]
 
         if line.startswith(b"__attribute__((deprecated("):
             continue
         elif b"__declspec(deprecated(" in line:
             continue
 
         lines.append(line)
 
     return b"\n".join(lines)
 
 
 ffi = cffi.FFI()
 # zstd.h uses a possible undefined MIN(). Define it until
 # https://github.com/facebook/zstd/issues/976 is fixed.
 # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
 # when cffi uses the function. Since we statically link against zstd, even
 # if we use the deprecated functions it shouldn't be a huge problem.
 ffi.set_source(
     "_zstd_cffi",
     """
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define ZSTD_STATIC_LINKING_ONLY
 #include <zstd.h>
 #define ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_DISABLE_DEPRECATE_WARNINGS
 #include <zdict.h>
 """,
     sources=SOURCES,
     include_dirs=INCLUDE_DIRS,
     extra_compile_args=["-DZSTD_MULTITHREAD"],
 )
 
 DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
 
 sources = []
 
 # Feed normalized preprocessor output for headers into the cdef parser.
 for header in HEADERS:
     preprocessed = preprocess(header)
     sources.append(normalize_output(preprocessed))
 
     # #define's are effectively erased as part of going through preprocessor.
     # So perform a manual pass to re-add those to the cdef source.
     with open(header, "rb") as fh:
         for line in fh:
             line = line.strip()
             m = DEFINE.match(line)
             if not m:
                 continue
 
             if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
                 continue
 
             # The parser doesn't like some constants with complex values.
             if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
                 continue
 
             # The ... is magic syntax by the cdef parser to resolve the
             # value at compile time.
             sources.append(m.group(0) + b" ...")
 
 cdeflines = b"\n".join(sources).splitlines()
 cdeflines = [l for l in cdeflines if l.strip()]
 ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
 
 if __name__ == "__main__":
     ffi.compile()

contrib/python-zstandard/setup.py

@@ -1,120 +1,119 b''
 #!/usr/bin/env python
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
-from __future__ import print_function
 
 from distutils.version import LooseVersion
 import os
 import sys
 from setuptools import setup
 
 # Need change in 1.10 for ffi.from_buffer() to handle all buffer types
 # (like memoryview).
 # Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
 # garbage collection pitfalls.
 MINIMUM_CFFI_VERSION = "1.11"
 
 try:
     import cffi
 
     # PyPy (and possibly other distros) have CFFI distributed as part of
     # them. The install_requires for CFFI below won't work. We need to sniff
     # out the CFFI version here and reject CFFI if it is too old.
     cffi_version = LooseVersion(cffi.__version__)
     if cffi_version < LooseVersion(MINIMUM_CFFI_VERSION):
         print(
             "CFFI 1.11 or newer required (%s found); "
             "not building CFFI backend" % cffi_version,
             file=sys.stderr,
         )
         cffi = None
 
 except ImportError:
     cffi = None
 
 import setup_zstd
 
 SUPPORT_LEGACY = False
 SYSTEM_ZSTD = False
 WARNINGS_AS_ERRORS = False
 
 if os.environ.get("ZSTD_WARNINGS_AS_ERRORS", ""):
     WARNINGS_AS_ERRORS = True
 
 if "--legacy" in sys.argv:
     SUPPORT_LEGACY = True
     sys.argv.remove("--legacy")
 
 if "--system-zstd" in sys.argv:
     SYSTEM_ZSTD = True
     sys.argv.remove("--system-zstd")
 
 if "--warnings-as-errors" in sys.argv:
     WARNINGS_AS_ERRORS = True
     sys.argv.remove("--warning-as-errors")
 
 # Code for obtaining the Extension instance is in its own module to
 # facilitate reuse in other projects.
 extensions = [
     setup_zstd.get_c_extension(
         name="zstd",
         support_legacy=SUPPORT_LEGACY,
         system_zstd=SYSTEM_ZSTD,
         warnings_as_errors=WARNINGS_AS_ERRORS,
     ),
 ]
 
 install_requires = []
 
 if cffi:
     import make_cffi
 
     extensions.append(make_cffi.ffi.distutils_extension())
     install_requires.append("cffi>=%s" % MINIMUM_CFFI_VERSION)
 
 version = None
 
 with open("c-ext/python-zstandard.h", "r") as fh:
     for line in fh:
         if not line.startswith("#define PYTHON_ZSTANDARD_VERSION"):
             continue
 
         version = line.split()[2][1:-1]
         break
 
 if not version:
     raise Exception(
         "could not resolve package version; " "this should never happen"
     )
 
 setup(
     name="zstandard",
     version=version,
     description="Zstandard bindings for Python",
     long_description=open("README.rst", "r").read(),
     url="https://github.com/indygreg/python-zstandard",
     author="Gregory Szorc",
     author_email="[e-mail]",
     license="BSD",
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: BSD License",
         "Programming Language :: C",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
     ],
     keywords="zstandard zstd compression",
     packages=["zstandard"],
     ext_modules=extensions,
     test_suite="tests",
     install_requires=install_requires,
     tests_require=["hypothesis"],
 )

contrib/python-zstandard/tests/test_module_attributes.py

@@ -1,70 +1,68 b''
-from __future__ import unicode_literals
-
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     TestCase,
 )
 
 
 @make_cffi
 class TestModuleAttributes(TestCase):
     def test_version(self):
         self.assertEqual(zstd.ZSTD_VERSION, (1, 4, 4))
 
         self.assertEqual(zstd.__version__, "0.13.0")
 
     def test_constants(self):
         self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22)
         self.assertEqual(zstd.FRAME_HEADER, b"\x28\xb5\x2f\xfd")
 
     def test_hasattr(self):
         attrs = (
             "CONTENTSIZE_UNKNOWN",
             "CONTENTSIZE_ERROR",
             "COMPRESSION_RECOMMENDED_INPUT_SIZE",
             "COMPRESSION_RECOMMENDED_OUTPUT_SIZE",
             "DECOMPRESSION_RECOMMENDED_INPUT_SIZE",
             "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE",
             "MAGIC_NUMBER",
             "FLUSH_BLOCK",
             "FLUSH_FRAME",
             "BLOCKSIZELOG_MAX",
             "BLOCKSIZE_MAX",
             "WINDOWLOG_MIN",
             "WINDOWLOG_MAX",
             "CHAINLOG_MIN",
             "CHAINLOG_MAX",
             "HASHLOG_MIN",
             "HASHLOG_MAX",
             "HASHLOG3_MAX",
             "MINMATCH_MIN",
             "MINMATCH_MAX",
             "SEARCHLOG_MIN",
             "SEARCHLOG_MAX",
             "SEARCHLENGTH_MIN",
             "SEARCHLENGTH_MAX",
             "TARGETLENGTH_MIN",
             "TARGETLENGTH_MAX",
             "LDM_MINMATCH_MIN",
             "LDM_MINMATCH_MAX",
             "LDM_BUCKETSIZELOG_MAX",
             "STRATEGY_FAST",
             "STRATEGY_DFAST",
             "STRATEGY_GREEDY",
             "STRATEGY_LAZY",
             "STRATEGY_LAZY2",
             "STRATEGY_BTLAZY2",
             "STRATEGY_BTOPT",
             "STRATEGY_BTULTRA",
             "STRATEGY_BTULTRA2",
             "DICT_TYPE_AUTO",
             "DICT_TYPE_RAWCONTENT",
             "DICT_TYPE_FULLDICT",
         )
 
         for a in attrs:
             self.assertTrue(hasattr(zstd, a), a)

contrib/python-zstandard/zstandard/__init__.py

@@ -1,75 +1,74 b''
 # Copyright (c) 2017-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 """Python interface to the Zstandard (zstd) compression library."""
 
-from __future__ import absolute_import, unicode_literals
 
 # This module serves 2 roles:
 #
 # 1) Export the C or CFFI "backend" through a central module.
 # 2) Implement additional functionality built on top of C or CFFI backend.
 
 import os
 import platform
 
 # Some Python implementations don't support C extensions. That's why we have
 # a CFFI implementation in the first place. The code here import one of our
 # "backends" then re-exports the symbols from this module. For convenience,
 # we support falling back to the CFFI backend if the C extension can't be
 # imported. But for performance reasons, we only do this on unknown Python
 # implementation. Notably, for CPython we require the C extension by default.
 # Because someone will inevitably want special behavior, the behavior is
 # configurable via an environment variable. A potentially better way to handle
 # this is to import a special ``__importpolicy__`` module or something
 # defining a variable and `setup.py` could write the file with whatever
 # policy was specified at build time. Until someone needs it, we go with
 # the hacky but simple environment variable approach.
 _module_policy = os.environ.get("PYTHON_ZSTANDARD_IMPORT_POLICY", "default")
 
 if _module_policy == "default":
     if platform.python_implementation() in ("CPython",):
         from zstd import *
 
         backend = "cext"
     elif platform.python_implementation() in ("PyPy",):
         from .cffi import *
 
         backend = "cffi"
     else:
         try:
             from zstd import *
 
             backend = "cext"
         except ImportError:
             from .cffi import *
 
             backend = "cffi"
 elif _module_policy == "cffi_fallback":
     try:
         from zstd import *
 
         backend = "cext"
     except ImportError:
         from .cffi import *
 
         backend = "cffi"
 elif _module_policy == "cext":
     from zstd import *
 
     backend = "cext"
 elif _module_policy == "cffi":
     from .cffi import *
 
     backend = "cffi"
 else:
     raise ImportError(
         "unknown module import policy: %s; use default, cffi_fallback, "
         "cext, or cffi" % _module_policy
     )
 
 # Keep this in sync with python-zstandard.h.
 __version__ = "0.13.0"

contrib/python-zstandard/zstandard/cffi.py

@@ -1,2769 +1,2768 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 """Python interface to the Zstandard (zstd) compression library."""
 
-from __future__ import absolute_import, unicode_literals
 
 # This should match what the C extension exports.
 __all__ = [
     #'BufferSegment',
     #'BufferSegments',
     #'BufferWithSegments',
     #'BufferWithSegmentsCollection',
     "CompressionParameters",
     "ZstdCompressionDict",
     "ZstdCompressionParameters",
     "ZstdCompressor",
     "ZstdError",
     "ZstdDecompressor",
     "FrameParameters",
     "estimate_decompression_context_size",
     "frame_content_size",
     "frame_header_size",
     "get_frame_parameters",
     "train_dictionary",
     # Constants.
     "FLUSH_BLOCK",
     "FLUSH_FRAME",
     "COMPRESSOBJ_FLUSH_FINISH",
     "COMPRESSOBJ_FLUSH_BLOCK",
     "ZSTD_VERSION",
     "FRAME_HEADER",
     "CONTENTSIZE_UNKNOWN",
     "CONTENTSIZE_ERROR",
     "MAX_COMPRESSION_LEVEL",
     "COMPRESSION_RECOMMENDED_INPUT_SIZE",
     "COMPRESSION_RECOMMENDED_OUTPUT_SIZE",
     "DECOMPRESSION_RECOMMENDED_INPUT_SIZE",
     "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE",
     "MAGIC_NUMBER",
     "BLOCKSIZELOG_MAX",
     "BLOCKSIZE_MAX",
     "WINDOWLOG_MIN",
     "WINDOWLOG_MAX",
     "CHAINLOG_MIN",
     "CHAINLOG_MAX",
     "HASHLOG_MIN",
     "HASHLOG_MAX",
     "HASHLOG3_MAX",
     "MINMATCH_MIN",
     "MINMATCH_MAX",
     "SEARCHLOG_MIN",
     "SEARCHLOG_MAX",
     "SEARCHLENGTH_MIN",
     "SEARCHLENGTH_MAX",
     "TARGETLENGTH_MIN",
     "TARGETLENGTH_MAX",
     "LDM_MINMATCH_MIN",
     "LDM_MINMATCH_MAX",
     "LDM_BUCKETSIZELOG_MAX",
     "STRATEGY_FAST",
     "STRATEGY_DFAST",
     "STRATEGY_GREEDY",
     "STRATEGY_LAZY",
     "STRATEGY_LAZY2",
     "STRATEGY_BTLAZY2",
     "STRATEGY_BTOPT",
     "STRATEGY_BTULTRA",
     "STRATEGY_BTULTRA2",
     "DICT_TYPE_AUTO",
     "DICT_TYPE_RAWCONTENT",
     "DICT_TYPE_FULLDICT",
     "FORMAT_ZSTD1",
     "FORMAT_ZSTD1_MAGICLESS",
 ]
 
 import io
 import os
 import sys
 
 from _zstd_cffi import (
     ffi,
     lib,
 )
 
 if sys.version_info[0] == 2:
     bytes_type = str
     int_type = long
 else:
     bytes_type = bytes
     int_type = int
 
 
 COMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_CStreamInSize()
 COMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_CStreamOutSize()
 DECOMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_DStreamInSize()
 DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_DStreamOutSize()
 
 new_nonzero = ffi.new_allocator(should_clear_after_alloc=False)
 
 
 MAX_COMPRESSION_LEVEL = lib.ZSTD_maxCLevel()
 MAGIC_NUMBER = lib.ZSTD_MAGICNUMBER
 FRAME_HEADER = b"\x28\xb5\x2f\xfd"
 CONTENTSIZE_UNKNOWN = lib.ZSTD_CONTENTSIZE_UNKNOWN
 CONTENTSIZE_ERROR = lib.ZSTD_CONTENTSIZE_ERROR
 ZSTD_VERSION = (
     lib.ZSTD_VERSION_MAJOR,
     lib.ZSTD_VERSION_MINOR,
     lib.ZSTD_VERSION_RELEASE,
 )
 
 BLOCKSIZELOG_MAX = lib.ZSTD_BLOCKSIZELOG_MAX
 BLOCKSIZE_MAX = lib.ZSTD_BLOCKSIZE_MAX
 WINDOWLOG_MIN = lib.ZSTD_WINDOWLOG_MIN
 WINDOWLOG_MAX = lib.ZSTD_WINDOWLOG_MAX
 CHAINLOG_MIN = lib.ZSTD_CHAINLOG_MIN
 CHAINLOG_MAX = lib.ZSTD_CHAINLOG_MAX
 HASHLOG_MIN = lib.ZSTD_HASHLOG_MIN
 HASHLOG_MAX = lib.ZSTD_HASHLOG_MAX
 HASHLOG3_MAX = lib.ZSTD_HASHLOG3_MAX
 MINMATCH_MIN = lib.ZSTD_MINMATCH_MIN
 MINMATCH_MAX = lib.ZSTD_MINMATCH_MAX
 SEARCHLOG_MIN = lib.ZSTD_SEARCHLOG_MIN
 SEARCHLOG_MAX = lib.ZSTD_SEARCHLOG_MAX
 SEARCHLENGTH_MIN = lib.ZSTD_MINMATCH_MIN
 SEARCHLENGTH_MAX = lib.ZSTD_MINMATCH_MAX
 TARGETLENGTH_MIN = lib.ZSTD_TARGETLENGTH_MIN
 TARGETLENGTH_MAX = lib.ZSTD_TARGETLENGTH_MAX
 LDM_MINMATCH_MIN = lib.ZSTD_LDM_MINMATCH_MIN
 LDM_MINMATCH_MAX = lib.ZSTD_LDM_MINMATCH_MAX
 LDM_BUCKETSIZELOG_MAX = lib.ZSTD_LDM_BUCKETSIZELOG_MAX
 
 STRATEGY_FAST = lib.ZSTD_fast
 STRATEGY_DFAST = lib.ZSTD_dfast
 STRATEGY_GREEDY = lib.ZSTD_greedy
 STRATEGY_LAZY = lib.ZSTD_lazy
 STRATEGY_LAZY2 = lib.ZSTD_lazy2
 STRATEGY_BTLAZY2 = lib.ZSTD_btlazy2
 STRATEGY_BTOPT = lib.ZSTD_btopt
 STRATEGY_BTULTRA = lib.ZSTD_btultra
 STRATEGY_BTULTRA2 = lib.ZSTD_btultra2
 
 DICT_TYPE_AUTO = lib.ZSTD_dct_auto
 DICT_TYPE_RAWCONTENT = lib.ZSTD_dct_rawContent
 DICT_TYPE_FULLDICT = lib.ZSTD_dct_fullDict
 
 FORMAT_ZSTD1 = lib.ZSTD_f_zstd1
 FORMAT_ZSTD1_MAGICLESS = lib.ZSTD_f_zstd1_magicless
 
 FLUSH_BLOCK = 0
 FLUSH_FRAME = 1
 
 COMPRESSOBJ_FLUSH_FINISH = 0
 COMPRESSOBJ_FLUSH_BLOCK = 1
 
 
 def _cpu_count():
     # os.cpu_count() was introducd in Python 3.4.
     try:
         return os.cpu_count() or 0
     except AttributeError:
         pass
 
     # Linux.
     try:
         if sys.version_info[0] == 2:
             return os.sysconf(b"SC_NPROCESSORS_ONLN")
         else:
             return os.sysconf("SC_NPROCESSORS_ONLN")
     except (AttributeError, ValueError):
         pass
 
     # TODO implement on other platforms.
     return 0
 
 
 class ZstdError(Exception):
     pass
 
 
 def _zstd_error(zresult):
     # Resolves to bytes on Python 2 and 3. We use the string for formatting
     # into error messages, which will be literal unicode. So convert it to
     # unicode.
     return ffi.string(lib.ZSTD_getErrorName(zresult)).decode("utf-8")
 
 
 def _make_cctx_params(params):
     res = lib.ZSTD_createCCtxParams()
     if res == ffi.NULL:
         raise MemoryError()
 
     res = ffi.gc(res, lib.ZSTD_freeCCtxParams)
 
     attrs = [
         (lib.ZSTD_c_format, params.format),
         (lib.ZSTD_c_compressionLevel, params.compression_level),
         (lib.ZSTD_c_windowLog, params.window_log),
         (lib.ZSTD_c_hashLog, params.hash_log),
         (lib.ZSTD_c_chainLog, params.chain_log),
         (lib.ZSTD_c_searchLog, params.search_log),
         (lib.ZSTD_c_minMatch, params.min_match),
         (lib.ZSTD_c_targetLength, params.target_length),
         (lib.ZSTD_c_strategy, params.compression_strategy),
         (lib.ZSTD_c_contentSizeFlag, params.write_content_size),
         (lib.ZSTD_c_checksumFlag, params.write_checksum),
         (lib.ZSTD_c_dictIDFlag, params.write_dict_id),
         (lib.ZSTD_c_nbWorkers, params.threads),
         (lib.ZSTD_c_jobSize, params.job_size),
         (lib.ZSTD_c_overlapLog, params.overlap_log),
         (lib.ZSTD_c_forceMaxWindow, params.force_max_window),
         (lib.ZSTD_c_enableLongDistanceMatching, params.enable_ldm),
         (lib.ZSTD_c_ldmHashLog, params.ldm_hash_log),
         (lib.ZSTD_c_ldmMinMatch, params.ldm_min_match),
         (lib.ZSTD_c_ldmBucketSizeLog, params.ldm_bucket_size_log),
         (lib.ZSTD_c_ldmHashRateLog, params.ldm_hash_rate_log),
     ]
 
     for param, value in attrs:
         _set_compression_parameter(res, param, value)
 
     return res
 
 
 class ZstdCompressionParameters(object):
     @staticmethod
     def from_level(level, source_size=0, dict_size=0, **kwargs):
         params = lib.ZSTD_getCParams(level, source_size, dict_size)
 
         args = {
             "window_log": "windowLog",
             "chain_log": "chainLog",
             "hash_log": "hashLog",
             "search_log": "searchLog",
             "min_match": "minMatch",
             "target_length": "targetLength",
             "compression_strategy": "strategy",
         }
 
         for arg, attr in args.items():
             if arg not in kwargs:
                 kwargs[arg] = getattr(params, attr)
 
         return ZstdCompressionParameters(**kwargs)
 
     def __init__(
         self,
         format=0,
         compression_level=0,
         window_log=0,
         hash_log=0,
         chain_log=0,
         search_log=0,
         min_match=0,
         target_length=0,
         strategy=-1,
         compression_strategy=-1,
         write_content_size=1,
         write_checksum=0,
         write_dict_id=0,
         job_size=0,
         overlap_log=-1,
         overlap_size_log=-1,
         force_max_window=0,
         enable_ldm=0,
         ldm_hash_log=0,
         ldm_min_match=0,
         ldm_bucket_size_log=0,
         ldm_hash_rate_log=-1,
         ldm_hash_every_log=-1,
         threads=0,
     ):
 
         params = lib.ZSTD_createCCtxParams()
         if params == ffi.NULL:
             raise MemoryError()
 
         params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
         self._params = params
 
         if threads < 0:
             threads = _cpu_count()
 
         # We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
         # because setting ZSTD_c_nbWorkers resets the other parameters.
         _set_compression_parameter(params, lib.ZSTD_c_nbWorkers, threads)
 
         _set_compression_parameter(params, lib.ZSTD_c_format, format)
         _set_compression_parameter(
             params, lib.ZSTD_c_compressionLevel, compression_level
         )
         _set_compression_parameter(params, lib.ZSTD_c_windowLog, window_log)
         _set_compression_parameter(params, lib.ZSTD_c_hashLog, hash_log)
         _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log)
         _set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log)
         _set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match)
         _set_compression_parameter(
             params, lib.ZSTD_c_targetLength, target_length
         )
 
         if strategy != -1 and compression_strategy != -1:
             raise ValueError(
                 "cannot specify both compression_strategy and strategy"
             )
 
         if compression_strategy != -1:
             strategy = compression_strategy
         elif strategy == -1:
             strategy = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_strategy, strategy)
         _set_compression_parameter(
             params, lib.ZSTD_c_contentSizeFlag, write_content_size
         )
         _set_compression_parameter(
             params, lib.ZSTD_c_checksumFlag, write_checksum
         )
         _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id)
         _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size)
 
         if overlap_log != -1 and overlap_size_log != -1:
             raise ValueError(
                 "cannot specify both overlap_log and overlap_size_log"
             )
 
         if overlap_size_log != -1:
             overlap_log = overlap_size_log
         elif overlap_log == -1:
             overlap_log = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log)
         _set_compression_parameter(
             params, lib.ZSTD_c_forceMaxWindow, force_max_window
         )
         _set_compression_parameter(
             params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm
         )
         _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log)
         _set_compression_parameter(
             params, lib.ZSTD_c_ldmMinMatch, ldm_min_match
         )
         _set_compression_parameter(
             params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log
         )
 
         if ldm_hash_rate_log != -1 and ldm_hash_every_log != -1:
             raise ValueError(
                 "cannot specify both ldm_hash_rate_log and ldm_hash_every_log"
             )
 
         if ldm_hash_every_log != -1:
             ldm_hash_rate_log = ldm_hash_every_log
         elif ldm_hash_rate_log == -1:
             ldm_hash_rate_log = 0
 
         _set_compression_parameter(
             params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log
         )
 
     @property
     def format(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_format)
 
     @property
     def compression_level(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_compressionLevel
         )
 
     @property
     def window_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_windowLog)
 
     @property
     def hash_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_hashLog)
 
     @property
     def chain_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_chainLog)
 
     @property
     def search_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_searchLog)
 
     @property
     def min_match(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_minMatch)
 
     @property
     def target_length(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_targetLength)
 
     @property
     def compression_strategy(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_strategy)
 
     @property
     def write_content_size(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_contentSizeFlag
         )
 
     @property
     def write_checksum(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_checksumFlag)
 
     @property
     def write_dict_id(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_dictIDFlag)
 
     @property
     def job_size(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_jobSize)
 
     @property
     def overlap_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_overlapLog)
 
     @property
     def overlap_size_log(self):
         return self.overlap_log
 
     @property
     def force_max_window(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_forceMaxWindow
         )
 
     @property
     def enable_ldm(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_enableLongDistanceMatching
         )
 
     @property
     def ldm_hash_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashLog)
 
     @property
     def ldm_min_match(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmMinMatch)
 
     @property
     def ldm_bucket_size_log(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_ldmBucketSizeLog
         )
 
     @property
     def ldm_hash_rate_log(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_ldmHashRateLog
         )
 
     @property
     def ldm_hash_every_log(self):
         return self.ldm_hash_rate_log
 
     @property
     def threads(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_nbWorkers)
 
     def estimated_compression_context_size(self):
         return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self._params)
 
 
 CompressionParameters = ZstdCompressionParameters
 
 
 def estimate_decompression_context_size():
     return lib.ZSTD_estimateDCtxSize()
 
 
 def _set_compression_parameter(params, param, value):
     zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value)
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
             "unable to set compression context parameter: %s"
             % _zstd_error(zresult)
         )
 
 
 def _get_compression_parameter(params, param):
     result = ffi.new("int *")
 
     zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result)
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
             "unable to get compression context parameter: %s"
             % _zstd_error(zresult)
         )
 
     return result[0]
 
 
 class ZstdCompressionWriter(object):
     def __init__(
         self, compressor, writer, source_size, write_size, write_return_read
     ):
         self._compressor = compressor
         self._writer = writer
         self._write_size = write_size
         self._write_return_read = bool(write_return_read)
         self._entered = False
         self._closed = False
         self._bytes_compressed = 0
 
         self._dst_buffer = ffi.new("char[]", write_size)
         self._out_buffer = ffi.new("ZSTD_outBuffer *")
         self._out_buffer.dst = self._dst_buffer
         self._out_buffer.size = len(self._dst_buffer)
         self._out_buffer.pos = 0
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, source_size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
     def __enter__(self):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._entered:
             raise ZstdError("cannot __enter__ multiple times")
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
 
         if not exc_type and not exc_value and not exc_tb:
             self.close()
 
         self._compressor = None
 
         return False
 
     def memory_size(self):
         return lib.ZSTD_sizeof_CCtx(self._compressor._cctx)
 
     def fileno(self):
         f = getattr(self._writer, "fileno", None)
         if f:
             return f()
         else:
             raise OSError("fileno not available on underlying writer")
 
     def close(self):
         if self._closed:
             return
 
         try:
             self.flush(FLUSH_FRAME)
         finally:
             self._closed = True
 
         # Call close() on underlying stream as well.
         f = getattr(self._writer, "close", None)
         if f:
             f()
 
     @property
     def closed(self):
         return self._closed
 
     def isatty(self):
         return False
 
     def readable(self):
         return False
 
     def readline(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readlines(self, hint=-1):
         raise io.UnsupportedOperation()
 
     def seek(self, offset, whence=None):
         raise io.UnsupportedOperation()
 
     def seekable(self):
         return False
 
     def truncate(self, size=None):
         raise io.UnsupportedOperation()
 
     def writable(self):
         return True
 
     def writelines(self, lines):
         raise NotImplementedError("writelines() is not yet implemented")
 
     def read(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readall(self):
         raise io.UnsupportedOperation()
 
     def readinto(self, b):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         if self._closed:
             raise ValueError("stream is closed")
 
         total_write = 0
 
         data_buffer = ffi.from_buffer(data)
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx,
                 out_buffer,
                 in_buffer,
                 lib.ZSTD_e_continue,
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd compress error: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 self._writer.write(
                     ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                 )
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
         if self._write_return_read:
             return in_buffer.pos
         else:
             return total_write
 
     def flush(self, flush_mode=FLUSH_BLOCK):
         if flush_mode == FLUSH_BLOCK:
             flush = lib.ZSTD_e_flush
         elif flush_mode == FLUSH_FRAME:
             flush = lib.ZSTD_e_end
         else:
             raise ValueError("unknown flush_mode: %r" % flush_mode)
 
         if self._closed:
             raise ValueError("stream is closed")
 
         total_write = 0
 
         out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, out_buffer, in_buffer, flush
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd compress error: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 self._writer.write(
                     ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                 )
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
             if not zresult:
                 break
 
         return total_write
 
     def tell(self):
         return self._bytes_compressed
 
 
 class ZstdCompressionObj(object):
     def compress(self, data):
         if self._finished:
             raise ZstdError("cannot call compress() after compressor finished")
 
         data_buffer = ffi.from_buffer(data)
         source = ffi.new("ZSTD_inBuffer *")
         source.src = data_buffer
         source.size = len(data_buffer)
         source.pos = 0
 
         chunks = []
 
         while source.pos < len(data):
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, source, lib.ZSTD_e_continue
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd compress error: %s" % _zstd_error(zresult)
                 )
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
                 self._out.pos = 0
 
         return b"".join(chunks)
 
     def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH):
         if flush_mode not in (
             COMPRESSOBJ_FLUSH_FINISH,
             COMPRESSOBJ_FLUSH_BLOCK,
         ):
             raise ValueError("flush mode not recognized")
 
         if self._finished:
             raise ZstdError("compressor object already finished")
 
         if flush_mode == COMPRESSOBJ_FLUSH_BLOCK:
             z_flush_mode = lib.ZSTD_e_flush
         elif flush_mode == COMPRESSOBJ_FLUSH_FINISH:
             z_flush_mode = lib.ZSTD_e_end
             self._finished = True
         else:
             raise ZstdError("unhandled flush mode")
 
         assert self._out.pos == 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         chunks = []
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, in_buffer, z_flush_mode
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "error ending compression stream: %s" % _zstd_error(zresult)
                 )
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
                 self._out.pos = 0
 
             if not zresult:
                 break
 
         return b"".join(chunks)
 
 
 class ZstdCompressionChunker(object):
     def __init__(self, compressor, chunk_size):
         self._compressor = compressor
         self._out = ffi.new("ZSTD_outBuffer *")
         self._dst_buffer = ffi.new("char[]", chunk_size)
         self._out.dst = self._dst_buffer
         self._out.size = chunk_size
         self._out.pos = 0
 
         self._in = ffi.new("ZSTD_inBuffer *")
         self._in.src = ffi.NULL
         self._in.size = 0
         self._in.pos = 0
         self._finished = False
 
     def compress(self, data):
         if self._finished:
             raise ZstdError("cannot call compress() after compression finished")
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
                 "cannot perform operation before consuming output "
                 "from previous operation"
             )
 
         data_buffer = ffi.from_buffer(data)
 
         if not len(data_buffer):
             return
 
         self._in.src = data_buffer
         self._in.size = len(data_buffer)
         self._in.pos = 0
 
         while self._in.pos < self._in.size:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_continue
             )
 
             if self._in.pos == self._in.size:
                 self._in.src = ffi.NULL
                 self._in.size = 0
                 self._in.pos = 0
 
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd compress error: %s" % _zstd_error(zresult)
                 )
 
             if self._out.pos == self._out.size:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
     def flush(self):
         if self._finished:
             raise ZstdError("cannot call flush() after compression finished")
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
                 "cannot call flush() before consuming output from "
                 "previous operation"
             )
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_flush
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd compress error: %s" % _zstd_error(zresult)
                 )
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
             if not zresult:
                 return
 
     def finish(self):
         if self._finished:
             raise ZstdError("cannot call finish() after compression finished")
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
                 "cannot call finish() before consuming output from "
                 "previous operation"
             )
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd compress error: %s" % _zstd_error(zresult)
                 )
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
             if not zresult:
                 self._finished = True
                 return
 
 
 class ZstdCompressionReader(object):
     def __init__(self, compressor, source, read_size):
         self._compressor = compressor
         self._source = source
         self._read_size = read_size
         self._entered = False
         self._closed = False
         self._bytes_compressed = 0
         self._finished_input = False
         self._finished_output = False
 
         self._in_buffer = ffi.new("ZSTD_inBuffer *")
         # Holds a ref so backing bytes in self._in_buffer stay alive.
         self._source_buffer = None
 
     def __enter__(self):
         if self._entered:
             raise ValueError("cannot __enter__ multiple times")
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self._closed = True
         self._source = None
         self._compressor = None
 
         return False
 
     def readable(self):
         return True
 
     def writable(self):
         return False
 
     def seekable(self):
         return False
 
     def readline(self):
         raise io.UnsupportedOperation()
 
     def readlines(self):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         raise OSError("stream is not writable")
 
     def writelines(self, ignored):
         raise OSError("stream is not writable")
 
     def isatty(self):
         return False
 
     def flush(self):
         return None
 
     def close(self):
         self._closed = True
         return None
 
     @property
     def closed(self):
         return self._closed
 
     def tell(self):
         return self._bytes_compressed
 
     def readall(self):
         chunks = []
 
         while True:
             chunk = self.read(1048576)
             if not chunk:
                 break
 
             chunks.append(chunk)
 
         return b"".join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
 
     def __next__(self):
         raise io.UnsupportedOperation()
 
     next = __next__
 
     def _read_input(self):
         if self._finished_input:
             return
 
         if hasattr(self._source, "read"):
             data = self._source.read(self._read_size)
 
             if not data:
                 self._finished_input = True
                 return
 
             self._source_buffer = ffi.from_buffer(data)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
         else:
             self._source_buffer = ffi.from_buffer(self._source)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
 
     def _compress_into_buffer(self, out_buffer):
         if self._in_buffer.pos >= self._in_buffer.size:
             return
 
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx,
             out_buffer,
             self._in_buffer,
             lib.ZSTD_e_continue,
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if self._in_buffer.pos == self._in_buffer.size:
             self._in_buffer.src = ffi.NULL
             self._in_buffer.pos = 0
             self._in_buffer.size = 0
             self._source_buffer = None
 
             if not hasattr(self._source, "read"):
                 self._finished_input = True
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError("zstd compress error: %s", _zstd_error(zresult))
 
         return out_buffer.pos and out_buffer.pos == out_buffer.size
 
     def read(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if size == -1:
             return self.readall()
 
         if self._finished_output or size == 0:
             return b""
 
         # Need a dedicated ref to dest buffer otherwise it gets collected.
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         if self._compress_into_buffer(out_buffer):
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
 
             if self._compress_into_buffer(out_buffer):
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         # EOF
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error ending compression stream: %s", _zstd_error(zresult)
             )
 
         if zresult == 0:
             self._finished_output = True
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def read1(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if self._finished_output or size == 0:
             return b""
 
         # -1 returns arbitrary number of bytes.
         if size == -1:
             size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         # read1() dictates that we can perform at most 1 call to the
         # underlying stream to get input. However, we can't satisfy this
         # restriction with compression because not all input generates output.
         # It is possible to perform a block flush in order to ensure output.
         # But this may not be desirable behavior. So we allow multiple read()
         # to the underlying stream. But unlike read(), we stop once we have
         # any output.
 
         self._compress_into_buffer(out_buffer)
         if out_buffer.pos:
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
 
             # If we've filled the output buffer, return immediately.
             if self._compress_into_buffer(out_buffer):
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
             # If we've populated the output buffer and we're not at EOF,
             # also return, as we've satisfied the read1() limits.
             if out_buffer.pos and not self._finished_input:
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
             # Else if we're at EOS and we have room left in the buffer,
             # fall through to below and try to add more data to the output.
 
         # EOF.
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error ending compression stream: %s" % _zstd_error(zresult)
             )
 
         if zresult == 0:
             self._finished_output = True
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         if self._compress_into_buffer(out_buffer):
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
             if self._compress_into_buffer(out_buffer):
                 return out_buffer.pos
 
         # EOF.
         old_pos = out_buffer.pos
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error ending compression stream: %s", _zstd_error(zresult)
             )
 
         if zresult == 0:
             self._finished_output = True
 
         return out_buffer.pos
 
     def readinto1(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         self._compress_into_buffer(out_buffer)
         if out_buffer.pos:
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
 
             if self._compress_into_buffer(out_buffer):
                 return out_buffer.pos
 
             if out_buffer.pos and not self._finished_input:
                 return out_buffer.pos
 
         # EOF.
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error ending compression stream: %s" % _zstd_error(zresult)
             )
 
         if zresult == 0:
             self._finished_output = True
 
         return out_buffer.pos
 
 
 class ZstdCompressor(object):
     def __init__(
         self,
         level=3,
         dict_data=None,
         compression_params=None,
         write_checksum=None,
         write_content_size=None,
         write_dict_id=None,
         threads=0,
     ):
         if level > lib.ZSTD_maxCLevel():
             raise ValueError(
                 "level must be less than %d" % lib.ZSTD_maxCLevel()
             )
 
         if threads < 0:
             threads = _cpu_count()
 
         if compression_params and write_checksum is not None:
             raise ValueError(
                 "cannot define compression_params and " "write_checksum"
             )
 
         if compression_params and write_content_size is not None:
             raise ValueError(
                 "cannot define compression_params and " "write_content_size"
             )
 
         if compression_params and write_dict_id is not None:
             raise ValueError(
                 "cannot define compression_params and " "write_dict_id"
             )
 
         if compression_params and threads:
             raise ValueError("cannot define compression_params and threads")
 
         if compression_params:
             self._params = _make_cctx_params(compression_params)
         else:
             if write_dict_id is None:
                 write_dict_id = True
 
             params = lib.ZSTD_createCCtxParams()
             if params == ffi.NULL:
                 raise MemoryError()
 
             self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
             _set_compression_parameter(
                 self._params, lib.ZSTD_c_compressionLevel, level
             )
 
             _set_compression_parameter(
                 self._params,
                 lib.ZSTD_c_contentSizeFlag,
                 write_content_size if write_content_size is not None else 1,
             )
 
             _set_compression_parameter(
                 self._params,
                 lib.ZSTD_c_checksumFlag,
                 1 if write_checksum else 0,
             )
 
             _set_compression_parameter(
                 self._params, lib.ZSTD_c_dictIDFlag, 1 if write_dict_id else 0
             )
 
             if threads:
                 _set_compression_parameter(
                     self._params, lib.ZSTD_c_nbWorkers, threads
                 )
 
         cctx = lib.ZSTD_createCCtx()
         if cctx == ffi.NULL:
             raise MemoryError()
 
         self._cctx = cctx
         self._dict_data = dict_data
 
         # We defer setting up garbage collection until after calling
         # _setup_cctx() to ensure the memory size estimate is more accurate.
         try:
             self._setup_cctx()
         finally:
             self._cctx = ffi.gc(
                 cctx, lib.ZSTD_freeCCtx, size=lib.ZSTD_sizeof_CCtx(cctx)
             )
 
     def _setup_cctx(self):
         zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(
             self._cctx, self._params
         )
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "could not set compression parameters: %s"
                 % _zstd_error(zresult)
             )
 
         dict_data = self._dict_data
 
         if dict_data:
             if dict_data._cdict:
                 zresult = lib.ZSTD_CCtx_refCDict(self._cctx, dict_data._cdict)
             else:
                 zresult = lib.ZSTD_CCtx_loadDictionary_advanced(
                     self._cctx,
                     dict_data.as_bytes(),
                     len(dict_data),
                     lib.ZSTD_dlm_byRef,
                     dict_data._dict_type,
                 )
 
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "could not load compression dictionary: %s"
                     % _zstd_error(zresult)
                 )
 
     def memory_size(self):
         return lib.ZSTD_sizeof_CCtx(self._cctx)
 
     def compress(self, data):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         data_buffer = ffi.from_buffer(data)
 
         dest_size = lib.ZSTD_compressBound(len(data_buffer))
         out = new_nonzero("char[]", dest_size)
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer))
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         in_buffer = ffi.new("ZSTD_inBuffer *")
 
         out_buffer.dst = out
         out_buffer.size = dest_size
         out_buffer.pos = 0
 
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_compressStream2(
             self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end
         )
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError("cannot compress: %s" % _zstd_error(zresult))
         elif zresult:
             raise ZstdError("unexpected partial frame flush")
 
         return ffi.buffer(out, out_buffer.pos)[:]
 
     def compressobj(self, size=-1):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
         cobj = ZstdCompressionObj()
         cobj._out = ffi.new("ZSTD_outBuffer *")
         cobj._dst_buffer = ffi.new(
             "char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE
         )
         cobj._out.dst = cobj._dst_buffer
         cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
         cobj._out.pos = 0
         cobj._compressor = self
         cobj._finished = False
 
         return cobj
 
     def chunker(self, size=-1, chunk_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
         return ZstdCompressionChunker(self, chunk_size=chunk_size)
 
     def copy_stream(
         self,
         ifh,
         ofh,
         size=-1,
         read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
     ):
 
         if not hasattr(ifh, "read"):
             raise ValueError("first argument must have a read() method")
         if not hasattr(ofh, "write"):
             raise ValueError("second argument must have a write() method")
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         total_read, total_write = 0, 0
 
         while True:
             data = ifh.read(read_size)
             if not data:
                 break
 
             data_buffer = ffi.from_buffer(data)
             total_read += len(data_buffer)
             in_buffer.src = data_buffer
             in_buffer.size = len(data_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_compressStream2(
                     self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
                 )
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError(
                         "zstd compress error: %s" % _zstd_error(zresult)
                     )
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                     total_write += out_buffer.pos
                     out_buffer.pos = 0
 
         # We've finished reading. Flush the compressor.
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "error ending compression stream: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
             if zresult == 0:
                 break
 
         return total_read, total_write
 
     def stream_reader(
         self, source, size=-1, read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE
     ):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         try:
             size = len(source)
         except Exception:
             pass
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
         return ZstdCompressionReader(self, source, read_size)
 
     def stream_writer(
         self,
         writer,
         size=-1,
         write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         write_return_read=False,
     ):
 
         if not hasattr(writer, "write"):
             raise ValueError("must pass an object with a write() method")
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         return ZstdCompressionWriter(
             self, writer, size, write_size, write_return_read
         )
 
     write_to = stream_writer
 
     def read_to_iter(
         self,
         reader,
         size=-1,
         read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
     ):
         if hasattr(reader, "read"):
             have_read = True
         elif hasattr(reader, "__getitem__"):
             have_read = False
             buffer_offset = 0
             size = len(reader)
         else:
             raise ValueError(
                 "must pass an object with a read() method or "
                 "conforms to buffer protocol"
             )
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error setting source size: %s" % _zstd_error(zresult)
             )
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         while True:
             # We should never have output data sitting around after a previous
             # iteration.
             assert out_buffer.pos == 0
 
             # Collect input data.
             if have_read:
                 read_result = reader.read(read_size)
             else:
                 remaining = len(reader) - buffer_offset
                 slice_size = min(remaining, read_size)
                 read_result = reader[buffer_offset : buffer_offset + slice_size]
                 buffer_offset += slice_size
 
             # No new input data. Break out of the read loop.
             if not read_result:
                 break
 
             # Feed all read data into the compressor and emit output until
             # exhausted.
             read_buffer = ffi.from_buffer(read_result)
             in_buffer.src = read_buffer
             in_buffer.size = len(read_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_compressStream2(
                     self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
                 )
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError(
                         "zstd compress error: %s" % _zstd_error(zresult)
                     )
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                     out_buffer.pos = 0
                     yield data
 
             assert out_buffer.pos == 0
 
             # And repeat the loop to collect more data.
             continue
 
         # If we get here, input is exhausted. End the stream and emit what
         # remains.
         while True:
             assert out_buffer.pos == 0
             zresult = lib.ZSTD_compressStream2(
                 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "error ending compression stream: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                 out_buffer.pos = 0
                 yield data
 
             if zresult == 0:
                 break
 
     read_from = read_to_iter
 
     def frame_progression(self):
         progression = lib.ZSTD_getFrameProgression(self._cctx)
 
         return progression.ingested, progression.consumed, progression.produced
 
 
 class FrameParameters(object):
     def __init__(self, fparams):
         self.content_size = fparams.frameContentSize
         self.window_size = fparams.windowSize
         self.dict_id = fparams.dictID
         self.has_checksum = bool(fparams.checksumFlag)
 
 
 def frame_content_size(data):
     data_buffer = ffi.from_buffer(data)
 
     size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
 
     if size == lib.ZSTD_CONTENTSIZE_ERROR:
         raise ZstdError("error when determining content size")
     elif size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
         return -1
     else:
         return size
 
 
 def frame_header_size(data):
     data_buffer = ffi.from_buffer(data)
 
     zresult = lib.ZSTD_frameHeaderSize(data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
             "could not determine frame header size: %s" % _zstd_error(zresult)
         )
 
     return zresult
 
 
 def get_frame_parameters(data):
     params = ffi.new("ZSTD_frameHeader *")
 
     data_buffer = ffi.from_buffer(data)
     zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
             "cannot get frame parameters: %s" % _zstd_error(zresult)
         )
 
     if zresult:
         raise ZstdError(
             "not enough data for frame parameters; need %d bytes" % zresult
         )
 
     return FrameParameters(params[0])
 
 
 class ZstdCompressionDict(object):
     def __init__(self, data, dict_type=DICT_TYPE_AUTO, k=0, d=0):
         assert isinstance(data, bytes_type)
         self._data = data
         self.k = k
         self.d = d
 
         if dict_type not in (
             DICT_TYPE_AUTO,
             DICT_TYPE_RAWCONTENT,
             DICT_TYPE_FULLDICT,
         ):
             raise ValueError(
                 "invalid dictionary load mode: %d; must use "
                 "DICT_TYPE_* constants"
             )
 
         self._dict_type = dict_type
         self._cdict = None
 
     def __len__(self):
         return len(self._data)
 
     def dict_id(self):
         return int_type(lib.ZDICT_getDictID(self._data, len(self._data)))
 
     def as_bytes(self):
         return self._data
 
     def precompute_compress(self, level=0, compression_params=None):
         if level and compression_params:
             raise ValueError(
                 "must only specify one of level or " "compression_params"
             )
 
         if not level and not compression_params:
             raise ValueError("must specify one of level or compression_params")
 
         if level:
             cparams = lib.ZSTD_getCParams(level, 0, len(self._data))
         else:
             cparams = ffi.new("ZSTD_compressionParameters")
             cparams.chainLog = compression_params.chain_log
             cparams.hashLog = compression_params.hash_log
             cparams.minMatch = compression_params.min_match
             cparams.searchLog = compression_params.search_log
             cparams.strategy = compression_params.compression_strategy
             cparams.targetLength = compression_params.target_length
             cparams.windowLog = compression_params.window_log
 
         cdict = lib.ZSTD_createCDict_advanced(
             self._data,
             len(self._data),
             lib.ZSTD_dlm_byRef,
             self._dict_type,
             cparams,
             lib.ZSTD_defaultCMem,
         )
         if cdict == ffi.NULL:
             raise ZstdError("unable to precompute dictionary")
 
         self._cdict = ffi.gc(
             cdict, lib.ZSTD_freeCDict, size=lib.ZSTD_sizeof_CDict(cdict)
         )
 
     @property
     def _ddict(self):
         ddict = lib.ZSTD_createDDict_advanced(
             self._data,
             len(self._data),
             lib.ZSTD_dlm_byRef,
             self._dict_type,
             lib.ZSTD_defaultCMem,
         )
 
         if ddict == ffi.NULL:
             raise ZstdError("could not create decompression dict")
 
         ddict = ffi.gc(
             ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict)
         )
         self.__dict__["_ddict"] = ddict
 
         return ddict
 
 
 def train_dictionary(
     dict_size,
     samples,
     k=0,
     d=0,
     notifications=0,
     dict_id=0,
     level=0,
     steps=0,
     threads=0,
 ):
     if not isinstance(samples, list):
         raise TypeError("samples must be a list")
 
     if threads < 0:
         threads = _cpu_count()
 
     total_size = sum(map(len, samples))
 
     samples_buffer = new_nonzero("char[]", total_size)
     sample_sizes = new_nonzero("size_t[]", len(samples))
 
     offset = 0
     for i, sample in enumerate(samples):
         if not isinstance(sample, bytes_type):
             raise ValueError("samples must be bytes")
 
         l = len(sample)
         ffi.memmove(samples_buffer + offset, sample, l)
         offset += l
         sample_sizes[i] = l
 
     dict_data = new_nonzero("char[]", dict_size)
 
     dparams = ffi.new("ZDICT_cover_params_t *")[0]
     dparams.k = k
     dparams.d = d
     dparams.steps = steps
     dparams.nbThreads = threads
     dparams.zParams.notificationLevel = notifications
     dparams.zParams.dictID = dict_id
     dparams.zParams.compressionLevel = level
 
     if (
         not dparams.k
         and not dparams.d
         and not dparams.steps
         and not dparams.nbThreads
         and not dparams.zParams.notificationLevel
         and not dparams.zParams.dictID
         and not dparams.zParams.compressionLevel
     ):
         zresult = lib.ZDICT_trainFromBuffer(
             ffi.addressof(dict_data),
             dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0),
             len(samples),
         )
     elif dparams.steps or dparams.nbThreads:
         zresult = lib.ZDICT_optimizeTrainFromBuffer_cover(
             ffi.addressof(dict_data),
             dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0),
             len(samples),
             ffi.addressof(dparams),
         )
     else:
         zresult = lib.ZDICT_trainFromBuffer_cover(
             ffi.addressof(dict_data),
             dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0),
             len(samples),
             dparams,
         )
 
     if lib.ZDICT_isError(zresult):
         msg = ffi.string(lib.ZDICT_getErrorName(zresult)).decode("utf-8")
         raise ZstdError("cannot train dict: %s" % msg)
 
     return ZstdCompressionDict(
         ffi.buffer(dict_data, zresult)[:],
         dict_type=DICT_TYPE_FULLDICT,
         k=dparams.k,
         d=dparams.d,
     )
 
 
 class ZstdDecompressionObj(object):
     def __init__(self, decompressor, write_size):
         self._decompressor = decompressor
         self._write_size = write_size
         self._finished = False
 
     def decompress(self, data):
         if self._finished:
             raise ZstdError("cannot use a decompressobj multiple times")
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         data_buffer = ffi.from_buffer(data)
 
         if len(data_buffer) == 0:
             return b""
 
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         dst_buffer = ffi.new("char[]", self._write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         chunks = []
 
         while True:
             zresult = lib.ZSTD_decompressStream(
                 self._decompressor._dctx, out_buffer, in_buffer
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd decompressor error: %s" % _zstd_error(zresult)
                 )
 
             if zresult == 0:
                 self._finished = True
                 self._decompressor = None
 
             if out_buffer.pos:
                 chunks.append(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
 
             if zresult == 0 or (
                 in_buffer.pos == in_buffer.size and out_buffer.pos == 0
             ):
                 break
 
             out_buffer.pos = 0
 
         return b"".join(chunks)
 
     def flush(self, length=0):
         pass
 
 
 class ZstdDecompressionReader(object):
     def __init__(self, decompressor, source, read_size, read_across_frames):
         self._decompressor = decompressor
         self._source = source
         self._read_size = read_size
         self._read_across_frames = bool(read_across_frames)
         self._entered = False
         self._closed = False
         self._bytes_decompressed = 0
         self._finished_input = False
         self._finished_output = False
         self._in_buffer = ffi.new("ZSTD_inBuffer *")
         # Holds a ref to self._in_buffer.src.
         self._source_buffer = None
 
     def __enter__(self):
         if self._entered:
             raise ValueError("cannot __enter__ multiple times")
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self._closed = True
         self._source = None
         self._decompressor = None
 
         return False
 
     def readable(self):
         return True
 
     def writable(self):
         return False
 
     def seekable(self):
         return True
 
     def readline(self):
         raise io.UnsupportedOperation()
 
     def readlines(self):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         raise io.UnsupportedOperation()
 
     def writelines(self, lines):
         raise io.UnsupportedOperation()
 
     def isatty(self):
         return False
 
     def flush(self):
         return None
 
     def close(self):
         self._closed = True
         return None
 
     @property
     def closed(self):
         return self._closed
 
     def tell(self):
         return self._bytes_decompressed
 
     def readall(self):
         chunks = []
 
         while True:
             chunk = self.read(1048576)
             if not chunk:
                 break
 
             chunks.append(chunk)
 
         return b"".join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
 
     def __next__(self):
         raise io.UnsupportedOperation()
 
     next = __next__
 
     def _read_input(self):
         # We have data left over in the input buffer. Use it.
         if self._in_buffer.pos < self._in_buffer.size:
             return
 
         # All input data exhausted. Nothing to do.
         if self._finished_input:
             return
 
         # Else populate the input buffer from our source.
         if hasattr(self._source, "read"):
             data = self._source.read(self._read_size)
 
             if not data:
                 self._finished_input = True
                 return
 
             self._source_buffer = ffi.from_buffer(data)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
         else:
             self._source_buffer = ffi.from_buffer(self._source)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
 
     def _decompress_into_buffer(self, out_buffer):
         """Decompress available input into an output buffer.
 
         Returns True if data in output buffer should be emitted.
         """
         zresult = lib.ZSTD_decompressStream(
             self._decompressor._dctx, out_buffer, self._in_buffer
         )
 
         if self._in_buffer.pos == self._in_buffer.size:
             self._in_buffer.src = ffi.NULL
             self._in_buffer.pos = 0
             self._in_buffer.size = 0
             self._source_buffer = None
 
             if not hasattr(self._source, "read"):
                 self._finished_input = True
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
 
         # Emit data if there is data AND either:
         # a) output buffer is full (read amount is satisfied)
         # b) we're at end of a frame and not in frame spanning mode
         return out_buffer.pos and (
             out_buffer.pos == out_buffer.size
             or zresult == 0
             and not self._read_across_frames
         )
 
     def read(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if size == -1:
             # This is recursive. But it gets the job done.
             return self.readall()
 
         if self._finished_output or size == 0:
             return b""
 
         # We /could/ call into readinto() here. But that introduces more
         # overhead.
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         self._read_input()
         if self._decompress_into_buffer(out_buffer):
             self._bytes_decompressed += out_buffer.pos
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
             if self._decompress_into_buffer(out_buffer):
                 self._bytes_decompressed += out_buffer.pos
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         self._read_input()
         if self._decompress_into_buffer(out_buffer):
             self._bytes_decompressed += out_buffer.pos
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
             if self._decompress_into_buffer(out_buffer):
                 self._bytes_decompressed += out_buffer.pos
                 return out_buffer.pos
 
         self._bytes_decompressed += out_buffer.pos
         return out_buffer.pos
 
     def read1(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if self._finished_output or size == 0:
             return b""
 
         # -1 returns arbitrary number of bytes.
         if size == -1:
             size = DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         # read1() dictates that we can perform at most 1 call to underlying
         # stream to get input. However, we can't satisfy this restriction with
         # decompression because not all input generates output. So we allow
         # multiple read(). But unlike read(), we stop once we have any output.
         while not self._finished_input:
             self._read_input()
             self._decompress_into_buffer(out_buffer)
 
             if out_buffer.pos:
                 break
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto1(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         while not self._finished_input and not self._finished_output:
             self._read_input()
             self._decompress_into_buffer(out_buffer)
 
             if out_buffer.pos:
                 break
 
         self._bytes_decompressed += out_buffer.pos
         return out_buffer.pos
 
     def seek(self, pos, whence=os.SEEK_SET):
         if self._closed:
             raise ValueError("stream is closed")
 
         read_amount = 0
 
         if whence == os.SEEK_SET:
             if pos < 0:
                 raise ValueError(
                     "cannot seek to negative position with SEEK_SET"
                 )
 
             if pos < self._bytes_decompressed:
                 raise ValueError(
                     "cannot seek zstd decompression stream " "backwards"
                 )
 
             read_amount = pos - self._bytes_decompressed
 
         elif whence == os.SEEK_CUR:
             if pos < 0:
                 raise ValueError(
                     "cannot seek zstd decompression stream " "backwards"
                 )
 
             read_amount = pos
         elif whence == os.SEEK_END:
             raise ValueError(
                 "zstd decompression streams cannot be seeked " "with SEEK_END"
             )
 
         while read_amount:
             result = self.read(
                 min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
             )
 
             if not result:
                 break
 
             read_amount -= len(result)
 
         return self._bytes_decompressed
 
 
 class ZstdDecompressionWriter(object):
     def __init__(self, decompressor, writer, write_size, write_return_read):
         decompressor._ensure_dctx()
 
         self._decompressor = decompressor
         self._writer = writer
         self._write_size = write_size
         self._write_return_read = bool(write_return_read)
         self._entered = False
         self._closed = False
 
     def __enter__(self):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._entered:
             raise ZstdError("cannot __enter__ multiple times")
 
         self._entered = True
 
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self.close()
 
     def memory_size(self):
         return lib.ZSTD_sizeof_DCtx(self._decompressor._dctx)
 
     def close(self):
         if self._closed:
             return
 
         try:
             self.flush()
         finally:
             self._closed = True
 
         f = getattr(self._writer, "close", None)
         if f:
             f()
 
     @property
     def closed(self):
         return self._closed
 
     def fileno(self):
         f = getattr(self._writer, "fileno", None)
         if f:
             return f()
         else:
             raise OSError("fileno not available on underlying writer")
 
     def flush(self):
         if self._closed:
             raise ValueError("stream is closed")
 
         f = getattr(self._writer, "flush", None)
         if f:
             return f()
 
     def isatty(self):
         return False
 
     def readable(self):
         return False
 
     def readline(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readlines(self, hint=-1):
         raise io.UnsupportedOperation()
 
     def seek(self, offset, whence=None):
         raise io.UnsupportedOperation()
 
     def seekable(self):
         return False
 
     def tell(self):
         raise io.UnsupportedOperation()
 
     def truncate(self, size=None):
         raise io.UnsupportedOperation()
 
     def writable(self):
         return True
 
     def writelines(self, lines):
         raise io.UnsupportedOperation()
 
     def read(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readall(self):
         raise io.UnsupportedOperation()
 
     def readinto(self, b):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         if self._closed:
             raise ValueError("stream is closed")
 
         total_write = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         data_buffer = ffi.from_buffer(data)
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         dst_buffer = ffi.new("char[]", self._write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         dctx = self._decompressor._dctx
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "zstd decompress error: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 self._writer.write(
                     ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                 )
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
         if self._write_return_read:
             return in_buffer.pos
         else:
             return total_write
 
 
 class ZstdDecompressor(object):
     def __init__(self, dict_data=None, max_window_size=0, format=FORMAT_ZSTD1):
         self._dict_data = dict_data
         self._max_window_size = max_window_size
         self._format = format
 
         dctx = lib.ZSTD_createDCtx()
         if dctx == ffi.NULL:
             raise MemoryError()
 
         self._dctx = dctx
 
         # Defer setting up garbage collection until full state is loaded so
         # the memory size is more accurate.
         try:
             self._ensure_dctx()
         finally:
             self._dctx = ffi.gc(
                 dctx, lib.ZSTD_freeDCtx, size=lib.ZSTD_sizeof_DCtx(dctx)
             )
 
     def memory_size(self):
         return lib.ZSTD_sizeof_DCtx(self._dctx)
 
     def decompress(self, data, max_output_size=0):
         self._ensure_dctx()
 
         data_buffer = ffi.from_buffer(data)
 
         output_size = lib.ZSTD_getFrameContentSize(
             data_buffer, len(data_buffer)
         )
 
         if output_size == lib.ZSTD_CONTENTSIZE_ERROR:
             raise ZstdError("error determining content size from frame header")
         elif output_size == 0:
             return b""
         elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             if not max_output_size:
                 raise ZstdError(
                     "could not determine content size in frame header"
                 )
 
             result_buffer = ffi.new("char[]", max_output_size)
             result_size = max_output_size
             output_size = 0
         else:
             result_buffer = ffi.new("char[]", output_size)
             result_size = output_size
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = result_buffer
         out_buffer.size = result_size
         out_buffer.pos = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError("decompression error: %s" % _zstd_error(zresult))
         elif zresult:
             raise ZstdError(
                 "decompression error: did not decompress full frame"
             )
         elif output_size and out_buffer.pos != output_size:
             raise ZstdError(
                 "decompression error: decompressed %d bytes; expected %d"
                 % (zresult, output_size)
             )
 
         return ffi.buffer(result_buffer, out_buffer.pos)[:]
 
     def stream_reader(
         self,
         source,
         read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         read_across_frames=False,
     ):
         self._ensure_dctx()
         return ZstdDecompressionReader(
             self, source, read_size, read_across_frames
         )
 
     def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if write_size < 1:
             raise ValueError("write_size must be positive")
 
         self._ensure_dctx()
         return ZstdDecompressionObj(self, write_size=write_size)
 
     def read_to_iter(
         self,
         reader,
         read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         skip_bytes=0,
     ):
         if skip_bytes >= read_size:
             raise ValueError("skip_bytes must be smaller than read_size")
 
         if hasattr(reader, "read"):
             have_read = True
         elif hasattr(reader, "__getitem__"):
             have_read = False
             buffer_offset = 0
             size = len(reader)
         else:
             raise ValueError(
                 "must pass an object with a read() method or "
                 "conforms to buffer protocol"
             )
 
         if skip_bytes:
             if have_read:
                 reader.read(skip_bytes)
             else:
                 if skip_bytes > size:
                     raise ValueError("skip_bytes larger than first input chunk")
 
                 buffer_offset = skip_bytes
 
         self._ensure_dctx()
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         while True:
             assert out_buffer.pos == 0
 
             if have_read:
                 read_result = reader.read(read_size)
             else:
                 remaining = size - buffer_offset
                 slice_size = min(remaining, read_size)
                 read_result = reader[buffer_offset : buffer_offset + slice_size]
                 buffer_offset += slice_size
 
             # No new input. Break out of read loop.
             if not read_result:
                 break
 
             # Feed all read data into decompressor and emit output until
             # exhausted.
             read_buffer = ffi.from_buffer(read_result)
             in_buffer.src = read_buffer
             in_buffer.size = len(read_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 assert out_buffer.pos == 0
 
                 zresult = lib.ZSTD_decompressStream(
                     self._dctx, out_buffer, in_buffer
                 )
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError(
                         "zstd decompress error: %s" % _zstd_error(zresult)
                     )
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                     out_buffer.pos = 0
                     yield data
 
                 if zresult == 0:
                     return
 
             # Repeat loop to collect more input data.
             continue
 
         # If we get here, input is exhausted.
 
     read_from = read_to_iter
 
     def stream_writer(
         self,
         writer,
         write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         write_return_read=False,
     ):
         if not hasattr(writer, "write"):
             raise ValueError("must pass an object with a write() method")
 
         return ZstdDecompressionWriter(
             self, writer, write_size, write_return_read
         )
 
     write_to = stream_writer
 
     def copy_stream(
         self,
         ifh,
         ofh,
         read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
     ):
         if not hasattr(ifh, "read"):
             raise ValueError("first argument must have a read() method")
         if not hasattr(ofh, "write"):
             raise ValueError("second argument must have a write() method")
 
         self._ensure_dctx()
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         total_read, total_write = 0, 0
 
         # Read all available input.
         while True:
             data = ifh.read(read_size)
             if not data:
                 break
 
             data_buffer = ffi.from_buffer(data)
             total_read += len(data_buffer)
             in_buffer.src = data_buffer
             in_buffer.size = len(data_buffer)
             in_buffer.pos = 0
 
             # Flush all read data to output.
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_decompressStream(
                     self._dctx, out_buffer, in_buffer
                 )
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError(
                         "zstd decompressor error: %s" % _zstd_error(zresult)
                     )
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                     total_write += out_buffer.pos
                     out_buffer.pos = 0
 
             # Continue loop to keep reading.
 
         return total_read, total_write
 
     def decompress_content_dict_chain(self, frames):
         if not isinstance(frames, list):
             raise TypeError("argument must be a list")
 
         if not frames:
             raise ValueError("empty input chain")
 
         # First chunk should not be using a dictionary. We handle it specially.
         chunk = frames[0]
         if not isinstance(chunk, bytes_type):
             raise ValueError("chunk 0 must be bytes")
 
         # All chunks should be zstd frames and should have content size set.
         chunk_buffer = ffi.from_buffer(chunk)
         params = ffi.new("ZSTD_frameHeader *")
         zresult = lib.ZSTD_getFrameHeader(
             params, chunk_buffer, len(chunk_buffer)
         )
         if lib.ZSTD_isError(zresult):
             raise ValueError("chunk 0 is not a valid zstd frame")
         elif zresult:
             raise ValueError("chunk 0 is too small to contain a zstd frame")
 
         if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             raise ValueError("chunk 0 missing content size in frame")
 
         self._ensure_dctx(load_dict=False)
 
         last_buffer = ffi.new("char[]", params.frameContentSize)
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = last_buffer
         out_buffer.size = len(last_buffer)
         out_buffer.pos = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = chunk_buffer
         in_buffer.size = len(chunk_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "could not decompress chunk 0: %s" % _zstd_error(zresult)
             )
         elif zresult:
             raise ZstdError("chunk 0 did not decompress full frame")
 
         # Special case of chain length of 1
         if len(frames) == 1:
             return ffi.buffer(last_buffer, len(last_buffer))[:]
 
         i = 1
         while i < len(frames):
             chunk = frames[i]
             if not isinstance(chunk, bytes_type):
                 raise ValueError("chunk %d must be bytes" % i)
 
             chunk_buffer = ffi.from_buffer(chunk)
             zresult = lib.ZSTD_getFrameHeader(
                 params, chunk_buffer, len(chunk_buffer)
             )
             if lib.ZSTD_isError(zresult):
                 raise ValueError("chunk %d is not a valid zstd frame" % i)
             elif zresult:
                 raise ValueError(
                     "chunk %d is too small to contain a zstd frame" % i
                 )
 
             if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
                 raise ValueError("chunk %d missing content size in frame" % i)
 
             dest_buffer = ffi.new("char[]", params.frameContentSize)
 
             out_buffer.dst = dest_buffer
             out_buffer.size = len(dest_buffer)
             out_buffer.pos = 0
 
             in_buffer.src = chunk_buffer
             in_buffer.size = len(chunk_buffer)
             in_buffer.pos = 0
 
             zresult = lib.ZSTD_decompressStream(
                 self._dctx, out_buffer, in_buffer
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "could not decompress chunk %d: %s" % _zstd_error(zresult)
                 )
             elif zresult:
                 raise ZstdError("chunk %d did not decompress full frame" % i)
 
             last_buffer = dest_buffer
             i += 1
 
         return ffi.buffer(last_buffer, len(last_buffer))[:]
 
     def _ensure_dctx(self, load_dict=True):
         lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only)
 
         if self._max_window_size:
             zresult = lib.ZSTD_DCtx_setMaxWindowSize(
                 self._dctx, self._max_window_size
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "unable to set max window size: %s" % _zstd_error(zresult)
                 )
 
         zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format)
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "unable to set decoding format: %s" % _zstd_error(zresult)
             )
 
         if self._dict_data and load_dict:
             zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "unable to reference prepared dictionary: %s"
                     % _zstd_error(zresult)
                 )

contrib/python3-ratchet.py

@@ -1,163 +1,161 b''
 # Copyright 2012 Facebook
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """Find tests that newly pass under Python 3.
 
 The approach is simple: we maintain a whitelist of Python 3 passing
 tests in the repository, and periodically run all the /other/ tests
 and look for new passes. Any newly passing tests get automatically
 added to the whitelist.
 
 You probably want to run it like this:
 
   $ cd tests
   $ python3 ../contrib/python3-ratchet.py \
   >   --working-tests=../contrib/python3-whitelist
 """
-from __future__ import print_function
-from __future__ import absolute_import
 
 import argparse
 import json
 import os
 import subprocess
 import sys
 
 _hgenv = dict(os.environ)
 _hgenv.update(
     {
         'HGPLAIN': '1',
     }
 )
 
 _HG_FIRST_CHANGE = '9117c6561b0bd7792fa13b50d28239d51b78e51f'
 
 
 def _runhg(*args):
     return subprocess.check_output(args, env=_hgenv)
 
 
 def _is_hg_repo(path):
     return (
         _runhg('hg', 'log', '-R', path, '-r0', '--template={node}').strip()
         == _HG_FIRST_CHANGE
     )
 
 
 def _py3default():
     if sys.version_info[0] >= 3:
         return sys.executable
     return 'python3'
 
 
 def main(argv=()):
     p = argparse.ArgumentParser()
     p.add_argument(
         '--working-tests', help='List of tests that already work in Python 3.'
     )
     p.add_argument(
         '--commit-to-repo',
         help='If set, commit newly fixed tests to the given repo',
     )
     p.add_argument(
         '-j',
         default=os.sysconf('SC_NPROCESSORS_ONLN'),
         type=int,
         help='Number of parallel tests to run.',
     )
     p.add_argument(
         '--python3',
         default=_py3default(),
         help='python3 interpreter to use for test run',
     )
     p.add_argument(
         '--commit-user',
         default='[e-mail]',
         help='Username to specify when committing to a repo.',
     )
     opts = p.parse_args(argv)
     if opts.commit_to_repo:
         if not _is_hg_repo(opts.commit_to_repo):
             print('abort: specified repository is not the hg repository')
             sys.exit(1)
     if not opts.working_tests or not os.path.isfile(opts.working_tests):
         print(
             'abort: --working-tests must exist and be a file (got %r)'
             % opts.working_tests
         )
         sys.exit(1)
     elif opts.commit_to_repo:
         root = _runhg('hg', 'root').strip()
         if not opts.working_tests.startswith(root):
             print(
                 'abort: if --commit-to-repo is given, '
                 '--working-tests must be from that repo'
             )
             sys.exit(1)
     try:
         subprocess.check_call(
             [
                 opts.python3,
                 '-c',
                 'import sys ; '
                 'assert ((3, 5) <= sys.version_info < (3, 6) '
                 'or sys.version_info >= (3, 6, 2))',
             ]
         )
     except subprocess.CalledProcessError:
         print(
             'warning: Python 3.6.0 and 3.6.1 have '
             'a bug which breaks Mercurial'
         )
         print('(see https://bugs.python.org/issue29714 for details)')
         sys.exit(1)
 
     rt = subprocess.Popen(
         [
             opts.python3,
             'run-tests.py',
             '-j',
             str(opts.j),
             '--blacklist',
             opts.working_tests,
             '--json',
         ]
     )
     rt.wait()
     with open('report.json') as f:
         data = f.read()
     report = json.loads(data.split('=', 1)[1])
     newpass = set()
     for test, result in report.items():
         if result['result'] != 'success':
             continue
         # A new passing test! Huzzah!
         newpass.add(test)
     if newpass:
         # We already validated the repo, so we can just dive right in
         # and commit.
         if opts.commit_to_repo:
             print(len(newpass), 'new passing tests on Python 3!')
             with open(opts.working_tests) as f:
                 oldpass = {l for l in f.read().splitlines() if l}
             with open(opts.working_tests, 'w') as f:
                 for p in sorted(oldpass | newpass):
                     f.write('%s\n' % p)
             _runhg(
                 'hg',
                 'commit',
                 '-R',
                 opts.commit_to_repo,
                 '--user',
                 opts.commit_user,
                 '--message',
                 'python3: expand list of passing tests',
             )
         else:
             print('Newly passing tests:', '\n'.join(sorted(newpass)))
             sys.exit(2)
 
 
 if __name__ == '__main__':
     main(sys.argv[1:])

contrib/revsetbenchmarks.py

@@ -1,389 +1,388 b''
 #!/usr/bin/env python3
 
 # Measure the performance of a list of revsets against multiple revisions
 # defined by parameter. Checkout one by one and run perfrevset with every
 # revset in the list to benchmark its performance.
 #
 # You should run this from the root of your mercurial repository.
 #
 # call with --help for details
 
-from __future__ import absolute_import, print_function
 import math
 import optparse  # cannot use argparse, python 2.7 only
 import os
 import re
 import subprocess
 import sys
 
 DEFAULTVARIANTS = [
     'plain',
     'min',
     'max',
     'first',
     'last',
     'reverse',
     'reverse+first',
     'reverse+last',
     'sort',
     'sort+first',
     'sort+last',
 ]
 
 
 def check_output(*args, **kwargs):
     kwargs.setdefault('stderr', subprocess.PIPE)
     kwargs.setdefault('stdout', subprocess.PIPE)
     proc = subprocess.Popen(*args, **kwargs)
     output, error = proc.communicate()
     if proc.returncode != 0:
         raise subprocess.CalledProcessError(proc.returncode, ' '.join(args[0]))
     return output
 
 
 def update(rev):
     """update the repo to a revision"""
     try:
         subprocess.check_call(['hg', 'update', '--quiet', '--check', str(rev)])
         check_output(
             ['make', 'local'], stderr=None
         )  # suppress output except for error/warning
     except subprocess.CalledProcessError as exc:
         print('update to revision %s failed, aborting' % rev, file=sys.stderr)
         sys.exit(exc.returncode)
 
 
 def hg(cmd, repo=None):
     """run a mercurial command
 
     <cmd> is the list of command + argument,
     <repo> is an optional repository path to run this command in."""
     fullcmd = ['./hg']
     if repo is not None:
         fullcmd += ['-R', repo]
     fullcmd += [
         '--config',
         'extensions.perf=' + os.path.join(contribdir, 'perf.py'),
     ]
     fullcmd += cmd
     return check_output(fullcmd, stderr=subprocess.STDOUT)
 
 
 def perf(revset, target=None, contexts=False):
     """run benchmark for this very revset"""
     try:
         args = ['perfrevset']
         if contexts:
             args.append('--contexts')
         args.append('--')
         args.append(revset)
         output = hg(args, repo=target)
         return parseoutput(output)
     except subprocess.CalledProcessError as exc:
         print(
             'abort: cannot run revset benchmark: %s' % exc.cmd, file=sys.stderr
         )
         if getattr(exc, 'output', None) is None:  # no output before 2.7
             print('(no output)', file=sys.stderr)
         else:
             print(exc.output, file=sys.stderr)
         return None
 
 
 outputre = re.compile(
     br'! wall (\d+.\d+) comb (\d+.\d+) user (\d+.\d+) '
     br'sys (\d+.\d+) \(best of (\d+)\)'
 )
 
 
 def parseoutput(output):
     """parse a textual output into a dict
 
     We cannot just use json because we want to compare with old
     versions of Mercurial that may not support json output.
     """
     match = outputre.search(output)
     if not match:
         print('abort: invalid output:', file=sys.stderr)
         print(output, file=sys.stderr)
         sys.exit(1)
     return {
         'comb': float(match.group(2)),
         'count': int(match.group(5)),
         'sys': float(match.group(3)),
         'user': float(match.group(4)),
         'wall': float(match.group(1)),
     }
 
 
 def printrevision(rev):
     """print data about a revision"""
     sys.stdout.write("Revision ")
     sys.stdout.flush()
     subprocess.check_call(
         [
             'hg',
             'log',
             '--rev',
             str(rev),
             '--template',
             '{if(tags, " ({tags})")} ' '{rev}:{node|short}: {desc|firstline}\n',
         ]
     )
 
 
 def idxwidth(nbidx):
     """return the max width of number used for index
 
     This is similar to log10(nbidx), but we use custom code here
     because we start with zero and we'd rather not deal with all the
     extra rounding business that log10 would imply.
     """
     nbidx -= 1  # starts at 0
     idxwidth = 0
     while nbidx:
         idxwidth += 1
         nbidx //= 10
     if not idxwidth:
         idxwidth = 1
     return idxwidth
 
 
 def getfactor(main, other, field, sensitivity=0.05):
     """return the relative factor between values for 'field' in main and other
 
     Return None if the factor is insignificant (less than <sensitivity>
     variation)."""
     factor = 1
     if main is not None:
         factor = other[field] / main[field]
     low, high = 1 - sensitivity, 1 + sensitivity
     if low < factor < high:
         return None
     return factor
 
 
 def formatfactor(factor):
     """format a factor into a 4 char string
 
      22%
     156%
     x2.4
      x23
     x789
     x1e4
     x5x7
 
     """
     if factor is None:
         return '    '
     elif factor < 2:
         return '%3i%%' % (factor * 100)
     elif factor < 10:
         return 'x%3.1f' % factor
     elif factor < 1000:
         return '%4s' % ('x%i' % factor)
     else:
         order = int(math.log(factor)) + 1
         while math.log(factor) > 1:
             factor //= 0
         return 'x%ix%i' % (factor, order)
 
 
 def formattiming(value):
     """format a value to strictly 8 char, dropping some precision if needed"""
     if value < 10 ** 7:
         return ('%.6f' % value)[:8]
     else:
         # value is HUGE very unlikely to happen (4+ month run)
         return '%i' % value
 
 
 _marker = object()
 
 
 def printresult(variants, idx, data, maxidx, verbose=False, reference=_marker):
     """print a line of result to stdout"""
     mask = '%%0%ii) %%s' % idxwidth(maxidx)
 
     out = []
     for var in variants:
         if data[var] is None:
             out.append('error   ')
             out.append(' ' * 4)
             continue
         out.append(formattiming(data[var]['wall']))
         if reference is not _marker:
             factor = None
             if reference is not None:
                 factor = getfactor(reference[var], data[var], 'wall')
             out.append(formatfactor(factor))
         if verbose:
             out.append(formattiming(data[var]['comb']))
             out.append(formattiming(data[var]['user']))
             out.append(formattiming(data[var]['sys']))
             out.append('%6d' % data[var]['count'])
     print(mask % (idx, ' '.join(out)))
 
 
 def printheader(variants, maxidx, verbose=False, relative=False):
     header = [' ' * (idxwidth(maxidx) + 1)]
     for var in variants:
         if not var:
             var = 'iter'
         if len(var) > 8:
             var = var[:3] + '..' + var[-3:]
         header.append('%-8s' % var)
         if relative:
             header.append('    ')
         if verbose:
             header.append('%-8s' % 'comb')
             header.append('%-8s' % 'user')
             header.append('%-8s' % 'sys')
             header.append('%6s' % 'count')
     print(' '.join(header))
 
 
 def getrevs(spec):
     """get the list of rev matched by a revset"""
     try:
         out = check_output(['hg', 'log', '--template={rev}\n', '--rev', spec])
     except subprocess.CalledProcessError as exc:
         print("abort, can't get revision from %s" % spec, file=sys.stderr)
         sys.exit(exc.returncode)
     return [r for r in out.split() if r]
 
 
 def applyvariants(revset, variant):
     if variant == 'plain':
         return revset
     for var in variant.split('+'):
         revset = '%s(%s)' % (var, revset)
     return revset
 
 
 helptext = """This script will run multiple variants of provided revsets using
 different revisions in your mercurial repository. After the benchmark are run
 summary output is provided. Use it to demonstrate speed improvements or pin
 point regressions. Revsets to run are specified in a file (or from stdin), one
 revsets per line. Line starting with '#' will be ignored, allowing insertion of
 comments."""
 parser = optparse.OptionParser(
     usage="usage: %prog [options] <revs>", description=helptext
 )
 parser.add_option(
     "-f",
     "--file",
     help="read revset from FILE (stdin if omitted)",
     metavar="FILE",
 )
 parser.add_option("-R", "--repo", help="run benchmark on REPO", metavar="REPO")
 
 parser.add_option(
     "-v",
     "--verbose",
     action='store_true',
     help="display all timing data (not just best total time)",
 )
 
 parser.add_option(
     "",
     "--variants",
     default=','.join(DEFAULTVARIANTS),
     help="comma separated list of variant to test "
     "(eg: plain,min,sorted) (plain = no modification)",
 )
 parser.add_option(
     '',
     '--contexts',
     action='store_true',
     help='obtain changectx from results instead of integer revs',
 )
 
 (options, args) = parser.parse_args()
 
 if not args:
     parser.print_help()
     sys.exit(255)
 
 # the directory where both this script and the perf.py extension live.
 contribdir = os.path.dirname(__file__)
 
 revsetsfile = sys.stdin
 if options.file:
     revsetsfile = open(options.file)
 
 revsets = [l.strip() for l in revsetsfile if not l.startswith('#')]
 revsets = [l for l in revsets if l]
 
 print("Revsets to benchmark")
 print("----------------------------")
 
 for idx, rset in enumerate(revsets):
     print("%i) %s" % (idx, rset))
 
 print("----------------------------")
 print()
 
 revs = []
 for a in args:
     revs.extend(getrevs(a))
 
 variants = options.variants.split(',')
 
 results = []
 for r in revs:
     print("----------------------------")
     printrevision(r)
     print("----------------------------")
     update(r)
     res = []
     results.append(res)
     printheader(variants, len(revsets), verbose=options.verbose)
     for idx, rset in enumerate(revsets):
         varres = {}
         for var in variants:
             varrset = applyvariants(rset, var)
             data = perf(varrset, target=options.repo, contexts=options.contexts)
             varres[var] = data
         res.append(varres)
         printresult(
             variants, idx, varres, len(revsets), verbose=options.verbose
         )
         sys.stdout.flush()
     print("----------------------------")
 
 
 print(
     """
 
 Result by revset
 ================
 """
 )
 
 print('Revision:')
 for idx, rev in enumerate(revs):
     sys.stdout.write('%i) ' % idx)
     sys.stdout.flush()
     printrevision(rev)
 
 print()
 print()
 
 for ridx, rset in enumerate(revsets):
 
     print("revset #%i: %s" % (ridx, rset))
     printheader(variants, len(results), verbose=options.verbose, relative=True)
     ref = None
     for idx, data in enumerate(results):
         printresult(
             variants,
             idx,
             data[ridx],
             len(results),
             verbose=options.verbose,
             reference=ref,
         )
         ref = data[ridx]
     print()

contrib/showstack.py

@@ -1,31 +1,30 b''
 # showstack.py - extension to dump a Python stack trace on signal
 #
 # binds to both SIGQUIT (Ctrl-\) and SIGINFO (Ctrl-T on BSDs)
 r"""dump stack trace when receiving SIGQUIT (Ctrl-\) or SIGINFO (Ctrl-T on BSDs)
 """
 
-from __future__ import absolute_import, print_function
 import signal
 import sys
 import traceback
 
 
 def sigshow(*args):
     sys.stderr.write("\n")
     traceback.print_stack(args[1], limit=10, file=sys.stderr)
     sys.stderr.write("----\n")
 
 
 def sigexit(*args):
     sigshow(*args)
     print('alarm!')
     sys.exit(1)
 
 
 def extsetup(ui):
     signal.signal(signal.SIGQUIT, sigshow)
     signal.signal(signal.SIGALRM, sigexit)
     try:
         signal.signal(signal.SIGINFO, sigshow)
     except AttributeError:
         pass

contrib/simplemerge

@@ -1,141 +1,140 b''
 #!/usr/bin/env python3
-from __future__ import absolute_import
 
 import getopt
 import sys
 
 import hgdemandimport
 
 hgdemandimport.enable()
 
 from mercurial.i18n import _
 from mercurial import (
     context,
     error,
     fancyopts,
     simplemerge,
     ui as uimod,
     util,
 )
 from mercurial.utils import procutil, stringutil
 
 options = [
     (b'L', b'label', [], _(b'labels to use on conflict markers')),
     (b'a', b'text', None, _(b'treat all files as text')),
     (b'p', b'print', None, _(b'print results instead of overwriting LOCAL')),
     (b'', b'no-minimal', None, _(b'no effect (DEPRECATED)')),
     (b'h', b'help', None, _(b'display help and exit')),
     (b'q', b'quiet', None, _(b'suppress output')),
 ]
 
 usage = _(
     b'''simplemerge [OPTS] LOCAL BASE OTHER
 
     Simple three-way file merge utility with a minimal feature set.
 
     Apply to LOCAL the changes necessary to go from BASE to OTHER.
 
     By default, LOCAL is overwritten with the results of this operation.
 '''
 )
 
 
 class ParseError(Exception):
     """Exception raised on errors in parsing the command line."""
 
 
 def showhelp():
     procutil.stdout.write(usage)
     procutil.stdout.write(b'\noptions:\n')
 
     out_opts = []
     for shortopt, longopt, default, desc in options:
         out_opts.append(
             (
                 b'%2s%s'
                 % (
                     shortopt and b'-%s' % shortopt,
                     longopt and b' --%s' % longopt,
                 ),
                 b'%s' % desc,
             )
         )
     opts_len = max([len(opt[0]) for opt in out_opts])
     for first, second in out_opts:
         procutil.stdout.write(b' %-*s  %s\n' % (opts_len, first, second))
 
 
 def _verifytext(input, ui, quiet=False, allow_binary=False):
     """verifies that text is non-binary (unless opts[text] is passed,
     then we just warn)"""
     if stringutil.binary(input.text()):
         msg = _(b"%s looks like a binary file.") % input.fctx.path()
         if not quiet:
             ui.warn(_(b'warning: %s\n') % msg)
         if not allow_binary:
             sys.exit(1)
 
 
 try:
     for fp in (sys.stdin, procutil.stdout, sys.stderr):
         procutil.setbinary(fp)
 
     opts = {}
     try:
         bargv = [a.encode('utf8') for a in sys.argv[1:]]
         args = fancyopts.fancyopts(bargv, options, opts)
     except getopt.GetoptError as e:
         raise ParseError(e)
     if opts[b'help']:
         showhelp()
         sys.exit(0)
     if len(args) != 3:
         raise ParseError(_(b'wrong number of arguments').decode('utf8'))
     mode = b'merge'
     if len(opts[b'label']) > 2:
         mode = b'merge3'
     local, base, other = args
     overrides = opts[b'label']
     if len(overrides) > 3:
         raise error.InputError(b'can only specify three labels.')
     labels = [local, other, base]
     labels[: len(overrides)] = overrides
     local_input = simplemerge.MergeInput(
         context.arbitraryfilectx(local), labels[0]
     )
     other_input = simplemerge.MergeInput(
         context.arbitraryfilectx(other), labels[1]
     )
     base_input = simplemerge.MergeInput(
         context.arbitraryfilectx(base), labels[2]
     )
 
     quiet = opts.get(b'quiet')
     allow_binary = opts.get(b'text')
     ui = uimod.ui.load()
     _verifytext(local_input, ui, quiet=quiet, allow_binary=allow_binary)
     _verifytext(base_input, ui, quiet=quiet, allow_binary=allow_binary)
     _verifytext(other_input, ui, quiet=quiet, allow_binary=allow_binary)
 
     merged_text, conflicts = simplemerge.simplemerge(
         local_input,
         base_input,
         other_input,
         mode,
         allow_binary=allow_binary,
     )
     if opts.get(b'print'):
         ui.fout.write(merged_text)
     else:
         util.writefile(local, merged_text)
     sys.exit(1 if conflicts else 0)
 except ParseError as e:
     e = stringutil.forcebytestr(e)
     procutil.stdout.write(b"%s: %s\n" % (sys.argv[0].encode('utf8'), e))
     showhelp()
     sys.exit(1)
 except error.Abort as e:
     procutil.stderr.write(b"abort: %s\n" % e)
     sys.exit(255)
 except KeyboardInterrupt:
     sys.exit(255)

contrib/synthrepo.py

@@ -1,565 +1,564 b''
 # synthrepo.py - repo synthesis
 #
 # Copyright 2012 Facebook
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''synthesize structurally interesting change history
 
 This extension is useful for creating a repository with properties
 that are statistically similar to an existing repository. During
 analysis, a simple probability table is constructed from the history
 of an existing repository.  During synthesis, these properties are
 reconstructed.
 
 Properties that are analyzed and synthesized include the following:
 
 - Lines added or removed when an existing file is modified
 - Number and sizes of files added
 - Number of files removed
 - Line lengths
 - Topological distance to parent changeset(s)
 - Probability of a commit being a merge
 - Probability of a newly added file being added to a new directory
 - Interarrival time, and time zone, of commits
 - Number of files in each directory
 
 A few obvious properties that are not currently handled realistically:
 
 - Merges are treated as regular commits with two parents, which is not
   realistic
 - Modifications are not treated as operations on hunks of lines, but
   as insertions and deletions of randomly chosen single lines
 - Committer ID (always random)
 - Executability of files
 - Symlinks and binary files are ignored
 '''
 
-from __future__ import absolute_import
 import bisect
 import collections
 import itertools
 import json
 import os
 import random
 import sys
 import time
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     nullrev,
     short,
 )
 from mercurial import (
     context,
     diffutil,
     error,
     hg,
     logcmdutil,
     patch,
     pycompat,
     registrar,
 )
 from mercurial.utils import dateutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 newfile = {'new fi', 'rename', 'copy f', 'copy t'}
 
 
 def zerodict():
     return collections.defaultdict(lambda: 0)
 
 
 def roundto(x, k):
     if x > k * 2:
         return int(round(x / float(k)) * k)
     return int(round(x))
 
 
 def parsegitdiff(lines):
     filename, mar, lineadd, lineremove = None, None, zerodict(), 0
     binary = False
     for line in lines:
         start = line[:6]
         if start == 'diff -':
             if filename:
                 yield filename, mar, lineadd, lineremove, binary
             mar, lineadd, lineremove, binary = 'm', zerodict(), 0, False
             filename = patch.gitre.match(line).group(1)
         elif start in newfile:
             mar = 'a'
         elif start == 'GIT bi':
             binary = True
         elif start == 'delete':
             mar = 'r'
         elif start:
             s = start[0]
             if s == '-' and not line.startswith('--- '):
                 lineremove += 1
             elif s == '+' and not line.startswith('+++ '):
                 lineadd[roundto(len(line) - 1, 5)] += 1
     if filename:
         yield filename, mar, lineadd, lineremove, binary
 
 
 @command(
     'analyze',
     [
         ('o', 'output', '', _('write output to given file'), _('FILE')),
         ('r', 'rev', [], _('analyze specified revisions'), _('REV')),
     ],
     _('hg analyze'),
     optionalrepo=True,
 )
 def analyze(ui, repo, *revs, **opts):
     """create a simple model of a repository to use for later synthesis
 
     This command examines every changeset in the given range (or all
     of history if none are specified) and creates a simple statistical
     model of the history of the repository. It also measures the directory
     structure of the repository as checked out.
 
     The model is written out to a JSON file, and can be used by
     :hg:`synthesize` to create or augment a repository with synthetic
     commits that have a structure that is statistically similar to the
     analyzed repository.
     """
     root = repo.root
     if not root.endswith(os.path.sep):
         root += os.path.sep
 
     revs = list(revs)
     revs.extend(opts['rev'])
     if not revs:
         revs = [':']
 
     output = opts['output']
     if not output:
         output = os.path.basename(root) + '.json'
 
     if output == '-':
         fp = sys.stdout
     else:
         fp = open(output, 'w')
 
     # Always obtain file counts of each directory in the given root directory.
     def onerror(e):
         ui.warn(_('error walking directory structure: %s\n') % e)
 
     dirs = {}
     rootprefixlen = len(root)
     for dirpath, dirnames, filenames in os.walk(root, onerror=onerror):
         dirpathfromroot = dirpath[rootprefixlen:]
         dirs[dirpathfromroot] = len(filenames)
         if '.hg' in dirnames:
             dirnames.remove('.hg')
 
     lineschanged = zerodict()
     children = zerodict()
     p1distance = zerodict()
     p2distance = zerodict()
     linesinfilesadded = zerodict()
     fileschanged = zerodict()
     filesadded = zerodict()
     filesremoved = zerodict()
     linelengths = zerodict()
     interarrival = zerodict()
     parents = zerodict()
     dirsadded = zerodict()
     tzoffset = zerodict()
 
     # If a mercurial repo is available, also model the commit history.
     if repo:
         revs = logcmdutil.revrange(repo, revs)
         revs.sort()
 
         progress = ui.makeprogress(
             _('analyzing'), unit=_('changesets'), total=len(revs)
         )
         for i, rev in enumerate(revs):
             progress.update(i)
             ctx = repo[rev]
             pl = ctx.parents()
             pctx = pl[0]
             prev = pctx.rev()
             children[prev] += 1
             p1distance[rev - prev] += 1
             parents[len(pl)] += 1
             tzoffset[ctx.date()[1]] += 1
             if len(pl) > 1:
                 p2distance[rev - pl[1].rev()] += 1
             if prev == rev - 1:
                 lastctx = pctx
             else:
                 lastctx = repo[rev - 1]
             if lastctx.rev() != nullrev:
                 timedelta = ctx.date()[0] - lastctx.date()[0]
                 interarrival[roundto(timedelta, 300)] += 1
             diffopts = diffutil.diffallopts(ui, {'git': True})
             diff = sum(
                 (d.splitlines() for d in ctx.diff(pctx, opts=diffopts)), []
             )
             fileadds, diradds, fileremoves, filechanges = 0, 0, 0, 0
             for filename, mar, lineadd, lineremove, isbin in parsegitdiff(diff):
                 if isbin:
                     continue
                 added = sum(pycompat.itervalues(lineadd), 0)
                 if mar == 'm':
                     if added and lineremove:
                         lineschanged[
                             roundto(added, 5), roundto(lineremove, 5)
                         ] += 1
                         filechanges += 1
                 elif mar == 'a':
                     fileadds += 1
                     if '/' in filename:
                         filedir = filename.rsplit('/', 1)[0]
                         if filedir not in pctx.dirs():
                             diradds += 1
                     linesinfilesadded[roundto(added, 5)] += 1
                 elif mar == 'r':
                     fileremoves += 1
                 for length, count in lineadd.iteritems():
                     linelengths[length] += count
             fileschanged[filechanges] += 1
             filesadded[fileadds] += 1
             dirsadded[diradds] += 1
             filesremoved[fileremoves] += 1
         progress.complete()
 
     invchildren = zerodict()
 
     for rev, count in children.iteritems():
         invchildren[count] += 1
 
     if output != '-':
         ui.status(_('writing output to %s\n') % output)
 
     def pronk(d):
         return sorted(d.iteritems(), key=lambda x: x[1], reverse=True)
 
     json.dump(
         {
             'revs': len(revs),
             'initdirs': pronk(dirs),
             'lineschanged': pronk(lineschanged),
             'children': pronk(invchildren),
             'fileschanged': pronk(fileschanged),
             'filesadded': pronk(filesadded),
             'linesinfilesadded': pronk(linesinfilesadded),
             'dirsadded': pronk(dirsadded),
             'filesremoved': pronk(filesremoved),
             'linelengths': pronk(linelengths),
             'parents': pronk(parents),
             'p1distance': pronk(p1distance),
             'p2distance': pronk(p2distance),
             'interarrival': pronk(interarrival),
             'tzoffset': pronk(tzoffset),
         },
         fp,
     )
     fp.close()
 
 
 @command(
     'synthesize',
     [
         ('c', 'count', 0, _('create given number of commits'), _('COUNT')),
         ('', 'dict', '', _('path to a dictionary of words'), _('FILE')),
         ('', 'initfiles', 0, _('initial file count to create'), _('COUNT')),
     ],
     _('hg synthesize [OPTION].. DESCFILE'),
 )
 def synthesize(ui, repo, descpath, **opts):
     """synthesize commits based on a model of an existing repository
 
     The model must have been generated by :hg:`analyze`. Commits will
     be generated randomly according to the probabilities described in
     the model. If --initfiles is set, the repository will be seeded with
     the given number files following the modeled repository's directory
     structure.
 
     When synthesizing new content, commit descriptions, and user
     names, words will be chosen randomly from a dictionary that is
     presumed to contain one word per line. Use --dict to specify the
     path to an alternate dictionary to use.
     """
     try:
         fp = hg.openpath(ui, descpath)
     except Exception as err:
         raise error.Abort('%s: %s' % (descpath, err[0].strerror))
     desc = json.load(fp)
     fp.close()
 
     def cdf(l):
         if not l:
             return [], []
         vals, probs = zip(*sorted(l, key=lambda x: x[1], reverse=True))
         t = float(sum(probs, 0))
         s, cdfs = 0, []
         for v in probs:
             s += v
             cdfs.append(s / t)
         return vals, cdfs
 
     lineschanged = cdf(desc['lineschanged'])
     fileschanged = cdf(desc['fileschanged'])
     filesadded = cdf(desc['filesadded'])
     dirsadded = cdf(desc['dirsadded'])
     filesremoved = cdf(desc['filesremoved'])
     linelengths = cdf(desc['linelengths'])
     parents = cdf(desc['parents'])
     p1distance = cdf(desc['p1distance'])
     p2distance = cdf(desc['p2distance'])
     interarrival = cdf(desc['interarrival'])
     linesinfilesadded = cdf(desc['linesinfilesadded'])
     tzoffset = cdf(desc['tzoffset'])
 
     dictfile = opts.get('dict') or '/usr/share/dict/words'
     try:
         fp = open(dictfile, 'rU')
     except IOError as err:
         raise error.Abort('%s: %s' % (dictfile, err.strerror))
     words = fp.read().splitlines()
     fp.close()
 
     initdirs = {}
     if desc['initdirs']:
         for k, v in desc['initdirs']:
             initdirs[k.encode('utf-8').replace('.hg', '_hg')] = v
         initdirs = renamedirs(initdirs, words)
     initdirscdf = cdf(initdirs)
 
     def pick(cdf):
         return cdf[0][bisect.bisect_left(cdf[1], random.random())]
 
     def pickpath():
         return os.path.join(pick(initdirscdf), random.choice(words))
 
     def makeline(minimum=0):
         total = max(minimum, pick(linelengths))
         c, l = 0, []
         while c < total:
             w = random.choice(words)
             c += len(w) + 1
             l.append(w)
         return ' '.join(l)
 
     wlock = repo.wlock()
     lock = repo.lock()
 
     nevertouch = {'.hgsub', '.hgignore', '.hgtags'}
 
     _synthesizing = _('synthesizing')
     _files = _('initial files')
     _changesets = _('changesets')
 
     # Synthesize a single initial revision adding files to the repo according
     # to the modeled directory structure.
     initcount = int(opts['initfiles'])
     if initcount and initdirs:
         pctx = repo['.']
         dirs = set(pctx.dirs())
         files = {}
 
         def validpath(path):
             # Don't pick filenames which are already directory names.
             if path in dirs:
                 return False
             # Don't pick directories which were used as file names.
             while path:
                 if path in files:
                     return False
                 path = os.path.dirname(path)
             return True
 
         progress = ui.makeprogress(_synthesizing, unit=_files, total=initcount)
         for i in pycompat.xrange(0, initcount):
             progress.update(i)
 
             path = pickpath()
             while not validpath(path):
                 path = pickpath()
             data = '%s contents\n' % path
             files[path] = data
             dir = os.path.dirname(path)
             while dir and dir not in dirs:
                 dirs.add(dir)
                 dir = os.path.dirname(dir)
 
         def filectxfn(repo, memctx, path):
             return context.memfilectx(repo, memctx, path, files[path])
 
         progress.complete()
         message = 'synthesized wide repo with %d files' % (len(files),)
         mc = context.memctx(
             repo,
             [pctx.node(), nullid],
             message,
             files,
             filectxfn,
             ui.username(),
             '%d %d' % dateutil.makedate(),
         )
         initnode = mc.commit()
         if ui.debugflag:
             hexfn = hex
         else:
             hexfn = short
         ui.status(
             _('added commit %s with %d files\n') % (hexfn(initnode), len(files))
         )
 
     # Synthesize incremental revisions to the repository, adding repo depth.
     count = int(opts['count'])
     heads = set(map(repo.changelog.rev, repo.heads()))
     progress = ui.makeprogress(_synthesizing, unit=_changesets, total=count)
     for i in pycompat.xrange(count):
         progress.update(i)
 
         node = repo.changelog.node
         revs = len(repo)
 
         def pickhead(heads, distance):
             if heads:
                 lheads = sorted(heads)
                 rev = revs - min(pick(distance), revs)
                 if rev < lheads[-1]:
                     rev = lheads[bisect.bisect_left(lheads, rev)]
                 else:
                     rev = lheads[-1]
                 return rev, node(rev)
             return nullrev, nullid
 
         r1 = revs - min(pick(p1distance), revs)
         p1 = node(r1)
 
         # the number of heads will grow without bound if we use a pure
         # model, so artificially constrain their proliferation
         toomanyheads = len(heads) > random.randint(1, 20)
         if p2distance[0] and (pick(parents) == 2 or toomanyheads):
             r2, p2 = pickhead(heads.difference([r1]), p2distance)
         else:
             r2, p2 = nullrev, nullid
 
         pl = [p1, p2]
         pctx = repo[r1]
         mf = pctx.manifest()
         mfk = mf.keys()
         changes = {}
         if mfk:
             for __ in pycompat.xrange(pick(fileschanged)):
                 for __ in pycompat.xrange(10):
                     fctx = pctx.filectx(random.choice(mfk))
                     path = fctx.path()
                     if not (
                         path in nevertouch
                         or fctx.isbinary()
                         or 'l' in fctx.flags()
                     ):
                         break
                 lines = fctx.data().splitlines()
                 add, remove = pick(lineschanged)
                 for __ in pycompat.xrange(remove):
                     if not lines:
                         break
                     del lines[random.randrange(0, len(lines))]
                 for __ in pycompat.xrange(add):
                     lines.insert(random.randint(0, len(lines)), makeline())
                 path = fctx.path()
                 changes[path] = '\n'.join(lines) + '\n'
             for __ in pycompat.xrange(pick(filesremoved)):
                 for __ in pycompat.xrange(10):
                     path = random.choice(mfk)
                     if path not in changes:
                         break
         if filesadded:
             dirs = list(pctx.dirs())
             dirs.insert(0, '')
         for __ in pycompat.xrange(pick(filesadded)):
             pathstr = ''
             while pathstr in dirs:
                 path = [random.choice(dirs)]
                 if pick(dirsadded):
                     path.append(random.choice(words))
                 path.append(random.choice(words))
                 pathstr = '/'.join(filter(None, path))
             data = (
                 '\n'.join(
                     makeline()
                     for __ in pycompat.xrange(pick(linesinfilesadded))
                 )
                 + '\n'
             )
             changes[pathstr] = data
 
         def filectxfn(repo, memctx, path):
             if path not in changes:
                 return None
             return context.memfilectx(repo, memctx, path, changes[path])
 
         if not changes:
             continue
         if revs:
             date = repo['tip'].date()[0] + pick(interarrival)
         else:
             date = time.time() - (86400 * count)
         # dates in mercurial must be positive, fit in 32-bit signed integers.
         date = min(0x7FFFFFFF, max(0, date))
         user = random.choice(words) + '@' + random.choice(words)
         mc = context.memctx(
             repo,
             pl,
             makeline(minimum=2),
             sorted(changes),
             filectxfn,
             user,
             '%d %d' % (date, pick(tzoffset)),
         )
         newnode = mc.commit()
         heads.add(repo.changelog.rev(newnode))
         heads.discard(r1)
         heads.discard(r2)
     progress.complete()
 
     lock.release()
     wlock.release()
 
 
 def renamedirs(dirs, words):
     '''Randomly rename the directory names in the per-dir file count dict.'''
     wordgen = itertools.cycle(words)
     replacements = {'': ''}
 
     def rename(dirpath):
         """Recursively rename the directory and all path prefixes.
 
         The mapping from path to renamed path is stored for all path prefixes
         as in dynamic programming, ensuring linear runtime and consistent
         renaming regardless of iteration order through the model.
         """
         if dirpath in replacements:
             return replacements[dirpath]
         head, _ = os.path.split(dirpath)
         if head:
             head = rename(head)
         else:
             head = ''
         renamed = os.path.join(head, next(wordgen))
         replacements[dirpath] = renamed
         return renamed
 
     result = []
     for dirpath, count in dirs.iteritems():
         result.append([rename(dirpath.lstrip(os.sep)), count])
     return result

contrib/testparseutil.py

@@ -1,668 +1,667 b''
 # testparseutil.py - utilities to parse test script for check tools
 #
 #  Copyright 2018 FUJIWARA Katsunori <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import, print_function
 
 import abc
 import re
 import sys
 
 ####################
 # for Python3 compatibility (almost comes from mercurial/pycompat.py)
 
 ispy3 = sys.version_info[0] >= 3
 
 
 def identity(a):
     return a
 
 
 def _rapply(f, xs):
     if xs is None:
         # assume None means non-value of optional data
         return xs
     if isinstance(xs, (list, set, tuple)):
         return type(xs)(_rapply(f, x) for x in xs)
     if isinstance(xs, dict):
         return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items())
     return f(xs)
 
 
 def rapply(f, xs):
     if f is identity:
         # fast path mainly for py2
         return xs
     return _rapply(f, xs)
 
 
 if ispy3:
     import builtins
 
     def bytestr(s):
         # tiny version of pycompat.bytestr
         return s.encode('latin1')
 
     def sysstr(s):
         if isinstance(s, builtins.str):
             return s
         return s.decode('latin-1')
 
     def opentext(f):
         return open(f, 'r')
 
 
 else:
     bytestr = str
     sysstr = identity
 
     opentext = open
 
 
 def b2s(x):
     # convert BYTES elements in "x" to SYSSTR recursively
     return rapply(sysstr, x)
 
 
 def writeout(data):
     # write "data" in BYTES into stdout
     sys.stdout.write(data)
 
 
 def writeerr(data):
     # write "data" in BYTES into stderr
     sys.stderr.write(data)
 
 
 ####################
 
 
 class embeddedmatcher(object):  # pytype: disable=ignored-metaclass
     """Base class to detect embedded code fragments in *.t test script"""
 
     __metaclass__ = abc.ABCMeta
 
     def __init__(self, desc):
         self.desc = desc
 
     @abc.abstractmethod
     def startsat(self, line):
         """Examine whether embedded code starts at line
 
         This can return arbitrary object, and it is used as 'ctx' for
         subsequent method invocations.
         """
 
     @abc.abstractmethod
     def endsat(self, ctx, line):
         """Examine whether embedded code ends at line"""
 
     @abc.abstractmethod
     def isinside(self, ctx, line):
         """Examine whether line is inside embedded code, if not yet endsat"""
 
     @abc.abstractmethod
     def ignores(self, ctx):
         """Examine whether detected embedded code should be ignored"""
 
     @abc.abstractmethod
     def filename(self, ctx):
         """Return filename of embedded code
 
         If filename isn't specified for embedded code explicitly, this
         returns None.
         """
 
     @abc.abstractmethod
     def codeatstart(self, ctx, line):
         """Return actual code at the start line of embedded code
 
         This might return None, if the start line doesn't contain
         actual code.
         """
 
     @abc.abstractmethod
     def codeatend(self, ctx, line):
         """Return actual code at the end line of embedded code
 
         This might return None, if the end line doesn't contain actual
         code.
         """
 
     @abc.abstractmethod
     def codeinside(self, ctx, line):
         """Return actual code at line inside embedded code"""
 
 
 def embedded(basefile, lines, errors, matchers):
     """pick embedded code fragments up from given lines
 
     This is common parsing logic, which examines specified matchers on
     given lines.
 
     :basefile: a name of a file, from which lines to be parsed come.
     :lines: to be parsed (might be a value returned by "open(basefile)")
     :errors: an array, into which messages for detected error are stored
     :matchers: an array of embeddedmatcher objects
 
     This function yields '(filename, starts, ends, code)' tuple.
 
     :filename: a name of embedded code, if it is explicitly specified
                (e.g.  "foobar" of "cat >> foobar <<EOF").
                Otherwise, this is None
     :starts: line number (1-origin), at which embedded code starts (inclusive)
     :ends: line number (1-origin), at which embedded code ends (exclusive)
     :code: extracted embedded code, which is single-stringified
 
     >>> class ambigmatcher(object):
     ...     # mock matcher class to examine implementation of
     ...     # "ambiguous matching" corner case
     ...     def __init__(self, desc, matchfunc):
     ...         self.desc = desc
     ...         self.matchfunc = matchfunc
     ...     def startsat(self, line):
     ...         return self.matchfunc(line)
     >>> ambig1 = ambigmatcher('ambiguous #1',
     ...                       lambda l: l.startswith('  $ cat '))
     >>> ambig2 = ambigmatcher('ambiguous #2',
     ...                       lambda l: l.endswith('<< EOF\\n'))
     >>> lines = ['  $ cat > foo.py << EOF\\n']
     >>> errors = []
     >>> matchers = [ambig1, ambig2]
     >>> list(t for t in embedded('<dummy>', lines, errors, matchers))
     []
     >>> b2s(errors)
     ['<dummy>:1: ambiguous line for "ambiguous #1", "ambiguous #2"']
 
     """
     matcher = None
     ctx = filename = code = startline = None  # for pyflakes
 
     for lineno, line in enumerate(lines, 1):
         if not line.endswith('\n'):
             line += '\n'  # to normalize EOF line
         if matcher:  # now, inside embedded code
             if matcher.endsat(ctx, line):
                 codeatend = matcher.codeatend(ctx, line)
                 if codeatend is not None:
                     code.append(codeatend)
                 if not matcher.ignores(ctx):
                     yield (filename, startline, lineno, ''.join(code))
                 matcher = None
                 # DO NOT "continue", because line might start next fragment
             elif not matcher.isinside(ctx, line):
                 # this is an error of basefile
                 # (if matchers are implemented correctly)
                 errors.append(
                     '%s:%d: unexpected line for "%s"'
                     % (basefile, lineno, matcher.desc)
                 )
                 # stop extracting embedded code by current 'matcher',
                 # because appearance of unexpected line might mean
                 # that expected end-of-embedded-code line might never
                 # appear
                 matcher = None
                 # DO NOT "continue", because line might start next fragment
             else:
                 code.append(matcher.codeinside(ctx, line))
                 continue
 
         # examine whether current line starts embedded code or not
         assert not matcher
 
         matched = []
         for m in matchers:
             ctx = m.startsat(line)
             if ctx:
                 matched.append((m, ctx))
         if matched:
             if len(matched) > 1:
                 # this is an error of matchers, maybe
                 errors.append(
                     '%s:%d: ambiguous line for %s'
                     % (
                         basefile,
                         lineno,
                         ', '.join(['"%s"' % m.desc for m, c in matched]),
                     )
                 )
                 # omit extracting embedded code, because choosing
                 # arbitrary matcher from matched ones might fail to
                 # detect the end of embedded code as expected.
                 continue
             matcher, ctx = matched[0]
             filename = matcher.filename(ctx)
             code = []
             codeatstart = matcher.codeatstart(ctx, line)
             if codeatstart is not None:
                 code.append(codeatstart)
                 startline = lineno
             else:
                 startline = lineno + 1
 
     if matcher:
         # examine whether EOF ends embedded code, because embedded
         # code isn't yet ended explicitly
         if matcher.endsat(ctx, '\n'):
             codeatend = matcher.codeatend(ctx, '\n')
             if codeatend is not None:
                 code.append(codeatend)
             if not matcher.ignores(ctx):
                 yield (filename, startline, lineno + 1, ''.join(code))
         else:
             # this is an error of basefile
             # (if matchers are implemented correctly)
             errors.append(
                 '%s:%d: unexpected end of file for "%s"'
                 % (basefile, lineno, matcher.desc)
             )
 
 
 # heredoc limit mark to ignore embedded code at check-code.py or so
 heredocignorelimit = 'NO_CHECK_EOF'
 
 # the pattern to match against cases below, and to return a limit mark
 # string as 'lname' group
 #
 # - << LIMITMARK
 # - << "LIMITMARK"
 # - << 'LIMITMARK'
 heredoclimitpat = r'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)'
 
 
 class fileheredocmatcher(embeddedmatcher):
     """Detect "cat > FILE << LIMIT" style embedded code
 
     >>> matcher = fileheredocmatcher('heredoc .py file', r'[^<]+\\.py')
     >>> b2s(matcher.startsat('  $ cat > file.py << EOF\\n'))
     ('file.py', '  > EOF\\n')
     >>> b2s(matcher.startsat('  $ cat   >>file.py   <<EOF\\n'))
     ('file.py', '  > EOF\\n')
     >>> b2s(matcher.startsat('  $ cat>  \\x27any file.py\\x27<<  "EOF"\\n'))
     ('any file.py', '  > EOF\\n')
     >>> b2s(matcher.startsat("  $ cat > file.py << 'ANYLIMIT'\\n"))
     ('file.py', '  > ANYLIMIT\\n')
     >>> b2s(matcher.startsat('  $ cat<<ANYLIMIT>"file.py"\\n'))
     ('file.py', '  > ANYLIMIT\\n')
     >>> start = '  $ cat > file.py << EOF\\n'
     >>> ctx = matcher.startsat(start)
     >>> matcher.codeatstart(ctx, start)
     >>> b2s(matcher.filename(ctx))
     'file.py'
     >>> matcher.ignores(ctx)
     False
     >>> inside = '  > foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> end = '  > EOF\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     >>> matcher.endsat(ctx, '  > EOFEOF\\n')
     False
     >>> ctx = matcher.startsat('  $ cat > file.py << NO_CHECK_EOF\\n')
     >>> matcher.ignores(ctx)
     True
     """
 
     _prefix = '  > '
 
     def __init__(self, desc, namepat):
         super(fileheredocmatcher, self).__init__(desc)
 
         # build the pattern to match against cases below (and ">>"
         # variants), and to return a target filename string as 'name'
         # group
         #
         # - > NAMEPAT
         # - > "NAMEPAT"
         # - > 'NAMEPAT'
         namepat = (
             r'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)' % namepat
         )
         self._fileres = [
             # "cat > NAME << LIMIT" case
             re.compile(r' {2}\$ \s*cat' + namepat + heredoclimitpat),
             # "cat << LIMIT > NAME" case
             re.compile(r' {2}\$ \s*cat' + heredoclimitpat + namepat),
         ]
 
     def startsat(self, line):
         # ctx is (filename, END-LINE-OF-EMBEDDED-CODE) tuple
         for filere in self._fileres:
             matched = filere.match(line)
             if matched:
                 return (
                     matched.group('name'),
                     '  > %s\n' % matched.group('limit'),
                 )
 
     def endsat(self, ctx, line):
         return ctx[1] == line
 
     def isinside(self, ctx, line):
         return line.startswith(self._prefix)
 
     def ignores(self, ctx):
         return '  > %s\n' % heredocignorelimit == ctx[1]
 
     def filename(self, ctx):
         return ctx[0]
 
     def codeatstart(self, ctx, line):
         return None  # no embedded code at start line
 
     def codeatend(self, ctx, line):
         return None  # no embedded code at end line
 
     def codeinside(self, ctx, line):
         return line[len(self._prefix) :]  # strip prefix
 
 
 ####
 # for embedded python script
 
 
 class pydoctestmatcher(embeddedmatcher):
     """Detect ">>> code" style embedded python code
 
     >>> matcher = pydoctestmatcher()
     >>> startline = '  >>> foo = 1\\n'
     >>> matcher.startsat(startline)
     True
     >>> matcher.startsat('  ... foo = 1\\n')
     False
     >>> ctx = matcher.startsat(startline)
     >>> matcher.filename(ctx)
     >>> matcher.ignores(ctx)
     False
     >>> b2s(matcher.codeatstart(ctx, startline))
     'foo = 1\\n'
     >>> inside = '  >>> foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> inside = '  ... foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> inside = '  expected output\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     '\\n'
     >>> inside = '  \\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     '\\n'
     >>> end = '  $ foo bar\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     >>> end = '\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     """
 
     _prefix = '  >>> '
     _prefixre = re.compile(r' {2}(>>>|\.\.\.) ')
 
     # If a line matches against not _prefixre but _outputre, that line
     # is "an expected output line" (= not a part of code fragment).
     #
     # Strictly speaking, a line matching against "(#if|#else|#endif)"
     # is also treated similarly in "inline python code" semantics by
     # run-tests.py. But "directive line inside inline python code"
     # should be rejected by Mercurial reviewers. Therefore, this
     # regexp does not matche against such directive lines.
     _outputre = re.compile(r' {2}$| {2}[^$]')
 
     def __init__(self):
         super(pydoctestmatcher, self).__init__("doctest style python code")
 
     def startsat(self, line):
         # ctx is "True"
         return line.startswith(self._prefix)
 
     def endsat(self, ctx, line):
         return not (self._prefixre.match(line) or self._outputre.match(line))
 
     def isinside(self, ctx, line):
         return True  # always true, if not yet ended
 
     def ignores(self, ctx):
         return False  # should be checked always
 
     def filename(self, ctx):
         return None  # no filename
 
     def codeatstart(self, ctx, line):
         return line[len(self._prefix) :]  # strip prefix '  >>> '/'  ... '
 
     def codeatend(self, ctx, line):
         return None  # no embedded code at end line
 
     def codeinside(self, ctx, line):
         if self._prefixre.match(line):
             return line[len(self._prefix) :]  # strip prefix '  >>> '/'  ... '
         return '\n'  # an expected output line is treated as an empty line
 
 
 class pyheredocmatcher(embeddedmatcher):
     """Detect "python << LIMIT" style embedded python code
 
     >>> matcher = pyheredocmatcher()
     >>> b2s(matcher.startsat('  $ python << EOF\\n'))
     '  > EOF\\n'
     >>> b2s(matcher.startsat('  $ $PYTHON   <<EOF\\n'))
     '  > EOF\\n'
     >>> b2s(matcher.startsat('  $ "$PYTHON"<<  "EOF"\\n'))
     '  > EOF\\n'
     >>> b2s(matcher.startsat("  $ $PYTHON << 'ANYLIMIT'\\n"))
     '  > ANYLIMIT\\n'
     >>> matcher.startsat('  $ "$PYTHON" < EOF\\n')
     >>> start = '  $ python << EOF\\n'
     >>> ctx = matcher.startsat(start)
     >>> matcher.codeatstart(ctx, start)
     >>> matcher.filename(ctx)
     >>> matcher.ignores(ctx)
     False
     >>> inside = '  > foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> end = '  > EOF\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     >>> matcher.endsat(ctx, '  > EOFEOF\\n')
     False
     >>> ctx = matcher.startsat('  $ python << NO_CHECK_EOF\\n')
     >>> matcher.ignores(ctx)
     True
     """
 
     _prefix = '  > '
 
     _startre = re.compile(
         r' {2}\$ (\$PYTHON|"\$PYTHON"|python).*' + heredoclimitpat
     )
 
     def __init__(self):
         super(pyheredocmatcher, self).__init__("heredoc python invocation")
 
     def startsat(self, line):
         # ctx is END-LINE-OF-EMBEDDED-CODE
         matched = self._startre.match(line)
         if matched:
             return '  > %s\n' % matched.group('limit')
 
     def endsat(self, ctx, line):
         return ctx == line
 
     def isinside(self, ctx, line):
         return line.startswith(self._prefix)
 
     def ignores(self, ctx):
         return '  > %s\n' % heredocignorelimit == ctx
 
     def filename(self, ctx):
         return None  # no filename
 
     def codeatstart(self, ctx, line):
         return None  # no embedded code at start line
 
     def codeatend(self, ctx, line):
         return None  # no embedded code at end line
 
     def codeinside(self, ctx, line):
         return line[len(self._prefix) :]  # strip prefix
 
 
 _pymatchers = [
     pydoctestmatcher(),
     pyheredocmatcher(),
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
     fileheredocmatcher('heredoc .py file', r'[^<]+\.py'),
 ]
 
 
 def pyembedded(basefile, lines, errors):
     return embedded(basefile, lines, errors, _pymatchers)
 
 
 ####
 # for embedded shell script
 
 _shmatchers = [
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
     fileheredocmatcher('heredoc .sh file', r'[^<]+\.sh'),
 ]
 
 
 def shembedded(basefile, lines, errors):
     return embedded(basefile, lines, errors, _shmatchers)
 
 
 ####
 # for embedded hgrc configuration
 
 _hgrcmatchers = [
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
     fileheredocmatcher(
         'heredoc hgrc file', r'(([^/<]+/)+hgrc|\$HGRCPATH|\${HGRCPATH})'
     ),
 ]
 
 
 def hgrcembedded(basefile, lines, errors):
     return embedded(basefile, lines, errors, _hgrcmatchers)
 
 
 ####
 
 if __name__ == "__main__":
     import optparse
     import sys
 
     def showembedded(basefile, lines, embeddedfunc, opts):
         errors = []
         for name, starts, ends, code in embeddedfunc(basefile, lines, errors):
             if not name:
                 name = '<anonymous>'
             writeout("%s:%d: %s starts\n" % (basefile, starts, name))
             if opts.verbose and code:
                 writeout("  |%s\n" % "\n  |".join(l for l in code.splitlines()))
             writeout("%s:%d: %s ends\n" % (basefile, ends, name))
         for e in errors:
             writeerr("%s\n" % e)
         return len(errors)
 
     def applyembedded(args, embeddedfunc, opts):
         ret = 0
         if args:
             for f in args:
                 with opentext(f) as fp:
                     if showembedded(f, fp, embeddedfunc, opts):
                         ret = 1
         else:
             lines = [l for l in sys.stdin.readlines()]
             if showembedded('<stdin>', lines, embeddedfunc, opts):
                 ret = 1
         return ret
 
     commands = {}
 
     def command(name, desc):
         def wrap(func):
             commands[name] = (desc, func)
 
         return wrap
 
     @command("pyembedded", "detect embedded python script")
     def pyembeddedcmd(args, opts):
         return applyembedded(args, pyembedded, opts)
 
     @command("shembedded", "detect embedded shell script")
     def shembeddedcmd(args, opts):
         return applyembedded(args, shembedded, opts)
 
     @command("hgrcembedded", "detect embedded hgrc configuration")
     def hgrcembeddedcmd(args, opts):
         return applyembedded(args, hgrcembedded, opts)
 
     availablecommands = "\n".join(
         ["  - %s: %s" % (key, value[0]) for key, value in commands.items()]
     )
 
     parser = optparse.OptionParser(
         """%prog COMMAND [file ...]
 
 Pick up embedded code fragments from given file(s) or stdin, and list
 up start/end lines of them in standard compiler format
 ("FILENAME:LINENO:").
 
 Available commands are:
 """
         + availablecommands
         + """
 """
     )
     parser.add_option(
         "-v",
         "--verbose",
         help="enable additional output (e.g. actual code)",
         action="store_true",
     )
     (opts, args) = parser.parse_args()
 
     if not args or args[0] not in commands:
         parser.print_help()
         sys.exit(255)
 
     sys.exit(commands[args[0]][1](args[1:], opts))

contrib/undumprevlog

@@ -1,57 +1,56 b''
 #!/usr/bin/env python3
 # Undump a dump from dumprevlog
 # $ hg init
 # $ undumprevlog < repo.dump
 
-from __future__ import absolute_import, print_function
 
 import sys
 from mercurial.node import bin
 from mercurial import (
     encoding,
     revlog,
     transaction,
     vfs as vfsmod,
 )
 from mercurial.utils import procutil
 
 from mercurial.revlogutils import (
     constants as revlog_constants,
 )
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     procutil.setbinary(fp)
 
 opener = vfsmod.vfs(b'.', False)
 tr = transaction.transaction(
     sys.stderr.write, opener, {b'store': opener}, b"undump.journal"
 )
 while True:
     l = sys.stdin.readline()
     if not l:
         break
     if l.startswith("file:"):
         f = encoding.strtolocal(l[6:-1])
         assert f.endswith(b'.i')
         r = revlog.revlog(
             opener,
             target=(revlog_constants.KIND_OTHER, b'undump-revlog'),
             radix=f[:-2],
         )
         procutil.stdout.write(b'%s\n' % f)
     elif l.startswith("node:"):
         n = bin(l[6:-1])
     elif l.startswith("linkrev:"):
         lr = int(l[9:-1])
     elif l.startswith("parents:"):
         p = l[9:-1].split()
         p1 = bin(p[0])
         p2 = bin(p[1])
     elif l.startswith("length:"):
         length = int(l[8:-1])
         sys.stdin.readline()  # start marker
         d = encoding.strtolocal(sys.stdin.read(length))
         sys.stdin.readline()  # end marker
         r.addrevision(d, tr, lr, p1, p2)
 
 tr.close()

contrib/win32/hgwebdir_wsgi.py

@@ -1,140 +1,139 b''
 # An example WSGI script for IIS/isapi-wsgi to export multiple hgweb repos
 # Copyright 2010-2016 Sune Foldager <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # Requirements:
 # - Python 2.7, preferably 64 bit
 # - Mercurial installed from source (python setup.py install) or download the
 #   python module installer from https://www.mercurial-scm.org/wiki/Download
 # - IIS 7 or newer
 #
 #
 # Installation and use:
 #
 # - Download or clone the isapi-wsgi source and run python setup.py install.
 #   https://github.com/hexdump42/isapi-wsgi
 #
 # - Create a directory to hold the shim dll, config files etc. This can reside
 #   inside the standard IIS directory, C:\inetpub, or anywhere else. Copy this
 #   script there.
 #
 # - Run this script (i.e. python hgwebdir_wsgi.py) to get a shim dll. The
 #   shim is identical for all scripts, so you can just copy and rename one
 #   from an earlier run, if you wish. The shim needs to reside in the same
 #   directory as this script.
 #
 # - Start IIS manager and create a new app pool:
 #   .NET CLR Version: No Managed Code
 #   Advanced Settings: Enable 32 Bit Applications, if using 32 bit Python.
 #   You can adjust the identity and maximum worker processes if you wish. This
 #   setup works fine with multiple worker processes.
 #
 # - Create an IIS application where your hgwebdir is to be served from.
 #   Assign it the app pool you just created and point its physical path to the
 #   directory you created.
 #
 # - In the application, remove all handler mappings and setup a wildcard script
 #   handler mapping of type IsapiModule with the shim dll as its executable.
 #   This file MUST reside in the same directory as the shim. The easiest way
 #   to do all this is to close IIS manager, place a web.config file in your
 #   directory and start IIS manager again. The file should contain:
 #
 #   <?xml version="1.0" encoding="UTF-8"?>
 #   <configuration>
 #       <system.webServer>
 #           <handlers accessPolicy="Read, Script">
 #               <clear />
 #               <add name="hgwebdir" path="*" verb="*" modules="IsapiModule"
 #                    scriptProcessor="C:\your\directory\_hgwebdir_wsgi.dll"
 #                    resourceType="Unspecified" requireAccess="None"
 #                    preCondition="bitness64" />
 #           </handlers>
 #       </system.webServer>
 #   </configuration>
 #
 #   Where "bitness64" should be replaced with "bitness32" for 32 bit Python.
 #
 # - Edit ISAPI And CGI Restrictions on the web server (global setting). Add a
 #   restriction pointing to your shim dll and allow it to run.
 #
 # - Create a configuration file in your directory and adjust the configuration
 #   variables below to match your needs. Example configuration:
 #
 #   [web]
 #   style = gitweb
 #   push_ssl = false
 #   allow_push = *
 #   encoding = utf8
 #
 #   [server]
 #   validate = true
 #
 #   [paths]
 #   repo1 = c:\your\directory\repo1
 #   repo2 = c:\your\directory\repo2
 #
 # - Restart the web server and see if things are running.
 #
 
-from __future__ import absolute_import
 
 # Configuration file location
 hgweb_config = r'c:\your\directory\wsgi.config'
 
 # Global settings for IIS path translation
 path_strip = 0  # Strip this many path elements off (when using url rewrite)
 path_prefix = 1  # This many path elements are prefixes (depends on the
 # virtual path of the IIS application).
 
 import sys
 
 # Adjust python path if this is not a system-wide install
 # sys.path.insert(0, r'C:\your\custom\hg\build\lib.win32-2.7')
 
 # Enable tracing. Run 'python -m win32traceutil' to debug
 if getattr(sys, 'isapidllhandle', None) is not None:
     import win32traceutil
 
     win32traceutil.SetupForPrint  # silence unused import warning
 
 import isapi_wsgi
 from mercurial.hgweb.hgwebdir_mod import hgwebdir
 
 # Example tweak: Replace isapi_wsgi's handler to provide better error message
 # Other stuff could also be done here, like logging errors etc.
 class WsgiHandler(isapi_wsgi.IsapiWsgiHandler):
     error_status = '500 Internal Server Error'  # less silly error message
 
 
 isapi_wsgi.IsapiWsgiHandler = WsgiHandler
 
 # Only create the hgwebdir instance once
 application = hgwebdir(hgweb_config)
 
 
 def handler(environ, start_response):
 
     # Translate IIS's weird URLs
     url = environ['SCRIPT_NAME'] + environ['PATH_INFO']
     paths = url[1:].split('/')[path_strip:]
     script_name = '/' + '/'.join(paths[:path_prefix])
     path_info = '/'.join(paths[path_prefix:])
     if path_info:
         path_info = '/' + path_info
     environ['SCRIPT_NAME'] = script_name
     environ['PATH_INFO'] = path_info
 
     return application(environ, start_response)
 
 
 def __ExtensionFactory__():
     return isapi_wsgi.ISAPISimpleHandler(handler)
 
 
 if __name__ == '__main__':
     from isapi.install import ISAPIParameters, HandleCommandLine
 
     params = ISAPIParameters()
     HandleCommandLine(params)

doc/check-seclevel.py

@@ -1,233 +1,232 b''
 #!/usr/bin/env python3
 #
 # checkseclevel - checking section title levels in each online help document
 
-from __future__ import absolute_import
 
 import optparse
 import os
 import sys
 
 # import from the live mercurial repo
 os.environ['HGMODULEPOLICY'] = 'py'
 sys.path.insert(0, "..")
 from mercurial import demandimport
 
 demandimport.enable()
 from mercurial import (
     commands,
     extensions,
     help,
     minirst,
     ui as uimod,
 )
 
 table = commands.table
 helptable = help.helptable
 
 level2mark = [b'"', b'=', b'-', b'.', b'#']
 reservedmarks = [b'"']
 
 mark2level = {}
 for m, l in zip(level2mark, range(len(level2mark))):
     if m not in reservedmarks:
         mark2level[m] = l
 
 initlevel_topic = 0
 initlevel_cmd = 1
 initlevel_ext = 1
 initlevel_ext_cmd = 3
 
 
 def showavailables(ui, initlevel):
     avail = '    available marks and order of them in this help: %s\n' % (
         ', '.join(['%r' % (m * 4) for m in level2mark[initlevel + 1 :]])
     )
     ui.warn(avail.encode('utf-8'))
 
 
 def checkseclevel(ui, doc, name, initlevel):
     ui.notenoi18n('checking "%s"\n' % name)
     if not isinstance(doc, bytes):
         doc = doc.encode('utf-8')
     blocks, pruned = minirst.parse(doc, 0, ['verbose'])
     errorcnt = 0
     curlevel = initlevel
     for block in blocks:
         if block[b'type'] != b'section':
             continue
         mark = block[b'underline']
         title = block[b'lines'][0]
         if (mark not in mark2level) or (mark2level[mark] <= initlevel):
             ui.warn(
                 (
                     'invalid section mark %r for "%s" of %s\n'
                     % (mark * 4, title, name)
                 ).encode('utf-8')
             )
             showavailables(ui, initlevel)
             errorcnt += 1
             continue
         nextlevel = mark2level[mark]
         if curlevel < nextlevel and curlevel + 1 != nextlevel:
             ui.warnnoi18n(
                 'gap of section level at "%s" of %s\n' % (title, name)
             )
             showavailables(ui, initlevel)
             errorcnt += 1
             continue
         ui.notenoi18n(
             'appropriate section level for "%s %s"\n'
             % (mark * (nextlevel * 2), title)
         )
         curlevel = nextlevel
 
     return errorcnt
 
 
 def checkcmdtable(ui, cmdtable, namefmt, initlevel):
     errorcnt = 0
     for k, entry in cmdtable.items():
         name = k.split(b"|")[0].lstrip(b"^")
         if not entry[0].__doc__:
             ui.notenoi18n(
                 'skip checking %s: no help document\n' % (namefmt % name)
             )
             continue
         errorcnt += checkseclevel(
             ui, entry[0].__doc__, namefmt % name, initlevel
         )
     return errorcnt
 
 
 def checkhghelps(ui):
     errorcnt = 0
     for h in helptable:
         names, sec, doc = h[0:3]
         if callable(doc):
             doc = doc(ui)
         errorcnt += checkseclevel(
             ui, doc, '%s help topic' % names[0], initlevel_topic
         )
 
     errorcnt += checkcmdtable(ui, table, '%s command', initlevel_cmd)
 
     for name in sorted(
         list(extensions.enabled()) + list(extensions.disabled())
     ):
         mod = extensions.load(ui, name, None)
         if not mod.__doc__:
             ui.notenoi18n(
                 'skip checking %s extension: no help document\n' % name
             )
             continue
         errorcnt += checkseclevel(
             ui, mod.__doc__, '%s extension' % name, initlevel_ext
         )
 
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
             errorcnt += checkcmdtable(
                 ui,
                 cmdtable,
                 '%%s command of %s extension' % name,
                 initlevel_ext_cmd,
             )
     return errorcnt
 
 
 def checkfile(ui, filename, initlevel):
     if filename == '-':
         filename = 'stdin'
         doc = sys.stdin.read()
     else:
         with open(filename) as fp:
             doc = fp.read()
 
     ui.notenoi18n(
         'checking input from %s with initlevel %d\n' % (filename, initlevel)
     )
     return checkseclevel(ui, doc, 'input from %s' % filename, initlevel)
 
 
 def main():
     optparser = optparse.OptionParser(
         """%prog [options]
 
 This checks all help documents of Mercurial (topics, commands,
 extensions and commands of them), if no file is specified by --file
 option.
 """
     )
     optparser.add_option(
         "-v", "--verbose", help="enable additional output", action="store_true"
     )
     optparser.add_option(
         "-d", "--debug", help="debug mode", action="store_true"
     )
     optparser.add_option(
         "-f",
         "--file",
         help="filename to read in (or '-' for stdin)",
         action="store",
         default="",
     )
 
     optparser.add_option(
         "-t",
         "--topic",
         help="parse file as help topic",
         action="store_const",
         dest="initlevel",
         const=0,
     )
     optparser.add_option(
         "-c",
         "--command",
         help="parse file as help of core command",
         action="store_const",
         dest="initlevel",
         const=1,
     )
     optparser.add_option(
         "-e",
         "--extension",
         help="parse file as help of extension",
         action="store_const",
         dest="initlevel",
         const=1,
     )
     optparser.add_option(
         "-C",
         "--extension-command",
         help="parse file as help of extension command",
         action="store_const",
         dest="initlevel",
         const=3,
     )
 
     optparser.add_option(
         "-l",
         "--initlevel",
         help="set initial section level manually",
         action="store",
         type="int",
         default=0,
     )
 
     (options, args) = optparser.parse_args()
 
     ui = uimod.ui.load()
     ui.setconfig(b'ui', b'verbose', options.verbose, b'--verbose')
     ui.setconfig(b'ui', b'debug', options.debug, b'--debug')
 
     if options.file:
         if checkfile(ui, options.file, options.initlevel):
             sys.exit(1)
     else:
         if checkhghelps(ui):
             sys.exit(1)
 
 
 if __name__ == "__main__":
     main()

doc/docchecker

@@ -1,84 +1,83 b''
 #!/usr/bin/env python3
 #
 # docchecker - look for problematic markup
 #
 # Copyright 2016 timeless <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import, print_function
 
 import os
 import re
 import sys
 
 try:
     import msvcrt
 
     msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
     msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
 except ImportError:
     pass
 
 stdout = getattr(sys.stdout, 'buffer', sys.stdout)
 
 leadingline = re.compile(br'(^\s*)(\S.*)$')
 
 checks = [
     (
         br""":hg:`[^`]*'[^`]*`""",
         b"""warning: please avoid nesting ' in :hg:`...`""",
     ),
     (br'\w:hg:`', b'warning: please have a space before :hg:'),
     (
         br"""(?:[^a-z][^'.])hg ([^,;"`]*'(?!hg)){2}""",
         b'''warning: please use " instead of ' for hg ... "..."''',
     ),
 ]
 
 
 def check(line):
     messages = []
     for match, msg in checks:
         if re.search(match, line):
             messages.append(msg)
     if messages:
         stdout.write(b'%s\n' % line)
         for msg in messages:
             stdout.write(b'%s\n' % msg)
 
 
 def work(file):
     (llead, lline) = (b'', b'')
 
     for line in file:
         # this section unwraps lines
         match = leadingline.match(line)
         if not match:
             check(lline)
             (llead, lline) = (b'', b'')
             continue
 
         lead, line = match.group(1), match.group(2)
         if lead == llead:
             if lline != b'':
                 lline += b' ' + line
             else:
                 lline = line
         else:
             check(lline)
             (llead, lline) = (lead, line)
     check(lline)
 
 
 def main():
     for f in sys.argv[1:]:
         try:
             with open(f, 'rb') as file:
                 work(file)
         except BaseException as e:
             sys.stdout.write(r"failed to process %s: %s\n" % (f, e))
 
 
 main()

doc/gendoc.py

@@ -1,346 +1,345 b''
 #!/usr/bin/env python3
 """usage: %s DOC ...
 
 where DOC is the name of a document
 """
 
-from __future__ import absolute_import
 
 import os
 import sys
 import textwrap
 
 try:
     import msvcrt
 
     msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
     msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
 except ImportError:
     pass
 
 # This script is executed during installs and may not have C extensions
 # available. Relax C module requirements.
 os.environ['HGMODULEPOLICY'] = 'allow'
 # import from the live mercurial repo
 sys.path.insert(0, "..")
 from mercurial import demandimport
 
 demandimport.enable()
 
 from mercurial import (
     commands,
     encoding,
     extensions,
     fancyopts,
     help,
     minirst,
     pycompat,
     ui as uimod,
 )
 from mercurial.i18n import (
     gettext,
     _,
 )
 from mercurial.utils import stringutil
 
 table = commands.table
 globalopts = commands.globalopts
 helptable = help.helptable
 loaddoc = help.loaddoc
 
 
 def get_desc(docstr):
     if not docstr:
         return b"", b""
     # sanitize
     docstr = docstr.strip(b"\n")
     docstr = docstr.rstrip()
     shortdesc = docstr.splitlines()[0].strip()
 
     i = docstr.find(b"\n")
     if i != -1:
         desc = docstr[i + 2 :]
     else:
         desc = shortdesc
 
     desc = textwrap.dedent(desc.decode('latin1')).encode('latin1')
 
     return (shortdesc, desc)
 
 
 def get_opts(opts):
     for opt in opts:
         if len(opt) == 5:
             shortopt, longopt, default, desc, optlabel = opt
         else:
             shortopt, longopt, default, desc = opt
             optlabel = _(b"VALUE")
         allopts = []
         if shortopt:
             allopts.append(b"-%s" % shortopt)
         if longopt:
             allopts.append(b"--%s" % longopt)
         if isinstance(default, list):
             allopts[-1] += b" <%s[+]>" % optlabel
         elif (default is not None) and not isinstance(default, bool):
             allopts[-1] += b" <%s>" % optlabel
         if b'\n' in desc:
             # only remove line breaks and indentation
             desc = b' '.join(l.lstrip() for l in desc.split(b'\n'))
         if isinstance(default, fancyopts.customopt):
             default = default.getdefaultvalue()
         if default:
             default = stringutil.forcebytestr(default)
             desc += _(b" (default: %s)") % default
         yield (b", ".join(allopts), desc)
 
 
 def get_cmd(cmd, cmdtable):
     d = {}
     attr = cmdtable[cmd]
     cmds = cmd.lstrip(b"^").split(b"|")
 
     d[b'cmd'] = cmds[0]
     d[b'aliases'] = cmd.split(b"|")[1:]
     d[b'desc'] = get_desc(gettext(pycompat.getdoc(attr[0])))
     d[b'opts'] = list(get_opts(attr[1]))
 
     s = b'hg ' + cmds[0]
     if len(attr) > 2:
         if not attr[2].startswith(b'hg'):
             s += b' ' + attr[2]
         else:
             s = attr[2]
     d[b'synopsis'] = s.strip()
 
     return d
 
 
 def showdoc(ui):
     # print options
     ui.write(minirst.section(_(b"Options")))
     multioccur = False
     for optstr, desc in get_opts(globalopts):
         ui.write(b"%s\n    %s\n\n" % (optstr, desc))
         if optstr.endswith(b"[+]>"):
             multioccur = True
     if multioccur:
         ui.write(_(b"\n[+] marked option can be specified multiple times\n"))
         ui.write(b"\n")
 
     # print cmds
     ui.write(minirst.section(_(b"Commands")))
     commandprinter(ui, table, minirst.subsection, minirst.subsubsection)
 
     # print help topics
     # The config help topic is included in the hgrc.5 man page.
     helpprinter(ui, helptable, minirst.section, exclude=[b'config'])
 
     ui.write(minirst.section(_(b"Extensions")))
     ui.write(
         _(
             b"This section contains help for extensions that are "
             b"distributed together with Mercurial. Help for other "
             b"extensions is available in the help system."
         )
     )
     ui.write(
         (
             b"\n\n"
             b".. contents::\n"
             b"   :class: htmlonly\n"
             b"   :local:\n"
             b"   :depth: 1\n\n"
         )
     )
 
     for extensionname in sorted(allextensionnames()):
         mod = extensions.load(ui, extensionname, None)
         ui.write(minirst.subsection(extensionname))
         ui.write(b"%s\n\n" % gettext(pycompat.getdoc(mod)))
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
             ui.write(minirst.subsubsection(_(b'Commands')))
             commandprinter(
                 ui,
                 cmdtable,
                 minirst.subsubsubsection,
                 minirst.subsubsubsubsection,
             )
 
 
 def showtopic(ui, topic):
     extrahelptable = [
         ([b"common"], b'', loaddoc(b'common'), help.TOPIC_CATEGORY_MISC),
         ([b"hg.1"], b'', loaddoc(b'hg.1'), help.TOPIC_CATEGORY_CONFIG),
         ([b"hg-ssh.8"], b'', loaddoc(b'hg-ssh.8'), help.TOPIC_CATEGORY_CONFIG),
         (
             [b"hgignore.5"],
             b'',
             loaddoc(b'hgignore.5'),
             help.TOPIC_CATEGORY_CONFIG,
         ),
         ([b"hgrc.5"], b'', loaddoc(b'hgrc.5'), help.TOPIC_CATEGORY_CONFIG),
         (
             [b"hgignore.5.gendoc"],
             b'',
             loaddoc(b'hgignore'),
             help.TOPIC_CATEGORY_CONFIG,
         ),
         (
             [b"hgrc.5.gendoc"],
             b'',
             loaddoc(b'config'),
             help.TOPIC_CATEGORY_CONFIG,
         ),
     ]
     helpprinter(ui, helptable + extrahelptable, None, include=[topic])
 
 
 def helpprinter(ui, helptable, sectionfunc, include=[], exclude=[]):
     for h in helptable:
         names, sec, doc = h[0:3]
         if exclude and names[0] in exclude:
             continue
         if include and names[0] not in include:
             continue
         for name in names:
             ui.write(b".. _%s:\n" % name)
         ui.write(b"\n")
         if sectionfunc:
             ui.write(sectionfunc(sec))
         if callable(doc):
             doc = doc(ui)
         ui.write(doc)
         ui.write(b"\n")
 
 
 def commandprinter(ui, cmdtable, sectionfunc, subsectionfunc):
     """Render restructuredtext describing a list of commands and their
     documentations, grouped by command category.
 
     Args:
       ui: UI object to write the output to
       cmdtable: a dict that maps a string of the command name plus its aliases
         (separated with pipes) to a 3-tuple of (the command's function, a list
         of its option descriptions, and a string summarizing available
         options). Example, with aliases added for demonstration purposes:
 
           'phase|alias1|alias2': (
              <function phase at 0x7f0816b05e60>,
              [ ('p', 'public', False, 'set changeset phase to public'),
                ...,
                ('r', 'rev', [], 'target revision', 'REV')],
              '[-p|-d|-s] [-f] [-r] [REV...]'
           )
       sectionfunc: minirst function to format command category headers
       subsectionfunc: minirst function to format command headers
     """
     h = {}
     for c, attr in cmdtable.items():
         f = c.split(b"|")[0]
         f = f.lstrip(b"^")
         h[f] = c
     cmds = h.keys()
 
     def helpcategory(cmd):
         """Given a canonical command name from `cmds` (above), retrieve its
         help category. If helpcategory is None, default to CATEGORY_NONE.
         """
         fullname = h[cmd]
         details = cmdtable[fullname]
         helpcategory = details[0].helpcategory
         return helpcategory or help.registrar.command.CATEGORY_NONE
 
     cmdsbycategory = {category: [] for category in help.CATEGORY_ORDER}
     for cmd in cmds:
         # If a command category wasn't registered, the command won't get
         # rendered below, so we raise an AssertionError.
         if helpcategory(cmd) not in cmdsbycategory:
             raise AssertionError(
                 "The following command did not register its (category) in "
                 "help.CATEGORY_ORDER: %s (%s)" % (cmd, helpcategory(cmd))
             )
         cmdsbycategory[helpcategory(cmd)].append(cmd)
 
     # Print the help for each command. We present the commands grouped by
     # category, and we use help.CATEGORY_ORDER as a guide for a helpful order
     # in which to present the categories.
     for category in help.CATEGORY_ORDER:
         categorycmds = cmdsbycategory[category]
         if not categorycmds:
             # Skip empty categories
             continue
         # Print a section header for the category.
         # For now, the category header is at the same level as the headers for
         # the commands in the category; this is fixed in the next commit.
         ui.write(sectionfunc(help.CATEGORY_NAMES[category]))
         # Print each command in the category
         for f in sorted(categorycmds):
             if f.startswith(b"debug"):
                 continue
             d = get_cmd(h[f], cmdtable)
             ui.write(subsectionfunc(d[b'cmd']))
             # short description
             ui.write(d[b'desc'][0])
             # synopsis
             ui.write(b"::\n\n")
             synopsislines = d[b'synopsis'].splitlines()
             for line in synopsislines:
                 # some commands (such as rebase) have a multi-line
                 # synopsis
                 ui.write(b"   %s\n" % line)
             ui.write(b'\n')
             # description
             ui.write(b"%s\n\n" % d[b'desc'][1])
             # options
             opt_output = list(d[b'opts'])
             if opt_output:
                 opts_len = max([len(line[0]) for line in opt_output])
                 ui.write(_(b"Options:\n\n"))
                 multioccur = False
                 for optstr, desc in opt_output:
                     if desc:
                         s = b"%-*s  %s" % (opts_len, optstr, desc)
                     else:
                         s = optstr
                     ui.write(b"%s\n" % s)
                     if optstr.endswith(b"[+]>"):
                         multioccur = True
                 if multioccur:
                     ui.write(
                         _(
                             b"\n[+] marked option can be specified"
                             b" multiple times\n"
                         )
                     )
                 ui.write(b"\n")
             # aliases
             if d[b'aliases']:
                 # Note the empty comment, this is required to separate this
                 # (which should be a blockquote) from any preceding things (such
                 # as a definition list).
                 ui.write(
                     _(b"..\n\n    aliases: %s\n\n") % b" ".join(d[b'aliases'])
                 )
 
 
 def allextensionnames():
     return set(extensions.enabled().keys()) | set(extensions.disabled().keys())
 
 
 if __name__ == "__main__":
     doc = b'hg.1.gendoc'
     if len(sys.argv) > 1:
         doc = encoding.strtolocal(sys.argv[1])
 
     ui = uimod.ui.load()
     # Trigger extensions to load. This is disabled by default because it uses
     # the current user's configuration, which is often not what is wanted.
     if encoding.environ.get(b'GENDOC_LOAD_CONFIGURED_EXTENSIONS', b'0') != b'0':
         extensions.loadall(ui)
 
     if doc == b'hg.1.gendoc':
         showdoc(ui)
     else:
         showtopic(ui, encoding.strtolocal(sys.argv[1]))