merge default into stable for 4.9 release

contrib/discovery-helper.sh

@@ -0,0 +1,64 b''
+#!/bin/bash
+#
+# produces two repositories with different common and missing subsets
+#
+#   $ discovery-helper.sh REPO NBHEADS DEPT
+#
+# The Goal is to produce two repositories with some common part and some
+# exclusive part on each side. Provide a source repository REPO, it will
+# produce two repositories REPO-left and REPO-right.
+#
+# Each repository will be missing some revisions exclusive to NBHEADS of the
+# repo topological heads. These heads and revisions exclusive to them (up to
+# DEPTH depth) are stripped.
+#
+# The "left" repository will use the NBHEADS first heads (sorted by
+# description). The "right" use the last NBHEADS one.
+#
+# To find out how many topological heads a repo has, use:
+#
+#   $ hg heads -t -T '{rev}\n' | wc -l
+#
+# Example:
+#
+#  The `pypy-2018-09-01` repository has 192 heads. To produce two repositories
+#  with 92 common heads and ~50 exclusive heads on each side.
+#
+#    $ ./discovery-helper.sh pypy-2018-08-01 50 10
+
+set -euo pipefail
+
+if [ $# -lt 3 ]; then
+     echo "usage: `basename $0` REPO NBHEADS DEPTH"
+     exit 64
+fi
+
+repo="$1"
+shift
+
+nbheads="$1"
+shift
+
+depth="$1"
+shift
+
+leftrepo="${repo}-left"
+rightrepo="${repo}-right"
+
+left="first(sort(heads(all()), 'desc'), $nbheads)"
+right="last(sort(heads(all()), 'desc'), $nbheads)"
+
+leftsubset="ancestors($left, $depth) and only($left, heads(all() - $left))"
+rightsubset="ancestors($right, $depth) and only($right, heads(all() - $right))"
+
+echo '### building left repository:' $left-repo
+echo '# cloning'
+hg clone --noupdate "${repo}" "${leftrepo}"
+echo '# stripping' '"'${leftsubset}'"'
+hg -R "${leftrepo}" --config extensions.strip= strip --rev "$leftsubset" --no-backup
+
+echo '### building right repository:' $right-repo
+echo '# cloning'
+hg clone --noupdate "${repo}" "${rightrepo}"
+echo '# stripping:' '"'${rightsubset}'"'
+hg -R "${rightrepo}" --config extensions.strip= strip --rev "$rightsubset" --no-backup

contrib/fuzz/dirstate.cc

@@ -0,0 +1,48 b''
+#include <Python.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "pyutil.h"
+
+extern "C" {
+
+static PyCodeObject *code;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+	contrib::initpy(*argv[0]);
+	code = (PyCodeObject *)Py_CompileString(R"py(
+from parsers import parse_dirstate
+try:
+    dmap = {}
+    copymap = {}
+    p = parse_dirstate(dmap, copymap, data)
+except Exception as e:
+    pass
+    # uncomment this print if you're editing this Python code
+    # to debug failures.
+    # print e
+)py",
+	                                        "fuzzer", Py_file_input);
+	return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+	PyObject *text =
+	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
+	PyObject *locals = PyDict_New();
+	PyDict_SetItemString(locals, "data", text);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
+	if (!res) {
+		PyErr_Print();
+	}
+	Py_XDECREF(res);
+	Py_DECREF(locals);
+	Py_DECREF(text);
+	return 0; // Non-zero return values are reserved for future use.
+}
+}

contrib/fuzz/dirstate_corpus.py

@@ -0,0 +1,18 b''
+from __future__ import absolute_import, print_function
+
+import argparse
+import os
+import zipfile
+
+ap = argparse.ArgumentParser()
+ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
+args = ap.parse_args()
+
+reporoot = os.path.normpath(os.path.join(os.path.dirname(__file__),
+                                         '..', '..'))
+dirstate = os.path.join(reporoot, '.hg', 'dirstate')
+
+with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
+    if os.path.exists(dirstate):
+        with open(dirstate) as f:
+            zf.writestr("dirstate", f.read())

contrib/fuzz/fm1readmarkers.cc

@@ -0,0 +1,60 b''
+#include <Python.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "pyutil.h"
+
+extern "C" {
+
+static PyCodeObject *code;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+	contrib::initpy(*argv[0]);
+	code = (PyCodeObject *)Py_CompileString(R"py(
+from parsers import fm1readmarkers
+def maybeint(s, default):
+    try:
+        return int(s)
+    except ValueError:
+        return default
+try:
+    parts = data.split('\0', 2)
+    if len(parts) == 3:
+        offset, stop, data = parts
+    elif len(parts) == 2:
+        stop, data = parts
+        offset = 0
+    else:
+        offset = stop = 0
+    offset, stop = maybeint(offset, 0), maybeint(stop, len(data))
+    fm1readmarkers(data, offset, stop)
+except Exception as e:
+    pass
+    # uncomment this print if you're editing this Python code
+    # to debug failures.
+    # print e
+)py",
+	                                        "fuzzer", Py_file_input);
+	return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+	PyObject *text =
+	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
+	PyObject *locals = PyDict_New();
+	PyDict_SetItemString(locals, "data", text);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
+	if (!res) {
+		PyErr_Print();
+	}
+	Py_XDECREF(res);
+	Py_DECREF(locals);
+	Py_DECREF(text);
+	return 0; // Non-zero return values are reserved for future use.
+}
+}

contrib/fuzz/fm1readmarkers_corpus.py

@@ -0,0 +1,36 b''
+from __future__ import absolute_import, print_function
+
+import argparse
+import zipfile
+
+ap = argparse.ArgumentParser()
+ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
+args = ap.parse_args()
+
+with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
+    zf.writestr(
+        'smallish_obsstore',
+        (
+            # header: fm1readmarkers should start at offset 1, and
+            # read until byte 597.
+            '1\x00597\x00'
+            # body of obsstore file
+            '\x01\x00\x00\x00vA\xd7\x02+C\x1a<)\x01,\x00\x00\x01\x03\x03\xe6'
+            '\x92\xde)x\x16\xd1Xph\xc7\xa7[\xe5\xe2\x1a\xab\x1e6e\xaf\xc2\xae'
+            '\xe7\xbc\x83\xe1\x88\xa5\xda\xce>O\xbd\x04\xe9\x03\xc4o\xeb\x03'
+            '\x01\t\x05\x04\x1fef18operationamenduserAugie Fackler <raf@duri'
+            'n42.com>\x00\x00\x00vA\xd7\x02-\x8aD\xaf-\x01,\x00\x00\x01\x03\x03'
+            '\x17*\xca\x8f\x9e}i\xe0i\xbb\xdf\x9fb\x03\xd2XG?\xd3h\x98\x89\x1a'
+            '=2\xeb\xc3\xc5<\xb3\x9e\xcc\x0e;#\xee\xc3\x10ux\x03\x01\t\x05\x04'
+            '\x1fef18operationamenduserAugie Fackler <[e-mail]>\x00\x00'
+            '\x00vA\xd7\x02Mn\xd9%\xea\x01,\x00\x00\x01\x03\x03\x98\x89\x1a='
+            '2\xeb\xc3\xc5<\xb3\x9e\xcc\x0e;#\xee\xc3\x10ux\xe0*\xcaT\x86Z8J'
+            '\x85)\x97\xff7\xcc)\xc1\x7f\x19\x0c\x01\x03\x01\t\x05\x04\x1fef'
+            '18operationamenduserAugie Fackler <[e-mail]>\x00\x00\x00'
+            'yA\xd7\x02MtA\xbfj\x01,\x00\x00\x01\x03\x03\xe0*\xcaT\x86Z8J\x85'
+            ')\x97\xff7\xcc)\xc1\x7f\x19\x0c\x01\x00\x94\x01\xa9\n\xf80\x92\xa3'
+            'j\xc5X\xb1\xc9:\xd51\xb8*\xa9\x03\x01\t\x08\x04\x1fef11operatio'
+            'nhistedituserAugie Fackler <[e-mail]>\x00\x00\x00yA\xd7\x02'
+            'MtA\xd4\xe1\x01,\x00\x00\x01\x03\x03"\xa5\xcb\x86\xb6\xf4\xbaO\xa0'
+            'sH\xe7?\xcb\x9b\xc2n\xcfI\x9e\x14\xf0D\xf0!\x18DN\xcd\x97\x016\xa5'
+            '\xef\xa06\xcb\x884\x8a\x03\x01\t\x08\x04\x1fef14operationhisted'))

contrib/fuzz/pyutil.cc

@@ -0,0 +1,49 b''
+#include "pyutil.h"
+
+#include <string>
+
+namespace contrib
+{
+
+static char cpypath[8192] = "\0";
+
+static PyObject *mainmod;
+static PyObject *globals;
+
+/* TODO: use Python 3 for this fuzzing? */
+PyMODINIT_FUNC initparsers(void);
+
+void initpy(const char *cselfpath)
+{
+	const std::string subdir = "/sanpy/lib/python2.7";
+	/* HACK ALERT: we need a full Python installation built without
+	   pymalloc and with ASAN, so we dump one in
+	   $OUT/sanpy/lib/python2.7. This helps us wire that up. */
+	std::string selfpath(cselfpath);
+	std::string pypath;
+	auto pos = selfpath.rfind("/");
+	if (pos == std::string::npos) {
+		char wd[8192];
+		getcwd(wd, 8192);
+		pypath = std::string(wd) + subdir;
+	} else {
+		pypath = selfpath.substr(0, pos) + subdir;
+	}
+	strncpy(cpypath, pypath.c_str(), pypath.size());
+	setenv("PYTHONPATH", cpypath, 1);
+	setenv("PYTHONNOUSERSITE", "1", 1);
+	/* prevent Python from looking up users in the fuzz environment */
+	setenv("PYTHONUSERBASE", cpypath, 1);
+	Py_SetPythonHome(cpypath);
+	Py_InitializeEx(0);
+	mainmod = PyImport_AddModule("__main__");
+	globals = PyModule_GetDict(mainmod);
+	initparsers();
+}
+
+PyObject *pyglobals()
+{
+	return globals;
+}
+
+} // namespace contrib

contrib/fuzz/pyutil.h

@@ -0,0 +1,9 b''
+#include <Python.h>
+
+namespace contrib
+{
+
+void initpy(const char *cselfpath);
+PyObject *pyglobals();
+
+} /* namespace contrib */

contrib/fuzz/revlog.cc

@@ -0,0 +1,47 b''
+#include <Python.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "pyutil.h"
+
+extern "C" {
+
+static PyCodeObject *code;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+	contrib::initpy(*argv[0]);
+	code = (PyCodeObject *)Py_CompileString(R"py(
+from parsers import parse_index2
+for inline in (True, False):
+    try:
+        index, cache = parse_index2(data, inline)
+    except Exception as e:
+        pass
+        # uncomment this print if you're editing this Python code
+        # to debug failures.
+        # print e
+)py",
+	                                        "fuzzer", Py_file_input);
+	return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+	PyObject *text =
+	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
+	PyObject *locals = PyDict_New();
+	PyDict_SetItemString(locals, "data", text);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
+	if (!res) {
+		PyErr_Print();
+	}
+	Py_XDECREF(res);
+	Py_DECREF(locals);
+	Py_DECREF(text);
+	return 0; // Non-zero return values are reserved for future use.
+}
+}

contrib/fuzz/revlog_corpus.py

@@ -0,0 +1,28 b''
+from __future__ import absolute_import, print_function
+
+import argparse
+import os
+import zipfile
+
+ap = argparse.ArgumentParser()
+ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
+args = ap.parse_args()
+
+reporoot = os.path.normpath(os.path.join(os.path.dirname(__file__),
+                                         '..', '..'))
+# typically a standalone index
+changelog = os.path.join(reporoot, '.hg', 'store', '00changelog.i')
+# an inline revlog with only a few revisions
+contributing = os.path.join(
+    reporoot, '.hg', 'store', 'data', 'contrib', 'fuzz', 'mpatch.cc.i')
+
+print(changelog, os.path.exists(changelog))
+print(contributing, os.path.exists(contributing))
+
+with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
+    if os.path.exists(changelog):
+        with open(changelog) as f:
+            zf.writestr("00changelog.i", f.read())
+    if os.path.exists(contributing):
+        with open(contributing) as f:
+            zf.writestr("contributing.i", f.read())

contrib/perf-utils/perf-revlog-write-plot.py

@@ -0,0 +1,127 b''
+#!/usr/bin/env python
+#
+#  Copyright 2018 Paul Morelle <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+#
+# This script use the output of `hg perfrevlogwrite -T json --details` to draw
+# various plot related to write performance in a revlog
+#
+# usage: perf-revlog-write-plot.py details.json
+from __future__ import absolute_import, print_function
+import json
+import re
+
+import numpy as np
+import scipy.signal
+
+from matplotlib import (
+    pyplot as plt,
+    ticker as mticker,
+)
+
+
+def plot(data, title=None):
+    items = {}
+    re_title = re.compile(r'^revisions #\d+ of \d+, rev (\d+)$')
+    for item in data:
+        m = re_title.match(item['title'])
+        if m is None:
+            continue
+
+        rev = int(m.group(1))
+        items[rev] = item
+
+    min_rev = min(items.keys())
+    max_rev = max(items.keys())
+    ary = np.empty((2, max_rev - min_rev + 1))
+    for rev, item in items.items():
+        ary[0][rev - min_rev] = rev
+        ary[1][rev - min_rev] = item['wall']
+
+    fig = plt.figure()
+    comb_plt = fig.add_subplot(211)
+    other_plt = fig.add_subplot(212)
+
+    comb_plt.plot(ary[0],
+                  np.cumsum(ary[1]),
+                  color='red',
+                  linewidth=1,
+                  label='comb')
+
+    plots = []
+    p = other_plt.plot(ary[0],
+                       ary[1],
+                       color='red',
+                       linewidth=1,
+                       label='wall')
+    plots.append(p)
+
+    colors = {
+        10: ('green', 'xkcd:grass green'),
+        100: ('blue', 'xkcd:bright blue'),
+        1000: ('purple', 'xkcd:dark pink'),
+    }
+    for n, color in colors.items():
+        avg_n = np.convolve(ary[1], np.full(n, 1. / n), 'valid')
+        p = other_plt.plot(ary[0][n - 1:],
+                           avg_n,
+                           color=color[0],
+                           linewidth=1,
+                           label='avg time last %d' % n)
+        plots.append(p)
+
+        med_n = scipy.signal.medfilt(ary[1], n + 1)
+        p = other_plt.plot(ary[0],
+                           med_n,
+                           color=color[1],
+                           linewidth=1,
+                           label='median time last %d' % n)
+        plots.append(p)
+
+    formatter = mticker.ScalarFormatter()
+    formatter.set_scientific(False)
+    formatter.set_useOffset(False)
+
+    comb_plt.grid()
+    comb_plt.xaxis.set_major_formatter(formatter)
+    comb_plt.legend()
+
+    other_plt.grid()
+    other_plt.xaxis.set_major_formatter(formatter)
+    leg = other_plt.legend()
+    leg2plot = {}
+    for legline, plot in zip(leg.get_lines(), plots):
+        legline.set_picker(5)
+        leg2plot[legline] = plot
+
+    def onpick(event):
+        legline = event.artist
+        plot = leg2plot[legline]
+        visible = not plot[0].get_visible()
+        for l in plot:
+            l.set_visible(visible)
+
+        if visible:
+            legline.set_alpha(1.0)
+        else:
+            legline.set_alpha(0.2)
+        fig.canvas.draw()
+    if title is not None:
+        fig.canvas.set_window_title(title)
+    fig.canvas.mpl_connect('pick_event', onpick)
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    import sys
+
+    if len(sys.argv) > 1:
+        print('reading from %r' % sys.argv[1])
+        with open(sys.argv[1], 'r') as fp:
+            plot(json.load(fp), title=sys.argv[1])
+    else:
+        print('reading from stdin')
+        plot(json.load(sys.stdin))

hgext/bookflow.py

@@ -0,0 +1,104 b''
+"""implements bookmark-based branching (EXPERIMENTAL)
+
+ - Disables creation of new branches (config: enable_branches=False).
+ - Requires an active bookmark on commit (config: require_bookmark=True).
+ - Doesn't move the active bookmark on update, only on commit.
+ - Requires '--rev' for moving an existing bookmark.
+ - Protects special bookmarks (config: protect=@).
+
+ flow related commands
+
+    :hg book NAME: create a new bookmark
+    :hg book NAME -r REV: move bookmark to revision (fast-forward)
+    :hg up|co NAME: switch to bookmark
+    :hg push -B .: push active bookmark
+"""
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+from mercurial import (
+    bookmarks,
+    commands,
+    error,
+    extensions,
+    registrar,
+)
+
+MY_NAME = 'bookflow'
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem(MY_NAME, 'protect', ['@'])
+configitem(MY_NAME, 'require-bookmark', True)
+configitem(MY_NAME, 'enable-branches', False)
+
+cmdtable = {}
+command = registrar.command(cmdtable)
+
+def commit_hook(ui, repo, **kwargs):
+    active = repo._bookmarks.active
+    if active:
+        if active in ui.configlist(MY_NAME, 'protect'):
+            raise error.Abort(
+                _('cannot commit, bookmark %s is protected') % active)
+        if not cwd_at_bookmark(repo, active):
+            raise error.Abort(
+       _('cannot commit, working directory out of sync with active bookmark'),
+                hint=_("run 'hg up %s'") % active)
+    elif ui.configbool(MY_NAME, 'require-bookmark', True):
+        raise error.Abort(_('cannot commit without an active bookmark'))
+    return 0
+
+def bookmarks_update(orig, repo, parents, node):
+    if len(parents) == 2:
+        # called during commit
+        return orig(repo, parents, node)
+    else:
+        # called during update
+        return False
+
+def bookmarks_addbookmarks(
+        orig, repo, tr, names, rev=None, force=False, inactive=False):
+    if not rev:
+        marks = repo._bookmarks
+        for name in names:
+            if name in marks:
+                raise error.Abort(_(
+                    "bookmark %s already exists, to move use the --rev option"
+                    ) % name)
+    return orig(repo, tr, names, rev, force, inactive)
+
+def commands_commit(orig, ui, repo, *args, **opts):
+    commit_hook(ui, repo)
+    return orig(ui, repo, *args, **opts)
+
+def commands_pull(orig, ui, repo, *args, **opts):
+    rc = orig(ui, repo, *args, **opts)
+    active = repo._bookmarks.active
+    if active and not cwd_at_bookmark(repo, active):
+        ui.warn(_(
+            "working directory out of sync with active bookmark, run "
+            "'hg up %s'"
+        ) % active)
+    return rc
+
+def commands_branch(orig, ui, repo, label=None, **opts):
+    if label and not opts.get(r'clean') and not opts.get(r'rev'):
+        raise error.Abort(
+         _("creating named branches is disabled and you should use bookmarks"),
+            hint="see 'hg help bookflow'")
+    return orig(ui, repo, label, **opts)
+
+def cwd_at_bookmark(repo, mark):
+    mark_id = repo._bookmarks[mark]
+    cur_id = repo.lookup('.')
+    return cur_id == mark_id
+
+def uisetup(ui):
+    extensions.wrapfunction(bookmarks, 'update', bookmarks_update)
+    extensions.wrapfunction(bookmarks, 'addbookmarks', bookmarks_addbookmarks)
+    extensions.wrapcommand(commands.table, 'commit', commands_commit)
+    extensions.wrapcommand(commands.table, 'pull', commands_pull)
+    if not ui.configbool(MY_NAME, 'enable-branches'):
+        extensions.wrapcommand(commands.table, 'branch', commands_branch)

Makefile

@@ -1,244 +1,248 b''
 # If you want to change PREFIX, do not just edit it below. The changed
 # value wont get passed on to recursive make calls. You should instead
 # override the variable on the command like:
 #
 # % make PREFIX=/opt/ install
 
 export PREFIX=/usr/local
 PYTHON=python
 $(eval HGROOT := $(shell pwd))
 HGPYTHONS ?= $(HGROOT)/build/pythons
 PURE=
 PYFILESCMD=find mercurial hgext doc -name '*.py'
 PYFILES:=$(shell $(PYFILESCMD))
 DOCFILES=mercurial/help/*.txt
 export LANGUAGE=C
 export LC_ALL=C
 TESTFLAGS ?= $(shell echo $$HGTESTFLAGS)
 OSXVERSIONFLAGS ?= $(shell echo $$OSXVERSIONFLAGS)
 
 # Set this to e.g. "mingw32" to use a non-default compiler.
 COMPILER=
 
 COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
 help:
 	@echo 'Commonly used make targets:'
 	@echo '  all          - build program and documentation'
 	@echo '  install      - install program and man pages to $$PREFIX ($(PREFIX))'
 	@echo '  install-home - install with setup.py install --home=$$HOME ($(HOME))'
 	@echo '  local        - build for inplace usage'
 	@echo '  tests        - run all tests in the automatic test suite'
 	@echo '  test-foo     - run only specified tests (e.g. test-merge1.t)'
 	@echo '  dist         - run all tests and create a source tarball in dist/'
 	@echo '  clean        - remove files created by other targets'
 	@echo '                 (except installed files or dist source tarball)'
 	@echo '  update-pot   - update i18n/hg.pot'
 	@echo
 	@echo 'Example for a system-wide installation under /usr/local:'
 	@echo '  make all && su -c "make install" && hg version'
 	@echo
 	@echo 'Example for a local installation (usable in this directory):'
 	@echo '  make local && ./hg version'
 
 all: build doc
 
 local:
 	$(PYTHON) setup.py $(PURE) \
 	  build_py -c -d . \
 	  build_ext $(COMPILERFLAG) -i \
 	  build_hgexe $(COMPILERFLAG) -i \
 	  build_mo
 	env HGRCPATH= $(PYTHON) hg version
 
 build:
 	$(PYTHON) setup.py $(PURE) build $(COMPILERFLAG)
 
 wheel:
 	FORCE_SETUPTOOLS=1 $(PYTHON) setup.py $(PURE) bdist_wheel $(COMPILERFLAG)
 
 doc:
 	$(MAKE) -C doc
 
 cleanbutpackages:
 	-$(PYTHON) setup.py clean --all # ignore errors from this command
 	find contrib doc hgext hgext3rd i18n mercurial tests hgdemandimport \
 		\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'
 	rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err
 	rm -f mercurial/__modulepolicy__.py
 	if test -d .hg; then rm -f mercurial/__version__.py; fi
 	rm -rf build mercurial/locale
 	$(MAKE) -C doc clean
 	$(MAKE) -C contrib/chg distclean
+	rm -rf rust/target
+	rm -f mercurial/rustext.so
 
 clean: cleanbutpackages
 	rm -rf packages
 
 install: install-bin install-doc
 
 install-bin: build
 	$(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force
 
 install-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) install
 
 install-home: install-home-bin install-home-doc
 
 install-home-bin: build
 	$(PYTHON) setup.py $(PURE) install --home="$(HOME)" --prefix="" --force
 
 install-home-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) PREFIX="$(HOME)" install
 
 MANIFEST-doc:
 	$(MAKE) -C doc MANIFEST
 
 MANIFEST.in: MANIFEST-doc
 	hg manifest | sed -e 's/^/include /' > MANIFEST.in
 	echo include mercurial/__version__.py >> MANIFEST.in
 	sed -e 's/^/include /' < doc/MANIFEST >> MANIFEST.in
 
 dist:	tests dist-notests
 
 dist-notests:	doc MANIFEST.in
 	TAR_OPTIONS="--owner=root --group=root --mode=u+w,go-w,a+rX-s" $(PYTHON) setup.py -q sdist
 
 check: tests
 
 tests:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS)
 
 test-%:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS) $@
 
 testpy-%:
 	@echo Looking for Python $* in $(HGPYTHONS)
 	[ -e $(HGPYTHONS)/$*/bin/python ] || ( \
 	cd $$(mktemp --directory --tmpdir) && \
         $(MAKE) -f $(HGROOT)/contrib/Makefile.python PYTHONVER=$* PREFIX=$(HGPYTHONS)/$* python )
 	cd tests && $(HGPYTHONS)/$*/bin/python run-tests.py $(TESTFLAGS)
 
 check-code:
 	hg manifest | xargs python contrib/check-code.py
 
 format-c:
 	clang-format --style file -i \
 	  `hg files 'set:(**.c or **.cc or **.h) and not "listfile:contrib/clang-format-ignorelist"'`
 
 update-pot: i18n/hg.pot
 
 i18n/hg.pot: $(PYFILES) $(DOCFILES) i18n/posplit i18n/hggettext
 	$(PYTHON) i18n/hggettext mercurial/commands.py \
 	  hgext/*.py hgext/*/__init__.py \
 	  mercurial/fileset.py mercurial/revset.py \
 	  mercurial/templatefilters.py \
 	  mercurial/templatefuncs.py \
 	  mercurial/templatekw.py \
 	  mercurial/filemerge.py \
 	  mercurial/hgweb/webcommands.py \
 	  mercurial/util.py \
 	  $(DOCFILES) > i18n/hg.pot.tmp
         # All strings marked for translation in Mercurial contain
         # ASCII characters only. But some files contain string
         # literals like this '\037\213'. xgettext thinks it has to
         # parse them even though they are not marked for translation.
         # Extracting with an explicit encoding of ISO-8859-1 will make
         # xgettext "parse" and ignore them.
 	$(PYFILESCMD) | xargs \
 	  xgettext --package-name "Mercurial" \
 	  --msgid-bugs-address "<[e-mail]>" \
 	  --copyright-holder "Matt Mackall <[e-mail]> and others" \
 	  --from-code ISO-8859-1 --join --sort-by-file --add-comments=i18n: \
 	  -d hg -p i18n -o hg.pot.tmp
 	$(PYTHON) i18n/posplit i18n/hg.pot.tmp
         # The target file is not created before the last step. So it never is in
         # an intermediate state.
 	mv -f i18n/hg.pot.tmp i18n/hg.pot
 
 %.po: i18n/hg.pot
         # work on a temporary copy for never having a half completed target
 	cp $@ $@.tmp
 	msgmerge --no-location --update $@.tmp $^
 	mv -f $@.tmp $@
 
 # Packaging targets
 
 packaging_targets := \
   centos5 \
   centos6 \
   centos7 \
   deb \
   docker-centos5 \
   docker-centos6 \
   docker-centos7 \
   docker-debian-jessie \
   docker-debian-stretch \
   docker-fedora20 \
   docker-fedora21 \
   docker-fedora28 \
+  docker-fedora29 \
   docker-ubuntu-trusty \
   docker-ubuntu-trusty-ppa \
   docker-ubuntu-xenial \
   docker-ubuntu-xenial-ppa \
   docker-ubuntu-artful \
   docker-ubuntu-artful-ppa \
   docker-ubuntu-bionic \
   docker-ubuntu-bionic-ppa \
   fedora20 \
   fedora21 \
   fedora28 \
+  fedora29 \
   linux-wheels \
   linux-wheels-x86_64 \
   linux-wheels-i686 \
   ppa
 
 # Forward packaging targets for convenience.
 $(packaging_targets):
 	$(MAKE) -C contrib/packaging $@
 
 osx:
 	rm -rf build/mercurial
 	/usr/bin/python2.7 setup.py install --optimize=1 \
 	  --root=build/mercurial/ --prefix=/usr/local/ \
 	  --install-lib=/Library/Python/2.7/site-packages/
 	make -C doc all install DESTDIR="$(PWD)/build/mercurial/"
         # Place a bogon .DS_Store file in the target dir so we can be
         # sure it doesn't get included in the final package.
 	touch build/mercurial/.DS_Store
         # install zsh completions - this location appears to be
         # searched by default as of macOS Sierra.
 	install -d build/mercurial/usr/local/share/zsh/site-functions/
 	install -m 0644 contrib/zsh_completion build/mercurial/usr/local/share/zsh/site-functions/_hg
         # install bash completions - there doesn't appear to be a
         # place that's searched by default for bash, so we'll follow
         # the lead of Apple's git install and just put it in a
         # location of our own.
 	install -d build/mercurial/usr/local/hg/contrib/
 	install -m 0644 contrib/bash_completion build/mercurial/usr/local/hg/contrib/hg-completion.bash
 	make -C contrib/chg \
 	  HGPATH=/usr/local/bin/hg \
 	  PYTHON=/usr/bin/python2.7 \
 	  HGEXTDIR=/Library/Python/2.7/site-packages/hgext \
 	  DESTDIR=../../build/mercurial \
 	  PREFIX=/usr/local \
 	  clean install
 	mkdir -p $${OUTPUTDIR:-dist}
 	HGVER=$$(python contrib/genosxversion.py $(OSXVERSIONFLAGS) build/mercurial/Library/Python/2.7/site-packages/mercurial/__version__.py) && \
 	OSXVER=$$(sw_vers -productVersion | cut -d. -f1,2) && \
 	pkgbuild --filter \\.DS_Store --root build/mercurial/ \
 	  --identifier org.mercurial-scm.mercurial \
 	  --version "$${HGVER}" \
 	  build/mercurial.pkg && \
 	productbuild --distribution contrib/packaging/macosx/distribution.xml \
 	  --package-path build/ \
 	  --version "$${HGVER}" \
 	  --resources contrib/packaging/macosx/ \
 	  "$${OUTPUTDIR:-dist/}"/Mercurial-"$${HGVER}"-macosx"$${OSXVER}".pkg
 
 .PHONY: help all local build doc cleanbutpackages clean install install-bin \
 	install-doc install-home install-home-bin install-home-doc \
 	dist dist-notests check tests check-code format-c update-pot \
 	$(packaging_targets) \
 	osx

contrib/all-revsets.txt

@@ -1,141 +1,156 b''
 # All revsets ever used with revsetbenchmarks.py script
 #
 # The goal of this file is to gather all revsets ever used for benchmarking
 # revset's performance. It should be used to gather revsets that test a
 # specific usecase or a specific implementation of revset predicates.
 # If you are working on the smartset implementation itself, check
 # 'base-revsets.txt'.
 #
 # Please update this file with any revsets you use for benchmarking a change so
 # that future contributors can easily find and retest it when doing further
 # modification. Feel free to highlight interesting variants if needed.
 
 
 ## Revset from this section are all extracted from changelog when this file was
 # created. Feel free to dig and improve documentation.
 
 # Used in revision da05fe01170b
 (20000::) - (20000)
 # Used in revision 95af98616aa7
 parents(20000)
 # Used in revision 186fd06283b4
 (_intlist('20000\x0020001')) and merge()
 # Used in revision 911f5a6579d1
 p1(20000)
 p2(10000)
 # Used in revision b6dc3b79bb25
 0::
 # Used in revision faf4f63533ff
 bookmark()
 # Used in revision 22ba2c0825da
 tip~25
 # Used in revision 0cf46b8298fe
 bisect(range)
 # Used in revision 5b65429721d5
 divergent()
 # Used in revision 6261b9c549a2
 file(COPYING)
 # Used in revision 44f471102f3a
 follow(COPYING)
 # Used in revision 8040a44aab1c
 origin(tip)
 # Used in revision bbf4f3dfd700
 rev(25)
 # Used in revision a428db9ab61d
 p1()
 # Used in revision c1546d7400ef
 min(0::)
 # Used in revision 546fa6576815
 author(lmoscovicz) or author(mpm)
 author(mpm) or author(lmoscovicz)
 # Used in revision 9bfe68357c01
 public() and id("d82e2223f132")
 # Used in revision ba89f7b542c9
 rev(25)
 # Used in revision eb763217152a
 rev(210000)
 # Used in revision 69524a05a7fa
 10:100
 parents(10):parents(100)
 # Used in revision 6f1b8b3f12fd
 100~5
 parents(100)~5
 (100~5)~5
 # Used in revision 7a42e5d4c418
 children(tip~100)
 # Used in revision 7e8737e6ab08
 100^1
 parents(100)^1
 (100^1)^1
 # Used in revision 30e0dcd7c5ff
 matching(100)
 matching(parents(100))
 # Used in revision aafeaba22826
 0|1|2|3|4|5|6|7|8|9
 # Used in revision 33c7a94d4dd0
 tip:0
 # Used in revision 7d369fae098e
 (0:100000)
 # Used in revision b333ca94403d
 0 + 1 + 2 + ... + 200
 0 + 1 + 2 + ... + 1000
 sort(0 + 1 + 2 + ... + 200)
 sort(0 + 1 + 2 + ... + 1000)
 # Used in revision 7fbef7932af9
 first(0 + 1 + 2 + ... + 1000)
 # Used in revision ceaf04bb14ff
 0:1000
 # Used in revision 262e6ad93885
 not public()
 (tip~1000::) - public()
 not public() and branch("default")
 # Used in revision 15412bba5a68
 0::tip
 
 ## all the revsets from this section have been taken from the former central file
 # for revset's benchmarking, they are undocumented for this reason.
 all()
 draft()
 ::tip
 draft() and ::tip
 ::tip and draft()
 author(lmoscovicz)
 author(mpm)
 ::p1(p1(tip))::
 public()
 :10000 and public()
 :10000 and draft()
 (not public() - obsolete())
 
 # The one below is used by rebase
 (children(ancestor(tip~5, tip)) and ::(tip~5))::
 
 # those two `roots(...)` inputs are close to what phase movement use.
 roots((tip~100::) - (tip~100::tip))
 roots((0::) - (0::tip))
 
 # more roots testing
 roots(tip~100:)
 roots(:42)
 roots(not public())
 roots((0:tip)::)
 roots(0::tip)
 42:68 and roots(42:tip)
 # Used in revision f140d6207cca
 roots(0:tip)
 # test disjoint set with multiple roots
 roots((:42) + (tip~42:))
 
 # Testing the behavior of "head()" in various situations
 head()
 head() - public()
 draft() and head()
 head() and author("mpm")
 
 # testing the mutable phases set
 draft()
 secret()
 
 # test finding common ancestors
 heads(commonancestors(last(head(), 2)))
 heads(commonancestors(head()))
+
+# more heads testing
+heads(all())
+heads(-10000:-1)
+(-5000:-1000) and heads(-10000:-1)
+heads(matching(tip, "author"))
+heads(matching(tip, "author")) and -10000:-1
+(-10000:-1) and heads(matching(tip, "author"))
+# more roots testing
+roots(all())
+roots(-10000:-1)
+(-5000:-1000) and roots(-10000:-1)
+roots(matching(tip, "author"))
+roots(matching(tip, "author")) and -10000:-1
+(-10000:-1) and roots(matching(tip, "author"))

contrib/catapipe.py

@@ -1,90 +1,98 b''
 #!/usr/bin/env python3
 #
 # Copyright 2018 Google LLC.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """Tool read primitive events from a pipe to produce a catapult trace.
 
+Usage:
+    Terminal 1: $ catapipe.py /tmp/mypipe /tmp/trace.json
+    Terminal 2: $ HGCATAPULTSERVERPIPE=/tmp/mypipe hg root
+    <ctrl-c catapipe.py in Terminal 1>
+    $ catapult/tracing/bin/trace2html /tmp/trace.json  # produce /tmp/trace.html
+    <open trace.html in your browser of choice; the WASD keys are very useful>
+    (catapult is located at https://github.com/catapult-project/catapult)
+
 For now the event stream supports
 
   START $SESSIONID ...
 
 and
 
   END $SESSIONID ...
 
 events. Everything after the SESSIONID (which must not contain spaces)
 is used as a label for the event. Events are timestamped as of when
 they arrive in this process and are then used to produce catapult
 traces that can be loaded in Chrome's about:tracing utility. It's
 important that the event stream *into* this process stay simple,
 because we have to emit it from the shell scripts produced by
 run-tests.py.
 
 Typically you'll want to place the path to the named pipe in the
 HGCATAPULTSERVERPIPE environment variable, which both run-tests and hg
-understand.
+understand. To trace *only* run-tests, use HGTESTCATAPULTSERVERPIPE instead.
 """
 from __future__ import absolute_import, print_function
 
 import argparse
 import json
 import os
 import timeit
 
 _TYPEMAP = {
     'START': 'B',
     'END': 'E',
 }
 
 _threadmap = {}
 
 # Timeit already contains the whole logic about which timer to use based on
 # Python version and OS
 timer = timeit.default_timer
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('pipe', type=str, nargs=1,
                         help='Path of named pipe to create and listen on.')
     parser.add_argument('output', default='trace.json', type=str, nargs='?',
                         help='Path of json file to create where the traces '
                              'will be stored.')
     parser.add_argument('--debug', default=False, action='store_true',
                         help='Print useful debug messages')
     args = parser.parse_args()
     fn = args.pipe[0]
     os.mkfifo(fn)
     try:
         with open(fn) as f, open(args.output, 'w') as out:
             out.write('[\n')
             start = timer()
             while True:
                 ev = f.readline().strip()
                 if not ev:
                     continue
                 now = timer()
                 if args.debug:
                     print(ev)
                 verb, session, label = ev.split(' ', 2)
                 if session not in _threadmap:
                     _threadmap[session] = len(_threadmap)
                 pid = _threadmap[session]
                 ts_micros = (now - start) * 1000000
                 out.write(json.dumps(
                     {
                         "name": label,
                         "cat": "misc",
                         "ph": _TYPEMAP[verb],
                         "ts": ts_micros,
                         "pid": pid,
                         "tid": 1,
                         "args": {}
                     }))
                 out.write(',\n')
     finally:
         os.unlink(fn)
 
 if __name__ == '__main__':
     main()

contrib/check-commit

@@ -1,109 +1,109 b''
 #!/usr/bin/env python
 #
 # Copyright 2014 Matt Mackall <[e-mail]>
 #
 # A tool/hook to run basic sanity checks on commits/patches for
 # submission to Mercurial. Install by adding the following to your
 # .hg/hgrc:
 #
 # [hooks]
 # pretxncommit = contrib/check-commit
 #
 # The hook can be temporarily bypassed with:
 #
 # $ BYPASS= hg commit
 #
 # See also: https://mercurial-scm.org/wiki/ContributingChanges
 
 from __future__ import absolute_import, print_function
 
 import os
 import re
 import sys
 
 commitheader = r"^(?:# [^\n]*\n)*"
 afterheader = commitheader + r"(?!#)"
 beforepatch = afterheader + r"(?!\n(?!@@))"
 
 errors = [
     (beforepatch + r".*[(]bc[)]", "(BC) needs to be uppercase"),
     (beforepatch + r".*[(]issue \d\d\d",
      "no space allowed between issue and number"),
     (beforepatch + r".*[(]bug(\d|\s)", "use (issueDDDD) instead of bug"),
     (commitheader + r"# User [^@\n]+\n", "username is not an email address"),
     (commitheader + r"(?!merge with )[^#]\S+[^:] ",
      "summary line doesn't start with 'topic: '"),
     (afterheader + r"[A-Z][a-z]\S+", "don't capitalize summary lines"),
-    (afterheader + r"[^\n]*: *[A-Z][a-z]\S+", "don't capitalize summary lines"),
+    (afterheader + r"^\S+: *[A-Z][a-z]\S+", "don't capitalize summary lines"),
     (afterheader + r"\S*[^A-Za-z0-9-_]\S*: ",
      "summary keyword should be most user-relevant one-word command or topic"),
     (afterheader + r".*\.\s*\n", "don't add trailing period on summary line"),
     (afterheader + r".{79,}", "summary line too long (limit is 78)"),
     # Forbid "_" in function name.
     #
     # We skip the check for cffi related functions. They use names mapping the
     # name of the C function. C function names may contain "_".
     (r"\n\+[ \t]+def (?!cffi)[a-z]+_[a-z]",
      "adds a function with foo_bar naming"),
 ]
 
 word = re.compile('\S')
 def nonempty(first, second):
     if word.search(first):
         return first
     return second
 
 def checkcommit(commit, node=None):
     exitcode = 0
     printed = node is None
     hits = []
     signtag = (afterheader +
           r'Added (tag [^ ]+|signature) for changeset [a-f0-9]{12}')
     if re.search(signtag, commit):
         return 0
     for exp, msg in errors:
         for m in re.finditer(exp, commit):
             end = m.end()
             trailing = re.search(r'(\\n)+$', exp)
             if trailing:
                 end -= len(trailing.group()) / 2
             hits.append((end, exp, msg))
     if hits:
         hits.sort()
         pos = 0
         last = ''
         for n, l in enumerate(commit.splitlines(True)):
             pos += len(l)
             while len(hits):
                 end, exp, msg = hits[0]
                 if pos < end:
                     break
                 if not printed:
                     printed = True
                     print("node: %s" % node)
                 print("%d: %s" % (n, msg))
                 print(" %s" % nonempty(l, last)[:-1])
                 if "BYPASS" not in os.environ:
                     exitcode = 1
                 del hits[0]
             last = nonempty(l, last)
 
     return exitcode
 
 def readcommit(node):
     return os.popen("hg export %s" % node).read()
 
 if __name__ == "__main__":
     exitcode = 0
     node = os.environ.get("HG_NODE")
 
     if node:
         commit = readcommit(node)
         exitcode = checkcommit(commit)
     elif sys.argv[1:]:
         for node in sys.argv[1:]:
             exitcode |= checkcommit(readcommit(node), node)
     else:
         commit = sys.stdin.read()
         exitcode = checkcommit(commit)
     sys.exit(exitcode)

contrib/clang-format-ignorelist

@@ -1,95 +1,94 b''
 # Files that just need to be migrated to the formatter.
 # Do not add new files here!
 mercurial/cext/dirs.c
 mercurial/cext/manifest.c
 mercurial/cext/osutil.c
-mercurial/cext/revlog.c
 # Vendored code that we should never format:
 contrib/python-zstandard/c-ext/bufferutil.c
 contrib/python-zstandard/c-ext/compressionchunker.c
 contrib/python-zstandard/c-ext/compressiondict.c
 contrib/python-zstandard/c-ext/compressionparams.c
 contrib/python-zstandard/c-ext/compressionreader.c
 contrib/python-zstandard/c-ext/compressionwriter.c
 contrib/python-zstandard/c-ext/compressobj.c
 contrib/python-zstandard/c-ext/compressor.c
 contrib/python-zstandard/c-ext/compressoriterator.c
 contrib/python-zstandard/c-ext/constants.c
 contrib/python-zstandard/c-ext/decompressionreader.c
 contrib/python-zstandard/c-ext/decompressionwriter.c
 contrib/python-zstandard/c-ext/decompressobj.c
 contrib/python-zstandard/c-ext/decompressor.c
 contrib/python-zstandard/c-ext/decompressoriterator.c
 contrib/python-zstandard/c-ext/frameparams.c
 contrib/python-zstandard/c-ext/python-zstandard.h
 contrib/python-zstandard/zstd.c
 contrib/python-zstandard/zstd/common/bitstream.h
 contrib/python-zstandard/zstd/common/compiler.h
 contrib/python-zstandard/zstd/common/cpu.h
 contrib/python-zstandard/zstd/common/debug.c
 contrib/python-zstandard/zstd/common/debug.h
 contrib/python-zstandard/zstd/common/entropy_common.c
 contrib/python-zstandard/zstd/common/error_private.c
 contrib/python-zstandard/zstd/common/error_private.h
 contrib/python-zstandard/zstd/common/fse_decompress.c
 contrib/python-zstandard/zstd/common/fse.h
 contrib/python-zstandard/zstd/common/huf.h
 contrib/python-zstandard/zstd/common/mem.h
 contrib/python-zstandard/zstd/common/pool.c
 contrib/python-zstandard/zstd/common/pool.h
 contrib/python-zstandard/zstd/common/threading.c
 contrib/python-zstandard/zstd/common/threading.h
 contrib/python-zstandard/zstd/common/xxhash.c
 contrib/python-zstandard/zstd/common/xxhash.h
 contrib/python-zstandard/zstd/common/zstd_common.c
 contrib/python-zstandard/zstd/common/zstd_errors.h
 contrib/python-zstandard/zstd/common/zstd_internal.h
 contrib/python-zstandard/zstd/compress/fse_compress.c
 contrib/python-zstandard/zstd/compress/hist.c
 contrib/python-zstandard/zstd/compress/hist.h
 contrib/python-zstandard/zstd/compress/huf_compress.c
 contrib/python-zstandard/zstd/compress/zstd_compress.c
 contrib/python-zstandard/zstd/compress/zstd_compress_internal.h
 contrib/python-zstandard/zstd/compress/zstd_double_fast.c
 contrib/python-zstandard/zstd/compress/zstd_double_fast.h
 contrib/python-zstandard/zstd/compress/zstd_fast.c
 contrib/python-zstandard/zstd/compress/zstd_fast.h
 contrib/python-zstandard/zstd/compress/zstd_lazy.c
 contrib/python-zstandard/zstd/compress/zstd_lazy.h
 contrib/python-zstandard/zstd/compress/zstd_ldm.c
 contrib/python-zstandard/zstd/compress/zstd_ldm.h
 contrib/python-zstandard/zstd/compress/zstdmt_compress.c
 contrib/python-zstandard/zstd/compress/zstdmt_compress.h
 contrib/python-zstandard/zstd/compress/zstd_opt.c
 contrib/python-zstandard/zstd/compress/zstd_opt.h
 contrib/python-zstandard/zstd/decompress/huf_decompress.c
 contrib/python-zstandard/zstd/decompress/zstd_decompress.c
 contrib/python-zstandard/zstd/deprecated/zbuff_common.c
 contrib/python-zstandard/zstd/deprecated/zbuff_compress.c
 contrib/python-zstandard/zstd/deprecated/zbuff_decompress.c
 contrib/python-zstandard/zstd/deprecated/zbuff.h
 contrib/python-zstandard/zstd/dictBuilder/cover.c
 contrib/python-zstandard/zstd/dictBuilder/cover.h
 contrib/python-zstandard/zstd/dictBuilder/divsufsort.c
 contrib/python-zstandard/zstd/dictBuilder/divsufsort.h
 contrib/python-zstandard/zstd/dictBuilder/fastcover.c
 contrib/python-zstandard/zstd/dictBuilder/zdict.c
 contrib/python-zstandard/zstd/dictBuilder/zdict.h
 contrib/python-zstandard/zstd/zstd.h
 hgext/fsmonitor/pywatchman/bser.c
 mercurial/thirdparty/xdiff/xdiff.h
 mercurial/thirdparty/xdiff/xdiffi.c
 mercurial/thirdparty/xdiff/xdiffi.h
 mercurial/thirdparty/xdiff/xemit.c
 mercurial/thirdparty/xdiff/xemit.h
 mercurial/thirdparty/xdiff/xhistogram.c
 mercurial/thirdparty/xdiff/xinclude.h
 mercurial/thirdparty/xdiff/xmacros.h
 mercurial/thirdparty/xdiff/xmerge.c
 mercurial/thirdparty/xdiff/xpatience.c
 mercurial/thirdparty/xdiff/xprepare.c
 mercurial/thirdparty/xdiff/xprepare.h
 mercurial/thirdparty/xdiff/xtypes.h
 mercurial/thirdparty/xdiff/xutils.c
 mercurial/thirdparty/xdiff/xutils.h
 mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c

contrib/fuzz/Makefile

@@ -1,128 +1,160 b''
 CC = clang
 CXX = clang++
 
 all: bdiff mpatch xdiff
 
 fuzzutil.o: fuzzutil.cc fuzzutil.h
-	$(CXX) $(CXXFLAGS) -g -O1 -fsanitize=fuzzer-no-link,address \
+	$(CXX) $(CXXFLAGS) -g -O1 \
 	  -std=c++17 \
 	  -I../../mercurial -c -o fuzzutil.o fuzzutil.cc
 
 fuzzutil-oss-fuzz.o: fuzzutil.cc fuzzutil.h
 	$(CXX) $(CXXFLAGS) -std=c++17 \
 	  -I../../mercurial -c -o fuzzutil-oss-fuzz.o fuzzutil.cc
 
+pyutil.o: pyutil.cc pyutil.h
+	$(CXX) $(CXXFLAGS) -g -O1 \
+	  `$$OUT/sanpy/bin/python-config --cflags` \
+	  -I../../mercurial -c -o pyutil.o pyutil.cc
+
 bdiff.o: ../../mercurial/bdiff.c
 	$(CC) $(CFLAGS) -fsanitize=fuzzer-no-link,address -c -o bdiff.o \
 	  ../../mercurial/bdiff.c
 
 bdiff: bdiff.cc bdiff.o fuzzutil.o
 	$(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
 	  -std=c++17 \
 	  -I../../mercurial bdiff.cc bdiff.o fuzzutil.o -o bdiff
 
 bdiff-oss-fuzz.o: ../../mercurial/bdiff.c
 	$(CC) $(CFLAGS) -c -o bdiff-oss-fuzz.o ../../mercurial/bdiff.c
 
 bdiff_fuzzer: bdiff.cc bdiff-oss-fuzz.o fuzzutil-oss-fuzz.o
 	$(CXX) $(CXXFLAGS) -std=c++17 -I../../mercurial bdiff.cc \
 	  bdiff-oss-fuzz.o fuzzutil-oss-fuzz.o -lFuzzingEngine -o \
 	  $$OUT/bdiff_fuzzer
 
 mpatch.o: ../../mercurial/mpatch.c
 	$(CC) -g -O1 -fsanitize=fuzzer-no-link,address -c -o mpatch.o \
 	  ../../mercurial/mpatch.c
 
 mpatch: CXXFLAGS += -std=c++17
 mpatch: mpatch.cc mpatch.o fuzzutil.o
 	$(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
 	  -I../../mercurial mpatch.cc mpatch.o fuzzutil.o -o mpatch
 
 mpatch-oss-fuzz.o: ../../mercurial/mpatch.c
 	$(CC) $(CFLAGS) -c -o mpatch-oss-fuzz.o ../../mercurial/mpatch.c
 
 mpatch_fuzzer: mpatch.cc mpatch-oss-fuzz.o fuzzutil-oss-fuzz.o
 	$(CXX) $(CXXFLAGS) -std=c++17 -I../../mercurial mpatch.cc \
 	  mpatch-oss-fuzz.o fuzzutil-oss-fuzz.o -lFuzzingEngine -o \
 	  $$OUT/mpatch_fuzzer
 
 mpatch_corpus.zip:
 	python mpatch_corpus.py $$OUT/mpatch_fuzzer_seed_corpus.zip
 
 x%.o: ../../mercurial/thirdparty/xdiff/x%.c ../../mercurial/thirdparty/xdiff/*.h
 	$(CC) -g -O1 -fsanitize=fuzzer-no-link,address -c \
 	  -o $@ \
 	  $<
 
 xdiff: CXXFLAGS += -std=c++17
 xdiff: xdiff.cc xdiffi.o xprepare.o xutils.o fuzzutil.o
 	$(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
 	  -I../../mercurial xdiff.cc \
 	  xdiffi.o xprepare.o xutils.o fuzzutil.o -o xdiff
 
 fuzz-x%.o: ../../mercurial/thirdparty/xdiff/x%.c ../../mercurial/thirdparty/xdiff/*.h
 	$(CC) $(CFLAGS) -c \
 	  -o $@ \
 	  $<
 
 xdiff_fuzzer: xdiff.cc fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o fuzzutil-oss-fuzz.o
 	$(CXX) $(CXXFLAGS) -std=c++17 -I../../mercurial xdiff.cc \
 	  fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o fuzzutil-oss-fuzz.o \
 	  -lFuzzingEngine -o $$OUT/xdiff_fuzzer
 
-# TODO use the $OUT env var instead of hardcoding /out
-/out/sanpy/bin/python:
-	cd /Python-2.7.15/ && ./configure --without-pymalloc --prefix=$$OUT/sanpy CFLAGS='-O1 -fno-omit-frame-pointer -g -fwrapv -fstack-protector-strong' LDFLAGS=-lasan  && ASAN_OPTIONS=detect_leaks=0 make && make install
-
-sanpy: /out/sanpy/bin/python
-
-manifest.o: sanpy ../../mercurial/cext/manifest.c
+manifest.o: ../../mercurial/cext/manifest.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o manifest.o ../../mercurial/cext/manifest.c
 
-charencode.o: sanpy ../../mercurial/cext/charencode.c
+charencode.o: ../../mercurial/cext/charencode.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o charencode.o ../../mercurial/cext/charencode.c
 
-parsers.o: sanpy ../../mercurial/cext/parsers.c
+parsers.o: ../../mercurial/cext/parsers.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o parsers.o ../../mercurial/cext/parsers.c
 
-dirs.o: sanpy ../../mercurial/cext/dirs.c
+dirs.o: ../../mercurial/cext/dirs.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o dirs.o ../../mercurial/cext/dirs.c
 
-pathencode.o: sanpy ../../mercurial/cext/pathencode.c
+pathencode.o: ../../mercurial/cext/pathencode.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o pathencode.o ../../mercurial/cext/pathencode.c
 
-revlog.o: sanpy ../../mercurial/cext/revlog.c
+revlog.o: ../../mercurial/cext/revlog.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o revlog.o ../../mercurial/cext/revlog.c
 
-manifest_fuzzer: sanpy manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o
+manifest_fuzzer: manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
 	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -Wno-register -Wno-macro-redefined \
 	  -I../../mercurial manifest.cc \
-	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
 	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
 	  -o $$OUT/manifest_fuzzer
 
 manifest_corpus.zip:
 	python manifest_corpus.py $$OUT/manifest_fuzzer_seed_corpus.zip
 
+revlog_fuzzer: revlog.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
+	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
+	  -Wno-register -Wno-macro-redefined \
+	  -I../../mercurial revlog.cc \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
+	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
+	  -o $$OUT/revlog_fuzzer
+
+revlog_corpus.zip:
+	python revlog_corpus.py $$OUT/revlog_fuzzer_seed_corpus.zip
+
+dirstate_fuzzer: dirstate.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
+	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
+	  -Wno-register -Wno-macro-redefined \
+	  -I../../mercurial dirstate.cc \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
+	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
+	  -o $$OUT/dirstate_fuzzer
+
+dirstate_corpus.zip:
+	python dirstate_corpus.py $$OUT/dirstate_fuzzer_seed_corpus.zip
+
+fm1readmarkers_fuzzer: fm1readmarkers.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
+	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
+	  -Wno-register -Wno-macro-redefined \
+	  -I../../mercurial fm1readmarkers.cc \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
+	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
+	  -o $$OUT/fm1readmarkers_fuzzer
+
+fm1readmarkers_corpus.zip:
+	python fm1readmarkers_corpus.py $$OUT/fm1readmarkers_fuzzer_seed_corpus.zip
+
 clean:
 	$(RM) *.o *_fuzzer \
 	  bdiff \
 	  mpatch \
 	  xdiff
 
-oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip
+oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip revlog_fuzzer revlog_corpus.zip dirstate_fuzzer dirstate_corpus.zip fm1readmarkers_fuzzer fm1readmarkers_corpus.zip
 
-.PHONY: all clean oss-fuzz sanpy
+.PHONY: all clean oss-fuzz

contrib/fuzz/manifest.cc

@@ -1,83 +1,55 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
+#include "pyutil.h"
+
 #include <string>
 
 extern "C" {
 
-/* TODO: use Python 3 for this fuzzing? */
-PyMODINIT_FUNC initparsers(void);
-
-static char cpypath[8192] = "\0";
-
 static PyCodeObject *code;
-static PyObject *mainmod;
-static PyObject *globals;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
-	const std::string subdir = "/sanpy/lib/python2.7";
-	/* HACK ALERT: we need a full Python installation built without
-	   pymalloc and with ASAN, so we dump one in
-	   $OUT/sanpy/lib/python2.7. This helps us wire that up. */
-	std::string selfpath(*argv[0]);
-	std::string pypath;
-	auto pos = selfpath.rfind("/");
-	if (pos == std::string::npos) {
-		char wd[8192];
-		getcwd(wd, 8192);
-		pypath = std::string(wd) + subdir;
-	} else {
-		pypath = selfpath.substr(0, pos) + subdir;
-	}
-	strncpy(cpypath, pypath.c_str(), pypath.size());
-	setenv("PYTHONPATH", cpypath, 1);
-	setenv("PYTHONNOUSERSITE", "1", 1);
-	/* prevent Python from looking up users in the fuzz environment */
-	setenv("PYTHONUSERBASE", cpypath, 1);
-	Py_SetPythonHome(cpypath);
-	Py_InitializeEx(0);
-	initparsers();
+	contrib::initpy(*argv[0]);
 	code = (PyCodeObject *)Py_CompileString(R"py(
 from parsers import lazymanifest
 try:
   lm = lazymanifest(mdata)
   # iterate the whole thing, which causes the code to fully parse
   # every line in the manifest
   list(lm.iterentries())
   lm[b'xyzzy'] = (b'\0' * 20, 'x')
   # do an insert, text should change
   assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
   del lm[b'xyzzy']
   # should be back to the same
   assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata)
 except Exception as e:
   pass
   # uncomment this print if you're editing this Python code
   # to debug failures.
   # print e
 )py",
 	                                        "fuzzer", Py_file_input);
-	mainmod = PyImport_AddModule("__main__");
-	globals = PyModule_GetDict(mainmod);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	PyObject *mtext =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "mdata", mtext);
-	PyObject *res = PyEval_EvalCode(code, globals, locals);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(mtext);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/xdiff.cc

@@ -1,58 +1,63 b''
 /*
  * xdiff.cc - fuzzer harness for thirdparty/xdiff
  *
  * Copyright 2018, Google Inc.
  *
  * This software may be used and distributed according to the terms of
  * the GNU General Public License, incorporated herein by reference.
  */
 #include "thirdparty/xdiff/xdiff.h"
 #include <inttypes.h>
 #include <stdlib.h>
 
 #include "fuzzutil.h"
 
 extern "C" {
 
 int hunk_consumer(long a1, long a2, long b1, long b2, void *priv)
 {
 	// TODO: probably also test returning -1 from this when things break?
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
+	// Don't allow fuzzer inputs larger than 100k, since we'll just bog
+	// down and not accomplish much.
+	if (Size > 100000) {
+		return 0;
+	}
 	auto maybe_inputs = SplitInputs(Data, Size);
 	if (!maybe_inputs) {
 		return 0;
 	}
 	auto inputs = std::move(maybe_inputs.value());
 	mmfile_t a, b;
 
 	a.ptr = inputs.left.get();
 	a.size = inputs.left_size;
 	b.ptr = inputs.right.get();
 	b.size = inputs.right_size;
 	xpparam_t xpp = {
 	    XDF_INDENT_HEURISTIC, /* flags */
 	};
 	xdemitconf_t xecfg = {
 	    XDL_EMIT_BDIFFHUNK, /* flags */
 	    hunk_consumer,      /* hunk_consume_func */
 	};
 	xdemitcb_t ecb = {
 	    NULL, /* priv */
 	};
 	xdl_diff(&a, &b, &xpp, &xecfg, &ecb);
 	return 0; // Non-zero return values are reserved for future use.
 }
 
 #ifdef HG_FUZZER_INCLUDE_MAIN
 int main(int argc, char **argv)
 {
 	const char data[] = "asdf";
 	return LLVMFuzzerTestOneInput((const uint8_t *)data, 4);
 }
 #endif
 
 } // extern "C"

contrib/hgclient.py

@@ -1,134 +1,147 b''
 # A minimal client for Mercurial's command server
 
 from __future__ import absolute_import, print_function
 
 import io
 import os
 import re
 import signal
 import socket
 import struct
 import subprocess
 import sys
 import time
 
 if sys.version_info[0] >= 3:
     stdout = sys.stdout.buffer
     stderr = sys.stderr.buffer
     stringio = io.BytesIO
     def bprint(*args):
         # remove b'' as well for ease of test migration
         pargs = [re.sub(br'''\bb(['"])''', br'\1', b'%s' % a) for a in args]
         stdout.write(b' '.join(pargs) + b'\n')
 else:
     import cStringIO
     stdout = sys.stdout
     stderr = sys.stderr
     stringio = cStringIO.StringIO
     bprint = print
 
-def connectpipe(path=None):
+def connectpipe(path=None, extraargs=()):
     cmdline = [b'hg', b'serve', b'--cmdserver', b'pipe']
     if path:
         cmdline += [b'-R', path]
+    cmdline.extend(extraargs)
 
-    server = subprocess.Popen(cmdline, stdin=subprocess.PIPE,
+    def tonative(cmdline):
+        if os.name != r'nt':
+            return cmdline
+        return [arg.decode("utf-8") for arg in cmdline]
+
+    server = subprocess.Popen(tonative(cmdline), stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE)
 
     return server
 
 class unixconnection(object):
     def __init__(self, sockpath):
         self.sock = sock = socket.socket(socket.AF_UNIX)
         sock.connect(sockpath)
         self.stdin = sock.makefile('wb')
         self.stdout = sock.makefile('rb')
 
     def wait(self):
         self.stdin.close()
         self.stdout.close()
         self.sock.close()
 
 class unixserver(object):
     def __init__(self, sockpath, logpath=None, repopath=None):
         self.sockpath = sockpath
         cmdline = [b'hg', b'serve', b'--cmdserver', b'unix', b'-a', sockpath]
         if repopath:
             cmdline += [b'-R', repopath]
         if logpath:
             stdout = open(logpath, 'a')
             stderr = subprocess.STDOUT
         else:
             stdout = stderr = None
         self.server = subprocess.Popen(cmdline, stdout=stdout, stderr=stderr)
         # wait for listen()
         while self.server.poll() is None:
             if os.path.exists(sockpath):
                 break
             time.sleep(0.1)
 
     def connect(self):
         return unixconnection(self.sockpath)
 
     def shutdown(self):
         os.kill(self.server.pid, signal.SIGTERM)
         self.server.wait()
 
 def writeblock(server, data):
     server.stdin.write(struct.pack(b'>I', len(data)))
     server.stdin.write(data)
     server.stdin.flush()
 
 def readchannel(server):
     data = server.stdout.read(5)
     if not data:
         raise EOFError
     channel, length = struct.unpack('>cI', data)
     if channel in b'IL':
         return channel, length
     else:
         return channel, server.stdout.read(length)
 
 def sep(text):
     return text.replace(b'\\', b'/')
 
 def runcommand(server, args, output=stdout, error=stderr, input=None,
                outfilter=lambda x: x):
     bprint(b'*** runcommand', b' '.join(args))
     stdout.flush()
     server.stdin.write(b'runcommand\n')
     writeblock(server, b'\0'.join(args))
 
     if not input:
         input = stringio()
 
     while True:
         ch, data = readchannel(server)
         if ch == b'o':
             output.write(outfilter(data))
             output.flush()
         elif ch == b'e':
             error.write(data)
             error.flush()
         elif ch == b'I':
             writeblock(server, input.read(data))
         elif ch == b'L':
             writeblock(server, input.readline(data))
+        elif ch == b'm':
+            bprint(b"message: %r" % data)
         elif ch == b'r':
             ret, = struct.unpack('>i', data)
             if ret != 0:
                 bprint(b' [%d]' % ret)
             return ret
         else:
             bprint(b"unexpected channel %c: %r" % (ch, data))
             if ch.isupper():
                 return
 
 def check(func, connect=connectpipe):
     stdout.flush()
     server = connect()
     try:
         return func(server)
     finally:
         server.stdin.close()
         server.wait()
+
+def checkwith(connect=connectpipe, **kwargs):
+    def wrap(func):
+        return check(func, lambda: connect(**kwargs))
+    return wrap

contrib/import-checker.py

@@ -1,745 +1,747 b''
 #!/usr/bin/env python
 
 from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import os
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
 if True: # disable lexical sorting checks
     try:
         import BaseHTTPServer as basehttpserver
     except ImportError:
         basehttpserver = None
     import zlib
 
 import testparseutil
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
     'bzrlib',
     'hgclient',
     'mercurial',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
     'mercurial.node',
     # for revlog to re-export constant to extensions
     'mercurial.revlogutils.constants',
     # for cffi modules to re-export pure functions
     'mercurial.pure.base85',
     'mercurial.pure.bdiff',
     'mercurial.pure.mpatch',
     'mercurial.pure.osutil',
     'mercurial.pure.parsers',
     # third-party imports should be directly imported
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
-    'mercurial.thirdparty.cbor',
-    'mercurial.thirdparty.cbor.cbor2',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
 )
 
 # Whitelist of symbols that can be directly imported.
 directsymbols = (
     'demandimport',
 )
 
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 def usingabsolute(root):
     """Whether absolute imports are being used."""
     if sys.version_info[0] >= 3:
         return True
 
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 for n in node.names:
                     if n.name == 'absolute_import':
                         return True
 
     return False
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 def dotted_name_of_path(path):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     return '.'.join(parts)
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
     paths of locally defined (= Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__', 'foo.foo1',
     ...              'foo.bar.__init__', 'foo.bar.bar1',
     ...              'baz.__init__', 'baz.baz1'}
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
     >>> fromlocal('foo1', 1)
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
     >>> fromlocal2('bar2', 1)
     False
     >>> fromlocal2('bar', 2)
     ('foo.bar', 'foo.bar.__init__', True)
     """
     if not isinstance(modulename, str):
         modulename = modulename.decode('ascii')
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
     def fromlocal(name, level=0):
         # name is false value when relative imports are used.
         if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
             if not level:
                 # Check relative name first.
                 candidates = [prefix + name, name]
             else:
                 candidates = ['.'.join(modulename.split('.')[:-level]) +
                               '.' + name]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
     return fromlocal
 
 def populateextmods(localmods):
     """Populate C extension modules based on pure modules"""
     newlocalmods = set(localmods)
     for n in localmods:
         if n.startswith('mercurial.pure.'):
             m = n[len('mercurial.pure.'):]
             newlocalmods.add('mercurial.cext.' + m)
             newlocalmods.add('mercurial.cffi._' + m)
     return newlocalmods
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> py3 = sys.version_info[0] >= 3
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods or py3
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods or py3
     True
 
     >>> 'cffi' in mods
     True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield '__builtin__'
     yield 'builtins' # python3 only
     yield 'importlib.abc' # python3 only
     yield 'importlib.machinery' # python3 only
     yield 'importlib.util' # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
         yield m
     for m in ['cffi']:
         yield m
     stdlib_prefixes = {sys.prefix, sys.exec_prefix}
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (basehttpserver, zlib):
         if mod is None:
             continue
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
+    sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
     for libpath in sys.path:
-        # We want to walk everything in sys.path that starts with
-        # something in stdlib_prefixes.
-        if not any(libpath.startswith(p) for p in stdlib_prefixes):
+        # We want to walk everything in sys.path that starts with something in
+        # stdlib_prefixes, but not directories from the hg sources.
+        if (os.path.abspath(libpath).startswith(sourceroot)
+            or not any(libpath.startswith(p) for p in stdlib_prefixes)):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
                 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath and d in ('hgdemandimport', 'hgext',
                                                 'mercurial')):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1:]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 stdlib_modules = set(list_stdlib_modules())
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: set of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
             if modnotfound:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
                 yield dottedpath
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention.
 
     We have 2 conventions: legacy and modern. The modern convention is in
     effect when using absolute imports.
 
     The legacy convention only looks for mixed imports. The modern convention
     is much more thorough.
     """
     root = ast.parse(source)
     absolute = usingabsolute(root)
 
     if absolute:
         return verify_modern_convention(module, root, localmods)
     else:
         return verify_stdlib_on_own_line(root)
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
       must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed, unless that
       library is also a local module.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     if not isinstance(module, str):
         module = module.decode('ascii')
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
     # Whether a local/non-stdlib, non-symbol import has been seen.
     seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     laststdlib = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(module, node, localmods,
                                               node.col_offset + 4):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg('multiple imported names: %s',
                           ', '.join(n.name for n in node.names))
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             stdlib = name in stdlib_modules
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname and laststdlib == stdlib:
                     yield msg('imports not lexically sorted: %s < %s',
                               name, lastname)
 
             lastname = name
             laststdlib = stdlib
 
             # stdlib imports should be before local imports.
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg('stdlib import "%s" follows local import: %s',
                           name, seenlocal)
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg('%s module must be "as" aliased to %s',
                           name, requirealias[name])
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[:-node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or (
                     fullname in stdlib_modules
                     and fullname not in localmods
                     and fullname + '.__init__' not in localmods):
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
             found = fromlocal(node.module, node.level)
             if found and found[2]:  # node.module is a package
                 prefix = found[0] + '.'
                 symbols = (n.name for n in node.names
                            if not fromlocal(prefix + n.name))
             else:
                 symbols = (n.name for n in node.names)
             symbols = [sym for sym in symbols if sym not in directsymbols]
             if node.module and node.col_offset == root_col_offset:
                 if symbols and fullname not in allowsymbolimports:
                     yield msg('direct symbol import %s from %s',
                               ', '.join(symbols), fullname)
 
                 if symbols and seennonsymbollocal:
                     yield msg('symbol import follows non-symbol import: %s',
                               fullname)
             if not symbols and fullname not in stdlib_modules:
                 seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
 
                 # Only allow 1 group per level.
                 if (node.level in seenlevels
                     and node.col_offset == root_col_offset):
                     yield msg('multiple "from %s import" statements',
                               '.' * node.level)
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg('higher-level import should come first: %s',
                               fullname)
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg('imports from %s not lexically sorted: %s < %s',
                               fullname, n.name, lastentryname)
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg('%s from %s must be "as" aliased to %s',
                               n.name, fullname, requirealias[n.name])
 
 def verify_stdlib_on_own_line(root):
     """Given some python source, verify that stdlib imports are done
     in separate statements from relative local module imports.
 
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
     [('mixed imports\\n   stdlib:    sys\\n   relative:  foo', 1)]
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
     []
     >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
     []
     """
     for node in ast.walk(root):
         if isinstance(node, ast.Import):
             from_stdlib = {False: [], True: []}
             for n in node.names:
                 from_stdlib[n.name in stdlib_modules].append(n.name)
             if from_stdlib[True] and from_stdlib[False]:
                 yield ('mixed imports\n   stdlib:    %s\n   relative:  %s' %
                        (', '.join(sorted(from_stdlib[True])),
                         ', '.join(sorted(from_stdlib[False]))), node.lineno)
 
 class CircularImport(Exception):
     pass
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 def _cycle_sortkey(c):
     return len(c), c
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def _forcestr(thing):
     ...     if not isinstance(thing, str):
     ...         return thing.decode('ascii')
     ...     return thing
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, b"example", lines):
     ...         print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
     ...         print(repr(_forcestr(s)))
     >>> lines = [
     ...   b'comment',
     ...   b'  >>> from __future__ import print_function',
     ...   b"  >>> ' multiline",
     ...   b"  ... string'",
     ...   b'  ',
     ...   b'comment',
     ...   b'  $ cat > foo.py <<EOF',
     ...   b'  > from __future__ import print_function',
     ...   b'  > EOF',
     ... ]
     >>> test(b"example.t", lines)
     example[2] doctest.py 1
     "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
     example[8] foo.py 7
     'from __future__ import print_function\\n'
     """
     errors = []
     for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
         if not name:
             # use 'doctest.py', in order to make already existing
             # doctest above pass instantly
             name = 'doctest.py'
         # "starts" is "line number" (1-origin), but embedded() is
         # expected to return "line offset" (0-origin). Therefore, this
         # yields "starts - 1".
+        if not isinstance(modname, str):
+            modname = modname.decode('utf8')
         yield code, "%s[%d]" % (modname, starts), name, starts - 1
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
     if not f.endswith('.t'):
         with open(f, 'rb') as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         with open(f, 'rb') as src:
             for script, modname, t, line in embedded(f, modname, src):
-                yield script, modname, t, line
+                yield script, modname.encode('utf8'), t, line
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmodpaths = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
         modname = dotted_name_of_path(source_path)
         localmodpaths[modname] = source_path
     localmods = populateextmods(localmodpaths)
     for localmodname, source_path in sorted(localmodpaths.items()):
         if not isinstance(localmodname, bytes):
             # This is only safe because all hg's files are ascii
             localmodname = localmodname.encode('ascii')
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(src, modname, name, localmods,
                                      ignore_nested=True))
                 for error, lineno in verify_import_convention(modname, src,
                                                               localmods):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print('%s:%d: SyntaxError: %s' %
                       (source_path, e.lineno + line, e))
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

contrib/packaging/Makefile

@@ -1,144 +1,145 b''
 $(eval HGROOT := $(shell cd ../..; pwd))
 
 DEBIAN_CODENAMES := \
   jessie \
   stretch \
   buster
 
 UBUNTU_CODENAMES := \
   trusty \
   xenial \
   artful \
   bionic \
 
 FEDORA_RELEASES := \
   20 \
   21 \
-  28
+  28 \
+  29
 
 CENTOS_RELEASES := \
   5 \
   6 \
   7
 
 # Build a Python for these CentOS releases.
 CENTOS_WITH_PYTHON_RELEASES := 5 6
 
 help:
 	@echo 'Packaging Make Targets'
 	@echo ''
 	@echo 'docker-centos{$(strip $(CENTOS_RELEASES))}'
 	@echo '   Build an RPM for a specific CentOS version using Docker.'
 	@echo ''
 	@echo 'docker-debian-{$(strip $(DEBIAN_CODENAMES))}'
 	@echo '   Build Debian packages specific to a Debian distro using Docker.'
 	@echo ''
 	@echo 'docker-fedora{$(strip $(FEDORA_RELEASES))}'
 	@echo '   Build an RPM for a specific Fedora version using Docker.'
 	@echo ''
 	@echo 'docker-ubuntu-{$(strip $(UBUNTU_CODENAMES))}'
 	@echo '   Build Debian package specific to an Ubuntu distro using Docker.'
 	@echo ''
 	@echo 'docker-ubuntu-{$(strip $(UBUNTU_CODENAMES))}-ppa'
 	@echo '   Build a source-only Debian package specific to an Ubuntu distro'
 	@echo '   using Docker.'
 	@echo ''
 	@echo 'linux-wheels'
 	@echo '   Build Linux manylinux wheels using Docker.'
 	@echo ''
 	@echo 'linux-wheels-{x86_64, i686}'
 	@echo '   Build Linux manylinux wheels for a specific architecture using Docker'
 	@echo ''
 	@echo 'deb'
 	@echo '   Build a Debian package locally targeting the current system'
 	@echo ''
 	@echo 'ppa'
 	@echo '   Build a Debian source package locally targeting the current system'
 	@echo ''
 	@echo 'centos{$(strip $(CENTOS_RELEASES))}'
 	@echo '   Build an RPM for a specific CentOS version locally'
 	@echo ''
 	@echo 'fedora{$(strip $(FEDORA_RELEASES))}'
 	@echo '   Build an RPM for a specific Fedora version locally'
 
 .PHONY: help
 
 .PHONY: deb
 deb:
 	./builddeb
 
 .PHONY: ppa
 ppa:
 	./builddeb --source-only
 
 # Debian targets.
 define debian_targets =
 .PHONY: docker-debian-$(1)
 docker-debian-$(1):
 	./dockerdeb debian $(1)
 
 endef
 
 $(foreach codename,$(DEBIAN_CODENAMES),$(eval $(call debian_targets,$(codename))))
 
 # Ubuntu targets.
 define ubuntu_targets =
 .PHONY: docker-ubuntu-$(1)
 docker-ubuntu-$(1):
 	./dockerdeb ubuntu $(1)
 
 .PHONY: docker-ubuntu-$(1)-ppa
 docker-ubuntu-$(1)-ppa:
 	./dockerdeb ubuntu $(1) --source-only
 
 endef
 
 $(foreach codename,$(UBUNTU_CODENAMES),$(eval $(call ubuntu_targets,$(codename))))
 
 # Fedora targets.
 define fedora_targets
 .PHONY: fedora$(1)
 fedora$(1):
 	mkdir -p $$(HGROOT)/packages/fedora$(1)
 	./buildrpm
 	cp $$(HGROOT)/contrib/packaging/rpmbuild/RPMS/*/* $$(HGROOT)/packages/fedora$(1)
 	cp $$(HGROOT)/contrib/packaging/rpmbuild/SRPMS/* $$(HGROOT)/packages/fedora$(1)
 	rm -rf $(HGROOT)/rpmbuild
 
 .PHONY: docker-fedora$(1)
 docker-fedora$(1):
 	mkdir -p $$(HGROOT)/packages/fedora$(1)
 	./dockerrpm fedora$(1)
 
 endef
 
 $(foreach release,$(FEDORA_RELEASES),$(eval $(call fedora_targets,$(release))))
 
 # CentOS targets.
 define centos_targets
 .PHONY: centos$(1)
 centos$(1):
 	mkdir -p $$(HGROOT)/packages/centos$(1)
 	./buildrpm $$(if $$(filter $(1),$$(CENTOS_WITH_PYTHON_RELEASES)),--withpython)
 	cp $$(HGROOT)/contrib/packaging/rpmbuild/RPMS/*/* $$(HGROOT)/packages/centos$(1)
 	cp $$(HGROOT)/contrib/packaging/rpmbuild/SRPMS/* $$(HGROOT)/packages/centos$(1)
 
 .PHONY: docker-centos$(1)
 docker-centos$(1):
 	mkdir -p $$(HGROOT)/packages/centos$(1)
 	./dockerrpm centos$(1) $$(if $$(filter $(1),$$(CENTOS_WITH_PYTHON_RELEASES)),--withpython)
 
 endef
 
 $(foreach release,$(CENTOS_RELEASES),$(eval $(call centos_targets,$(release))))
 
 .PHONY: linux-wheels
 linux-wheels: linux-wheels-x86_64 linux-wheels-i686
 
 .PHONY: linux-wheels-x86_64
 linux-wheels-x86_64:
 	docker run -e "HGTEST_JOBS=$(shell nproc)" --rm -ti -v `pwd`/../..:/src quay.io/pypa/manylinux1_x86_64 /src/contrib/packaging/build-linux-wheels.sh
 
 .PHONY: linux-wheels-i686
 linux-wheels-i686:
 	docker run -e "HGTEST_JOBS=$(shell nproc)" --rm -ti -v `pwd`/../..:/src quay.io/pypa/manylinux1_i686 linux32 /src/contrib/packaging/build-linux-wheels.sh

contrib/packaging/docker/centos5

@@ -1,27 +1,27 b''
 FROM centos:centos5
 
-RUN groupadd -g 1000 build && \
-    useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
 
 RUN \
 	sed -i 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/*.repo && \
 	sed -i 's/^#\(baseurl=\)http:\/\/mirror.centos.org\/centos/\1http:\/\/vault.centos.org/' /etc/yum.repos.d/*.repo && \
 	sed -i 's/\$releasever/5.11/' /etc/yum.repos.d/*.repo
 
 RUN yum install -y \
 	gcc \
 	gettext \
 	make \
 	python-devel \
 	python-docutils \
 	rpm-build \
 	tar
 
 # For creating repo meta data
 RUN yum install -y \
 	bzip2-devel \
 	createrepo \
 	ncurses-devel \
 	openssl-devel \
 	readline-devel \
 	zlib-devel

contrib/packaging/docker/centos6

@@ -1,24 +1,24 b''
 FROM centos:centos6
 
-RUN groupadd -g 1000 build && \
-    useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
 
 RUN yum install -y \
 	gcc \
 	gettext \
 	make \
 	python-devel \
 	python-docutils \
 	rpm-build \
 	tar
 
 # For creating repo meta data
 RUN yum install -y createrepo
 
 # For python
 RUN yum install -y \
 	bzip2-devel \
 	ncurses-devel \
 	openssl-devel \
 	readline-devel \
 	zlib-devel

contrib/packaging/docker/centos7

@@ -1,16 +1,16 b''
 FROM centos:centos7
 
-RUN groupadd -g 1000 build && \
-    useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
 
 RUN yum install -y \
 	gcc \
 	gettext \
 	make \
 	python-devel \
 	python-docutils \
 	rpm-build \
 	tar
 
 # For creating repo meta data
 RUN yum install -y createrepo

contrib/packaging/docker/fedora29

@@ -1,15 +1,15 b''
-FROM fedora:28
+FROM fedora:29
 
 RUN groupadd -g 1000 build && \
     useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
 
 RUN dnf install -y \
 	gcc \
 	gettext \
 	make \
 	python-devel \
 	python-docutils \
 	rpm-build
 
 # For creating repo meta data
 RUN dnf install -y createrepo

contrib/packaging/dockerrpm

@@ -1,39 +1,47 b''
 #!/bin/bash -e
 
 BUILDDIR=$(dirname $0)
 export ROOTDIR=$(cd $BUILDDIR/../..; pwd)
 
 PLATFORM="$1"
 shift # extra params are passed to buildrpm
 
 DOCKER=$($BUILDDIR/hg-docker docker-path)
 
 CONTAINER=hg-docker-$PLATFORM
 
-$BUILDDIR/hg-docker build $BUILDDIR/docker/$PLATFORM $CONTAINER
+if [[ -z "${HG_DOCKER_OWN_USER}" ]]; then
+    DOCKERUID=1000
+    DOCKERGID=1000
+else
+    DOCKERUID=$(id -u)
+    DOCKERGID=$(id -g)
+fi
+
+$BUILDDIR/hg-docker build --build-arg UID=$DOCKERUID --build-arg GID=$DOCKERGID $BUILDDIR/docker/$PLATFORM $CONTAINER
 
 RPMBUILDDIR=$ROOTDIR/packages/$PLATFORM
 $ROOTDIR/contrib/packaging/buildrpm --rpmbuilddir $RPMBUILDDIR --prepare $*
 
 DSHARED=/mnt/shared
 DBUILDUSER=build
 
 $DOCKER run -e http_proxy -e https_proxy -u $DBUILDUSER --rm -v $RPMBUILDDIR:$DSHARED $CONTAINER \
     rpmbuild --define "_topdir $DSHARED" -ba $DSHARED/SPECS/mercurial.spec --clean
 
 $DOCKER run -e http_proxy -e https_proxy -u $DBUILDUSER --rm -v $RPMBUILDDIR:$DSHARED $CONTAINER \
     createrepo $DSHARED
 
 cat << EOF > $RPMBUILDDIR/mercurial.repo
 # Place this file in /etc/yum.repos.d/mercurial.repo
 [mercurial]
 name=Mercurial packages for $PLATFORM
 # baseurl=file://$RPMBUILDDIR/
 baseurl=http://hg.example.com/build/$PLATFORM/
 skip_if_unavailable=True
 gpgcheck=0
 enabled=1
 EOF
 
 echo
 echo "Build complete - results can be found in $RPMBUILDDIR"

contrib/packaging/hg-docker

@@ -1,111 +1,116 b''
 #!/usr/bin/env python3
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import argparse
 import pathlib
 import shutil
 import subprocess
 import sys
 
 def get_docker() -> str:
     docker = shutil.which('docker.io') or shutil.which('docker')
     if not docker:
         print('could not find docker executable')
         return 1
 
     try:
         out = subprocess.check_output([docker, '-h'], stderr=subprocess.STDOUT)
 
         if b'Jansens' in out:
             print('%s is the Docking System Tray; try installing docker.io' %
                   docker)
             sys.exit(1)
     except subprocess.CalledProcessError as e:
         print('error calling `%s -h`: %s' % (docker, e.output))
         sys.exit(1)
 
     out = subprocess.check_output([docker, 'version'],
                                   stderr=subprocess.STDOUT)
 
     lines = out.splitlines()
     if not any(l.startswith((b'Client:', b'Client version:')) for l in lines):
         print('`%s version` does not look like Docker' % docker)
         sys.exit(1)
 
     if not any(l.startswith((b'Server:', b'Server version:')) for l in lines):
         print('`%s version` does not look like Docker' % docker)
         sys.exit(1)
 
     return docker
 
 def get_dockerfile(path: pathlib.Path, args: list) -> bytes:
     with path.open('rb') as fh:
         df = fh.read()
 
     for k, v in args:
-        df = df.replace(b'%%%s%%' % k, v)
+        df = df.replace(bytes('%%%s%%' % k.decode(), 'utf-8'), v)
 
     return df
 
 def build_docker_image(dockerfile: pathlib.Path, params: list, tag: str):
     """Build a Docker image from a templatized Dockerfile."""
     docker = get_docker()
 
     dockerfile_path = pathlib.Path(dockerfile)
 
     dockerfile = get_dockerfile(dockerfile_path, params)
 
     print('building Dockerfile:')
     print(dockerfile.decode('utf-8', 'replace'))
 
     args = [
         docker,
         'build',
         '--build-arg', 'http_proxy',
         '--build-arg', 'https_proxy',
         '--tag', tag,
         '-',
     ]
 
     print('executing: %r' % args)
-    subprocess.run(args, input=dockerfile, check=True)
+    p = subprocess.Popen(args, stdin=subprocess.PIPE)
+    p.communicate(input=dockerfile)
+    if p.returncode:
+        raise subprocess.CalledProcessException(
+                p.returncode, 'failed to build docker image: %s %s' \
+                % (p.stdout, p.stderr))
 
 def command_build(args):
     build_args = []
     for arg in args.build_arg:
         k, v = arg.split('=', 1)
         build_args.append((k.encode('utf-8'), v.encode('utf-8')))
 
     build_docker_image(pathlib.Path(args.dockerfile),
                        build_args,
                        args.tag)
 
 def command_docker(args):
     print(get_docker())
 
 def main() -> int:
     parser = argparse.ArgumentParser()
 
     subparsers = parser.add_subparsers(title='subcommands')
 
     build = subparsers.add_parser('build', help='Build a Docker image')
     build.set_defaults(func=command_build)
     build.add_argument('--build-arg', action='append', default=[],
                         help='Substitution to perform in Dockerfile; '
                              'format: key=value')
     build.add_argument('dockerfile', help='path to Dockerfile to use')
     build.add_argument('tag', help='Tag to apply to created image')
 
     docker = subparsers.add_parser('docker-path', help='Resolve path to Docker')
     docker.set_defaults(func=command_docker)
 
     args = parser.parse_args()
 
     return args.func(args)
 
 if __name__ == '__main__':
     sys.exit(main())

contrib/perf.py

@@ -1,2126 +1,2675 b''
 # perf.py - performance test routines
 '''helper extension to measure performance'''
 
 # "historical portability" policy of perf.py:
 #
 # We have to do:
 # - make perf.py "loadable" with as wide Mercurial version as possible
 #   This doesn't mean that perf commands work correctly with that Mercurial.
 #   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
 # - make historical perf command work correctly with as wide Mercurial
 #   version as possible
 #
 # We have to do, if possible with reasonable cost:
 # - make recent perf command for historical feature work correctly
 #   with early Mercurial
 #
 # We don't have to do:
 # - make perf command for recent feature work correctly with early
 #   Mercurial
 
 from __future__ import absolute_import
 import contextlib
 import functools
 import gc
 import os
 import random
+import shutil
 import struct
 import sys
+import tempfile
 import threading
 import time
 from mercurial import (
     changegroup,
     cmdutil,
     commands,
     copies,
     error,
     extensions,
+    hg,
     mdiff,
     merge,
     revlog,
     util,
 )
 
 # for "historical portability":
 # try to import modules separately (in dict order), and ignore
 # failure, because these aren't available with early Mercurial
 try:
     from mercurial import branchmap # since 2.5 (or bcee63733aad)
 except ImportError:
     pass
 try:
     from mercurial import obsolete # since 2.3 (or ad0d6c2b3279)
 except ImportError:
     pass
 try:
     from mercurial import registrar # since 3.7 (or 37d50250b696)
     dir(registrar) # forcibly load it
 except ImportError:
     registrar = None
 try:
     from mercurial import repoview # since 2.5 (or 3a6ddacb7198)
 except ImportError:
     pass
 try:
     from mercurial import scmutil # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
+try:
+    from mercurial import setdiscovery # since 1.9 (or cb98fed52495)
+except ImportError:
+    pass
+
 
 def identity(a):
     return a
 
 try:
     from mercurial import pycompat
     getargspec = pycompat.getargspec  # added to module after 4.5
     _byteskwargs = pycompat.byteskwargs  # since 4.1 (or fbc3f73dc802)
     _sysstr = pycompat.sysstr         # since 4.0 (or 2219f4f82ede)
     _xrange = pycompat.xrange         # since 4.8 (or 7eba8f83129b)
     fsencode = pycompat.fsencode      # since 3.9 (or f4a5e0e86a7e)
     if pycompat.ispy3:
         _maxint = sys.maxsize  # per py3 docs for replacing maxint
     else:
         _maxint = sys.maxint
 except (ImportError, AttributeError):
     import inspect
     getargspec = inspect.getargspec
     _byteskwargs = identity
     fsencode = identity               # no py3 support
     _maxint = sys.maxint              # no py3 support
     _sysstr = lambda x: x             # no py3 support
     _xrange = xrange
 
 try:
     # 4.7+
     queue = pycompat.queue.Queue
 except (AttributeError, ImportError):
     # <4.7.
     try:
         queue = pycompat.queue
     except (AttributeError, ImportError):
         queue = util.queue
 
 try:
     from mercurial import logcmdutil
     makelogtemplater = logcmdutil.maketemplater
 except (AttributeError, ImportError):
     try:
         makelogtemplater = cmdutil.makelogtemplater
     except (AttributeError, ImportError):
         makelogtemplater = None
 
 # for "historical portability":
 # define util.safehasattr forcibly, because util.safehasattr has been
 # available since 1.9.3 (or 94b200a11cf7)
 _undefined = object()
 def safehasattr(thing, attr):
     return getattr(thing, _sysstr(attr), _undefined) is not _undefined
 setattr(util, 'safehasattr', safehasattr)
 
 # for "historical portability":
 # define util.timer forcibly, because util.timer has been available
 # since ae5d60bb70c9
 if safehasattr(time, 'perf_counter'):
     util.timer = time.perf_counter
 elif os.name == b'nt':
     util.timer = time.clock
 else:
     util.timer = time.time
 
 # for "historical portability":
 # use locally defined empty option list, if formatteropts isn't
 # available, because commands.formatteropts has been available since
 # 3.2 (or 7a7eed5176a4), even though formatting itself has been
 # available since 2.2 (or ae5f92e154d3)
 formatteropts = getattr(cmdutil, "formatteropts",
                         getattr(commands, "formatteropts", []))
 
 # for "historical portability":
 # use locally defined option list, if debugrevlogopts isn't available,
 # because commands.debugrevlogopts has been available since 3.7 (or
 # 5606f7d0d063), even though cmdutil.openrevlog() has been available
 # since 1.9 (or a79fea6b3e77).
 revlogopts = getattr(cmdutil, "debugrevlogopts",
                      getattr(commands, "debugrevlogopts", [
         (b'c', b'changelog', False, (b'open changelog')),
         (b'm', b'manifest', False, (b'open manifest')),
         (b'', b'dir', False, (b'open directory manifest')),
         ]))
 
 cmdtable = {}
 
 # for "historical portability":
 # define parsealiases locally, because cmdutil.parsealiases has been
 # available since 1.5 (or 6252852b4332)
 def parsealiases(cmd):
     return cmd.split(b"|")
 
 if safehasattr(registrar, 'command'):
     command = registrar.command(cmdtable)
 elif safehasattr(cmdutil, 'command'):
     command = cmdutil.command(cmdtable)
     if b'norepo' not in getargspec(command).args:
         # for "historical portability":
         # wrap original cmdutil.command, because "norepo" option has
         # been available since 3.1 (or 75a96326cecb)
         _command = command
         def command(name, options=(), synopsis=None, norepo=False):
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return _command(name, list(options), synopsis)
 else:
     # for "historical portability":
     # define "@command" annotation locally, because cmdutil.command
     # has been available since 1.9 (or 2daa5179e73f)
     def command(name, options=(), synopsis=None, norepo=False):
         def decorator(func):
             if synopsis:
                 cmdtable[name] = func, list(options), synopsis
             else:
                 cmdtable[name] = func, list(options)
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return func
         return decorator
 
 try:
     import mercurial.registrar
     import mercurial.configitems
     configtable = {}
     configitem = mercurial.registrar.configitem(configtable)
     configitem(b'perf', b'presleep',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(b'perf', b'stub',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(b'perf', b'parentscount',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(b'perf', b'all-timing',
         default=mercurial.configitems.dynamicdefault,
     )
 except (ImportError, AttributeError):
     pass
 
 def getlen(ui):
     if ui.configbool(b"perf", b"stub", False):
         return lambda x: 1
     return len
 
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
     This function exists to gather the creation of formatter in a single
     place instead of duplicating it in all performance commands."""
 
     # enforce an idle period before execution to counteract power management
     # experimental config: perf.presleep
     time.sleep(getint(ui, b"perf", b"presleep", 1))
 
     if opts is None:
         opts = {}
     # redirect all to stderr unless buffer api is in use
     if not ui._buffers:
         ui = ui.copy()
         uifout = safeattrsetter(ui, b'fout', ignoremissing=True)
         if uifout:
             # for "historical portability":
             # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d)
             uifout.set(ui.ferr)
 
     # get a formatter
     uiformatter = getattr(ui, 'formatter', None)
     if uiformatter:
         fm = uiformatter(b'perf', opts)
     else:
         # for "historical portability":
         # define formatter locally, because ui.formatter has been
         # available since 2.2 (or ae5f92e154d3)
         from mercurial import node
         class defaultformatter(object):
             """Minimized composition of baseformatter and plainformatter
             """
             def __init__(self, ui, topic, opts):
                 self._ui = ui
                 if ui.debugflag:
                     self.hexfunc = node.hex
                 else:
                     self.hexfunc = node.short
             def __nonzero__(self):
                 return False
             __bool__ = __nonzero__
             def startitem(self):
                 pass
             def data(self, **data):
                 pass
             def write(self, fields, deftext, *fielddata, **opts):
                 self._ui.write(deftext % fielddata, **opts)
             def condwrite(self, cond, fields, deftext, *fielddata, **opts):
                 if cond:
                     self._ui.write(deftext % fielddata, **opts)
             def plain(self, text, **opts):
                 self._ui.write(text, **opts)
             def end(self):
                 pass
         fm = defaultformatter(ui, b'perf', opts)
 
     # stub function, runs code only once instead of in a loop
     # experimental config: perf.stub
     if ui.configbool(b"perf", b"stub", False):
         return functools.partial(stub_timer, fm), fm
 
     # experimental config: perf.all-timing
     displayall = ui.configbool(b"perf", b"all-timing", False)
     return functools.partial(_timer, fm, displayall=displayall), fm
 
-def stub_timer(fm, func, title=None):
+def stub_timer(fm, func, setup=None, title=None):
+    if setup is not None:
+        setup()
     func()
 
 @contextlib.contextmanager
 def timeone():
     r = []
     ostart = os.times()
     cstart = util.timer()
     yield r
     cstop = util.timer()
     ostop = os.times()
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
 
-def _timer(fm, func, title=None, displayall=False):
+def _timer(fm, func, setup=None, title=None, displayall=False):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
     while True:
+        if setup is not None:
+            setup()
         with timeone() as item:
             r = func()
         count += 1
         results.append(item[0])
         cstop = util.timer()
         if cstop - begin > 3 and count >= 100:
             break
         if cstop - begin > 10 and count >= 3:
             break
 
     formatone(fm, results, title=title, result=r,
               displayall=displayall)
 
 def formatone(fm, timings, title=None, result=None, displayall=False):
 
     count = len(timings)
 
     fm.startitem()
 
     if title:
         fm.write(b'title', b'! %s\n', title)
     if result:
         fm.write(b'result', b'! result: %s\n', result)
     def display(role, entry):
         prefix = b''
         if role != b'best':
             prefix = b'%s.' % role
         fm.plain(b'!')
         fm.write(prefix + b'wall', b' wall %f', entry[0])
         fm.write(prefix + b'comb', b' comb %f', entry[1] + entry[2])
         fm.write(prefix + b'user', b' user %f', entry[1])
         fm.write(prefix + b'sys',  b' sys %f', entry[2])
         fm.write(prefix + b'count',  b' (%s of %%d)' % role, count)
         fm.plain(b'\n')
     timings.sort()
     min_val = timings[0]
     display(b'best', min_val)
     if displayall:
         max_val = timings[-1]
         display(b'max', max_val)
         avg = tuple([sum(x) / count for x in zip(*timings)])
         display(b'avg', avg)
         median = timings[len(timings) // 2]
         display(b'median', median)
 
 # utilities for historical portability
 
 def getint(ui, section, name, default):
     # for "historical portability":
     # ui.configint has been available since 1.9 (or fa2b596db182)
     v = ui.config(section, name, None)
     if v is None:
         return default
     try:
         return int(v)
     except ValueError:
         raise error.ConfigError((b"%s.%s is not an integer ('%s')")
                                 % (section, name, v))
 
 def safeattrsetter(obj, name, ignoremissing=False):
     """Ensure that 'obj' has 'name' attribute before subsequent setattr
 
     This function is aborted, if 'obj' doesn't have 'name' attribute
     at runtime. This avoids overlooking removal of an attribute, which
     breaks assumption of performance measurement, in the future.
 
     This function returns the object to (1) assign a new value, and
     (2) restore an original value to the attribute.
 
     If 'ignoremissing' is true, missing 'name' attribute doesn't cause
     abortion, and this function returns None. This is useful to
     examine an attribute, which isn't ensured in all Mercurial
     versions.
     """
     if not util.safehasattr(obj, name):
         if ignoremissing:
             return None
         raise error.Abort((b"missing attribute %s of %s might break assumption"
                            b" of performance measurement") % (name, obj))
 
     origvalue = getattr(obj, _sysstr(name))
     class attrutil(object):
         def set(self, newvalue):
             setattr(obj, _sysstr(name), newvalue)
         def restore(self):
             setattr(obj, _sysstr(name), origvalue)
 
     return attrutil()
 
 # utilities to examine each internal API changes
 
 def getbranchmapsubsettable():
     # for "historical portability":
     # subsettable is defined in:
     # - branchmap since 2.9 (or 175c6fd8cacc)
     # - repoview since 2.5 (or 59a9f18d4587)
     for mod in (branchmap, repoview):
         subsettable = getattr(mod, 'subsettable', None)
         if subsettable:
             return subsettable
 
     # bisecting in bcee63733aad::59a9f18d4587 can reach here (both
     # branchmap and repoview modules exist, but subsettable attribute
     # doesn't)
     raise error.Abort((b"perfbranchmap not available with this Mercurial"),
                       hint=b"use 2.5 or later")
 
 def getsvfs(repo):
     """Return appropriate object to access files under .hg/store
     """
     # for "historical portability":
     # repo.svfs has been available since 2.3 (or 7034365089bf)
     svfs = getattr(repo, 'svfs', None)
     if svfs:
         return svfs
     else:
         return getattr(repo, 'sopener')
 
 def getvfs(repo):
     """Return appropriate object to access files under .hg
     """
     # for "historical portability":
     # repo.vfs has been available since 2.3 (or 7034365089bf)
     vfs = getattr(repo, 'vfs', None)
     if vfs:
         return vfs
     else:
         return getattr(repo, 'opener')
 
 def repocleartagscachefunc(repo):
     """Return the function to clear tags cache according to repo internal API
     """
     if util.safehasattr(repo, b'_tagscache'): # since 2.0 (or 9dca7653b525)
         # in this case, setattr(repo, '_tagscache', None) or so isn't
         # correct way to clear tags cache, because existing code paths
         # expect _tagscache to be a structured object.
         def clearcache():
             # _tagscache has been filteredpropertycache since 2.5 (or
             # 98c867ac1330), and delattr() can't work in such case
             if b'_tagscache' in vars(repo):
                 del repo.__dict__[b'_tagscache']
         return clearcache
 
     repotags = safeattrsetter(repo, b'_tags', ignoremissing=True)
     if repotags: # since 1.4 (or 5614a628d173)
         return lambda : repotags.set(None)
 
     repotagscache = safeattrsetter(repo, b'tagscache', ignoremissing=True)
     if repotagscache: # since 0.6 (or d7df759d0e97)
         return lambda : repotagscache.set(None)
 
     # Mercurial earlier than 0.6 (or d7df759d0e97) logically reaches
     # this point, but it isn't so problematic, because:
     # - repo.tags of such Mercurial isn't "callable", and repo.tags()
     #   in perftags() causes failure soon
     # - perf.py itself has been available since 1.1 (or eb240755386d)
     raise error.Abort((b"tags API of this hg command is unknown"))
 
 # utilities to clear cache
 
-def clearfilecache(repo, attrname):
-    unfi = repo.unfiltered()
-    if attrname in vars(unfi):
-        delattr(unfi, attrname)
-    unfi._filecache.pop(attrname, None)
+def clearfilecache(obj, attrname):
+    unfiltered = getattr(obj, 'unfiltered', None)
+    if unfiltered is not None:
+        obj = obj.unfiltered()
+    if attrname in vars(obj):
+        delattr(obj, attrname)
+    obj._filecache.pop(attrname, None)
+
+def clearchangelog(repo):
+    if repo is not repo.unfiltered():
+        object.__setattr__(repo, r'_clcachekey', None)
+        object.__setattr__(repo, r'_clcache', None)
+    clearfilecache(repo.unfiltered(), 'changelog')
 
 # perf commands
 
 @command(b'perfwalk', formatteropts)
 def perfwalk(ui, repo, *pats, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     m = scmutil.match(repo[None], pats, {})
     timer(lambda: len(list(repo.dirstate.walk(m, subrepos=[], unknown=True,
                                               ignored=False))))
     fm.end()
 
 @command(b'perfannotate', formatteropts)
 def perfannotate(ui, repo, f, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     fc = repo[b'.'][f]
     timer(lambda: len(fc.annotate(True)))
     fm.end()
 
 @command(b'perfstatus',
          [(b'u', b'unknown', False,
            b'ask status to look for unknown files')] + formatteropts)
 def perfstatus(ui, repo, **opts):
     opts = _byteskwargs(opts)
     #m = match.always(repo.root, repo.getcwd())
     #timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer, fm = gettimer(ui, opts)
     timer(lambda: sum(map(len, repo.status(unknown=opts[b'unknown']))))
     fm.end()
 
 @command(b'perfaddremove', formatteropts)
 def perfaddremove(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     try:
         oldquiet = repo.ui.quiet
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         opts[b'dry_run'] = True
         timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, b'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, b'_nodecache'):
         from mercurial.node import nullid, nullrev
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 @command(b'perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
     def d():
         len(cl.headrevs())
         clearcaches(cl)
     timer(d)
     fm.end()
 
-@command(b'perftags', formatteropts)
+@command(b'perftags', formatteropts+
+        [
+            (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
+        ])
 def perftags(ui, repo, **opts):
-    import mercurial.changelog
-    import mercurial.manifest
-
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
-    svfs = getsvfs(repo)
     repocleartagscache = repocleartagscachefunc(repo)
+    clearrevlogs = opts[b'clear_revlogs']
+    def s():
+        if clearrevlogs:
+            clearchangelog(repo)
+            clearfilecache(repo.unfiltered(), 'manifest')
+        repocleartagscache()
     def t():
-        repo.changelog = mercurial.changelog.changelog(svfs)
-        rootmanifest = mercurial.manifest.manifestrevlog(svfs)
-        repo.manifestlog = mercurial.manifest.manifestlog(svfs, repo,
-                                                          rootmanifest)
-        repocleartagscache()
         return len(repo.tags())
-    timer(t)
+    timer(t, setup=s)
     fm.end()
 
 @command(b'perfancestors', formatteropts)
 def perfancestors(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     heads = repo.changelog.headrevs()
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
     timer(d)
     fm.end()
 
 @command(b'perfancestorset', formatteropts)
 def perfancestorset(ui, repo, revset, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
     def d():
         s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
     timer(d)
     fm.end()
 
-@command(b'perfbookmarks', formatteropts)
+@command(b'perfdiscovery', formatteropts, b'PATH')
+def perfdiscovery(ui, repo, path, **opts):
+    """benchmark discovery between local repo and the peer at given path
+    """
+    repos = [repo, None]
+    timer, fm = gettimer(ui, opts)
+    path = ui.expandpath(path)
+
+    def s():
+        repos[1] = hg.peer(ui, opts, path)
+    def d():
+        setdiscovery.findcommonheads(ui, *repos)
+    timer(d, setup=s)
+    fm.end()
+
+@command(b'perfbookmarks', formatteropts +
+        [
+            (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
+        ])
 def perfbookmarks(ui, repo, **opts):
     """benchmark parsing bookmarks from disk to memory"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
-    def d():
+
+    clearrevlogs = opts[b'clear_revlogs']
+    def s():
+        if clearrevlogs:
+            clearchangelog(repo)
         clearfilecache(repo, b'_bookmarks')
+    def d():
         repo._bookmarks
-    timer(d)
+    timer(d, setup=s)
     fm.end()
 
 @command(b'perfbundleread', formatteropts, b'BUNDLE')
 def perfbundleread(ui, repo, bundlepath, **opts):
     """Benchmark reading of bundle files.
 
     This command is meant to isolate the I/O part of bundle reading as
     much as possible.
     """
     from mercurial import (
         bundle2,
         exchange,
         streamclone,
     )
 
     opts = _byteskwargs(opts)
 
     def makebench(fn):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 fn(bundle)
 
         return run
 
     def makereadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 while bundle.read(size):
                     pass
 
         return run
 
     def makestdioread(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 while fh.read(size):
                     pass
 
         return run
 
     # bundle1
 
     def deltaiter(bundle):
         for delta in bundle.deltaiter():
             pass
 
     def iterchunks(bundle):
         for chunk in bundle.getchunks():
             pass
 
     # bundle2
 
     def forwardchunks(bundle):
         for chunk in bundle._forwardchunks():
             pass
 
     def iterparts(bundle):
         for part in bundle.iterparts():
             pass
 
     def iterpartsseekable(bundle):
         for part in bundle.iterparts(seekable=True):
             pass
 
     def seek(bundle):
         for part in bundle.iterparts(seekable=True):
             part.seek(0, os.SEEK_END)
 
     def makepartreadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 for part in bundle.iterparts():
                     while part.read(size):
                         pass
 
         return run
 
     benches = [
         (makestdioread(8192), b'read(8k)'),
         (makestdioread(16384), b'read(16k)'),
         (makestdioread(32768), b'read(32k)'),
         (makestdioread(131072), b'read(128k)'),
     ]
 
     with open(bundlepath, b'rb') as fh:
         bundle = exchange.readbundle(ui, fh, bundlepath)
 
         if isinstance(bundle, changegroup.cg1unpacker):
             benches.extend([
                 (makebench(deltaiter), b'cg1 deltaiter()'),
                 (makebench(iterchunks), b'cg1 getchunks()'),
                 (makereadnbytes(8192), b'cg1 read(8k)'),
                 (makereadnbytes(16384), b'cg1 read(16k)'),
                 (makereadnbytes(32768), b'cg1 read(32k)'),
                 (makereadnbytes(131072), b'cg1 read(128k)'),
             ])
         elif isinstance(bundle, bundle2.unbundle20):
             benches.extend([
                 (makebench(forwardchunks), b'bundle2 forwardchunks()'),
                 (makebench(iterparts), b'bundle2 iterparts()'),
                 (makebench(iterpartsseekable), b'bundle2 iterparts() seekable'),
                 (makebench(seek), b'bundle2 part seek()'),
                 (makepartreadnbytes(8192), b'bundle2 part read(8k)'),
                 (makepartreadnbytes(16384), b'bundle2 part read(16k)'),
                 (makepartreadnbytes(32768), b'bundle2 part read(32k)'),
                 (makepartreadnbytes(131072), b'bundle2 part read(128k)'),
             ])
         elif isinstance(bundle, streamclone.streamcloneapplier):
             raise error.Abort(b'stream clone bundles not supported')
         else:
             raise error.Abort(b'unhandled bundle type: %s' % type(bundle))
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 @command(b'perfchangegroupchangelog', formatteropts +
-         [(b'', b'version', b'02', b'changegroup version'),
+         [(b'', b'cgversion', b'02', b'changegroup version'),
           (b'r', b'rev', b'', b'revisions to add to changegroup')])
-def perfchangegroupchangelog(ui, repo, version=b'02', rev=None, **opts):
+def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
     """Benchmark producing a changelog group for a changegroup.
 
     This measures the time spent processing the changelog during a
     bundle operation. This occurs during `hg bundle` and on a server
     processing a `getbundle` wire protocol request (handles clones
     and pull requests).
 
     By default, all revisions are added to the changegroup.
     """
     opts = _byteskwargs(opts)
     cl = repo.changelog
     nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
-    bundler = changegroup.getbundler(version, repo)
+    bundler = changegroup.getbundler(cgversion, repo)
 
     def d():
         state, chunks = bundler._generatechangelog(cl, nodes)
         for chunk in chunks:
             pass
 
     timer, fm = gettimer(ui, opts)
 
     # Terminal printing can interfere with timing. So disable it.
     with ui.configoverride({(b'progress', b'disable'): True}):
         timer(d)
 
     fm.end()
 
 @command(b'perfdirs', formatteropts)
 def perfdirs(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
     def d():
         dirstate.hasdir(b'a')
         del dirstate._map._dirs
     timer(d)
     fm.end()
 
 @command(b'perfdirstate', formatteropts)
 def perfdirstate(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     b"a" in repo.dirstate
     def d():
         repo.dirstate.invalidate()
         b"a" in repo.dirstate
     timer(d)
     fm.end()
 
 @command(b'perfdirstatedirs', formatteropts)
 def perfdirstatedirs(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     b"a" in repo.dirstate
     def d():
         repo.dirstate.hasdir(b"a")
         del repo.dirstate._map._dirs
     timer(d)
     fm.end()
 
 @command(b'perfdirstatefoldmap', formatteropts)
 def perfdirstatefoldmap(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
     def d():
         dirstate._map.filefoldmap.get(b'a')
         del dirstate._map.filefoldmap
     timer(d)
     fm.end()
 
 @command(b'perfdirfoldmap', formatteropts)
 def perfdirfoldmap(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
     def d():
         dirstate._map.dirfoldmap.get(b'a')
         del dirstate._map.dirfoldmap
         del dirstate._map._dirs
     timer(d)
     fm.end()
 
 @command(b'perfdirstatewrite', formatteropts)
 def perfdirstatewrite(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ds = repo.dirstate
     b"a" in ds
     def d():
         ds._dirty = True
         ds.write(repo.currenttransaction())
     timer(d)
     fm.end()
 
 @command(b'perfmergecalculate',
          [(b'r', b'rev', b'.', b'rev to merge against')] + formatteropts)
 def perfmergecalculate(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     wctx = repo[None]
     rctx = scmutil.revsingle(repo, rev, rev)
     ancestor = wctx.ancestor(rctx)
     # we don't want working dir files to be stat'd in the benchmark, so prime
     # that cache
     wctx.dirty()
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         merge.calculateupdates(repo, wctx, rctx, [ancestor], False, False,
                                acceptremote=True, followcopies=True)
     timer(d)
     fm.end()
 
 @command(b'perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
+    """benchmark the copy tracing logic"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
     ctx2 = scmutil.revsingle(repo, rev2, rev2)
     def d():
         copies.pathcopies(ctx1, ctx2)
     timer(d)
     fm.end()
 
 @command(b'perfphases',
          [(b'', b'full', False, b'include file reading time too'),
           ], b"")
 def perfphases(ui, repo, **opts):
     """benchmark phasesets computation"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     _phases = repo._phasecache
     full = opts.get(b'full')
     def d():
         phases = _phases
         if full:
             clearfilecache(repo, b'_phasecache')
             phases = repo._phasecache
         phases.invalidate()
         phases.loadphaserevs(repo)
     timer(d)
     fm.end()
 
 @command(b'perfphasesremote',
          [], b"[DEST]")
 def perfphasesremote(ui, repo, dest=None, **opts):
     """benchmark time needed to analyse phases of the remote server"""
     from mercurial.node import (
         bin,
     )
     from mercurial import (
         exchange,
         hg,
         phases,
     )
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     path = ui.paths.getpath(dest, default=(b'default-push', b'default'))
     if not path:
         raise error.Abort((b'default repository not configured!'),
                           hint=(b"see 'hg help config.paths'"))
     dest = path.pushloc or path.loc
     branches = (path.branch, opts.get(b'branch') or [])
     ui.status((b'analysing phase of %s\n') % util.hidepassword(dest))
     revs, checkout = hg.addbranchrevs(repo, repo, branches, opts.get(b'rev'))
     other = hg.peer(repo, opts, dest)
 
     # easier to perform discovery through the operation
     op = exchange.pushoperation(repo, other)
     exchange._pushdiscoverychangeset(op)
 
     remotesubset = op.fallbackheads
 
     with other.commandexecutor() as e:
         remotephases = e.callcommand(b'listkeys',
                        {b'namespace': b'phases'}).result()
     del other
     publishing = remotephases.get(b'publishing', False)
     if publishing:
         ui.status((b'publishing: yes\n'))
     else:
         ui.status((b'publishing: no\n'))
 
     nodemap = repo.changelog.nodemap
     nonpublishroots = 0
     for nhex, phase in remotephases.iteritems():
         if nhex == b'publishing': # ignore data related to publish option
             continue
         node = bin(nhex)
         if node in nodemap and int(phase):
             nonpublishroots += 1
     ui.status((b'number of roots: %d\n') % len(remotephases))
     ui.status((b'number of known non public roots: %d\n') % nonpublishroots)
     def d():
         phases.remotephasessummary(repo,
                                    remotesubset,
                                    remotephases)
     timer(d)
     fm.end()
 
 @command(b'perfmanifest',[
             (b'm', b'manifest-rev', False, b'Look up a manifest node revision'),
             (b'', b'clear-disk', False, b'clear on-disk caches too'),
          ] + formatteropts, b'REV|NODE')
 def perfmanifest(ui, repo, rev, manifest_rev=False, clear_disk=False, **opts):
     """benchmark the time to read a manifest from disk and return a usable
     dict-like object
 
     Manifest caches are cleared before retrieval."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     if not manifest_rev:
         ctx = scmutil.revsingle(repo, rev, rev)
         t = ctx.manifestnode()
     else:
         from mercurial.node import bin
 
         if len(rev) == 40:
             t = bin(rev)
         else:
             try:
                 rev = int(rev)
 
                 if util.safehasattr(repo.manifestlog, b'getstorage'):
                     t = repo.manifestlog.getstorage(b'').node(rev)
                 else:
                     t = repo.manifestlog._revlog.lookup(rev)
             except ValueError:
                 raise error.Abort(b'manifest revision must be integer or full '
                                   b'node')
     def d():
         repo.manifestlog.clearcaches(clear_persisted_data=clear_disk)
         repo.manifestlog[t].read()
     timer(d)
     fm.end()
 
 @command(b'perfchangeset', formatteropts)
 def perfchangeset(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     n = scmutil.revsingle(repo, rev).node()
     def d():
         repo.changelog.read(n)
         #repo.changelog._cache = None
     timer(d)
     fm.end()
 
-@command(b'perfindex', formatteropts)
+@command(b'perfignore', formatteropts)
+def perfignore(ui, repo, **opts):
+    """benchmark operation related to computing ignore"""
+    opts = _byteskwargs(opts)
+    timer, fm = gettimer(ui, opts)
+    dirstate = repo.dirstate
+
+    def setupone():
+        dirstate.invalidate()
+        clearfilecache(dirstate, b'_ignore')
+
+    def runone():
+        dirstate._ignore
+
+    timer(runone, setup=setupone, title=b"load")
+    fm.end()
+
+@command(b'perfindex', [
+            (b'', b'rev', b'', b'revision to be looked up (default tip)'),
+         ] + formatteropts)
 def perfindex(ui, repo, **opts):
     import mercurial.revlog
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
-    n = repo[b"tip"].node()
-    svfs = getsvfs(repo)
+    if opts[b'rev'] is None:
+        n = repo[b"tip"].node()
+    else:
+        rev = scmutil.revsingle(repo, opts[b'rev'])
+        n = repo[rev].node()
+
+    unfi = repo.unfiltered()
+    # find the filecache func directly
+    # This avoid polluting the benchmark with the filecache logic
+    makecl = unfi.__class__.changelog.func
+    def setup():
+        # probably not necessary, but for good measure
+        clearchangelog(unfi)
     def d():
-        cl = mercurial.revlog.revlog(svfs, b"00changelog.i")
+        cl = makecl(unfi)
         cl.rev(n)
-    timer(d)
+    timer(d, setup=setup)
     fm.end()
 
 @command(b'perfstartup', formatteropts)
 def perfstartup(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     def d():
         if os.name != r'nt':
             os.system(b"HGRCPATH= %s version -q > /dev/null" %
                       fsencode(sys.argv[0]))
         else:
             os.environ[r'HGRCPATH'] = r' '
             os.system(r"%s version -q > NUL" % sys.argv[0])
     timer(d)
     fm.end()
 
 @command(b'perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
     # experimental config: perf.parentscount
     count = getint(ui, b"perf", b"parentscount", 1000)
     if len(repo.changelog) < count:
         raise error.Abort(b"repo needs %d commits for this test" % count)
     repo = repo.unfiltered()
     nl = [repo.changelog.node(i) for i in _xrange(count)]
     def d():
         for n in nl:
             repo.changelog.parents(n)
     timer(d)
     fm.end()
 
 @command(b'perfctxfiles', formatteropts)
 def perfctxfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
     def d():
         len(repo[x].files())
     timer(d)
     fm.end()
 
 @command(b'perfrawfiles', formatteropts)
 def perfrawfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
     def d():
         len(cl.read(x)[3])
     timer(d)
     fm.end()
 
 @command(b'perflookup', formatteropts)
 def perflookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: len(repo.lookup(rev)))
     fm.end()
 
 @command(b'perflinelogedits',
          [(b'n', b'edits', 10000, b'number of edits'),
           (b'', b'max-hunk-lines', 10, b'max lines in a hunk'),
           ], norepo=True)
 def perflinelogedits(ui, **opts):
     from mercurial import linelog
 
     opts = _byteskwargs(opts)
 
     edits = opts[b'edits']
     maxhunklines = opts[b'max_hunk_lines']
 
     maxb1 = 100000
     random.seed(0)
     randint = random.randint
     currentlines = 0
     arglist = []
     for rev in _xrange(edits):
         a1 = randint(0, currentlines)
         a2 = randint(a1, min(currentlines, a1 + maxhunklines))
         b1 = randint(0, maxb1)
         b2 = randint(b1, b1 + maxhunklines)
         currentlines += (b2 - b1) - (a2 - a1)
         arglist.append((rev, a1, a2, b1, b2))
 
     def d():
         ll = linelog.linelog()
         for args in arglist:
             ll.replacelines(*args)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 @command(b'perfrevrange', formatteropts)
 def perfrevrange(ui, repo, *specs, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
     fm.end()
 
 @command(b'perfnodelookup', formatteropts)
 def perfnodelookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     import mercurial.revlog
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
     n = scmutil.revsingle(repo, rev).node()
     cl = mercurial.revlog.revlog(getsvfs(repo), b"00changelog.i")
     def d():
         cl.rev(n)
         clearcaches(cl)
     timer(d)
     fm.end()
 
 @command(b'perflog',
          [(b'', b'rename', False, b'ask log to follow renames')
          ] + formatteropts)
 def perflog(ui, repo, rev=None, **opts):
     opts = _byteskwargs(opts)
     if rev is None:
         rev=[]
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(lambda: commands.log(ui, repo, rev=rev, date=b'', user=b'',
                                copies=opts.get(b'rename')))
     ui.popbuffer()
     fm.end()
 
 @command(b'perfmoonwalk', formatteropts)
 def perfmoonwalk(ui, repo, **opts):
     """benchmark walking the changelog backwards
 
     This also loads the changelog data for each revision in the changelog.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     def moonwalk():
         for i in repo.changelog.revs(start=(len(repo) - 1), stop=-1):
             ctx = repo[i]
             ctx.branch() # read changelog data (in addition to the index)
     timer(moonwalk)
     fm.end()
 
 @command(b'perftemplating',
          [(b'r', b'rev', [], b'revisions to run the template on'),
           ] + formatteropts)
 def perftemplating(ui, repo, testedtemplate=None, **opts):
     """test the rendering time of a given template"""
     if makelogtemplater is None:
         raise error.Abort((b"perftemplating not available with this Mercurial"),
                           hint=b"use 4.3 or later")
 
     opts = _byteskwargs(opts)
 
     nullui = ui.copy()
     nullui.fout = open(os.devnull, r'wb')
     nullui.disablepager()
     revs = opts.get(b'rev')
     if not revs:
         revs = [b'all()']
     revs = list(scmutil.revrange(repo, revs))
 
     defaulttemplate = (b'{date|shortdate} [{rev}:{node|short}]'
                        b' {author|person}: {desc|firstline}\n')
     if testedtemplate is None:
         testedtemplate = defaulttemplate
     displayer = makelogtemplater(nullui, repo, testedtemplate)
     def format():
         for r in revs:
             ctx = repo[r]
             displayer.show(ctx)
             displayer.flush(ctx)
 
     timer, fm = gettimer(ui, opts)
     timer(format)
     fm.end()
 
+@command(b'perfhelper-pathcopies', formatteropts +
+         [
+          (b'r', b'revs', [], b'restrict search to these revisions'),
+          (b'', b'timing', False, b'provides extra data (costly)'),
+         ])
+def perfhelperpathcopies(ui, repo, revs=[], **opts):
+    """find statistic about potential parameters for the `perftracecopies`
+
+    This command find source-destination pair relevant for copytracing testing.
+    It report value for some of the parameters that impact copy tracing time.
+
+    If `--timing` is set, rename detection is run and the associated timing
+    will be reported. The extra details comes at the cost of a slower command
+    execution.
+
+    Since the rename detection is only run once, other factors might easily
+    affect the precision of the timing. However it should give a good
+    approximation of which revision pairs are very costly.
+    """
+    opts = _byteskwargs(opts)
+    fm = ui.formatter(b'perf', opts)
+    dotiming = opts[b'timing']
+
+    if dotiming:
+        header = '%12s %12s %12s %12s %12s %12s\n'
+        output = ("%(source)12s %(destination)12s "
+                  "%(nbrevs)12d %(nbmissingfiles)12d "
+                  "%(nbrenamedfiles)12d %(time)18.5f\n")
+        header_names = ("source", "destination", "nb-revs", "nb-files",
+                        "nb-renames", "time")
+        fm.plain(header % header_names)
+    else:
+        header = '%12s %12s %12s %12s\n'
+        output = ("%(source)12s %(destination)12s "
+                  "%(nbrevs)12d %(nbmissingfiles)12d\n")
+        fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
+
+    if not revs:
+        revs = ['all()']
+    revs = scmutil.revrange(repo, revs)
+
+    roi = repo.revs('merge() and %ld', revs)
+    for r in roi:
+        ctx = repo[r]
+        p1 = ctx.p1().rev()
+        p2 = ctx.p2().rev()
+        bases = repo.changelog._commonancestorsheads(p1, p2)
+        for p in (p1, p2):
+            for b in bases:
+                base = repo[b]
+                parent = repo[p]
+                missing = copies._computeforwardmissing(base, parent)
+                if not missing:
+                    continue
+                data = {
+                    b'source': base.hex(),
+                    b'destination': parent.hex(),
+                    b'nbrevs': len(repo.revs('%d::%d', b, p)),
+                    b'nbmissingfiles': len(missing),
+                }
+                if dotiming:
+                    begin = util.timer()
+                    renames = copies.pathcopies(base, parent)
+                    end = util.timer()
+                    # not very stable timing since we did only one run
+                    data['time'] = end - begin
+                    data['nbrenamedfiles'] = len(renames)
+                fm.startitem()
+                fm.data(**data)
+                out = data.copy()
+                out['source'] = fm.hexfunc(base.node())
+                out['destination'] = fm.hexfunc(parent.node())
+                fm.plain(output % out)
+
+    fm.end()
+
 @command(b'perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
     fm.end()
 
 @command(b'perffncacheload', formatteropts)
 def perffncacheload(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     def d():
         s.fncache._load()
     timer(d)
     fm.end()
 
 @command(b'perffncachewrite', formatteropts)
 def perffncachewrite(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     lock = repo.lock()
     s.fncache._load()
     tr = repo.transaction(b'perffncachewrite')
     tr.addbackup(b'fncache')
     def d():
         s.fncache._dirty = True
         s.fncache.write(tr)
     timer(d)
     tr.close()
     lock.release()
     fm.end()
 
 @command(b'perffncacheencode', formatteropts)
 def perffncacheencode(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
     def d():
         for p in s.fncache.entries:
             s.encode(p)
     timer(d)
     fm.end()
 
 def _bdiffworker(q, blocks, xdiff, ready, done):
     while not done.is_set():
         pair = q.get()
         while pair is not None:
             if xdiff:
                 mdiff.bdiff.xdiffblocks(*pair)
             elif blocks:
                 mdiff.bdiff.blocks(*pair)
             else:
                 mdiff.textdiff(*pair)
             q.task_done()
             pair = q.get()
         q.task_done() # for the None one
         with ready:
             ready.wait()
 
 def _manifestrevision(repo, mnode):
     ml = repo.manifestlog
 
     if util.safehasattr(ml, b'getstorage'):
         store = ml.getstorage(b'')
     else:
         store = ml._revlog
 
     return store.revision(mnode)
 
 @command(b'perfbdiff', revlogopts + formatteropts + [
     (b'', b'count', 1, b'number of revisions to test (when using --startrev)'),
     (b'', b'alldata', False, b'test bdiffs for all associated revisions'),
     (b'', b'threads', 0, b'number of thread to use (disable with 0)'),
     (b'', b'blocks', False, b'test computing diffs into blocks'),
     (b'', b'xdiff', False, b'use xdiff algorithm'),
     ],
 
     b'-c|-m|FILE REV')
 def perfbdiff(ui, repo, file_, rev=None, count=None, threads=0, **opts):
     """benchmark a bdiff between revisions
 
     By default, benchmark a bdiff between its delta parent and itself.
 
     With ``--count``, benchmark bdiffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure bdiffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
 
     if opts[b'xdiff'] and not opts[b'blocks']:
         raise error.CommandError(b'perfbdiff', b'--xdiff requires --blocks')
 
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfbdiff', b'invalid arguments')
 
     blocks = opts[b'blocks']
     xdiff = opts[b'xdiff']
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfbdiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     withthreads = threads > 0
     if not withthreads:
         def d():
             for pair in textpairs:
                 if xdiff:
                     mdiff.bdiff.xdiffblocks(*pair)
                 elif blocks:
                     mdiff.bdiff.blocks(*pair)
                 else:
                     mdiff.textdiff(*pair)
     else:
         q = queue()
         for i in _xrange(threads):
             q.put(None)
         ready = threading.Condition()
         done = threading.Event()
         for i in _xrange(threads):
             threading.Thread(target=_bdiffworker,
                              args=(q, blocks, xdiff, ready, done)).start()
         q.join()
         def d():
             for pair in textpairs:
                 q.put(pair)
             for i in _xrange(threads):
                 q.put(None)
             with ready:
                 ready.notify_all()
             q.join()
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
     if withthreads:
         done.set()
         for i in _xrange(threads):
             q.put(None)
         with ready:
             ready.notify_all()
 
 @command(b'perfunidiff', revlogopts + formatteropts + [
     (b'', b'count', 1, b'number of revisions to test (when using --startrev)'),
     (b'', b'alldata', False, b'test unidiffs for all associated revisions'),
     ], b'-c|-m|FILE REV')
 def perfunidiff(ui, repo, file_, rev=None, count=None, **opts):
     """benchmark a unified diff between revisions
 
     This doesn't include any copy tracing - it's just a unified diff
     of the texts.
 
     By default, benchmark a diff between its delta parent and itself.
 
     With ``--count``, benchmark diffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure diffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfunidiff', b'invalid arguments')
 
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfunidiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     def d():
         for left, right in textpairs:
             # The date strings don't matter, so we pass empty strings.
             headerlines, hunks = mdiff.unidiff(
                 left, b'', right, b'', b'left', b'right', binary=False)
             # consume iterators in roughly the way patch.py does
             b'\n'.join(headerlines)
             b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 @command(b'perfdiffwd', formatteropts)
 def perfdiffwd(ui, repo, **opts):
     """Profile diff of working directory changes"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
         }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
         opts = dict((options[c], b'1') for c in diffopt)
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
         diffopt = diffopt.encode('ascii')
         title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
-        timer(d, title)
+        timer(d, title=title)
     fm.end()
 
 @command(b'perfrevlogindex', revlogopts + formatteropts,
          b'-c|-m|FILE')
 def perfrevlogindex(ui, repo, file_=None, **opts):
     """Benchmark operations against a revlog index.
 
     This tests constructing a revlog instance, reading index data,
     parsing index data, and performing various operations related to
     index data.
     """
 
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
 
     opener = getattr(rl, 'opener')  # trick linter
     indexfile = rl.indexfile
     data = opener.read(indexfile)
 
     header = struct.unpack(b'>I', data[0:4])[0]
     version = header & 0xFFFF
     if version == 1:
         revlogio = revlog.revlogio()
         inline = header & (1 << 16)
     else:
         raise error.Abort((b'unsupported revlog version: %d') % version)
 
     rllen = len(rl)
 
     node0 = rl.node(0)
     node25 = rl.node(rllen // 4)
     node50 = rl.node(rllen // 2)
     node75 = rl.node(rllen // 4 * 3)
     node100 = rl.node(rllen - 1)
 
     allrevs = range(rllen)
     allrevsrev = list(reversed(allrevs))
     allnodes = [rl.node(rev) for rev in range(rllen)]
     allnodesrev = list(reversed(allnodes))
 
     def constructor():
         revlog.revlog(opener, indexfile)
 
     def read():
         with opener(indexfile) as fh:
             fh.read()
 
     def parseindex():
         revlogio.parseindex(data, inline)
 
     def getentry(revornode):
         index = revlogio.parseindex(data, inline)[0]
         index[revornode]
 
     def getentries(revs, count=1):
         index = revlogio.parseindex(data, inline)[0]
 
         for i in range(count):
             for rev in revs:
                 index[rev]
 
     def resolvenode(node):
         nodemap = revlogio.parseindex(data, inline)[1]
         # This only works for the C code.
         if nodemap is None:
             return
 
         try:
             nodemap[node]
         except error.RevlogError:
             pass
 
     def resolvenodes(nodes, count=1):
         nodemap = revlogio.parseindex(data, inline)[1]
         if nodemap is None:
             return
 
         for i in range(count):
             for node in nodes:
                 try:
                     nodemap[node]
                 except error.RevlogError:
                     pass
 
     benches = [
         (constructor, b'revlog constructor'),
         (read, b'read'),
         (parseindex, b'create index object'),
         (lambda: getentry(0), b'retrieve index entry for rev 0'),
         (lambda: resolvenode(b'a' * 20), b'look up missing node'),
         (lambda: resolvenode(node0), b'look up node at rev 0'),
         (lambda: resolvenode(node25), b'look up node at 1/4 len'),
         (lambda: resolvenode(node50), b'look up node at 1/2 len'),
         (lambda: resolvenode(node75), b'look up node at 3/4 len'),
         (lambda: resolvenode(node100), b'look up node at tip'),
         # 2x variation is to measure caching impact.
         (lambda: resolvenodes(allnodes),
          b'look up all nodes (forward)'),
         (lambda: resolvenodes(allnodes, 2),
          b'look up all nodes 2x (forward)'),
         (lambda: resolvenodes(allnodesrev),
          b'look up all nodes (reverse)'),
         (lambda: resolvenodes(allnodesrev, 2),
          b'look up all nodes 2x (reverse)'),
         (lambda: getentries(allrevs),
          b'retrieve all index entries (forward)'),
         (lambda: getentries(allrevs, 2),
          b'retrieve all index entries 2x (forward)'),
         (lambda: getentries(allrevsrev),
          b'retrieve all index entries (reverse)'),
         (lambda: getentries(allrevsrev, 2),
          b'retrieve all index entries 2x (reverse)'),
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 @command(b'perfrevlogrevisions', revlogopts + formatteropts +
          [(b'd', b'dist', 100, b'distance between the revisions'),
           (b's', b'startrev', 0, b'revision to start reading at'),
           (b'', b'reverse', False, b'read in reverse')],
          b'-c|-m|FILE')
 def perfrevlogrevisions(ui, repo, file_=None, startrev=0, reverse=False,
                         **opts):
     """Benchmark reading a series of revisions from a revlog.
 
     By default, we read every ``-d/--dist`` revision from 0 to tip of
     the specified revlog.
 
     The start revision can be defined via ``-s/--startrev``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogrevisions', file_, opts)
     rllen = getlen(ui)(rl)
 
     if startrev < 0:
         startrev = rllen + startrev
 
     def d():
         rl.clearcaches()
 
         beginrev = startrev
         endrev = rllen
         dist = opts[b'dist']
 
         if reverse:
-            beginrev, endrev = endrev, beginrev
+            beginrev, endrev = endrev - 1, beginrev - 1
             dist = -1 * dist
 
         for x in _xrange(beginrev, endrev, dist):
             # Old revisions don't support passing int.
             n = rl.node(x)
             rl.revision(n)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
+@command(b'perfrevlogwrite', revlogopts + formatteropts +
+         [(b's', b'startrev', 1000, b'revision to start writing at'),
+          (b'', b'stoprev', -1, b'last revision to write'),
+          (b'', b'count', 3, b'last revision to write'),
+          (b'', b'details', False, b'print timing for every revisions tested'),
+          (b'', b'source', b'full', b'the kind of data feed in the revlog'),
+          (b'', b'lazydeltabase', True, b'try the provided delta first'),
+          (b'', b'clear-caches', True, b'clear revlog cache between calls'),
+         ],
+         b'-c|-m|FILE')
+def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
+    """Benchmark writing a series of revisions to a revlog.
+
+    Possible source values are:
+    * `full`: add from a full text (default).
+    * `parent-1`: add from a delta to the first parent
+    * `parent-2`: add from a delta to the second parent if it exists
+                  (use a delta from the first parent otherwise)
+    * `parent-smallest`: add from the smallest delta (either p1 or p2)
+    * `storage`: add from the existing precomputed deltas
+    """
+    opts = _byteskwargs(opts)
+
+    rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
+    rllen = getlen(ui)(rl)
+    if startrev < 0:
+        startrev = rllen + startrev
+    if stoprev < 0:
+        stoprev = rllen + stoprev
+
+    lazydeltabase = opts['lazydeltabase']
+    source = opts['source']
+    clearcaches = opts['clear_caches']
+    validsource = (b'full', b'parent-1', b'parent-2', b'parent-smallest',
+                   b'storage')
+    if source not in validsource:
+        raise error.Abort('invalid source type: %s' % source)
+
+    ### actually gather results
+    count = opts['count']
+    if count <= 0:
+        raise error.Abort('invalide run count: %d' % count)
+    allresults = []
+    for c in range(count):
+        timing = _timeonewrite(ui, rl, source, startrev, stoprev, c + 1,
+                               lazydeltabase=lazydeltabase,
+                               clearcaches=clearcaches)
+        allresults.append(timing)
+
+    ### consolidate the results in a single list
+    results = []
+    for idx, (rev, t) in enumerate(allresults[0]):
+        ts = [t]
+        for other in allresults[1:]:
+            orev, ot = other[idx]
+            assert orev == rev
+            ts.append(ot)
+        results.append((rev, ts))
+    resultcount = len(results)
+
+    ### Compute and display relevant statistics
+
+    # get a formatter
+    fm = ui.formatter(b'perf', opts)
+    displayall = ui.configbool(b"perf", b"all-timing", False)
+
+    # print individual details if requested
+    if opts['details']:
+        for idx, item in enumerate(results, 1):
+            rev, data = item
+            title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
+            formatone(fm, data, title=title, displayall=displayall)
+
+    # sorts results by median time
+    results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
+    # list of (name, index) to display)
+    relevants = [
+        ("min", 0),
+        ("10%", resultcount * 10 // 100),
+        ("25%", resultcount * 25 // 100),
+        ("50%", resultcount * 70 // 100),
+        ("75%", resultcount * 75 // 100),
+        ("90%", resultcount * 90 // 100),
+        ("95%", resultcount * 95 // 100),
+        ("99%", resultcount * 99 // 100),
+        ("99.9%", resultcount * 999 // 1000),
+        ("99.99%", resultcount * 9999 // 10000),
+        ("99.999%", resultcount * 99999 // 100000),
+        ("max", -1),
+    ]
+    if not ui.quiet:
+        for name, idx in relevants:
+            data = results[idx]
+            title = '%s of %d, rev %d' % (name, resultcount, data[0])
+            formatone(fm, data[1], title=title, displayall=displayall)
+
+    # XXX summing that many float will not be very precise, we ignore this fact
+    # for now
+    totaltime = []
+    for item in allresults:
+        totaltime.append((sum(x[1][0] for x in item),
+                          sum(x[1][1] for x in item),
+                          sum(x[1][2] for x in item),)
+        )
+    formatone(fm, totaltime, title="total time (%d revs)" % resultcount,
+              displayall=displayall)
+    fm.end()
+
+class _faketr(object):
+    def add(s, x, y, z=None):
+        return None
+
+def _timeonewrite(ui, orig, source, startrev, stoprev, runidx=None,
+                  lazydeltabase=True, clearcaches=True):
+    timings = []
+    tr = _faketr()
+    with _temprevlog(ui, orig, startrev) as dest:
+        dest._lazydeltabase = lazydeltabase
+        revs = list(orig.revs(startrev, stoprev))
+        total = len(revs)
+        topic = 'adding'
+        if runidx is not None:
+            topic += ' (run #%d)' % runidx
+         # Support both old and new progress API
+        if util.safehasattr(ui, 'makeprogress'):
+            progress = ui.makeprogress(topic, unit='revs', total=total)
+            def updateprogress(pos):
+                progress.update(pos)
+            def completeprogress():
+                progress.complete()
+        else:
+            def updateprogress(pos):
+                ui.progress(topic, pos, unit='revs', total=total)
+            def completeprogress():
+                ui.progress(topic, None, unit='revs', total=total)
+
+        for idx, rev in enumerate(revs):
+            updateprogress(idx)
+            addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
+            if clearcaches:
+                dest.index.clearcaches()
+                dest.clearcaches()
+            with timeone() as r:
+                dest.addrawrevision(*addargs, **addkwargs)
+            timings.append((rev, r[0]))
+        updateprogress(total)
+        completeprogress()
+    return timings
+
+def _getrevisionseed(orig, rev, tr, source):
+    from mercurial.node import nullid
+
+    linkrev = orig.linkrev(rev)
+    node = orig.node(rev)
+    p1, p2 = orig.parents(node)
+    flags = orig.flags(rev)
+    cachedelta = None
+    text = None
+
+    if source == b'full':
+        text = orig.revision(rev)
+    elif source == b'parent-1':
+        baserev = orig.rev(p1)
+        cachedelta = (baserev, orig.revdiff(p1, rev))
+    elif source == b'parent-2':
+        parent = p2
+        if p2 == nullid:
+            parent = p1
+        baserev = orig.rev(parent)
+        cachedelta = (baserev, orig.revdiff(parent, rev))
+    elif source == b'parent-smallest':
+        p1diff = orig.revdiff(p1, rev)
+        parent = p1
+        diff = p1diff
+        if p2 != nullid:
+            p2diff = orig.revdiff(p2, rev)
+            if len(p1diff) > len(p2diff):
+                parent = p2
+                diff = p2diff
+        baserev = orig.rev(parent)
+        cachedelta = (baserev, diff)
+    elif source == b'storage':
+        baserev = orig.deltaparent(rev)
+        cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
+
+    return ((text, tr, linkrev, p1, p2),
+            {'node': node, 'flags': flags, 'cachedelta': cachedelta})
+
[e-mail]
+def _temprevlog(ui, orig, truncaterev):
+    from mercurial import vfs as vfsmod
+
+    if orig._inline:
+        raise error.Abort('not supporting inline revlog (yet)')
+
+    origindexpath = orig.opener.join(orig.indexfile)
+    origdatapath = orig.opener.join(orig.datafile)
+    indexname = 'revlog.i'
+    dataname = 'revlog.d'
+
+    tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
+    try:
+        # copy the data file in a temporary directory
+        ui.debug('copying data in %s\n' % tmpdir)
+        destindexpath = os.path.join(tmpdir, 'revlog.i')
+        destdatapath = os.path.join(tmpdir, 'revlog.d')
+        shutil.copyfile(origindexpath, destindexpath)
+        shutil.copyfile(origdatapath, destdatapath)
+
+        # remove the data we want to add again
+        ui.debug('truncating data to be rewritten\n')
+        with open(destindexpath, 'ab') as index:
+            index.seek(0)
+            index.truncate(truncaterev * orig._io.size)
+        with open(destdatapath, 'ab') as data:
+            data.seek(0)
+            data.truncate(orig.start(truncaterev))
+
+        # instantiate a new revlog from the temporary copy
+        ui.debug('truncating adding to be rewritten\n')
+        vfs = vfsmod.vfs(tmpdir)
+        vfs.options = getattr(orig.opener, 'options', None)
+
+        dest = revlog.revlog(vfs,
+                             indexfile=indexname,
+                             datafile=dataname)
+        if dest._inline:
+            raise error.Abort('not supporting inline revlog (yet)')
+        # make sure internals are initialized
+        dest.revision(len(dest) - 1)
+        yield dest
+        del dest, vfs
+    finally:
+        shutil.rmtree(tmpdir, True)
+
 @command(b'perfrevlogchunks', revlogopts + formatteropts +
          [(b'e', b'engines', b'', b'compression engines to use'),
           (b's', b'startrev', 0, b'revision to start at')],
          b'-c|-m|FILE')
 def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
     stored within each revlog are "chunks" of possibly compressed data. This
     data needs to be read and decompressed or compressed and written.
 
     This command measures the time it takes to read+decompress and recompress
     chunks in a revlog. It effectively isolates I/O and compression performance.
     For measurements of higher-level operations like resolving revisions,
     see ``perfrevlogrevisions`` and ``perfrevlogrevision``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogchunks', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = rl._getsegmentforrevs
     except AttributeError:
         segmentforrevs = rl._chunkraw
 
     # Verify engines argument.
     if engines:
         engines = set(e.strip() for e in engines.split(b','))
         for engine in engines:
             try:
                 util.compressionengines[engine]
             except KeyError:
                 raise error.Abort(b'unknown compression engine: %s' % engine)
     else:
         engines = []
         for e in util.compengines:
             engine = util.compengines[e]
             try:
                 if engine.available():
                     engine.revlogcompressor().compress(b'dummy')
                     engines.append(e)
             except NotImplementedError:
                 pass
 
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
         if rl._inline:
             return getsvfs(repo)(rl.indexfile)
         else:
             return getsvfs(repo)(rl.datafile)
 
     def doread():
         rl.clearcaches()
         for rev in revs:
             segmentforrevs(rev, rev)
 
     def doreadcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             segmentforrevs(rev, rev, df=fh)
 
     def doreadbatch():
         rl.clearcaches()
         segmentforrevs(revs[0], revs[-1])
 
     def doreadbatchcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         segmentforrevs(revs[0], revs[-1], df=fh)
 
     def dochunk():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             rl._chunk(rev, df=fh)
 
     chunks = [None]
 
     def dochunkbatch():
         rl.clearcaches()
         fh = rlfh(rl)
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
     def docompress(compressor):
         rl.clearcaches()
 
         try:
             # Swap in the requested compression engine.
             oldcompressor = rl._compressor
             rl._compressor = compressor
             for chunk in chunks[0]:
                 rl.compress(chunk)
         finally:
             rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), b'read'),
         (lambda: doreadcachedfh(), b'read w/ reused fd'),
         (lambda: doreadbatch(), b'read batch'),
         (lambda: doreadbatchcachedfh(), b'read batch w/ reused fd'),
         (lambda: dochunk(), b'chunk'),
         (lambda: dochunkbatch(), b'chunk batch'),
     ]
 
     for engine in sorted(engines):
         compressor = util.compengines[engine].revlogcompressor()
         benches.append((functools.partial(docompress, compressor),
                         b'compress w/ %s' % engine))
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 @command(b'perfrevlogrevision', revlogopts + formatteropts +
          [(b'', b'cache', False, b'use caches instead of clearing')],
          b'-c|-m|FILE REV')
 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
     """Benchmark obtaining a revlog revision.
 
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
-    2. Obtain the raw chunks for that delta chain
-    3. Decompress each raw chunk
-    4. Apply binary patches to obtain fulltext
-    5. Verify hash of fulltext
+    2. Slice the delta chain if applicable
+    3. Obtain the raw chunks for that delta chain
+    4. Decompress each raw chunk
+    5. Apply binary patches to obtain fulltext
+    6. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
     opts = _byteskwargs(opts)
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfrevlogrevision', b'invalid arguments')
 
     r = cmdutil.openrevlog(repo, b'perfrevlogrevision', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = r._getsegmentforrevs
     except AttributeError:
         segmentforrevs = r._chunkraw
 
     node = r.lookup(rev)
     rev = r.rev(node)
 
     def getrawchunks(data, chain):
         start = r.start
         length = r.length
         inline = r._inline
         iosize = r._io.size
         buffer = util.buffer
-        offset = start(chain[0])
 
         chunks = []
         ladd = chunks.append
-
-        for rev in chain:
-            chunkstart = start(rev)
-            if inline:
-                chunkstart += (rev + 1) * iosize
-            chunklength = length(rev)
-            ladd(buffer(data, chunkstart - offset, chunklength))
+        for idx, item in enumerate(chain):
+            offset = start(item[0])
+            bits = data[idx]
+            for rev in item:
+                chunkstart = start(rev)
+                if inline:
+                    chunkstart += (rev + 1) * iosize
+                chunklength = length(rev)
+                ladd(buffer(bits, chunkstart - offset, chunklength))
 
         return chunks
 
     def dodeltachain(rev):
         if not cache:
             r.clearcaches()
         r._deltachain(rev)
 
     def doread(chain):
         if not cache:
             r.clearcaches()
-        segmentforrevs(chain[0], chain[-1])
+        for item in slicedchain:
+            segmentforrevs(item[0], item[-1])
+
+    def doslice(r, chain, size):
+        for s in slicechunk(r, chain, targetsize=size):
+            pass
 
     def dorawchunks(data, chain):
         if not cache:
             r.clearcaches()
         getrawchunks(data, chain)
 
     def dodecompress(chunks):
         decomp = r.decompress
         for chunk in chunks:
             decomp(chunk)
 
     def dopatch(text, bins):
         if not cache:
             r.clearcaches()
         mdiff.patches(text, bins)
 
     def dohash(text):
         if not cache:
             r.clearcaches()
         r.checkhash(text, node, rev=rev)
 
     def dorevision():
         if not cache:
             r.clearcaches()
         r.revision(node)
 
+    try:
+        from mercurial.revlogutils.deltas import slicechunk
+    except ImportError:
+        slicechunk = getattr(revlog, '_slicechunk', None)
+
+    size = r.length(rev)
     chain = r._deltachain(rev)[0]
-    data = segmentforrevs(chain[0], chain[-1])[1]
-    rawchunks = getrawchunks(data, chain)
+    if not getattr(r, '_withsparseread', False):
+        slicedchain = (chain,)
+    else:
+        slicedchain = tuple(slicechunk(r, chain, targetsize=size))
+    data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
+    rawchunks = getrawchunks(data, slicedchain)
     bins = r._chunks(chain)
     text = bytes(bins[0])
     bins = bins[1:]
     text = mdiff.patches(text, bins)
 
     benches = [
         (lambda: dorevision(), b'full'),
         (lambda: dodeltachain(rev), b'deltachain'),
         (lambda: doread(chain), b'read'),
-        (lambda: dorawchunks(data, chain), b'rawchunks'),
+    ]
+
+    if getattr(r, '_withsparseread', False):
+        slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
+        benches.append(slicing)
+
+    benches.extend([
+        (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
         (lambda: dodecompress(rawchunks), b'decompress'),
         (lambda: dopatch(text, bins), b'patch'),
         (lambda: dohash(text), b'hash'),
-    ]
+    ])
 
+    timer, fm = gettimer(ui, opts)
     for fn, title in benches:
-        timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
-        fm.end()
+    fm.end()
 
 @command(b'perfrevset',
          [(b'C', b'clear', False, b'clear volatile cache between each call.'),
           (b'', b'contexts', False, b'obtain changectx for each revision')]
          + formatteropts, b"REVSET")
 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
     """benchmark the execution time of a revset
 
     Use the --clean option if need to evaluate the impact of build volatile
     revisions set cache on the revset execution. Volatile cache hold filtered
     and obsolete related cache."""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
     def d():
         if clear:
             repo.invalidatevolatilesets()
         if contexts:
             for ctx in repo.set(expr): pass
         else:
             for r in repo.revs(expr): pass
     timer(d)
     fm.end()
 
 @command(b'perfvolatilesets',
          [(b'', b'clear-obsstore', False, b'drop obsstore between each call.'),
           ] + formatteropts)
 def perfvolatilesets(ui, repo, *names, **opts):
     """benchmark the computation of various volatile set
 
     Volatile set computes element related to filtering and obsolescence."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo = repo.unfiltered()
 
     def getobs(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             obsolete.getrevs(repo, name)
         return d
 
     allobs = sorted(obsolete.cachefuncs)
     if names:
         allobs = [n for n in allobs if n in names]
 
     for name in allobs:
         timer(getobs(name), title=name)
 
     def getfiltered(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             repoview.filterrevs(repo, name)
         return d
 
     allfilter = sorted(repoview.filtertable)
     if names:
         allfilter = [n for n in allfilter if n in names]
 
     for name in allfilter:
         timer(getfiltered(name), title=name)
     fm.end()
 
 @command(b'perfbranchmap',
          [(b'f', b'full', False,
            b'Includes build time of subset'),
           (b'', b'clear-revbranch', False,
            b'purge the revbranch cache between computation'),
           ] + formatteropts)
 def perfbranchmap(ui, repo, *filternames, **opts):
     """benchmark the update of a branchmap
 
     This benchmarks the full repo.branchmap() call with read and write disabled
     """
     opts = _byteskwargs(opts)
     full = opts.get(b"full", False)
     clear_revbranch = opts.get(b"clear_revbranch", False)
     timer, fm = gettimer(ui, opts)
     def getbranchmap(filtername):
         """generate a benchmark function for the filtername"""
         if filtername is None:
             view = repo
         else:
             view = repo.filtered(filtername)
         def d():
             if clear_revbranch:
                 repo.revbranchcache()._clear()
             if full:
                 view._branchcaches.clear()
             else:
                 view._branchcaches.pop(filtername, None)
             view.branchmap()
         return d
     # add filter in smaller subset to bigger subset
     possiblefilters = set(repoview.filtertable)
     if filternames:
         possiblefilters &= set(filternames)
     subsettable = getbranchmapsubsettable()
     allfilters = []
     while possiblefilters:
         for name in possiblefilters:
             subset = subsettable.get(name)
             if subset not in possiblefilters:
                 break
         else:
             assert False, b'subset cycle %s!' % possiblefilters
         allfilters.append(name)
         possiblefilters.remove(name)
 
     # warm the cache
     if not full:
         for name in allfilters:
             repo.filtered(name).branchmap()
     if not filternames or b'unfiltered' in filternames:
         # add unfiltered
         allfilters.append(None)
 
     branchcacheread = safeattrsetter(branchmap, b'read')
     branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
     branchcacheread.set(lambda repo: None)
     branchcachewrite.set(lambda bc, repo: None)
     try:
         for name in allfilters:
             printname = name
             if name is None:
                 printname = b'unfiltered'
             timer(getbranchmap(name), title=str(printname))
     finally:
         branchcacheread.restore()
         branchcachewrite.restore()
     fm.end()
 
+@command(b'perfbranchmapupdate', [
+     (b'', b'base', [], b'subset of revision to start from'),
+     (b'', b'target', [], b'subset of revision to end with'),
+     (b'', b'clear-caches', False, b'clear cache between each runs')
+    ] + formatteropts)
+def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
+    """benchmark branchmap update from for <base> revs to <target> revs
+
+    If `--clear-caches` is passed, the following items will be reset before
+    each update:
+        * the changelog instance and associated indexes
+        * the rev-branch-cache instance
+
+    Examples:
+
+       # update for the one last revision
+       $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
+
+       $ update for change coming with a new branch
+       $ hg perfbranchmapupdate --base 'stable' --target 'default'
+    """
+    from mercurial import branchmap
+    from mercurial import repoview
+    opts = _byteskwargs(opts)
+    timer, fm = gettimer(ui, opts)
+    clearcaches = opts[b'clear_caches']
+    unfi = repo.unfiltered()
+    x = [None] # used to pass data between closure
+
+    # we use a `list` here to avoid possible side effect from smartset
+    baserevs = list(scmutil.revrange(repo, base))
+    targetrevs = list(scmutil.revrange(repo, target))
+    if not baserevs:
+        raise error.Abort(b'no revisions selected for --base')
+    if not targetrevs:
+        raise error.Abort(b'no revisions selected for --target')
+
+    # make sure the target branchmap also contains the one in the base
+    targetrevs = list(set(baserevs) | set(targetrevs))
+    targetrevs.sort()
+
+    cl = repo.changelog
+    allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
+    allbaserevs.sort()
+    alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
+
+    newrevs = list(alltargetrevs.difference(allbaserevs))
+    newrevs.sort()
+
+    allrevs = frozenset(unfi.changelog.revs())
+    basefilterrevs = frozenset(allrevs.difference(allbaserevs))
+    targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
+
+    def basefilter(repo, visibilityexceptions=None):
+        return basefilterrevs
+
+    def targetfilter(repo, visibilityexceptions=None):
+        return targetfilterrevs
+
+    msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
+    ui.status(msg % (len(allbaserevs), len(newrevs)))
+    if targetfilterrevs:
+        msg = b'(%d revisions still filtered)\n'
+        ui.status(msg % len(targetfilterrevs))
+
+    try:
+        repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
+        repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
+
+        baserepo = repo.filtered(b'__perf_branchmap_update_base')
+        targetrepo = repo.filtered(b'__perf_branchmap_update_target')
+
+        # try to find an existing branchmap to reuse
+        subsettable = getbranchmapsubsettable()
+        candidatefilter = subsettable.get(None)
+        while candidatefilter is not None:
+            candidatebm = repo.filtered(candidatefilter).branchmap()
+            if candidatebm.validfor(baserepo):
+                filtered = repoview.filterrevs(repo, candidatefilter)
+                missing = [r for r in allbaserevs if r in filtered]
+                base = candidatebm.copy()
+                base.update(baserepo, missing)
+                break
+            candidatefilter = subsettable.get(candidatefilter)
+        else:
+            # no suitable subset where found
+            base = branchmap.branchcache()
+            base.update(baserepo, allbaserevs)
+
+        def setup():
+            x[0] = base.copy()
+            if clearcaches:
+                unfi._revbranchcache = None
+                clearchangelog(repo)
+
+        def bench():
+            x[0].update(targetrepo, newrevs)
+
+        timer(bench, setup=setup)
+        fm.end()
+    finally:
+        repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
+        repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
+
 @command(b'perfbranchmapload', [
      (b'f', b'filter', b'', b'Specify repoview filter'),
      (b'', b'list', False, b'List brachmap filter caches'),
+     (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
+
     ] + formatteropts)
-def perfbranchmapread(ui, repo, filter=b'', list=False, **opts):
+def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
     """benchmark reading the branchmap"""
     opts = _byteskwargs(opts)
+    clearrevlogs = opts[b'clear_revlogs']
 
     if list:
         for name, kind, st in repo.cachevfs.readdir(stat=True):
             if name.startswith(b'branch2'):
                 filtername = name.partition(b'-')[2] or b'unfiltered'
                 ui.status(b'%s - %s\n'
                           % (filtername, util.bytecount(st.st_size)))
         return
-    if filter:
+    if not filter:
+        filter = None
+    subsettable = getbranchmapsubsettable()
+    if filter is None:
+        repo = repo.unfiltered()
+    else:
         repo = repoview.repoview(repo, filter)
-    else:
-        repo = repo.unfiltered()
+
+    repo.branchmap() # make sure we have a relevant, up to date branchmap
+
+    currentfilter = filter
     # try once without timer, the filter may not be cached
-    if branchmap.read(repo) is None:
-        raise error.Abort(b'No brachmap cached for %s repo'
-                          % (filter or b'unfiltered'))
+    while branchmap.read(repo) is None:
+        currentfilter = subsettable.get(currentfilter)
+        if currentfilter is None:
+            raise error.Abort(b'No branchmap cached for %s repo'
+                              % (filter or b'unfiltered'))
+        repo = repo.filtered(currentfilter)
     timer, fm = gettimer(ui, opts)
-    timer(lambda: branchmap.read(repo) and None)
+    def setup():
+        if clearrevlogs:
+            clearchangelog(repo)
+    def bench():
+        branchmap.read(repo)
+    timer(bench, setup=setup)
     fm.end()
 
 @command(b'perfloadmarkers')
 def perfloadmarkers(ui, repo):
     """benchmark the time to parse the on-disk markers for a repo
 
     Result is the number of markers in the repo."""
     timer, fm = gettimer(ui)
     svfs = getsvfs(repo)
     timer(lambda: len(obsolete.obsstore(svfs)))
     fm.end()
 
 @command(b'perflrucachedict', formatteropts +
     [(b'', b'costlimit', 0, b'maximum total cost of items in cache'),
      (b'', b'mincost', 0, b'smallest cost of items in cache'),
      (b'', b'maxcost', 100, b'maximum cost of items in cache'),
      (b'', b'size', 4, b'size of cache'),
      (b'', b'gets', 10000, b'number of key lookups'),
      (b'', b'sets', 10000, b'number of key sets'),
      (b'', b'mixed', 10000, b'number of mixed mode operations'),
      (b'', b'mixedgetfreq', 50, b'frequency of get vs set ops in mixed mode')],
     norepo=True)
 def perflrucache(ui, mincost=0, maxcost=100, costlimit=0, size=4,
                  gets=10000, sets=10000, mixed=10000, mixedgetfreq=50, **opts):
     opts = _byteskwargs(opts)
 
     def doinit():
         for i in _xrange(10000):
             util.lrucachedict(size)
 
     costrange = list(range(mincost, maxcost + 1))
 
     values = []
     for i in _xrange(size):
         values.append(random.randint(0, _maxint))
 
     # Get mode fills the cache and tests raw lookup performance with no
     # eviction.
     getseq = []
     for i in _xrange(gets):
         getseq.append(random.choice(values))
 
     def dogets():
         d = util.lrucachedict(size)
         for v in values:
             d[v] = v
         for key in getseq:
             value = d[key]
             value # silence pyflakes warning
 
     def dogetscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(values):
             d.insert(v, v, cost=costs[i])
         for key in getseq:
             try:
                 value = d[key]
                 value # silence pyflakes warning
             except KeyError:
                 pass
 
     # Set mode tests insertion speed with cache eviction.
     setseq = []
     costs = []
     for i in _xrange(sets):
         setseq.append(random.randint(0, _maxint))
         costs.append(random.choice(costrange))
 
     def doinserts():
         d = util.lrucachedict(size)
         for v in setseq:
             d.insert(v, v)
 
     def doinsertscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(setseq):
             d.insert(v, v, cost=costs[i])
 
     def dosets():
         d = util.lrucachedict(size)
         for v in setseq:
             d[v] = v
 
     # Mixed mode randomly performs gets and sets with eviction.
     mixedops = []
     for i in _xrange(mixed):
         r = random.randint(0, 100)
         if r < mixedgetfreq:
             op = 0
         else:
             op = 1
 
         mixedops.append((op,
                          random.randint(0, size * 2),
                          random.choice(costrange)))
 
     def domixed():
         d = util.lrucachedict(size)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d[v] = v
 
     def domixedcost():
         d = util.lrucachedict(size, maxcost=costlimit)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d.insert(v, v, cost=cost)
 
     benches = [
         (doinit, b'init'),
     ]
 
     if costlimit:
         benches.extend([
             (dogetscost, b'gets w/ cost limit'),
             (doinsertscost, b'inserts w/ cost limit'),
             (domixedcost, b'mixed w/ cost limit'),
         ])
     else:
         benches.extend([
             (dogets, b'gets'),
             (doinserts, b'inserts'),
             (dosets, b'sets'),
             (domixed, b'mixed')
         ])
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 @command(b'perfwrite', formatteropts)
 def perfwrite(ui, repo, **opts):
     """microbenchmark ui.write
     """
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
     def write():
         for i in range(100000):
             ui.write((b'Testing write performance\n'))
     timer(write)
     fm.end()
 
 def uisetup(ui):
     if (util.safehasattr(cmdutil, b'openrevlog') and
         not util.safehasattr(commands, b'debugrevlogopts')):
         # for "historical portability":
         # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
         # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
         # openrevlog() should cause failure, because it has been
         # available since 3.5 (or 49c583ca48c4).
         def openrevlog(orig, repo, cmd, file_, opts):
             if opts.get(b'dir') and not util.safehasattr(repo, b'dirlog'):
                 raise error.Abort(b"This version doesn't support --dir option",
                                   hint=b"use 3.5 or later")
             return orig(repo, cmd, file_, opts)
         extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
+
+@command(b'perfprogress', formatteropts + [
+    (b'', b'topic', b'topic', b'topic for progress messages'),
+    (b'c', b'total', 1000000, b'total value we are progressing to'),
+], norepo=True)
+def perfprogress(ui, topic=None, total=None, **opts):
+    """printing of progress bars"""
+    opts = _byteskwargs(opts)
+
+    timer, fm = gettimer(ui, opts)
+
+    def doprogress():
+        with ui.makeprogress(topic, total=total) as progress:
+            for i in pycompat.xrange(total):
+                progress.increment()
+
+    timer(doprogress)
+    fm.end()

contrib/python3-whitelist

@@ -1,642 +1,703 b''
 test-abort-checkin.t
+test-absorb-edit-lines.t
 test-absorb-filefixupstate.py
 test-absorb-phase.t
 test-absorb-rename.t
 test-absorb-strip.t
 test-absorb.t
 test-add.t
 test-addremove-similar.t
 test-addremove.t
 test-alias.t
 test-amend-subrepo.t
 test-amend.t
 test-ancestor.py
 test-annotate.py
 test-annotate.t
 test-archive-symlinks.t
 test-archive.t
 test-atomictempfile.py
 test-audit-path.t
 test-audit-subrepo.t
 test-automv.t
 test-backout.t
 test-backwards-remove.t
 test-bad-extension.t
 test-bad-pull.t
 test-basic.t
 test-bdiff.py
 test-bheads.t
 test-bisect.t
 test-bisect2.t
 test-bisect3.t
 test-blackbox.t
+test-bookflow.t
 test-bookmarks-current.t
 test-bookmarks-merge.t
 test-bookmarks-pushpull.t
 test-bookmarks-rebase.t
 test-bookmarks-strip.t
 test-bookmarks.t
 test-branch-change.t
 test-branch-option.t
 test-branch-tag-confict.t
 test-branches.t
 test-bundle-phases.t
 test-bundle-r.t
 test-bundle-type.t
 test-bundle-vs-outgoing.t
 test-bundle.t
 test-bundle2-exchange.t
 test-bundle2-format.t
 test-bundle2-multiple-changegroups.t
 test-bundle2-pushback.t
 test-bundle2-remote-changegroup.t
 test-cache-abuse.t
 test-cappedreader.py
 test-casecollision.t
 test-cat.t
 test-cbor.py
 test-censor.t
 test-changelog-exec.t
 test-check-code.t
 test-check-commit.t
 test-check-config.py
 test-check-config.t
 test-check-execute.t
+test-check-help.t
 test-check-interfaces.py
 test-check-module-imports.t
 test-check-py3-compat.t
 test-check-pyflakes.t
 test-check-pylint.t
 test-check-shbang.t
 test-children.t
 test-churn.t
 test-clone-cgi.t
 test-clone-pull-corruption.t
 test-clone-r.t
 test-clone-uncompressed.t
 test-clone-update-order.t
 test-clone.t
 test-clonebundles.t
 test-close-head.t
 test-commandserver.t
 test-commit-amend.t
 test-commit-interactive.t
 test-commit-multiple.t
 test-commit-unresolved.t
 test-commit.t
 test-committer.t
 test-completion.t
 test-config-env.py
 test-config.t
 test-conflict.t
 test-confused-revert.t
 test-context-metadata.t
 test-context.py
 test-contrib-check-code.t
 test-contrib-check-commit.t
 test-contrib-dumprevlog.t
 test-contrib-perf.t
 test-contrib-relnotes.t
 test-contrib-testparseutil.t
 test-contrib.t
 test-convert-authormap.t
 test-convert-clonebranches.t
 test-convert-cvs-branch.t
 test-convert-cvs-detectmerge.t
 test-convert-cvs-synthetic.t
 test-convert-cvs.t
 test-convert-cvsnt-mergepoints.t
 test-convert-datesort.t
 test-convert-filemap.t
 test-convert-hg-sink.t
 test-convert-hg-source.t
 test-convert-hg-startrev.t
 test-convert-splicemap.t
 test-convert-tagsbranch-topology.t
 test-copy-move-merge.t
 test-copy.t
 test-copytrace-heuristics.t
+test-custom-filters.t
 test-debugbuilddag.t
 test-debugbundle.t
 test-debugcommands.t
 test-debugextensions.t
 test-debugindexdot.t
 test-debugrename.t
 test-default-push.t
 test-diff-antipatience.t
 test-diff-binary-file.t
 test-diff-change.t
 test-diff-color.t
 test-diff-copy-depth.t
 test-diff-hashes.t
 test-diff-ignore-whitespace.t
 test-diff-indent-heuristic.t
 test-diff-issue2761.t
 test-diff-newlines.t
 test-diff-reverse.t
 test-diff-subdir.t
 test-diff-unified.t
 test-diff-upgrade.t
 test-diffdir.t
 test-diffstat.t
 test-directaccess.t
 test-dirstate-backup.t
 test-dirstate-nonnormalset.t
 test-dirstate-race.t
 test-dirstate.t
 test-dispatch.py
 test-doctest.py
 test-double-merge.t
 test-drawdag.t
 test-duplicateoptions.py
 test-editor-filename.t
 test-empty-dir.t
 test-empty-file.t
 test-empty-group.t
 test-empty.t
 test-encode.t
 test-encoding-func.py
 test-encoding-textwrap.t
 test-encoding.t
 test-eol-add.t
 test-eol-clone.t
 test-eol-hook.t
 test-eol-patch.t
 test-eol-tag.t
 test-eol-update.t
 test-eol.t
 test-eolfilename.t
 test-excessive-merge.t
 test-exchange-obsmarkers-case-A1.t
 test-exchange-obsmarkers-case-A2.t
 test-exchange-obsmarkers-case-A3.t
 test-exchange-obsmarkers-case-A4.t
 test-exchange-obsmarkers-case-A5.t
 test-exchange-obsmarkers-case-A6.t
 test-exchange-obsmarkers-case-A7.t
 test-exchange-obsmarkers-case-B1.t
 test-exchange-obsmarkers-case-B2.t
 test-exchange-obsmarkers-case-B3.t
 test-exchange-obsmarkers-case-B4.t
 test-exchange-obsmarkers-case-B5.t
 test-exchange-obsmarkers-case-B6.t
 test-exchange-obsmarkers-case-B7.t
 test-exchange-obsmarkers-case-C1.t
 test-exchange-obsmarkers-case-C2.t
 test-exchange-obsmarkers-case-C3.t
 test-exchange-obsmarkers-case-C4.t
 test-exchange-obsmarkers-case-D1.t
 test-exchange-obsmarkers-case-D2.t
 test-exchange-obsmarkers-case-D3.t
 test-exchange-obsmarkers-case-D4.t
 test-execute-bit.t
 test-export.t
 test-extdata.t
 test-extdiff.t
+test-extension-timing.t
 test-extensions-afterloaded.t
 test-extensions-wrapfunction.py
 test-extra-filelog-entry.t
+test-fastannotate-corrupt.t
+test-fastannotate-diffopts.t
+test-fastannotate-hg.t
+test-fastannotate-perfhack.t
+test-fastannotate-protocol.t
+test-fastannotate-renames.t
+test-fastannotate-revmap.py
+test-fastannotate.t
 test-fetch.t
 test-filebranch.t
 test-filecache.py
 test-filelog.py
 test-fileset-generated.t
 test-fileset.t
 test-fix-topology.t
 test-fix.t
 test-flags.t
 test-fncache.t
+test-gendoc-da.t
+test-gendoc-de.t
+test-gendoc-el.t
+test-gendoc-fr.t
+test-gendoc-it.t
+test-gendoc-ja.t
+test-gendoc-pt_BR.t
+test-gendoc-ro.t
+test-gendoc-ru.t
+test-gendoc-sv.t
+test-gendoc-zh_CN.t
+test-gendoc-zh_TW.t
+test-gendoc.t
 test-generaldelta.t
 test-getbundle.t
 test-git-export.t
 test-globalopts.t
 test-glog-beautifygraph.t
 test-glog-topological.t
 test-glog.t
 test-gpg.t
 test-graft.t
 test-grep.t
 test-hardlinks.t
+test-help-hide.t
 test-help.t
 test-hg-parseurl.py
 test-hghave.t
 test-hgignore.t
 test-hgk.t
 test-hgrc.t
 test-hgweb-annotate-whitespace.t
 test-hgweb-bundle.t
 test-hgweb-csp.t
 test-hgweb-descend-empties.t
 test-hgweb-diffs.t
 test-hgweb-empty.t
 test-hgweb-filelog.t
 test-hgweb-non-interactive.t
 test-hgweb-raw.t
 test-hgweb-removed.t
 test-hgweb.t
 test-hgwebdir-paths.py
 test-hgwebdirsym.t
 test-histedit-arguments.t
 test-histedit-base.t
 test-histedit-bookmark-motion.t
 test-histedit-commute.t
 test-histedit-drop.t
 test-histedit-edit.t
 test-histedit-fold-non-commute.t
 test-histedit-fold.t
 test-histedit-no-backup.t
 test-histedit-no-change.t
 test-histedit-non-commute-abort.t
 test-histedit-non-commute.t
 test-histedit-obsolete.t
 test-histedit-outgoing.t
 test-histedit-templates.t
 test-http-branchmap.t
 test-http-bundle1.t
 test-http-clone-r.t
 test-http-permissions.t
 test-http.t
 test-hybridencode.py
 test-i18n.t
 test-identify.t
 test-impexp-branch.t
 test-import-bypass.t
+test-import-context.t
 test-import-eol.t
 test-import-merge.t
 test-import-unknown.t
 test-import.t
 test-imports-checker.t
 test-incoming-outgoing.t
 test-infinitepush-bundlestore.t
 test-infinitepush-ci.t
 test-infinitepush.t
 test-inherit-mode.t
 test-init.t
 test-issue1089.t
 test-issue1102.t
 test-issue1175.t
 test-issue1306.t
 test-issue1438.t
 test-issue1502.t
 test-issue1802.t
 test-issue1877.t
 test-issue1993.t
 test-issue2137.t
 test-issue3084.t
 test-issue4074.t
 test-issue522.t
 test-issue586.t
 test-issue5979.t
 test-issue612.t
 test-issue619.t
 test-issue660.t
 test-issue672.t
 test-issue842.t
 test-journal-exists.t
 test-journal-share.t
 test-journal.t
 test-keyword.t
 test-known.t
 test-largefiles-cache.t
 test-largefiles-misc.t
 test-largefiles-small-disk.t
 test-largefiles-update.t
+test-largefiles-wireproto.t
 test-largefiles.t
+test-lfconvert.t
+test-lfs-bundle.t
 test-lfs-largefiles.t
 test-lfs-pointer.py
+test-lfs.t
 test-linelog.py
 test-linerange.py
 test-locate.t
 test-lock-badness.t
+test-log-exthook.t
 test-log-linerange.t
 test-log.t
 test-logexchange.t
+test-logtoprocess.t
 test-lrucachedict.py
 test-mactext.t
 test-mailmap.t
 test-manifest-merging.t
 test-manifest.py
 test-manifest.t
 test-match.py
 test-mdiff.py
 test-merge-changedelete.t
 test-merge-closedheads.t
 test-merge-commit.t
 test-merge-criss-cross.t
 test-merge-default.t
 test-merge-force.t
 test-merge-halt.t
 test-merge-internal-tools-pattern.t
 test-merge-local.t
 test-merge-no-file-change.t
 test-merge-remove.t
 test-merge-revert.t
 test-merge-revert2.t
 test-merge-subrepos.t
 test-merge-symlinks.t
 test-merge-tools.t
 test-merge-types.t
 test-merge1.t
 test-merge10.t
 test-merge2.t
 test-merge4.t
 test-merge5.t
 test-merge6.t
 test-merge7.t
 test-merge8.t
 test-merge9.t
 test-minifileset.py
 test-minirst.py
 test-mq-git.t
 test-mq-guards.t
 test-mq-header-date.t
 test-mq-header-from.t
 test-mq-merge.t
 test-mq-pull-from-bundle.t
 test-mq-qclone-http.t
 test-mq-qdelete.t
 test-mq-qdiff.t
 test-mq-qfold.t
 test-mq-qgoto.t
 test-mq-qimport-fail-cleanup.t
 test-mq-qnew.t
 test-mq-qpush-exact.t
 test-mq-qpush-fail.t
 test-mq-qqueue.t
 test-mq-qrefresh-interactive.t
 test-mq-qrefresh-replace-log-message.t
 test-mq-qrefresh.t
 test-mq-qrename.t
 test-mq-qsave.t
 test-mq-safety.t
 test-mq-subrepo.t
 test-mq-symlinks.t
 test-mq.t
 test-mv-cp-st-diff.t
 test-narrow-acl.t
 test-narrow-archive.t
 test-narrow-clone-no-ellipsis.t
 test-narrow-clone-non-narrow-server.t
 test-narrow-clone-nonlinear.t
 test-narrow-clone-stream.t
 test-narrow-clone.t
 test-narrow-commit.t
 test-narrow-copies.t
 test-narrow-debugcommands.t
 test-narrow-debugrebuilddirstate.t
 test-narrow-exchange-merges.t
 test-narrow-exchange.t
 test-narrow-expanddirstate.t
 test-narrow-merge.t
 test-narrow-patch.t
 test-narrow-patterns.t
 test-narrow-pull.t
 test-narrow-rebase.t
 test-narrow-shallow-merges.t
 test-narrow-shallow.t
+test-narrow-share.t
+test-narrow-sparse.t
 test-narrow-strip.t
 test-narrow-trackedcmd.t
 test-narrow-update.t
 test-narrow-widen-no-ellipsis.t
 test-narrow-widen.t
 test-narrow.t
 test-nested-repo.t
 test-newbranch.t
 test-newercgi.t
 test-nointerrupt.t
 test-obshistory.t
 test-obsmarker-template.t
 test-obsmarkers-effectflag.t
 test-obsolete-bounds-checking.t
 test-obsolete-bundle-strip.t
 test-obsolete-changeset-exchange.t
 test-obsolete-checkheads.t
 test-obsolete-distributed.t
 test-obsolete-divergent.t
 test-obsolete-tag-cache.t
 test-obsolete.t
 test-origbackup-conflict.t
 test-pager-legacy.t
 test-pager.t
 test-parents.t
 test-parse-date.t
 test-parseindex2.py
 test-patch-offset.t
 test-patch.t
 test-patchbomb-bookmark.t
 test-patchbomb-tls.t
 test-patchbomb.t
 test-pathconflicts-basic.t
 test-pathconflicts-merge.t
 test-pathconflicts-update.t
 test-pathencode.py
 test-pending.t
 test-permissions.t
 test-phases-exchange.t
 test-phases.t
 test-profile.t
 test-progress.t
 test-propertycache.py
 test-pull-branch.t
 test-pull-http.t
 test-pull-permission.t
 test-pull-pull-corruption.t
 test-pull-r.t
 test-pull-update.t
 test-pull.t
 test-purge.t
 test-push-cgi.t
 test-push-checkheads-partial-C1.t
 test-push-checkheads-partial-C2.t
 test-push-checkheads-partial-C3.t
 test-push-checkheads-partial-C4.t
 test-push-checkheads-pruned-B1.t
 test-push-checkheads-pruned-B2.t
 test-push-checkheads-pruned-B3.t
 test-push-checkheads-pruned-B4.t
 test-push-checkheads-pruned-B5.t
 test-push-checkheads-pruned-B6.t
 test-push-checkheads-pruned-B7.t
 test-push-checkheads-pruned-B8.t
 test-push-checkheads-superceed-A1.t
 test-push-checkheads-superceed-A2.t
 test-push-checkheads-superceed-A3.t
 test-push-checkheads-superceed-A4.t
 test-push-checkheads-superceed-A5.t
 test-push-checkheads-superceed-A6.t
 test-push-checkheads-superceed-A7.t
 test-push-checkheads-superceed-A8.t
 test-push-checkheads-unpushed-D1.t
 test-push-checkheads-unpushed-D2.t
 test-push-checkheads-unpushed-D3.t
 test-push-checkheads-unpushed-D4.t
 test-push-checkheads-unpushed-D5.t
 test-push-checkheads-unpushed-D6.t
 test-push-checkheads-unpushed-D7.t
 test-push-http.t
+test-push-race.t
 test-push-warn.t
 test-push.t
 test-pushvars.t
 test-qrecord.t
 test-rebase-abort.t
 test-rebase-backup.t
 test-rebase-base-flag.t
 test-rebase-bookmarks.t
 test-rebase-brute-force.t
 test-rebase-cache.t
 test-rebase-check-restore.t
 test-rebase-collapse.t
 test-rebase-conflicts.t
 test-rebase-dest.t
 test-rebase-detach.t
 test-rebase-emptycommit.t
 test-rebase-inmemory.t
 test-rebase-interruptions.t
 test-rebase-issue-noparam-single-rev.t
 test-rebase-legacy.t
 test-rebase-mq-skip.t
 test-rebase-mq.t
 test-rebase-named-branches.t
 test-rebase-newancestor.t
 test-rebase-obsolete.t
 test-rebase-parameters.t
 test-rebase-partial.t
 test-rebase-pull.t
 test-rebase-rename.t
 test-rebase-scenario-global.t
 test-rebase-templates.t
 test-rebase-transaction.t
 test-rebuildstate.t
 test-record.t
 test-releasenotes-formatting.t
 test-releasenotes-merging.t
 test-releasenotes-parsing.t
 test-relink.t
+test-remotefilelog-bad-configs.t
+test-remotefilelog-bgprefetch.t
+test-remotefilelog-blame.t
+test-remotefilelog-bundle2.t
+test-remotefilelog-bundles.t
+test-remotefilelog-cacheprocess.t
+test-remotefilelog-clone-tree.t
+test-remotefilelog-clone.t
+test-remotefilelog-gcrepack.t
+test-remotefilelog-http.t
+test-remotefilelog-keepset.t
+test-remotefilelog-local.t
+test-remotefilelog-log.t
+test-remotefilelog-partial-shallow.t
+test-remotefilelog-permissions.t
+test-remotefilelog-permisssions.t
+test-remotefilelog-prefetch.t
+test-remotefilelog-pull-noshallow.t
+test-remotefilelog-share.t
+test-remotefilelog-sparse.t
+test-remotefilelog-tags.t
+test-remotefilelog-wireproto.t
 test-remove.t
 test-removeemptydirs.t
 test-rename-after-merge.t
 test-rename-dir-merge.t
 test-rename-merge1.t
 test-rename-merge2.t
 test-rename.t
 test-repair-strip.t
 test-repo-compengines.t
 test-requires.t
 test-resolve.t
 test-revert-flags.t
 test-revert-interactive.t
 test-revert-unknown.t
 test-revert.t
 test-revisions.t
 test-revlog-ancestry.py
 test-revlog-group-emptyiter.t
 test-revlog-mmapindex.t
 test-revlog-packentry.t
 test-revlog-raw.py
 test-revlog-v2.t
 test-revlog.t
 test-revset-dirstate-parents.t
 test-revset-legacy-lookup.t
 test-revset-outgoing.t
 test-rollback.t
 test-run-tests.py
 test-run-tests.t
+test-rust-ancestor.py
 test-schemes.t
 test-serve.t
 test-setdiscovery.t
 test-share.t
 test-shelve.t
+test-shelve2.t
 test-show-stack.t
 test-show-work.t
 test-show.t
 test-simple-update.t
 test-simplekeyvaluefile.py
 test-simplemerge.py
 test-single-head.t
 test-sparse-clear.t
 test-sparse-clone.t
 test-sparse-import.t
 test-sparse-merges.t
 test-sparse-profiles.t
 test-sparse-requirement.t
 test-sparse-verbose-json.t
 test-sparse.t
 test-split.t
 test-ssh-bundle1.t
 test-ssh-clone-r.t
 test-ssh-proto-unbundle.t
 test-ssh-proto.t
 test-ssh-repoerror.t
 test-ssh.t
 test-sshserver.py
 test-stack.t
 test-status-color.t
 test-status-inprocess.py
 test-status-rev.t
 test-status-terse.t
 test-status.t
 test-storage.py
 test-stream-bundle-v2.t
 test-strict.t
 test-strip-cross.t
 test-strip.t
 test-subrepo-deep-nested-change.t
 test-subrepo-missing.t
 test-subrepo-paths.t
 test-subrepo-recursion.t
 test-subrepo-relative-path.t
 test-subrepo.t
 test-symlink-os-yes-fs-no.py
 test-symlink-placeholder.t
 test-symlinks.t
 test-tag.t
 test-tags.t
 test-template-basic.t
 test-template-functions.t
 test-template-keywords.t
 test-template-map.t
 test-tools.t
 test-transplant.t
 test-treemanifest.t
 test-ui-color.py
 test-ui-config.py
 test-ui-verbosity.py
 test-unamend.t
 test-unbundlehash.t
 test-uncommit.t
 test-unified-test.t
 test-unionrepo.t
 test-unrelated-pull.t
 test-up-local-change.t
 test-update-branches.t
 test-update-dest.t
 test-update-issue1456.t
 test-update-names.t
 test-update-reverse.t
 test-upgrade-repo.t
 test-url-download.t
 test-url-rev.t
 test-url.py
 test-username-newline.t
 test-util.py
 test-verify.t
 test-walk.t
 test-walkrepo.py
 test-websub.t
 test-win32text.t
 test-wireproto-clientreactor.py
 test-wireproto-command-branchmap.t
 test-wireproto-command-changesetdata.t
 test-wireproto-command-filedata.t
 test-wireproto-command-filesdata.t
 test-wireproto-command-heads.t
 test-wireproto-command-listkeys.t
 test-wireproto-command-lookup.t
 test-wireproto-command-manifestdata.t
 test-wireproto-command-pushkey.t
 test-wireproto-command-rawstorefiledata.t
 test-wireproto-framing.py
 test-wireproto-serverreactor.py
 test-wireproto.py
 test-wsgirequest.py
 test-xdg.t

contrib/revsetbenchmarks.py

@@ -1,325 +1,327 b''
 #!/usr/bin/env python
 
 # Measure the performance of a list of revsets against multiple revisions
 # defined by parameter. Checkout one by one and run perfrevset with every
 # revset in the list to benchmark its performance.
 #
 # You should run this from the root of your mercurial repository.
 #
 # call with --help for details
 
 from __future__ import absolute_import, print_function
 import math
 import optparse  # cannot use argparse, python 2.7 only
 import os
 import re
 import subprocess
 import sys
 
 DEFAULTVARIANTS = ['plain', 'min', 'max', 'first', 'last',
                    'reverse', 'reverse+first', 'reverse+last',
                    'sort', 'sort+first', 'sort+last']
 
 def check_output(*args, **kwargs):
     kwargs.setdefault('stderr', subprocess.PIPE)
     kwargs.setdefault('stdout', subprocess.PIPE)
     proc = subprocess.Popen(*args, **kwargs)
     output, error = proc.communicate()
     if proc.returncode != 0:
         raise subprocess.CalledProcessError(proc.returncode, ' '.join(args[0]))
     return output
 
 def update(rev):
     """update the repo to a revision"""
     try:
         subprocess.check_call(['hg', 'update', '--quiet', '--check', str(rev)])
         check_output(['make', 'local'],
                      stderr=None)  # suppress output except for error/warning
     except subprocess.CalledProcessError as exc:
         print('update to revision %s failed, aborting'%rev, file=sys.stderr)
         sys.exit(exc.returncode)
 
 
 def hg(cmd, repo=None):
     """run a mercurial command
 
     <cmd> is the list of command + argument,
     <repo> is an optional repository path to run this command in."""
     fullcmd = ['./hg']
     if repo is not None:
         fullcmd += ['-R', repo]
     fullcmd += ['--config',
                 'extensions.perf=' + os.path.join(contribdir, 'perf.py')]
     fullcmd += cmd
     return check_output(fullcmd, stderr=subprocess.STDOUT)
 
 def perf(revset, target=None, contexts=False):
     """run benchmark for this very revset"""
     try:
-        args = ['perfrevset', revset]
+        args = ['perfrevset']
         if contexts:
             args.append('--contexts')
+        args.append('--')
+        args.append(revset)
         output = hg(args, repo=target)
         return parseoutput(output)
     except subprocess.CalledProcessError as exc:
         print('abort: cannot run revset benchmark: %s'%exc.cmd, file=sys.stderr)
         if getattr(exc, 'output', None) is None: # no output before 2.7
             print('(no output)', file=sys.stderr)
         else:
             print(exc.output, file=sys.stderr)
         return None
 
 outputre = re.compile(r'! wall (\d+.\d+) comb (\d+.\d+) user (\d+.\d+) '
                       'sys (\d+.\d+) \(best of (\d+)\)')
 
 def parseoutput(output):
     """parse a textual output into a dict
 
     We cannot just use json because we want to compare with old
     versions of Mercurial that may not support json output.
     """
     match = outputre.search(output)
     if not match:
         print('abort: invalid output:', file=sys.stderr)
         print(output, file=sys.stderr)
         sys.exit(1)
     return {'comb': float(match.group(2)),
             'count': int(match.group(5)),
             'sys': float(match.group(3)),
             'user': float(match.group(4)),
             'wall': float(match.group(1)),
             }
 
 def printrevision(rev):
     """print data about a revision"""
     sys.stdout.write("Revision ")
     sys.stdout.flush()
     subprocess.check_call(['hg', 'log', '--rev', str(rev), '--template',
                            '{if(tags, " ({tags})")} '
                            '{rev}:{node|short}: {desc|firstline}\n'])
 
 def idxwidth(nbidx):
     """return the max width of number used for index
 
     This is similar to log10(nbidx), but we use custom code here
     because we start with zero and we'd rather not deal with all the
     extra rounding business that log10 would imply.
     """
     nbidx -= 1 # starts at 0
     idxwidth = 0
     while nbidx:
         idxwidth += 1
         nbidx //= 10
     if not idxwidth:
         idxwidth = 1
     return idxwidth
 
 def getfactor(main, other, field, sensitivity=0.05):
     """return the relative factor between values for 'field' in main and other
 
     Return None if the factor is insignificant (less than <sensitivity>
     variation)."""
     factor = 1
     if main is not None:
         factor = other[field] / main[field]
     low, high = 1 - sensitivity, 1 + sensitivity
     if (low < factor < high):
         return None
     return factor
 
 def formatfactor(factor):
     """format a factor into a 4 char string
 
      22%
     156%
     x2.4
      x23
     x789
     x1e4
     x5x7
 
     """
     if factor is None:
         return '    '
     elif factor < 2:
         return '%3i%%' % (factor * 100)
     elif factor < 10:
         return 'x%3.1f' % factor
     elif factor < 1000:
         return '%4s' % ('x%i' % factor)
     else:
         order = int(math.log(factor)) + 1
         while math.log(factor) > 1:
             factor //= 0
         return 'x%ix%i' % (factor, order)
 
 def formattiming(value):
     """format a value to strictly 8 char, dropping some precision if needed"""
     if value < 10**7:
         return ('%.6f' % value)[:8]
     else:
         # value is HUGE very unlikely to happen (4+ month run)
         return '%i' % value
 
 _marker = object()
 def printresult(variants, idx, data, maxidx, verbose=False, reference=_marker):
     """print a line of result to stdout"""
     mask = '%%0%ii) %%s' % idxwidth(maxidx)
 
     out = []
     for var in variants:
         if data[var] is None:
             out.append('error   ')
             out.append(' ' * 4)
             continue
         out.append(formattiming(data[var]['wall']))
         if reference is not _marker:
             factor = None
             if reference is not None:
                 factor = getfactor(reference[var], data[var], 'wall')
             out.append(formatfactor(factor))
         if verbose:
             out.append(formattiming(data[var]['comb']))
             out.append(formattiming(data[var]['user']))
             out.append(formattiming(data[var]['sys']))
             out.append('%6d'    % data[var]['count'])
     print(mask % (idx, ' '.join(out)))
 
 def printheader(variants, maxidx, verbose=False, relative=False):
     header = [' ' * (idxwidth(maxidx) + 1)]
     for var in variants:
         if not var:
             var = 'iter'
         if len(var) > 8:
             var = var[:3] + '..' + var[-3:]
         header.append('%-8s' % var)
         if relative:
             header.append('    ')
         if verbose:
             header.append('%-8s' % 'comb')
             header.append('%-8s' % 'user')
             header.append('%-8s' % 'sys')
             header.append('%6s' % 'count')
     print(' '.join(header))
 
 def getrevs(spec):
     """get the list of rev matched by a revset"""
     try:
         out = check_output(['hg', 'log', '--template={rev}\n', '--rev', spec])
     except subprocess.CalledProcessError as exc:
         print("abort, can't get revision from %s"%spec, file=sys.stderr)
         sys.exit(exc.returncode)
     return [r for r in out.split() if r]
 
 
 def applyvariants(revset, variant):
     if variant == 'plain':
         return revset
     for var in variant.split('+'):
         revset = '%s(%s)' % (var, revset)
     return revset
 
 helptext="""This script will run multiple variants of provided revsets using
 different revisions in your mercurial repository. After the benchmark are run
 summary output is provided. Use it to demonstrate speed improvements or pin
 point regressions. Revsets to run are specified in a file (or from stdin), one
 revsets per line. Line starting with '#' will be ignored, allowing insertion of
 comments."""
 parser = optparse.OptionParser(usage="usage: %prog [options] <revs>",
                                description=helptext)
 parser.add_option("-f", "--file",
                   help="read revset from FILE (stdin if omitted)",
                   metavar="FILE")
 parser.add_option("-R", "--repo",
                   help="run benchmark on REPO", metavar="REPO")
 
 parser.add_option("-v", "--verbose",
                   action='store_true',
                   help="display all timing data (not just best total time)")
 
 parser.add_option("", "--variants",
                   default=','.join(DEFAULTVARIANTS),
                   help="comma separated list of variant to test "
                        "(eg: plain,min,sorted) (plain = no modification)")
 parser.add_option('', '--contexts',
                   action='store_true',
                   help='obtain changectx from results instead of integer revs')
 
 (options, args) = parser.parse_args()
 
 if not args:
     parser.print_help()
     sys.exit(255)
 
 # the directory where both this script and the perf.py extension live.
 contribdir = os.path.dirname(__file__)
 
 revsetsfile = sys.stdin
 if options.file:
     revsetsfile = open(options.file)
 
 revsets = [l.strip() for l in revsetsfile if not l.startswith('#')]
 revsets = [l for l in revsets if l]
 
 print("Revsets to benchmark")
 print("----------------------------")
 
 for idx, rset in enumerate(revsets):
     print("%i) %s" % (idx, rset))
 
 print("----------------------------")
 print()
 
 revs = []
 for a in args:
     revs.extend(getrevs(a))
 
 variants = options.variants.split(',')
 
 results = []
 for r in revs:
     print("----------------------------")
     printrevision(r)
     print("----------------------------")
     update(r)
     res = []
     results.append(res)
     printheader(variants, len(revsets), verbose=options.verbose)
     for idx, rset in enumerate(revsets):
         varres = {}
         for var in variants:
             varrset = applyvariants(rset, var)
             data = perf(varrset, target=options.repo, contexts=options.contexts)
             varres[var] = data
         res.append(varres)
         printresult(variants, idx, varres, len(revsets),
                     verbose=options.verbose)
         sys.stdout.flush()
     print("----------------------------")
 
 
 print("""
 
 Result by revset
 ================
 """)
 
 print('Revision:')
 for idx, rev in enumerate(revs):
     sys.stdout.write('%i) ' % idx)
     sys.stdout.flush()
     printrevision(rev)
 
 print()
 print()
 
 for ridx, rset in enumerate(revsets):
 
     print("revset #%i: %s" % (ridx, rset))
     printheader(variants, len(results), verbose=options.verbose, relative=True)
     ref = None
     for idx, data in enumerate(results):
         printresult(variants, idx, data[ridx], len(results),
                     verbose=options.verbose, reference=ref)
         ref = data[ridx]
     print()

contrib/wix/help.wxs

@@ -1,63 +1,64 b''
 <?xml version="1.0" encoding="utf-8"?>
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
 
   <?include guids.wxi ?>
   <?include defines.wxi ?>
 
   <Fragment>
     <ComponentGroup Id='helpFolder'>
       <ComponentRef Id='help.root' />
       <ComponentRef Id='help.internals' />
     </ComponentGroup>
   </Fragment>
 
   <Fragment>
     <DirectoryRef Id="INSTALLDIR">
       <Directory Id="helpdir" Name="help" FileSource="$(var.SourceDir)">
         <Component Id="help.root" Guid="$(var.help.root.guid)" Win64='$(var.IsX64)'>
           <File Name="bundlespec.txt" />
           <File Name="color.txt" />
           <File Name="config.txt" KeyPath="yes" />
           <File Name="dates.txt" />
           <File Name="deprecated.txt" />
           <File Name="diffs.txt" />
           <File Name="environment.txt" />
           <File Name="extensions.txt" />
           <File Name="filesets.txt" />
           <File Name="flags.txt" />
           <File Name="glossary.txt" />
           <File Name="hgignore.txt" />
           <File Name="hgweb.txt" />
           <File Name="merge-tools.txt" />
           <File Name="pager.txt" />
           <File Name="patterns.txt" />
           <File Name="phases.txt" />
           <File Name="revisions.txt" />
           <File Name="scripting.txt" />
           <File Name="subrepos.txt" />
           <File Name="templates.txt" />
           <File Name="urls.txt" />
         </Component>
 
         <Directory Id="help.internaldir" Name="internals">
           <Component Id="help.internals" Guid="$(var.help.internals.guid)" Win64='$(var.IsX64)'>
             <File Id="internals.bundle2.txt"      Name="bundle2.txt" />
             <File Id="internals.bundles.txt"      Name="bundles.txt" KeyPath="yes" />
             <File Id="internals.cbor.txt"         Name="cbor.txt" />
             <File Id="internals.censor.txt"       Name="censor.txt" />
             <File Id="internals.changegroups.txt" Name="changegroups.txt" />
             <File Id="internals.config.txt"       Name="config.txt" />
+            <File Id="internals.extensions.txt"   Name="extensions.txt" />
             <File Id="internals.linelog.txt"      Name="linelog.txt" />
             <File Id="internals.requirements.txt" Name="requirements.txt" />
             <File Id="internals.revlogs.txt"      Name="revlogs.txt" />
             <File Id="internals.wireprotocol.txt" Name="wireprotocol.txt" />
             <File Id="internals.wireprotocolrpc.txt" Name="wireprotocolrpc.txt" />
             <File Id="internals.wireprotocolv2.txt" Name="wireprotocolv2.txt" />
           </Component>
         </Directory>
 
       </Directory>
     </DirectoryRef>
   </Fragment>
 
 </Wix>

doc/docchecker

@@ -1,66 +1,76 b''
 #!/usr/bin/env python
 #
 # docchecker - look for problematic markup
 #
 # Copyright 2016 timeless <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
+import os
 import re
 import sys
 
-leadingline = re.compile(r'(^\s*)(\S.*)$')
+try:
+    import msvcrt
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
+except ImportError:
+    pass
+
+stdout = getattr(sys.stdout, 'buffer', sys.stdout)
+
+leadingline = re.compile(br'(^\s*)(\S.*)$')
 
 checks = [
-  (r""":hg:`[^`]*'[^`]*`""",
-    """warning: please avoid nesting ' in :hg:`...`"""),
-  (r'\w:hg:`',
-    'warning: please have a space before :hg:'),
-  (r"""(?:[^a-z][^'.])hg ([^,;"`]*'(?!hg)){2}""",
-    '''warning: please use " instead of ' for hg ... "..."'''),
+  (br""":hg:`[^`]*'[^`]*`""",
+   b"""warning: please avoid nesting ' in :hg:`...`"""),
+  (br'\w:hg:`',
+   b'warning: please have a space before :hg:'),
+  (br"""(?:[^a-z][^'.])hg ([^,;"`]*'(?!hg)){2}""",
+   b'''warning: please use " instead of ' for hg ... "..."'''),
 ]
 
 def check(line):
     messages = []
     for match, msg in checks:
         if re.search(match, line):
             messages.append(msg)
     if messages:
-        print(line)
+        stdout.write(b'%s\n' % line)
         for msg in messages:
-            print(msg)
+            stdout.write(b'%s\n' % msg)
 
 def work(file):
-    (llead, lline) = ('', '')
+    (llead, lline) = (b'', b'')
 
     for line in file:
         # this section unwraps lines
         match = leadingline.match(line)
         if not match:
             check(lline)
-            (llead, lline) = ('', '')
+            (llead, lline) = (b'', b'')
             continue
 
         lead, line = match.group(1), match.group(2)
         if (lead == llead):
-            if (lline != ''):
-                lline += ' ' + line
+            if (lline != b''):
+                lline += b' ' + line
             else:
                 lline = line
         else:
             check(lline)
             (llead, lline) = (lead, line)
     check(lline)
 
 def main():
     for f in sys.argv[1:]:
         try:
-            with open(f) as file:
+            with open(f, 'rb') as file:
                 work(file)
         except BaseException as e:
-            print("failed to process %s: %s" % (f, e))
+            sys.stdout.write(r"failed to process %s: %s\n" % (f, e))
 
 main()

doc/gendoc.py

@@ -1,230 +1,240 b''
 #!/usr/bin/env python
 """usage: %s DOC ...
 
 where DOC is the name of a document
 """
 
 from __future__ import absolute_import
 
 import os
 import sys
 import textwrap
 
+try:
+    import msvcrt
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
+except ImportError:
+    pass
+
 # This script is executed during installs and may not have C extensions
 # available. Relax C module requirements.
-os.environ['HGMODULEPOLICY'] = 'allow'
+os.environ[r'HGMODULEPOLICY'] = r'allow'
 # import from the live mercurial repo
-sys.path.insert(0, "..")
+sys.path.insert(0, r"..")
 from mercurial import demandimport; demandimport.enable()
 # Load util so that the locale path is set by i18n.setdatapath() before
 # calling _().
 from mercurial import util
 util.datapath
 from mercurial import (
     commands,
+    encoding,
     extensions,
     help,
     minirst,
+    pycompat,
     ui as uimod,
 )
 from mercurial.i18n import (
     gettext,
     _,
 )
 
 table = commands.table
 globalopts = commands.globalopts
 helptable = help.helptable
 loaddoc = help.loaddoc
 
 def get_desc(docstr):
     if not docstr:
-        return "", ""
+        return b"", b""
     # sanitize
-    docstr = docstr.strip("\n")
+    docstr = docstr.strip(b"\n")
     docstr = docstr.rstrip()
     shortdesc = docstr.splitlines()[0].strip()
 
-    i = docstr.find("\n")
+    i = docstr.find(b"\n")
     if i != -1:
         desc = docstr[i + 2:]
     else:
         desc = shortdesc
 
-    desc = textwrap.dedent(desc)
+    desc = textwrap.dedent(desc.decode('latin1')).encode('latin1')
 
     return (shortdesc, desc)
 
 def get_opts(opts):
     for opt in opts:
         if len(opt) == 5:
             shortopt, longopt, default, desc, optlabel = opt
         else:
             shortopt, longopt, default, desc = opt
-            optlabel = _("VALUE")
+            optlabel = _(b"VALUE")
         allopts = []
         if shortopt:
-            allopts.append("-%s" % shortopt)
+            allopts.append(b"-%s" % shortopt)
         if longopt:
-            allopts.append("--%s" % longopt)
+            allopts.append(b"--%s" % longopt)
         if isinstance(default, list):
-            allopts[-1] += " <%s[+]>" % optlabel
+            allopts[-1] += b" <%s[+]>" % optlabel
         elif (default is not None) and not isinstance(default, bool):
-            allopts[-1] += " <%s>" % optlabel
-        if '\n' in desc:
+            allopts[-1] += b" <%s>" % optlabel
+        if b'\n' in desc:
             # only remove line breaks and indentation
-            desc = ' '.join(l.lstrip() for l in desc.split('\n'))
-        desc += default and _(" (default: %s)") % default or ""
-        yield (", ".join(allopts), desc)
+            desc = b' '.join(l.lstrip() for l in desc.split(b'\n'))
+        desc += default and _(b" (default: %s)") % bytes(default) or b""
+        yield (b", ".join(allopts), desc)
 
 def get_cmd(cmd, cmdtable):
     d = {}
     attr = cmdtable[cmd]
-    cmds = cmd.lstrip("^").split("|")
+    cmds = cmd.lstrip(b"^").split(b"|")
 
-    d['cmd'] = cmds[0]
-    d['aliases'] = cmd.split("|")[1:]
-    d['desc'] = get_desc(gettext(attr[0].__doc__))
-    d['opts'] = list(get_opts(attr[1]))
+    d[b'cmd'] = cmds[0]
+    d[b'aliases'] = cmd.split(b"|")[1:]
+    d[b'desc'] = get_desc(gettext(pycompat.getdoc(attr[0])))
+    d[b'opts'] = list(get_opts(attr[1]))
 
-    s = 'hg ' + cmds[0]
+    s = b'hg ' + cmds[0]
     if len(attr) > 2:
-        if not attr[2].startswith('hg'):
-            s += ' ' + attr[2]
+        if not attr[2].startswith(b'hg'):
+            s += b' ' + attr[2]
         else:
             s = attr[2]
-    d['synopsis'] = s.strip()
+    d[b'synopsis'] = s.strip()
 
     return d
 
 def showdoc(ui):
     # print options
-    ui.write(minirst.section(_("Options")))
+    ui.write(minirst.section(_(b"Options")))
     multioccur = False
     for optstr, desc in get_opts(globalopts):
-        ui.write("%s\n    %s\n\n" % (optstr, desc))
-        if optstr.endswith("[+]>"):
+        ui.write(b"%s\n    %s\n\n" % (optstr, desc))
+        if optstr.endswith(b"[+]>"):
             multioccur = True
     if multioccur:
-        ui.write(_("\n[+] marked option can be specified multiple times\n"))
-        ui.write("\n")
+        ui.write(_(b"\n[+] marked option can be specified multiple times\n"))
+        ui.write(b"\n")
 
     # print cmds
-    ui.write(minirst.section(_("Commands")))
+    ui.write(minirst.section(_(b"Commands")))
     commandprinter(ui, table, minirst.subsection)
 
     # print help topics
     # The config help topic is included in the hgrc.5 man page.
-    helpprinter(ui, helptable, minirst.section, exclude=['config'])
+    helpprinter(ui, helptable, minirst.section, exclude=[b'config'])
 
-    ui.write(minirst.section(_("Extensions")))
-    ui.write(_("This section contains help for extensions that are "
-               "distributed together with Mercurial. Help for other "
-               "extensions is available in the help system."))
-    ui.write(("\n\n"
-             ".. contents::\n"
-             "   :class: htmlonly\n"
-             "   :local:\n"
-             "   :depth: 1\n\n"))
+    ui.write(minirst.section(_(b"Extensions")))
+    ui.write(_(b"This section contains help for extensions that are "
+               b"distributed together with Mercurial. Help for other "
+               b"extensions is available in the help system."))
+    ui.write((b"\n\n"
+              b".. contents::\n"
+              b"   :class: htmlonly\n"
+              b"   :local:\n"
+              b"   :depth: 1\n\n"))
 
     for extensionname in sorted(allextensionnames()):
         mod = extensions.load(ui, extensionname, None)
         ui.write(minirst.subsection(extensionname))
-        ui.write("%s\n\n" % gettext(mod.__doc__))
+        ui.write(b"%s\n\n" % gettext(pycompat.getdoc(mod)))
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
-            ui.write(minirst.subsubsection(_('Commands')))
+            ui.write(minirst.subsubsection(_(b'Commands')))
             commandprinter(ui, cmdtable, minirst.subsubsubsection)
 
 def showtopic(ui, topic):
     extrahelptable = [
-        (["common"], '', loaddoc('common'), help.TOPIC_CATEGORY_MISC),
-        (["hg.1"], '', loaddoc('hg.1'), help.TOPIC_CATEGORY_CONFIG),
-        (["hg-ssh.8"], '', loaddoc('hg-ssh.8'), help.TOPIC_CATEGORY_CONFIG),
-        (["hgignore.5"], '', loaddoc('hgignore.5'), help.TOPIC_CATEGORY_CONFIG),
-        (["hgrc.5"], '', loaddoc('hgrc.5'), help.TOPIC_CATEGORY_CONFIG),
-        (["hgignore.5.gendoc"], '', loaddoc('hgignore'),
+        ([b"common"], b'', loaddoc(b'common'), help.TOPIC_CATEGORY_MISC),
+        ([b"hg.1"], b'', loaddoc(b'hg.1'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hg-ssh.8"], b'', loaddoc(b'hg-ssh.8'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgignore.5"], b'', loaddoc(b'hgignore.5'),
          help.TOPIC_CATEGORY_CONFIG),
-        (["hgrc.5.gendoc"], '', loaddoc('config'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgrc.5"], b'', loaddoc(b'hgrc.5'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgignore.5.gendoc"], b'', loaddoc(b'hgignore'),
+         help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgrc.5.gendoc"], b'', loaddoc(b'config'),
+         help.TOPIC_CATEGORY_CONFIG),
     ]
     helpprinter(ui, helptable + extrahelptable, None, include=[topic])
 
 def helpprinter(ui, helptable, sectionfunc, include=[], exclude=[]):
     for h in helptable:
         names, sec, doc = h[0:3]
         if exclude and names[0] in exclude:
             continue
         if include and names[0] not in include:
             continue
         for name in names:
-            ui.write(".. _%s:\n" % name)
-        ui.write("\n")
+            ui.write(b".. _%s:\n" % name)
+        ui.write(b"\n")
         if sectionfunc:
             ui.write(sectionfunc(sec))
         if callable(doc):
             doc = doc(ui)
         ui.write(doc)
-        ui.write("\n")
+        ui.write(b"\n")
 
 def commandprinter(ui, cmdtable, sectionfunc):
     h = {}
     for c, attr in cmdtable.items():
-        f = c.split("|")[0]
-        f = f.lstrip("^")
+        f = c.split(b"|")[0]
+        f = f.lstrip(b"^")
         h[f] = c
     cmds = h.keys()
-    cmds.sort()
 
-    for f in cmds:
-        if f.startswith("debug"):
+    for f in sorted(cmds):
+        if f.startswith(b"debug"):
             continue
         d = get_cmd(h[f], cmdtable)
-        ui.write(sectionfunc(d['cmd']))
+        ui.write(sectionfunc(d[b'cmd']))
         # short description
-        ui.write(d['desc'][0])
+        ui.write(d[b'desc'][0])
         # synopsis
-        ui.write("::\n\n")
-        synopsislines = d['synopsis'].splitlines()
+        ui.write(b"::\n\n")
+        synopsislines = d[b'synopsis'].splitlines()
         for line in synopsislines:
             # some commands (such as rebase) have a multi-line
             # synopsis
-            ui.write("   %s\n" % line)
-        ui.write('\n')
+            ui.write(b"   %s\n" % line)
+        ui.write(b'\n')
         # description
-        ui.write("%s\n\n" % d['desc'][1])
+        ui.write(b"%s\n\n" % d[b'desc'][1])
         # options
-        opt_output = list(d['opts'])
+        opt_output = list(d[b'opts'])
         if opt_output:
             opts_len = max([len(line[0]) for line in opt_output])
-            ui.write(_("Options:\n\n"))
+            ui.write(_(b"Options:\n\n"))
             multioccur = False
             for optstr, desc in opt_output:
                 if desc:
-                    s = "%-*s  %s" % (opts_len, optstr, desc)
+                    s = b"%-*s  %s" % (opts_len, optstr, desc)
                 else:
                     s = optstr
-                ui.write("%s\n" % s)
-                if optstr.endswith("[+]>"):
+                ui.write(b"%s\n" % s)
+                if optstr.endswith(b"[+]>"):
                     multioccur = True
             if multioccur:
-                ui.write(_("\n[+] marked option can be specified"
-                           " multiple times\n"))
-            ui.write("\n")
+                ui.write(_(b"\n[+] marked option can be specified"
+                           b" multiple times\n"))
+            ui.write(b"\n")
         # aliases
-        if d['aliases']:
-            ui.write(_("    aliases: %s\n\n") % " ".join(d['aliases']))
+        if d[b'aliases']:
+            ui.write(_(b"    aliases: %s\n\n") % b" ".join(d[b'aliases']))
 
 
 def allextensionnames():
-    return extensions.enabled().keys() + extensions.disabled().keys()
+    return set(extensions.enabled().keys()) | set(extensions.disabled().keys())
 
 if __name__ == "__main__":
-    doc = 'hg.1.gendoc'
+    doc = b'hg.1.gendoc'
     if len(sys.argv) > 1:
-        doc = sys.argv[1]
+        doc = encoding.strtolocal(sys.argv[1])
 
     ui = uimod.ui.load()
-    if doc == 'hg.1.gendoc':
+    if doc == b'hg.1.gendoc':
         showdoc(ui)
     else:
-        showtopic(ui, sys.argv[1])
+        showtopic(ui, encoding.strtolocal(sys.argv[1]))

hgext/absorb.py

@@ -1,1026 +1,1028 b''
 # absorb.py
 #
 # Copyright 2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """apply working directory changes to changesets (EXPERIMENTAL)
 
 The absorb extension provides a command to use annotate information to
 amend modified chunks into the corresponding non-public changesets.
 
 ::
 
     [absorb]
     # only check 50 recent non-public changesets at most
     max-stack-size = 50
     # whether to add noise to new commits to avoid obsolescence cycle
     add-noise = 1
     # make `amend --correlated` a shortcut to the main command
     amend-flag = correlated
 
     [color]
     absorb.description = yellow
     absorb.node = blue bold
     absorb.path = bold
 """
 
 # TODO:
 #  * Rename config items to [commands] namespace
 #  * Converge getdraftstack() with other code in core
 #  * move many attributes on fixupstate to be private
 
 from __future__ import absolute_import
 
 import collections
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     context,
     crecord,
     error,
     linelog,
     mdiff,
     node,
     obsolete,
     patch,
     phases,
     pycompat,
     registrar,
     repair,
     scmutil,
     util,
 )
 from mercurial.utils import (
     stringutil,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('absorb', 'add-noise', default=True)
 configitem('absorb', 'amend-flag', default=None)
 configitem('absorb', 'max-stack-size', default=50)
 
 colortable = {
     'absorb.description': 'yellow',
     'absorb.node': 'blue bold',
     'absorb.path': 'bold',
 }
 
 defaultdict = collections.defaultdict
 
 class nullui(object):
     """blank ui object doing nothing"""
     debugflag = False
     verbose = False
     quiet = True
 
     def __getitem__(name):
         def nullfunc(*args, **kwds):
             return
         return nullfunc
 
 class emptyfilecontext(object):
     """minimal filecontext representing an empty file"""
     def data(self):
         return ''
 
     def node(self):
         return node.nullid
 
 def uniq(lst):
     """list -> list. remove duplicated items without changing the order"""
     seen = set()
     result = []
     for x in lst:
         if x not in seen:
             seen.add(x)
             result.append(x)
     return result
 
 def getdraftstack(headctx, limit=None):
     """(ctx, int?) -> [ctx]. get a linear stack of non-public changesets.
 
     changesets are sorted in topo order, oldest first.
     return at most limit items, if limit is a positive number.
 
     merges are considered as non-draft as well. i.e. every commit
     returned has and only has 1 parent.
     """
     ctx = headctx
     result = []
     while ctx.phase() != phases.public:
         if limit and len(result) >= limit:
             break
         parents = ctx.parents()
         if len(parents) != 1:
             break
         result.append(ctx)
         ctx = parents[0]
     result.reverse()
     return result
 
 def getfilestack(stack, path, seenfctxs=None):
     """([ctx], str, set) -> [fctx], {ctx: fctx}
 
     stack is a list of contexts, from old to new. usually they are what
     "getdraftstack" returns.
 
     follows renames, but not copies.
 
     seenfctxs is a set of filecontexts that will be considered "immutable".
     they are usually what this function returned in earlier calls, useful
     to avoid issues that a file was "moved" to multiple places and was then
     modified differently, like: "a" was copied to "b", "a" was also copied to
     "c" and then "a" was deleted, then both "b" and "c" were "moved" from "a"
     and we enforce only one of them to be able to affect "a"'s content.
 
     return an empty list and an empty dict, if the specified path does not
     exist in stack[-1] (the top of the stack).
 
     otherwise, return a list of de-duplicated filecontexts, and the map to
     convert ctx in the stack to fctx, for possible mutable fctxs. the first item
     of the list would be outside the stack and should be considered immutable.
     the remaining items are within the stack.
 
     for example, given the following changelog and corresponding filelog
     revisions:
 
       changelog: 3----4----5----6----7
       filelog:   x    0----1----1----2 (x: no such file yet)
 
     - if stack = [5, 6, 7], returns ([0, 1, 2], {5: 1, 6: 1, 7: 2})
     - if stack = [3, 4, 5], returns ([e, 0, 1], {4: 0, 5: 1}), where "e" is a
       dummy empty filecontext.
     - if stack = [2], returns ([], {})
     - if stack = [7], returns ([1, 2], {7: 2})
     - if stack = [6, 7], returns ([1, 2], {6: 1, 7: 2}), although {6: 1} can be
       removed, since 1 is immutable.
     """
     if seenfctxs is None:
         seenfctxs = set()
     assert stack
 
     if path not in stack[-1]:
         return [], {}
 
     fctxs = []
     fctxmap = {}
 
     pctx = stack[0].p1() # the public (immutable) ctx we stop at
     for ctx in reversed(stack):
         if path not in ctx: # the file is added in the next commit
             pctx = ctx
             break
         fctx = ctx[path]
         fctxs.append(fctx)
         if fctx in seenfctxs: # treat fctx as the immutable one
             pctx = None # do not add another immutable fctx
             break
         fctxmap[ctx] = fctx # only for mutable fctxs
         renamed = fctx.renamed()
         if renamed:
             path = renamed[0] # follow rename
             if path in ctx: # but do not follow copy
                 pctx = ctx.p1()
                 break
 
     if pctx is not None: # need an extra immutable fctx
         if path in pctx:
             fctxs.append(pctx[path])
         else:
             fctxs.append(emptyfilecontext())
 
     fctxs.reverse()
     # note: we rely on a property of hg: filerev is not reused for linear
     # history. i.e. it's impossible to have:
     #   changelog:  4----5----6 (linear, no merges)
     #   filelog:    1----2----1
     #                         ^ reuse filerev (impossible)
     # because parents are part of the hash. if that's not true, we need to
     # remove uniq and find a different way to identify fctxs.
     return uniq(fctxs), fctxmap
 
 class overlaystore(patch.filestore):
     """read-only, hybrid store based on a dict and ctx.
     memworkingcopy: {path: content}, overrides file contents.
     """
     def __init__(self, basectx, memworkingcopy):
         self.basectx = basectx
         self.memworkingcopy = memworkingcopy
 
     def getfile(self, path):
         """comply with mercurial.patch.filestore.getfile"""
         if path not in self.basectx:
             return None, None, None
         fctx = self.basectx[path]
         if path in self.memworkingcopy:
             content = self.memworkingcopy[path]
         else:
             content = fctx.data()
         mode = (fctx.islink(), fctx.isexec())
         renamed = fctx.renamed() # False or (path, node)
         return content, mode, (renamed and renamed[0])
 
 def overlaycontext(memworkingcopy, ctx, parents=None, extra=None):
     """({path: content}, ctx, (p1node, p2node)?, {}?) -> memctx
     memworkingcopy overrides file contents.
     """
     # parents must contain 2 items: (node1, node2)
     if parents is None:
         parents = ctx.repo().changelog.parents(ctx.node())
     if extra is None:
         extra = ctx.extra()
     date = ctx.date()
     desc = ctx.description()
     user = ctx.user()
     files = set(ctx.files()).union(memworkingcopy)
     store = overlaystore(ctx, memworkingcopy)
     return context.memctx(
         repo=ctx.repo(), parents=parents, text=desc,
         files=files, filectxfn=store, user=user, date=date,
         branch=None, extra=extra)
 
 class filefixupstate(object):
     """state needed to apply fixups to a single file
 
     internally, it keeps file contents of several revisions and a linelog.
 
     the linelog uses odd revision numbers for original contents (fctxs passed
     to __init__), and even revision numbers for fixups, like:
 
         linelog rev 1: self.fctxs[0] (from an immutable "public" changeset)
         linelog rev 2: fixups made to self.fctxs[0]
         linelog rev 3: self.fctxs[1] (a child of fctxs[0])
         linelog rev 4: fixups made to self.fctxs[1]
         ...
 
     a typical use is like:
 
         1. call diffwith, to calculate self.fixups
         2. (optionally), present self.fixups to the user, or change it
         3. call apply, to apply changes
         4. read results from "finalcontents", or call getfinalcontent
     """
 
     def __init__(self, fctxs, path, ui=None, opts=None):
         """([fctx], ui or None) -> None
 
         fctxs should be linear, and sorted by topo order - oldest first.
         fctxs[0] will be considered as "immutable" and will not be changed.
         """
         self.fctxs = fctxs
         self.path = path
         self.ui = ui or nullui()
         self.opts = opts or {}
 
         # following fields are built from fctxs. they exist for perf reason
         self.contents = [f.data() for f in fctxs]
         self.contentlines = pycompat.maplist(mdiff.splitnewlines, self.contents)
         self.linelog = self._buildlinelog()
         if self.ui.debugflag:
             assert self._checkoutlinelog() == self.contents
 
         # following fields will be filled later
         self.chunkstats = [0, 0] # [adopted, total : int]
         self.targetlines = [] # [str]
         self.fixups = [] # [(linelog rev, a1, a2, b1, b2)]
         self.finalcontents = [] # [str]
         self.ctxaffected = set()
 
     def diffwith(self, targetfctx, fm=None):
         """calculate fixups needed by examining the differences between
         self.fctxs[-1] and targetfctx, chunk by chunk.
 
         targetfctx is the target state we move towards. we may or may not be
         able to get there because not all modified chunks can be amended into
         a non-public fctx unambiguously.
 
         call this only once, before apply().
 
         update self.fixups, self.chunkstats, and self.targetlines.
         """
         a = self.contents[-1]
         alines = self.contentlines[-1]
         b = targetfctx.data()
         blines = mdiff.splitnewlines(b)
         self.targetlines = blines
 
         self.linelog.annotate(self.linelog.maxrev)
         annotated = self.linelog.annotateresult # [(linelog rev, linenum)]
         assert len(annotated) == len(alines)
         # add a dummy end line to make insertion at the end easier
         if annotated:
             dummyendline = (annotated[-1][0], annotated[-1][1] + 1)
             annotated.append(dummyendline)
 
         # analyse diff blocks
         for chunk in self._alldiffchunks(a, b, alines, blines):
             newfixups = self._analysediffchunk(chunk, annotated)
             self.chunkstats[0] += bool(newfixups) # 1 or 0
             self.chunkstats[1] += 1
             self.fixups += newfixups
             if fm is not None:
                 self._showchanges(fm, alines, blines, chunk, newfixups)
 
     def apply(self):
         """apply self.fixups. update self.linelog, self.finalcontents.
 
         call this only once, before getfinalcontent(), after diffwith().
         """
         # the following is unnecessary, as it's done by "diffwith":
         #   self.linelog.annotate(self.linelog.maxrev)
         for rev, a1, a2, b1, b2 in reversed(self.fixups):
             blines = self.targetlines[b1:b2]
             if self.ui.debugflag:
                 idx = (max(rev - 1, 0)) // 2
                 self.ui.write(_('%s: chunk %d:%d -> %d lines\n')
                               % (node.short(self.fctxs[idx].node()),
                                  a1, a2, len(blines)))
             self.linelog.replacelines(rev, a1, a2, b1, b2)
         if self.opts.get('edit_lines', False):
             self.finalcontents = self._checkoutlinelogwithedits()
         else:
             self.finalcontents = self._checkoutlinelog()
 
     def getfinalcontent(self, fctx):
         """(fctx) -> str. get modified file content for a given filecontext"""
         idx = self.fctxs.index(fctx)
         return self.finalcontents[idx]
 
     def _analysediffchunk(self, chunk, annotated):
         """analyse a different chunk and return new fixups found
 
         return [] if no lines from the chunk can be safely applied.
 
         the chunk (or lines) cannot be safely applied, if, for example:
           - the modified (deleted) lines belong to a public changeset
             (self.fctxs[0])
           - the chunk is a pure insertion and the adjacent lines (at most 2
             lines) belong to different non-public changesets, or do not belong
             to any non-public changesets.
           - the chunk is modifying lines from different changesets.
             in this case, if the number of lines deleted equals to the number
             of lines added, assume it's a simple 1:1 map (could be wrong).
             otherwise, give up.
           - the chunk is modifying lines from a single non-public changeset,
             but other revisions touch the area as well. i.e. the lines are
             not continuous as seen from the linelog.
         """
         a1, a2, b1, b2 = chunk
         # find involved indexes from annotate result
         involved = annotated[a1:a2]
         if not involved and annotated: # a1 == a2 and a is not empty
             # pure insertion, check nearby lines. ignore lines belong
             # to the public (first) changeset (i.e. annotated[i][0] == 1)
             nearbylinenums = {a2, max(0, a1 - 1)}
             involved = [annotated[i]
                         for i in nearbylinenums if annotated[i][0] != 1]
         involvedrevs = list(set(r for r, l in involved))
         newfixups = []
         if len(involvedrevs) == 1 and self._iscontinuous(a1, a2 - 1, True):
             # chunk belongs to a single revision
             rev = involvedrevs[0]
             if rev > 1:
                 fixuprev = rev + 1
                 newfixups.append((fixuprev, a1, a2, b1, b2))
         elif a2 - a1 == b2 - b1 or b1 == b2:
             # 1:1 line mapping, or chunk was deleted
             for i in pycompat.xrange(a1, a2):
                 rev, linenum = annotated[i]
                 if rev > 1:
                     if b1 == b2: # deletion, simply remove that single line
                         nb1 = nb2 = 0
                     else: # 1:1 line mapping, change the corresponding rev
                         nb1 = b1 + i - a1
                         nb2 = nb1 + 1
                     fixuprev = rev + 1
                     newfixups.append((fixuprev, i, i + 1, nb1, nb2))
         return self._optimizefixups(newfixups)
 
     @staticmethod
     def _alldiffchunks(a, b, alines, blines):
         """like mdiff.allblocks, but only care about differences"""
         blocks = mdiff.allblocks(a, b, lines1=alines, lines2=blines)
         for chunk, btype in blocks:
             if btype != '!':
                 continue
             yield chunk
 
     def _buildlinelog(self):
         """calculate the initial linelog based on self.content{,line}s.
         this is similar to running a partial "annotate".
         """
         llog = linelog.linelog()
         a, alines = '', []
         for i in pycompat.xrange(len(self.contents)):
             b, blines = self.contents[i], self.contentlines[i]
             llrev = i * 2 + 1
             chunks = self._alldiffchunks(a, b, alines, blines)
             for a1, a2, b1, b2 in reversed(list(chunks)):
                 llog.replacelines(llrev, a1, a2, b1, b2)
             a, alines = b, blines
         return llog
 
     def _checkoutlinelog(self):
         """() -> [str]. check out file contents from linelog"""
         contents = []
         for i in pycompat.xrange(len(self.contents)):
             rev = (i + 1) * 2
             self.linelog.annotate(rev)
             content = ''.join(map(self._getline, self.linelog.annotateresult))
             contents.append(content)
         return contents
 
     def _checkoutlinelogwithedits(self):
         """() -> [str]. prompt all lines for edit"""
         alllines = self.linelog.getalllines()
         # header
         editortext = (_('HG: editing %s\nHG: "y" means the line to the right '
                         'exists in the changeset to the top\nHG:\n')
                       % self.fctxs[-1].path())
         # [(idx, fctx)]. hide the dummy emptyfilecontext
         visiblefctxs = [(i, f)
                         for i, f in enumerate(self.fctxs)
                         if not isinstance(f, emptyfilecontext)]
         for i, (j, f) in enumerate(visiblefctxs):
             editortext += (_('HG: %s/%s %s %s\n') %
                            ('|' * i, '-' * (len(visiblefctxs) - i + 1),
                             node.short(f.node()),
                             f.description().split('\n',1)[0]))
         editortext += _('HG: %s\n') % ('|' * len(visiblefctxs))
         # figure out the lifetime of a line, this is relatively inefficient,
         # but probably fine
         lineset = defaultdict(lambda: set()) # {(llrev, linenum): {llrev}}
         for i, f in visiblefctxs:
             self.linelog.annotate((i + 1) * 2)
             for l in self.linelog.annotateresult:
                 lineset[l].add(i)
         # append lines
         for l in alllines:
             editortext += ('    %s : %s' %
                            (''.join([('y' if i in lineset[l] else ' ')
                                      for i, _f in visiblefctxs]),
                             self._getline(l)))
         # run editor
         editedtext = self.ui.edit(editortext, '', action='absorb')
         if not editedtext:
             raise error.Abort(_('empty editor text'))
         # parse edited result
         contents = ['' for i in self.fctxs]
         leftpadpos = 4
         colonpos = leftpadpos + len(visiblefctxs) + 1
         for l in mdiff.splitnewlines(editedtext):
             if l.startswith('HG:'):
                 continue
             if l[colonpos - 1:colonpos + 2] != ' : ':
                 raise error.Abort(_('malformed line: %s') % l)
             linecontent = l[colonpos + 2:]
-            for i, ch in enumerate(l[leftpadpos:colonpos - 1]):
+            for i, ch in enumerate(
+                    pycompat.bytestr(l[leftpadpos:colonpos - 1])):
                 if ch == 'y':
                     contents[visiblefctxs[i][0]] += linecontent
         # chunkstats is hard to calculate if anything changes, therefore
         # set them to just a simple value (1, 1).
         if editedtext != editortext:
             self.chunkstats = [1, 1]
         return contents
 
     def _getline(self, lineinfo):
         """((rev, linenum)) -> str. convert rev+line number to line content"""
         rev, linenum = lineinfo
         if rev & 1: # odd: original line taken from fctxs
             return self.contentlines[rev // 2][linenum]
         else: # even: fixup line from targetfctx
             return self.targetlines[linenum]
 
     def _iscontinuous(self, a1, a2, closedinterval=False):
         """(a1, a2 : int) -> bool
 
         check if these lines are continuous. i.e. no other insertions or
         deletions (from other revisions) among these lines.
 
         closedinterval decides whether a2 should be included or not. i.e. is
         it [a1, a2), or [a1, a2] ?
         """
         if a1 >= a2:
             return True
         llog = self.linelog
         offset1 = llog.getoffset(a1)
         offset2 = llog.getoffset(a2) + int(closedinterval)
         linesinbetween = llog.getalllines(offset1, offset2)
         return len(linesinbetween) == a2 - a1 + int(closedinterval)
 
     def _optimizefixups(self, fixups):
         """[(rev, a1, a2, b1, b2)] -> [(rev, a1, a2, b1, b2)].
         merge adjacent fixups to make them less fragmented.
         """
         result = []
         pcurrentchunk = [[-1, -1, -1, -1, -1]]
 
         def pushchunk():
             if pcurrentchunk[0][0] != -1:
                 result.append(tuple(pcurrentchunk[0]))
 
         for i, chunk in enumerate(fixups):
             rev, a1, a2, b1, b2 = chunk
             lastrev = pcurrentchunk[0][0]
             lasta2 = pcurrentchunk[0][2]
             lastb2 = pcurrentchunk[0][4]
             if (a1 == lasta2 and b1 == lastb2 and rev == lastrev and
                     self._iscontinuous(max(a1 - 1, 0), a1)):
                 # merge into currentchunk
                 pcurrentchunk[0][2] = a2
                 pcurrentchunk[0][4] = b2
             else:
                 pushchunk()
                 pcurrentchunk[0] = list(chunk)
         pushchunk()
         return result
 
     def _showchanges(self, fm, alines, blines, chunk, fixups):
 
         def trim(line):
             if line.endswith('\n'):
                 line = line[:-1]
             return line
 
         # this is not optimized for perf but _showchanges only gets executed
         # with an extra command-line flag.
         a1, a2, b1, b2 = chunk
         aidxs, bidxs = [0] * (a2 - a1), [0] * (b2 - b1)
         for idx, fa1, fa2, fb1, fb2 in fixups:
             for i in pycompat.xrange(fa1, fa2):
                 aidxs[i - a1] = (max(idx, 1) - 1) // 2
             for i in pycompat.xrange(fb1, fb2):
                 bidxs[i - b1] = (max(idx, 1) - 1) // 2
 
         fm.startitem()
         fm.write('hunk', '        %s\n',
                  '@@ -%d,%d +%d,%d @@'
                  % (a1, a2 - a1, b1, b2 - b1), label='diff.hunk')
         fm.data(path=self.path, linetype='hunk')
 
         def writeline(idx, diffchar, line, linetype, linelabel):
             fm.startitem()
             node = ''
             if idx:
                 ctx = self.fctxs[idx]
                 fm.context(fctx=ctx)
                 node = ctx.hex()
                 self.ctxaffected.add(ctx.changectx())
             fm.write('node', '%-7.7s ', node, label='absorb.node')
             fm.write('diffchar ' + linetype, '%s%s\n', diffchar, line,
                      label=linelabel)
             fm.data(path=self.path, linetype=linetype)
 
         for i in pycompat.xrange(a1, a2):
             writeline(aidxs[i - a1], '-', trim(alines[i]), 'deleted',
                       'diff.deleted')
         for i in pycompat.xrange(b1, b2):
             writeline(bidxs[i - b1], '+', trim(blines[i]), 'inserted',
                       'diff.inserted')
 
 class fixupstate(object):
     """state needed to run absorb
 
     internally, it keeps paths and filefixupstates.
 
     a typical use is like filefixupstates:
 
         1. call diffwith, to calculate fixups
         2. (optionally), present fixups to the user, or edit fixups
         3. call apply, to apply changes to memory
         4. call commit, to commit changes to hg database
     """
 
     def __init__(self, stack, ui=None, opts=None):
         """([ctx], ui or None) -> None
 
         stack: should be linear, and sorted by topo order - oldest first.
         all commits in stack are considered mutable.
         """
         assert stack
         self.ui = ui or nullui()
         self.opts = opts or {}
         self.stack = stack
         self.repo = stack[-1].repo().unfiltered()
 
         # following fields will be filled later
         self.paths = [] # [str]
         self.status = None # ctx.status output
         self.fctxmap = {} # {path: {ctx: fctx}}
         self.fixupmap = {} # {path: filefixupstate}
         self.replacemap = {} # {oldnode: newnode or None}
         self.finalnode = None # head after all fixups
         self.ctxaffected = set() # ctx that will be absorbed into
 
     def diffwith(self, targetctx, match=None, fm=None):
         """diff and prepare fixups. update self.fixupmap, self.paths"""
         # only care about modified files
         self.status = self.stack[-1].status(targetctx, match)
         self.paths = []
         # but if --edit-lines is used, the user may want to edit files
         # even if they are not modified
         editopt = self.opts.get('edit_lines')
         if not self.status.modified and editopt and match:
             interestingpaths = match.files()
         else:
             interestingpaths = self.status.modified
         # prepare the filefixupstate
         seenfctxs = set()
         # sorting is necessary to eliminate ambiguity for the "double move"
         # case: "hg cp A B; hg cp A C; hg rm A", then only "B" can affect "A".
         for path in sorted(interestingpaths):
             self.ui.debug('calculating fixups for %s\n' % path)
             targetfctx = targetctx[path]
             fctxs, ctx2fctx = getfilestack(self.stack, path, seenfctxs)
             # ignore symbolic links or binary, or unchanged files
             if any(f.islink() or stringutil.binary(f.data())
                    for f in [targetfctx] + fctxs
                    if not isinstance(f, emptyfilecontext)):
                 continue
             if targetfctx.data() == fctxs[-1].data() and not editopt:
                 continue
             seenfctxs.update(fctxs[1:])
             self.fctxmap[path] = ctx2fctx
             fstate = filefixupstate(fctxs, path, ui=self.ui, opts=self.opts)
             if fm is not None:
                 fm.startitem()
                 fm.plain('showing changes for ')
                 fm.write('path', '%s\n', path, label='absorb.path')
                 fm.data(linetype='path')
             fstate.diffwith(targetfctx, fm)
             self.fixupmap[path] = fstate
             self.paths.append(path)
             self.ctxaffected.update(fstate.ctxaffected)
 
     def apply(self):
         """apply fixups to individual filefixupstates"""
         for path, state in self.fixupmap.iteritems():
             if self.ui.debugflag:
                 self.ui.write(_('applying fixups to %s\n') % path)
             state.apply()
 
     @property
     def chunkstats(self):
         """-> {path: chunkstats}. collect chunkstats from filefixupstates"""
         return dict((path, state.chunkstats)
                     for path, state in self.fixupmap.iteritems())
 
     def commit(self):
         """commit changes. update self.finalnode, self.replacemap"""
         with self.repo.wlock(), self.repo.lock():
             with self.repo.transaction('absorb') as tr:
                 self._commitstack()
                 self._movebookmarks(tr)
                 if self.repo['.'].node() in self.replacemap:
                     self._moveworkingdirectoryparent()
                 if self._useobsolete:
                     self._obsoleteoldcommits()
             if not self._useobsolete: # strip must be outside transactions
                 self._stripoldcommits()
         return self.finalnode
 
     def printchunkstats(self):
         """print things like '1 of 2 chunk(s) applied'"""
         ui = self.ui
         chunkstats = self.chunkstats
         if ui.verbose:
             # chunkstats for each file
             for path, stat in chunkstats.iteritems():
                 if stat[0]:
                     ui.write(_('%s: %d of %d chunk(s) applied\n')
                              % (path, stat[0], stat[1]))
         elif not ui.quiet:
             # a summary for all files
             stats = chunkstats.values()
             applied, total = (sum(s[i] for s in stats) for i in (0, 1))
             ui.write(_('%d of %d chunk(s) applied\n') % (applied, total))
 
     def _commitstack(self):
         """make new commits. update self.finalnode, self.replacemap.
         it is splitted from "commit" to avoid too much indentation.
         """
         # last node (20-char) committed by us
         lastcommitted = None
         # p1 which overrides the parent of the next commit, "None" means use
         # the original parent unchanged
         nextp1 = None
         for ctx in self.stack:
             memworkingcopy = self._getnewfilecontents(ctx)
             if not memworkingcopy and not lastcommitted:
                 # nothing changed, nothing commited
                 nextp1 = ctx
                 continue
             msg = ''
             if self._willbecomenoop(memworkingcopy, ctx, nextp1):
                 # changeset is no longer necessary
                 self.replacemap[ctx.node()] = None
                 msg = _('became empty and was dropped')
             else:
                 # changeset needs re-commit
                 nodestr = self._commitsingle(memworkingcopy, ctx, p1=nextp1)
                 lastcommitted = self.repo[nodestr]
                 nextp1 = lastcommitted
                 self.replacemap[ctx.node()] = lastcommitted.node()
                 if memworkingcopy:
                     msg = _('%d file(s) changed, became %s') % (
                         len(memworkingcopy), self._ctx2str(lastcommitted))
                 else:
                     msg = _('became %s') % self._ctx2str(lastcommitted)
             if self.ui.verbose and msg:
                 self.ui.write(_('%s: %s\n') % (self._ctx2str(ctx), msg))
         self.finalnode = lastcommitted and lastcommitted.node()
 
     def _ctx2str(self, ctx):
         if self.ui.debugflag:
             return '%d:%s' % (ctx.rev(), ctx.hex())
         else:
             return '%d:%s' % (ctx.rev(), node.short(ctx.node()))
 
     def _getnewfilecontents(self, ctx):
         """(ctx) -> {path: str}
 
         fetch file contents from filefixupstates.
         return the working copy overrides - files different from ctx.
         """
         result = {}
         for path in self.paths:
             ctx2fctx = self.fctxmap[path] # {ctx: fctx}
             if ctx not in ctx2fctx:
                 continue
             fctx = ctx2fctx[ctx]
             content = fctx.data()
             newcontent = self.fixupmap[path].getfinalcontent(fctx)
             if content != newcontent:
                 result[fctx.path()] = newcontent
         return result
 
     def _movebookmarks(self, tr):
         repo = self.repo
         needupdate = [(name, self.replacemap[hsh])
                       for name, hsh in repo._bookmarks.iteritems()
                       if hsh in self.replacemap]
         changes = []
         for name, hsh in needupdate:
             if hsh:
                 changes.append((name, hsh))
                 if self.ui.verbose:
                     self.ui.write(_('moving bookmark %s to %s\n')
                                   % (name, node.hex(hsh)))
             else:
                 changes.append((name, None))
                 if self.ui.verbose:
                     self.ui.write(_('deleting bookmark %s\n') % name)
         repo._bookmarks.applychanges(repo, tr, changes)
 
     def _moveworkingdirectoryparent(self):
         if not self.finalnode:
             # Find the latest not-{obsoleted,stripped} parent.
             revs = self.repo.revs('max(::. - %ln)', self.replacemap.keys())
             ctx = self.repo[revs.first()]
             self.finalnode = ctx.node()
         else:
             ctx = self.repo[self.finalnode]
 
         dirstate = self.repo.dirstate
         # dirstate.rebuild invalidates fsmonitorstate, causing "hg status" to
         # be slow. in absorb's case, no need to invalidate fsmonitorstate.
         noop = lambda: 0
         restore = noop
         if util.safehasattr(dirstate, '_fsmonitorstate'):
             bak = dirstate._fsmonitorstate.invalidate
             def restore():
                 dirstate._fsmonitorstate.invalidate = bak
             dirstate._fsmonitorstate.invalidate = noop
         try:
             with dirstate.parentchange():
                 dirstate.rebuild(ctx.node(), ctx.manifest(), self.paths)
         finally:
             restore()
 
     @staticmethod
     def _willbecomenoop(memworkingcopy, ctx, pctx=None):
         """({path: content}, ctx, ctx) -> bool. test if a commit will be noop
 
         if it will become an empty commit (does not change anything, after the
         memworkingcopy overrides), return True. otherwise return False.
         """
         if not pctx:
             parents = ctx.parents()
             if len(parents) != 1:
                 return False
             pctx = parents[0]
         # ctx changes more files (not a subset of memworkingcopy)
         if not set(ctx.files()).issubset(set(memworkingcopy)):
             return False
         for path, content in memworkingcopy.iteritems():
             if path not in pctx or path not in ctx:
                 return False
             fctx = ctx[path]
             pfctx = pctx[path]
             if pfctx.flags() != fctx.flags():
                 return False
             if pfctx.data() != content:
                 return False
         return True
 
     def _commitsingle(self, memworkingcopy, ctx, p1=None):
         """(ctx, {path: content}, node) -> node. make a single commit
 
         the commit is a clone from ctx, with a (optionally) different p1, and
         different file contents replaced by memworkingcopy.
         """
         parents = p1 and (p1, node.nullid)
         extra = ctx.extra()
         if self._useobsolete and self.ui.configbool('absorb', 'add-noise'):
             extra['absorb_source'] = ctx.hex()
         mctx = overlaycontext(memworkingcopy, ctx, parents, extra=extra)
         # preserve phase
         with mctx.repo().ui.configoverride({
             ('phases', 'new-commit'): ctx.phase()}):
             return mctx.commit()
 
     @util.propertycache
     def _useobsolete(self):
         """() -> bool"""
         return obsolete.isenabled(self.repo, obsolete.createmarkersopt)
 
     def _obsoleteoldcommits(self):
         relations = [(self.repo[k], v and (self.repo[v],) or ())
                      for k, v in self.replacemap.iteritems()]
         if relations:
             obsolete.createmarkers(self.repo, relations)
 
     def _stripoldcommits(self):
         nodelist = self.replacemap.keys()
         # make sure we don't strip innocent children
         revs = self.repo.revs('%ln - (::(heads(%ln::)-%ln))', nodelist,
                               nodelist, nodelist)
         tonode = self.repo.changelog.node
         nodelist = [tonode(r) for r in revs]
         if nodelist:
             repair.strip(self.repo.ui, self.repo, nodelist)
 
 def _parsechunk(hunk):
     """(crecord.uihunk or patch.recordhunk) -> (path, (a1, a2, [bline]))"""
     if type(hunk) not in (crecord.uihunk, patch.recordhunk):
         return None, None
     path = hunk.header.filename()
     a1 = hunk.fromline + len(hunk.before) - 1
     # remove before and after context
     hunk.before = hunk.after = []
     buf = util.stringio()
     hunk.write(buf)
     patchlines = mdiff.splitnewlines(buf.getvalue())
     # hunk.prettystr() will update hunk.removed
     a2 = a1 + hunk.removed
     blines = [l[1:] for l in patchlines[1:] if l[0] != '-']
     return path, (a1, a2, blines)
 
 def overlaydiffcontext(ctx, chunks):
     """(ctx, [crecord.uihunk]) -> memctx
 
     return a memctx with some [1] patches (chunks) applied to ctx.
     [1]: modifications are handled. renames, mode changes, etc. are ignored.
     """
     # sadly the applying-patch logic is hardly reusable, and messy:
     # 1. the core logic "_applydiff" is too heavy - it writes .rej files, it
     #    needs a file stream of a patch and will re-parse it, while we have
     #    structured hunk objects at hand.
     # 2. a lot of different implementations about "chunk" (patch.hunk,
     #    patch.recordhunk, crecord.uihunk)
     # as we only care about applying changes to modified files, no mode
     # change, no binary diff, and no renames, it's probably okay to
     # re-invent the logic using much simpler code here.
     memworkingcopy = {} # {path: content}
     patchmap = defaultdict(lambda: []) # {path: [(a1, a2, [bline])]}
     for path, info in map(_parsechunk, chunks):
         if not path or not info:
             continue
         patchmap[path].append(info)
     for path, patches in patchmap.iteritems():
         if path not in ctx or not patches:
             continue
         patches.sort(reverse=True)
         lines = mdiff.splitnewlines(ctx[path].data())
         for a1, a2, blines in patches:
             lines[a1:a2] = blines
         memworkingcopy[path] = ''.join(lines)
     return overlaycontext(memworkingcopy, ctx)
 
 def absorb(ui, repo, stack=None, targetctx=None, pats=None, opts=None):
     """pick fixup chunks from targetctx, apply them to stack.
 
     if targetctx is None, the working copy context will be used.
     if stack is None, the current draft stack will be used.
     return fixupstate.
     """
     if stack is None:
         limit = ui.configint('absorb', 'max-stack-size')
         stack = getdraftstack(repo['.'], limit)
         if limit and len(stack) >= limit:
             ui.warn(_('absorb: only the recent %d changesets will '
                       'be analysed\n')
                     % limit)
     if not stack:
         raise error.Abort(_('no mutable changeset to change'))
     if targetctx is None: # default to working copy
         targetctx = repo[None]
     if pats is None:
         pats = ()
     if opts is None:
         opts = {}
     state = fixupstate(stack, ui=ui, opts=opts)
     matcher = scmutil.match(targetctx, pats, opts)
     if opts.get('interactive'):
         diff = patch.diff(repo, stack[-1].node(), targetctx.node(), matcher)
         origchunks = patch.parsepatch(diff)
         chunks = cmdutil.recordfilter(ui, origchunks)[0]
         targetctx = overlaydiffcontext(stack[-1], chunks)
     fm = None
     if opts.get('print_changes') or not opts.get('apply_changes'):
         fm = ui.formatter('absorb', opts)
     state.diffwith(targetctx, matcher, fm)
     if fm is not None:
         fm.startitem()
         fm.write("count", "\n%d changesets affected\n", len(state.ctxaffected))
         fm.data(linetype='summary')
         for ctx in reversed(stack):
             if ctx not in state.ctxaffected:
                 continue
             fm.startitem()
             fm.context(ctx=ctx)
             fm.data(linetype='changeset')
             fm.write('node', '%-7.7s ', ctx.hex(), label='absorb.node')
             descfirstline = ctx.description().splitlines()[0]
             fm.write('descfirstline', '%s\n', descfirstline,
                      label='absorb.description')
         fm.end()
     if not opts.get('dry_run'):
-        if not opts.get('apply_changes'):
-            if ui.promptchoice("apply changes (yn)? $$ &Yes $$ &No", default=1):
-                raise error.Abort(_('absorb cancelled\n'))
+        if (not opts.get('apply_changes') and
+            state.ctxaffected and
+            ui.promptchoice("apply changes (yn)? $$ &Yes $$ &No", default=1)):
+            raise error.Abort(_('absorb cancelled\n'))
 
         state.apply()
         if state.commit():
             state.printchunkstats()
         elif not ui.quiet:
             ui.write(_('nothing applied\n'))
     return state
 
 @command('absorb',
          [('a', 'apply-changes', None,
            _('apply changes without prompting for confirmation')),
           ('p', 'print-changes', None,
            _('always print which changesets are modified by which changes')),
           ('i', 'interactive', None,
            _('interactively select which chunks to apply (EXPERIMENTAL)')),
           ('e', 'edit-lines', None,
            _('edit what lines belong to which changesets before commit '
              '(EXPERIMENTAL)')),
          ] + commands.dryrunopts + commands.templateopts + commands.walkopts,
          _('hg absorb [OPTION] [FILE]...'),
          helpcategory=command.CATEGORY_COMMITTING,
          helpbasic=True)
 def absorbcmd(ui, repo, *pats, **opts):
     """incorporate corrections into the stack of draft changesets
 
     absorb analyzes each change in your working directory and attempts to
     amend the changed lines into the changesets in your stack that first
     introduced those lines.
 
     If absorb cannot find an unambiguous changeset to amend for a change,
     that change will be left in the working directory, untouched. They can be
     observed by :hg:`status` or :hg:`diff` afterwards. In other words,
     absorb does not write to the working directory.
 
     Changesets outside the revset `::. and not public() and not merge()` will
     not be changed.
 
     Changesets that become empty after applying the changes will be deleted.
 
     By default, absorb will show what it plans to do and prompt for
     confirmation.  If you are confident that the changes will be absorbed
     to the correct place, run :hg:`absorb -a` to apply the changes
     immediately.
 
     Returns 0 on success, 1 if all chunks were ignored and nothing amended.
     """
     opts = pycompat.byteskwargs(opts)
     state = absorb(ui, repo, pats=pats, opts=opts)
     if sum(s[0] for s in state.chunkstats.values()) == 0:
         return 1

hgext/amend.py

@@ -1,58 +1,60 b''
 # amend.py - provide the amend command
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """provide the amend command (EXPERIMENTAL)
 
 This extension provides an ``amend`` command that is similar to
 ``commit --amend`` but does not prompt an editor.
 """
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     error,
     pycompat,
     registrar,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 @command('amend',
     [('A', 'addremove', None,
       _('mark new/missing files as added/removed before committing')),
      ('e', 'edit', None, _('invoke editor on commit messages')),
      ('i', 'interactive', None, _('use interactive mode')),
      ('n', 'note', '', _('store a note on the amend')),
+     ('D', 'currentdate', None,
+      _('record the current date as commit date')),
     ] + cmdutil.walkopts + cmdutil.commitopts + cmdutil.commitopts2,
     _('[OPTION]... [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     inferrepo=True)
 def amend(ui, repo, *pats, **opts):
     """amend the working copy parent with all or specified outstanding changes
 
     Similar to :hg:`commit --amend`, but reuse the commit message without
     invoking editor, unless ``--edit`` was set.
 
     See :hg:`help commit` for more details.
     """
     opts = pycompat.byteskwargs(opts)
     if len(opts['note']) > 255:
         raise error.Abort(_("cannot store a note of more than 255 bytes"))
     with repo.wlock(), repo.lock():
         if not opts.get('logfile'):
             opts['message'] = opts.get('message') or repo['.'].description()
         opts['amend'] = True
         return commands._docommit(ui, repo, *pats, **pycompat.strkwargs(opts))

hgext/beautifygraph.py

@@ -1,96 +1,94 b''
 # -*- coding: UTF-8 -*-
 # beautifygraph.py - improve graph output by using Unicode characters
 #
 # Copyright 2018 John Stiles <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''beautify log -G output by using Unicode characters (EXPERIMENTAL)
 
    A terminal with UTF-8 support and monospace narrow text are required.
 '''
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     extensions,
     graphmod,
     pycompat,
     templatekw,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 def prettyedge(before, edge, after):
     if edge == '~':
         return '\xE2\x95\xA7' # U+2567 ╧
-    if edge == 'X':
-        return '\xE2\x95\xB3' # U+2573 ╳
     if edge == '/':
         return '\xE2\x95\xB1' # U+2571 ╱
     if edge == '-':
         return '\xE2\x94\x80' # U+2500 ─
     if edge == '|':
         return '\xE2\x94\x82' # U+2502 │
     if edge == ':':
         return '\xE2\x94\x86' # U+2506 ┆
     if edge == '\\':
         return '\xE2\x95\xB2' # U+2572 ╲
     if edge == '+':
         if before == ' ' and not after  == ' ':
             return '\xE2\x94\x9C' # U+251C ├
         if after  == ' ' and not before == ' ':
             return '\xE2\x94\xA4' # U+2524 ┤
         return '\xE2\x94\xBC' # U+253C ┼
     return edge
 
 def convertedges(line):
     line = ' %s ' % line
     pretty = []
     for idx in pycompat.xrange(len(line) - 2):
         pretty.append(prettyedge(line[idx:idx + 1],
                                  line[idx + 1:idx + 2],
                                  line[idx + 2:idx + 3]))
     return ''.join(pretty)
 
 def getprettygraphnode(orig, *args, **kwargs):
     node = orig(*args, **kwargs)
     if node == 'o':
         return '\xE2\x97\x8B' # U+25CB ○
     if node == '@':
         return '\xE2\x97\x8D' # U+25CD ◍
     if node == '*':
         return '\xE2\x88\x97' # U+2217 ∗
     if node == 'x':
         return '\xE2\x97\x8C' # U+25CC ◌
     if node == '_':
         return '\xE2\x95\xA4' # U+2564 ╤
     return node
 
 def outputprettygraph(orig, ui, graph, *args, **kwargs):
     (edges, text) = zip(*graph)
     graph = zip([convertedges(e) for e in edges], text)
     return orig(ui, graph, *args, **kwargs)
 
 def extsetup(ui):
     if ui.plain('graph'):
         return
 
     if encoding.encoding != 'UTF-8':
         ui.warn(_('beautifygraph: unsupported encoding, UTF-8 required\n'))
         return
 
     if r'A' in encoding._wide:
         ui.warn(_('beautifygraph: unsupported terminal settings, '
                   'monospace narrow text required\n'))
         return
 
     extensions.wrapfunction(graphmod, 'outputgraph', outputprettygraph)
     extensions.wrapfunction(templatekw, 'getgraphnode', getprettygraphnode)

hgext/blackbox.py

@@ -1,257 +1,198 b''
 # blackbox.py - log repository events to a file for post-mortem debugging
 #
 # Copyright 2010 Nicolas Dumazet
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """log repository events to a blackbox for debugging
 
 Logs event information to .hg/blackbox.log to help debug and diagnose problems.
 The events that get logged can be configured via the blackbox.track config key.
 
 Examples::
 
   [blackbox]
   track = *
   # dirty is *EXPENSIVE* (slow);
   # each log entry indicates `+` if the repository is dirty, like :hg:`id`.
   dirty = True
   # record the source of log messages
   logsource = True
 
   [blackbox]
   track = command, commandfinish, commandexception, exthook, pythonhook
 
   [blackbox]
   track = incoming
 
   [blackbox]
   # limit the size of a log file
   maxsize = 1.5 MB
   # rotate up to N log files when the current one gets too big
   maxfiles = 3
 
+  [blackbox]
+  # Include nanoseconds in log entries with %f (see Python function
+  # datetime.datetime.strftime)
+  date-format = '%Y-%m-%d @ %H:%M:%S.%f'
+
 """
 
 from __future__ import absolute_import
 
-import errno
 import re
 
 from mercurial.i18n import _
 from mercurial.node import hex
 
 from mercurial import (
     encoding,
-    pycompat,
+    loggingutil,
     registrar,
-    ui as uimod,
-    util,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('blackbox', 'dirty',
     default=False,
 )
 configitem('blackbox', 'maxsize',
     default='1 MB',
 )
 configitem('blackbox', 'logsource',
     default=False,
 )
 configitem('blackbox', 'maxfiles',
     default=7,
 )
 configitem('blackbox', 'track',
     default=lambda: ['*'],
 )
+configitem('blackbox', 'date-format',
+    default='%Y/%m/%d %H:%M:%S',
+)
 
-lastui = None
+_lastlogger = loggingutil.proxylogger()
 
-def _openlogfile(ui, vfs):
-    def rotate(oldpath, newpath):
-        try:
-            vfs.unlink(newpath)
-        except OSError as err:
-            if err.errno != errno.ENOENT:
-                ui.debug("warning: cannot remove '%s': %s\n" %
-                         (newpath, err.strerror))
-        try:
-            if newpath:
-                vfs.rename(oldpath, newpath)
-        except OSError as err:
-            if err.errno != errno.ENOENT:
-                ui.debug("warning: cannot rename '%s' to '%s': %s\n" %
-                         (newpath, oldpath, err.strerror))
+class blackboxlogger(object):
+    def __init__(self, ui, repo):
+        self._repo = repo
+        self._trackedevents = set(ui.configlist('blackbox', 'track'))
+        self._maxfiles = ui.configint('blackbox', 'maxfiles')
+        self._maxsize = ui.configbytes('blackbox', 'maxsize')
+        self._inlog = False
 
-    maxsize = ui.configbytes('blackbox', 'maxsize')
-    name = 'blackbox.log'
-    if maxsize > 0:
+    def tracked(self, event):
+        return b'*' in self._trackedevents or event in self._trackedevents
+
+    def log(self, ui, event, msg, opts):
+        # self._log() -> ctx.dirty() may create new subrepo instance, which
+        # ui is derived from baseui. So the recursion guard in ui.log()
+        # doesn't work as it's local to the ui instance.
+        if self._inlog:
+            return
+        self._inlog = True
         try:
-            st = vfs.stat(name)
-        except OSError:
-            pass
-        else:
-            if st.st_size >= maxsize:
-                path = vfs.join(name)
-                maxfiles = ui.configint('blackbox', 'maxfiles')
-                for i in pycompat.xrange(maxfiles - 1, 1, -1):
-                    rotate(oldpath='%s.%d' % (path, i - 1),
-                           newpath='%s.%d' % (path, i))
-                rotate(oldpath=path,
-                       newpath=maxfiles > 0 and path + '.1')
-    return vfs(name, 'a')
-
-def wrapui(ui):
-    class blackboxui(ui.__class__):
-        @property
-        def _bbvfs(self):
-            vfs = None
-            repo = getattr(self, '_bbrepo', None)
-            if repo:
-                vfs = repo.vfs
-                if not vfs.isdir('.'):
-                    vfs = None
-            return vfs
-
-        @util.propertycache
-        def track(self):
-            return self.configlist('blackbox', 'track')
-
-        def debug(self, *msg, **opts):
-            super(blackboxui, self).debug(*msg, **opts)
-            if self.debugflag:
-                self.log('debug', '%s', ''.join(msg))
-
-        def log(self, event, *msg, **opts):
-            global lastui
-            super(blackboxui, self).log(event, *msg, **opts)
+            self._log(ui, event, msg, opts)
+        finally:
+            self._inlog = False
 
-            if not '*' in self.track and not event in self.track:
-                return
-
-            if self._bbvfs:
-                ui = self
-            else:
-                # certain ui instances exist outside the context of
-                # a repo, so just default to the last blackbox that
-                # was seen.
-                ui = lastui
-
-            if not ui:
-                return
-            vfs = ui._bbvfs
-            if not vfs:
-                return
+    def _log(self, ui, event, msg, opts):
+        default = ui.configdate('devel', 'default-date')
+        date = dateutil.datestr(default, ui.config('blackbox', 'date-format'))
+        user = procutil.getuser()
+        pid = '%d' % procutil.getpid()
+        rev = '(unknown)'
+        changed = ''
+        ctx = self._repo[None]
+        parents = ctx.parents()
+        rev = ('+'.join([hex(p.node()) for p in parents]))
+        if (ui.configbool('blackbox', 'dirty') and
+            ctx.dirty(missing=True, merge=False, branch=False)):
+            changed = '+'
+        if ui.configbool('blackbox', 'logsource'):
+            src = ' [%s]' % event
+        else:
+            src = ''
+        try:
+            fmt = '%s %s @%s%s (%s)%s> %s'
+            args = (date, user, rev, changed, pid, src, msg)
+            with loggingutil.openlogfile(
+                    ui, self._repo.vfs, name='blackbox.log',
+                    maxfiles=self._maxfiles, maxsize=self._maxsize) as fp:
+                fp.write(fmt % args)
+        except (IOError, OSError) as err:
+            # deactivate this to avoid failed logging again
+            self._trackedevents.clear()
+            ui.debug('warning: cannot write to blackbox.log: %s\n' %
+                     encoding.strtolocal(err.strerror))
+            return
+        _lastlogger.logger = self
 
-            repo = getattr(ui, '_bbrepo', None)
-            if not lastui or repo:
-                lastui = ui
-            if getattr(ui, '_bbinlog', False):
-                # recursion and failure guard
-                return
-            ui._bbinlog = True
-            default = self.configdate('devel', 'default-date')
-            date = dateutil.datestr(default, '%Y/%m/%d %H:%M:%S')
-            user = procutil.getuser()
-            pid = '%d' % procutil.getpid()
-            formattedmsg = msg[0] % msg[1:]
-            rev = '(unknown)'
-            changed = ''
-            if repo:
-                ctx = repo[None]
-                parents = ctx.parents()
-                rev = ('+'.join([hex(p.node()) for p in parents]))
-                if (ui.configbool('blackbox', 'dirty') and
-                    ctx.dirty(missing=True, merge=False, branch=False)):
-                    changed = '+'
-            if ui.configbool('blackbox', 'logsource'):
-                src = ' [%s]' % event
-            else:
-                src = ''
-            try:
-                fmt = '%s %s @%s%s (%s)%s> %s'
-                args = (date, user, rev, changed, pid, src, formattedmsg)
-                with _openlogfile(ui, vfs) as fp:
-                    fp.write(fmt % args)
-            except (IOError, OSError) as err:
-                self.debug('warning: cannot write to blackbox.log: %s\n' %
-                           encoding.strtolocal(err.strerror))
-                # do not restore _bbinlog intentionally to avoid failed
-                # logging again
-            else:
-                ui._bbinlog = False
-
-        def setrepo(self, repo):
-            self._bbrepo = repo
-
-    ui.__class__ = blackboxui
-    uimod.ui = blackboxui
-
-def uisetup(ui):
-    wrapui(ui)
+def uipopulate(ui):
+    ui.setlogger(b'blackbox', _lastlogger)
 
 def reposetup(ui, repo):
     # During 'hg pull' a httppeer repo is created to represent the remote repo.
     # It doesn't have a .hg directory to put a blackbox in, so we don't do
     # the blackbox setup for it.
     if not repo.local():
         return
 
-    if util.safehasattr(ui, 'setrepo'):
-        ui.setrepo(repo)
+    # Since blackbox.log is stored in the repo directory, the logger should be
+    # instantiated per repository.
+    logger = blackboxlogger(ui, repo)
+    ui.setlogger(b'blackbox', logger)
 
-        # Set lastui even if ui.log is not called. This gives blackbox a
-        # fallback place to log.
-        global lastui
-        if lastui is None:
-            lastui = ui
+    # Set _lastlogger even if ui.log is not called. This gives blackbox a
+    # fallback place to log
+    if _lastlogger.logger is None:
+        _lastlogger.logger = logger
 
     repo._wlockfreeprefix.add('blackbox.log')
 
 @command('blackbox',
     [('l', 'limit', 10, _('the number of events to show')),
     ],
     _('hg blackbox [OPTION]...'),
     helpcategory=command.CATEGORY_MAINTENANCE,
     helpbasic=True)
 def blackbox(ui, repo, *revs, **opts):
     '''view the recent repository events
     '''
 
     if not repo.vfs.exists('blackbox.log'):
         return
 
     limit = opts.get(r'limit')
     fp = repo.vfs('blackbox.log', 'r')
     lines = fp.read().split('\n')
 
     count = 0
     output = []
     for line in reversed(lines):
         if count >= limit:
             break
 
         # count the commands by matching lines like: 2013/01/23 19:13:36 root>
         if re.match('^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} .*> .*', line):
             count += 1
         output.append(line)
 
     ui.status('\n'.join(reversed(output)))

hgext/convert/filemap.py

@@ -1,465 +1,479 b''
 # Copyright 2007 Bryan O'Sullivan <[e-mail]>
 # Copyright 2007 Alexis S. L. Carvalho <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import posixpath
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     pycompat,
 )
 from . import common
 SKIPREV = common.SKIPREV
 
 def rpairs(path):
     '''Yield tuples with path split at '/', starting with the full path.
     No leading, trailing or double '/', please.
     >>> for x in rpairs(b'foo/bar/baz'): print(x)
     ('foo/bar/baz', '')
     ('foo/bar', 'baz')
     ('foo', 'bar/baz')
     ('.', 'foo/bar/baz')
     '''
     i = len(path)
     while i != -1:
         yield path[:i], path[i + 1:]
         i = path.rfind('/', 0, i)
     yield '.', path
 
 def normalize(path):
     ''' We use posixpath.normpath to support cross-platform path format.
     However, it doesn't handle None input. So we wrap it up. '''
     if path is None:
         return None
     return posixpath.normpath(path)
 
 class filemapper(object):
     '''Map and filter filenames when importing.
     A name can be mapped to itself, a new name, or None (omit from new
     repository).'''
 
     def __init__(self, ui, path=None):
         self.ui = ui
         self.include = {}
         self.exclude = {}
         self.rename = {}
         self.targetprefixes = None
         if path:
             if self.parse(path):
                 raise error.Abort(_('errors in filemap'))
 
     def parse(self, path):
         errs = 0
         def check(name, mapping, listname):
             if not name:
                 self.ui.warn(_('%s:%d: path to %s is missing\n') %
                              (lex.infile, lex.lineno, listname))
                 return 1
             if name in mapping:
                 self.ui.warn(_('%s:%d: %r already in %s list\n') %
                              (lex.infile, lex.lineno, name, listname))
                 return 1
             if (name.startswith('/') or
                 name.endswith('/') or
                 '//' in name):
                 self.ui.warn(_('%s:%d: superfluous / in %s %r\n') %
                              (lex.infile, lex.lineno, listname,
                               pycompat.bytestr(name)))
                 return 1
             return 0
         lex = common.shlexer(
             filepath=path, wordchars='!@#$%^&*()-=+[]{}|;:,./<>?')
         cmd = lex.get_token()
         while cmd:
             if cmd == 'include':
                 name = normalize(lex.get_token())
                 errs += check(name, self.exclude, 'exclude')
                 self.include[name] = name
             elif cmd == 'exclude':
                 name = normalize(lex.get_token())
                 errs += check(name, self.include, 'include')
                 errs += check(name, self.rename, 'rename')
                 self.exclude[name] = name
             elif cmd == 'rename':
                 src = normalize(lex.get_token())
                 dest = normalize(lex.get_token())
                 errs += check(src, self.exclude, 'exclude')
                 self.rename[src] = dest
             elif cmd == 'source':
                 errs += self.parse(normalize(lex.get_token()))
             else:
                 self.ui.warn(_('%s:%d: unknown directive %r\n') %
                              (lex.infile, lex.lineno, pycompat.bytestr(cmd)))
                 errs += 1
             cmd = lex.get_token()
         return errs
 
     def lookup(self, name, mapping):
         name = normalize(name)
         for pre, suf in rpairs(name):
             try:
                 return mapping[pre], pre, suf
             except KeyError:
                 pass
         return '', name, ''
 
     def istargetfile(self, filename):
         """Return true if the given target filename is covered as a destination
         of the filemap. This is useful for identifying what parts of the target
         repo belong to the source repo and what parts don't."""
         if self.targetprefixes is None:
             self.targetprefixes = set()
             for before, after in self.rename.iteritems():
                 self.targetprefixes.add(after)
 
         # If "." is a target, then all target files are considered from the
         # source.
         if not self.targetprefixes or '.' in self.targetprefixes:
             return True
 
         filename = normalize(filename)
         for pre, suf in rpairs(filename):
             # This check is imperfect since it doesn't account for the
             # include/exclude list, but it should work in filemaps that don't
             # apply include/exclude to the same source directories they are
             # renaming.
             if pre in self.targetprefixes:
                 return True
         return False
 
     def __call__(self, name):
         if self.include:
             inc = self.lookup(name, self.include)[0]
         else:
             inc = name
         if self.exclude:
             exc = self.lookup(name, self.exclude)[0]
         else:
             exc = ''
         if (not self.include and exc) or (len(inc) <= len(exc)):
             return None
         newpre, pre, suf = self.lookup(name, self.rename)
         if newpre:
             if newpre == '.':
                 return suf
             if suf:
                 if newpre.endswith('/'):
                     return newpre + suf
                 return newpre + '/' + suf
             return newpre
         return name
 
     def active(self):
         return bool(self.include or self.exclude or self.rename)
 
 # This class does two additional things compared to a regular source:
 #
 # - Filter and rename files.  This is mostly wrapped by the filemapper
 #   class above. We hide the original filename in the revision that is
 #   returned by getchanges to be able to find things later in getfile.
 #
 # - Return only revisions that matter for the files we're interested in.
 #   This involves rewriting the parents of the original revision to
 #   create a graph that is restricted to those revisions.
 #
 #   This set of revisions includes not only revisions that directly
 #   touch files we're interested in, but also merges that merge two
 #   or more interesting revisions.
 
 class filemap_source(common.converter_source):
     def __init__(self, ui, baseconverter, filemap):
         super(filemap_source, self).__init__(ui, baseconverter.repotype)
         self.base = baseconverter
         self.filemapper = filemapper(ui, filemap)
         self.commits = {}
         # if a revision rev has parent p in the original revision graph, then
         # rev will have parent self.parentmap[p] in the restricted graph.
         self.parentmap = {}
         # self.wantedancestors[rev] is the set of all ancestors of rev that
         # are in the restricted graph.
         self.wantedancestors = {}
         self.convertedorder = None
         self._rebuilt = False
         self.origparents = {}
         self.children = {}
         self.seenchildren = {}
         # experimental config: convert.ignoreancestorcheck
         self.ignoreancestorcheck = self.ui.configbool('convert',
                                                       'ignoreancestorcheck')
 
     def before(self):
         self.base.before()
 
     def after(self):
         self.base.after()
 
     def setrevmap(self, revmap):
         # rebuild our state to make things restartable
         #
         # To avoid calling getcommit for every revision that has already
         # been converted, we rebuild only the parentmap, delaying the
         # rebuild of wantedancestors until we need it (i.e. until a
         # merge).
         #
         # We assume the order argument lists the revisions in
         # topological order, so that we can infer which revisions were
         # wanted by previous runs.
         self._rebuilt = not revmap
         seen = {SKIPREV: SKIPREV}
         dummyset = set()
         converted = []
         for rev in revmap.order:
             mapped = revmap[rev]
             wanted = mapped not in seen
             if wanted:
                 seen[mapped] = rev
                 self.parentmap[rev] = rev
             else:
                 self.parentmap[rev] = seen[mapped]
             self.wantedancestors[rev] = dummyset
             arg = seen[mapped]
             if arg == SKIPREV:
                 arg = None
             converted.append((rev, wanted, arg))
         self.convertedorder = converted
         return self.base.setrevmap(revmap)
 
     def rebuild(self):
         if self._rebuilt:
             return True
         self._rebuilt = True
         self.parentmap.clear()
         self.wantedancestors.clear()
         self.seenchildren.clear()
         for rev, wanted, arg in self.convertedorder:
             if rev not in self.origparents:
                 try:
                     self.origparents[rev] = self.getcommit(rev).parents
                 except error.RepoLookupError:
                     self.ui.debug("unknown revmap source: %s\n" % rev)
                     continue
             if arg is not None:
                 self.children[arg] = self.children.get(arg, 0) + 1
 
         for rev, wanted, arg in self.convertedorder:
             try:
                 parents = self.origparents[rev]
             except KeyError:
                 continue # unknown revmap source
             if wanted:
                 self.mark_wanted(rev, parents)
             else:
                 self.mark_not_wanted(rev, arg)
             self._discard(arg, *parents)
 
         return True
 
     def getheads(self):
         return self.base.getheads()
 
     def getcommit(self, rev):
         # We want to save a reference to the commit objects to be able
         # to rewrite their parents later on.
         c = self.commits[rev] = self.base.getcommit(rev)
         for p in c.parents:
             self.children[p] = self.children.get(p, 0) + 1
         return c
 
+    def numcommits(self):
+        return self.base.numcommits()
+
     def _cachedcommit(self, rev):
         if rev in self.commits:
             return self.commits[rev]
         return self.base.getcommit(rev)
 
     def _discard(self, *revs):
         for r in revs:
             if r is None:
                 continue
             self.seenchildren[r] = self.seenchildren.get(r, 0) + 1
             if self.seenchildren[r] == self.children[r]:
                 self.wantedancestors.pop(r, None)
                 self.parentmap.pop(r, None)
                 del self.seenchildren[r]
                 if self._rebuilt:
                     del self.children[r]
 
     def wanted(self, rev, i):
         # Return True if we're directly interested in rev.
         #
         # i is an index selecting one of the parents of rev (if rev
         # has no parents, i is None).  getchangedfiles will give us
         # the list of files that are different in rev and in the parent
         # indicated by i.  If we're interested in any of these files,
         # we're interested in rev.
         try:
             files = self.base.getchangedfiles(rev, i)
         except NotImplementedError:
             raise error.Abort(_("source repository doesn't support --filemap"))
         for f in files:
             if self.filemapper(f):
                 return True
-        return False
+
+        # The include directive is documented to include nothing else (though
+        # valid branch closes are included).
+        if self.filemapper.include:
+            return False
+
+        # Allow empty commits in the source revision through.  The getchanges()
+        # method doesn't even bother calling this if it determines that the
+        # close marker is significant (i.e. all of the branch ancestors weren't
+        # eliminated).  Therefore if there *is* a close marker, getchanges()
+        # doesn't consider it significant, and this revision should be dropped.
+        return not files and 'close' not in self.commits[rev].extra
 
     def mark_not_wanted(self, rev, p):
         # Mark rev as not interesting and update data structures.
 
         if p is None:
             # A root revision. Use SKIPREV to indicate that it doesn't
             # map to any revision in the restricted graph.  Put SKIPREV
             # in the set of wanted ancestors to simplify code elsewhere
             self.parentmap[rev] = SKIPREV
             self.wantedancestors[rev] = {SKIPREV}
             return
 
         # Reuse the data from our parent.
         self.parentmap[rev] = self.parentmap[p]
         self.wantedancestors[rev] = self.wantedancestors[p]
 
     def mark_wanted(self, rev, parents):
         # Mark rev ss wanted and update data structures.
 
         # rev will be in the restricted graph, so children of rev in
         # the original graph should still have rev as a parent in the
         # restricted graph.
         self.parentmap[rev] = rev
 
         # The set of wanted ancestors of rev is the union of the sets
         # of wanted ancestors of its parents. Plus rev itself.
         wrev = set()
         for p in parents:
             if p in self.wantedancestors:
                 wrev.update(self.wantedancestors[p])
             else:
                 self.ui.warn(_('warning: %s parent %s is missing\n') %
                              (rev, p))
         wrev.add(rev)
         self.wantedancestors[rev] = wrev
 
     def getchanges(self, rev, full):
         parents = self.commits[rev].parents
         if len(parents) > 1 and not self.ignoreancestorcheck:
             self.rebuild()
 
         # To decide whether we're interested in rev we:
         #
         # - calculate what parents rev will have if it turns out we're
         #   interested in it.  If it's going to have more than 1 parent,
         #   we're interested in it.
         #
         # - otherwise, we'll compare it with the single parent we found.
         #   If any of the files we're interested in is different in the
         #   the two revisions, we're interested in rev.
 
         # A parent p is interesting if its mapped version (self.parentmap[p]):
         # - is not SKIPREV
         # - is still not in the list of parents (we don't want duplicates)
         # - is not an ancestor of the mapped versions of the other parents or
         #   there is no parent in the same branch than the current revision.
         mparents = []
         knownparents = set()
         branch = self.commits[rev].branch
         hasbranchparent = False
         for i, p1 in enumerate(parents):
             mp1 = self.parentmap[p1]
             if mp1 == SKIPREV or mp1 in knownparents:
                 continue
 
             isancestor = (not self.ignoreancestorcheck and
                           any(p2 for p2 in parents
                               if p1 != p2 and mp1 != self.parentmap[p2]
                                  and mp1 in self.wantedancestors[p2]))
             if not isancestor and not hasbranchparent and len(parents) > 1:
                 # This could be expensive, avoid unnecessary calls.
                 if self._cachedcommit(p1).branch == branch:
                     hasbranchparent = True
             mparents.append((p1, mp1, i, isancestor))
             knownparents.add(mp1)
         # Discard parents ancestors of other parents if there is a
         # non-ancestor one on the same branch than current revision.
         if hasbranchparent:
             mparents = [p for p in mparents if not p[3]]
         wp = None
         if mparents:
             wp = max(p[2] for p in mparents)
             mparents = [p[1] for p in mparents]
         elif parents:
             wp = 0
 
         self.origparents[rev] = parents
 
         closed = False
         if 'close' in self.commits[rev].extra:
             # A branch closing revision is only useful if one of its
             # parents belong to the branch being closed
             pbranches = [self._cachedcommit(p).branch for p in mparents]
             if branch in pbranches:
                 closed = True
 
         if len(mparents) < 2 and not closed and not self.wanted(rev, wp):
             # We don't want this revision.
             # Update our state and tell the convert process to map this
             # revision to the same revision its parent as mapped to.
             p = None
             if parents:
                 p = parents[wp]
             self.mark_not_wanted(rev, p)
             self.convertedorder.append((rev, False, p))
             self._discard(*parents)
             return self.parentmap[rev]
 
         # We want this revision.
         # Rewrite the parents of the commit object
         self.commits[rev].parents = mparents
         self.mark_wanted(rev, parents)
         self.convertedorder.append((rev, True, None))
         self._discard(*parents)
 
         # Get the real changes and do the filtering/mapping. To be
         # able to get the files later on in getfile, we hide the
         # original filename in the rev part of the return value.
         changes, copies, cleanp2 = self.base.getchanges(rev, full)
         files = {}
         ncleanp2 = set(cleanp2)
         for f, r in changes:
             newf = self.filemapper(f)
             if newf and (newf != f or newf not in files):
                 files[newf] = (f, r)
                 if newf != f:
                     ncleanp2.discard(f)
         files = sorted(files.items())
 
         ncopies = {}
         for c in copies:
             newc = self.filemapper(c)
             if newc:
                 newsource = self.filemapper(copies[c])
                 if newsource:
                     ncopies[newc] = newsource
 
         return files, ncopies, ncleanp2
 
     def targetfilebelongstosource(self, targetfilename):
         return self.filemapper.istargetfile(targetfilename)
 
     def getfile(self, name, rev):
         realname, realrev = rev
         return self.base.getfile(realname, realrev)
 
     def gettags(self):
         return self.base.gettags()
 
     def hasnativeorder(self):
         return self.base.hasnativeorder()
 
     def lookuprev(self, rev):
         return self.base.lookuprev(rev)
 
     def getbookmarks(self):
         return self.base.getbookmarks()
 
     def converted(self, rev, sinkrev):
         self.base.converted(rev, sinkrev)

hgext/convert/hg.py

@@ -1,652 +1,655 b''
 # hg.py - hg backend for convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Notes for hg->hg conversion:
 #
 # * Old versions of Mercurial didn't trim the whitespace from the ends
 #   of commit messages, but new versions do.  Changesets created by
 #   those older versions, then converted, may thus have different
 #   hashes for changesets that are otherwise identical.
 #
 # * Using "--config convert.hg.saverev=true" will make the source
 #   identifier to be stored in the converted revision. This will cause
 #   the converted revision to have a different identity than the
 #   source.
 from __future__ import absolute_import
 
 import os
 import re
 import time
 
 from mercurial.i18n import _
 from mercurial import (
     bookmarks,
     context,
     error,
     exchange,
     hg,
     lock as lockmod,
     merge as mergemod,
     node as nodemod,
     phases,
     scmutil,
     util,
 )
 from mercurial.utils import dateutil
 stringio = util.stringio
 
 from . import common
 mapfile = common.mapfile
 NoRepo = common.NoRepo
 
 sha1re = re.compile(br'\b[0-9a-f]{12,40}\b')
 
 class mercurial_sink(common.converter_sink):
     def __init__(self, ui, repotype, path):
         common.converter_sink.__init__(self, ui, repotype, path)
         self.branchnames = ui.configbool('convert', 'hg.usebranchnames')
         self.clonebranches = ui.configbool('convert', 'hg.clonebranches')
         self.tagsbranch = ui.config('convert', 'hg.tagsbranch')
         self.lastbranch = None
         if os.path.isdir(path) and len(os.listdir(path)) > 0:
             try:
                 self.repo = hg.repository(self.ui, path)
                 if not self.repo.local():
                     raise NoRepo(_('%s is not a local Mercurial repository')
                                  % path)
             except error.RepoError as err:
                 ui.traceback()
                 raise NoRepo(err.args[0])
         else:
             try:
                 ui.status(_('initializing destination %s repository\n') % path)
                 self.repo = hg.repository(self.ui, path, create=True)
                 if not self.repo.local():
                     raise NoRepo(_('%s is not a local Mercurial repository')
                                  % path)
                 self.created.append(path)
             except error.RepoError:
                 ui.traceback()
                 raise NoRepo(_("could not create hg repository %s as sink")
                              % path)
         self.lock = None
         self.wlock = None
         self.filemapmode = False
         self.subrevmaps = {}
 
     def before(self):
         self.ui.debug('run hg sink pre-conversion action\n')
         self.wlock = self.repo.wlock()
         self.lock = self.repo.lock()
 
     def after(self):
         self.ui.debug('run hg sink post-conversion action\n')
         if self.lock:
             self.lock.release()
         if self.wlock:
             self.wlock.release()
 
     def revmapfile(self):
         return self.repo.vfs.join("shamap")
 
     def authorfile(self):
         return self.repo.vfs.join("authormap")
 
     def setbranch(self, branch, pbranches):
         if not self.clonebranches:
             return
 
         setbranch = (branch != self.lastbranch)
         self.lastbranch = branch
         if not branch:
             branch = 'default'
         pbranches = [(b[0], b[1] and b[1] or 'default') for b in pbranches]
         if pbranches:
             pbranch = pbranches[0][1]
         else:
             pbranch = 'default'
 
         branchpath = os.path.join(self.path, branch)
         if setbranch:
             self.after()
             try:
                 self.repo = hg.repository(self.ui, branchpath)
             except Exception:
                 self.repo = hg.repository(self.ui, branchpath, create=True)
             self.before()
 
         # pbranches may bring revisions from other branches (merge parents)
         # Make sure we have them, or pull them.
         missings = {}
         for b in pbranches:
             try:
                 self.repo.lookup(b[0])
             except Exception:
                 missings.setdefault(b[1], []).append(b[0])
 
         if missings:
             self.after()
             for pbranch, heads in sorted(missings.iteritems()):
                 pbranchpath = os.path.join(self.path, pbranch)
                 prepo = hg.peer(self.ui, {}, pbranchpath)
                 self.ui.note(_('pulling from %s into %s\n') % (pbranch, branch))
                 exchange.pull(self.repo, prepo,
                               [prepo.lookup(h) for h in heads])
             self.before()
 
     def _rewritetags(self, source, revmap, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(' ', 1)
             if len(s) != 2:
                 self.ui.warn(_('invalid tag entry: "%s"\n') % line)
                 fp.write('%s\n' % line)  # Bogus, but keep for hash stability
                 continue
             revid = revmap.get(source.lookuprev(s[0]))
             if not revid:
                 if s[0] == nodemod.nullhex:
                     revid = s[0]
                 else:
                     # missing, but keep for hash stability
                     self.ui.warn(_('missing tag entry: "%s"\n') % line)
                     fp.write('%s\n' % line)
                     continue
             fp.write('%s %s\n' % (revid, s[1]))
         return fp.getvalue()
 
     def _rewritesubstate(self, source, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(' ', 1)
             if len(s) != 2:
                 continue
 
             revid = s[0]
             subpath = s[1]
             if revid != nodemod.nullhex:
                 revmap = self.subrevmaps.get(subpath)
                 if revmap is None:
                     revmap = mapfile(self.ui,
                                      self.repo.wjoin(subpath, '.hg/shamap'))
                     self.subrevmaps[subpath] = revmap
 
                     # It is reasonable that one or more of the subrepos don't
                     # need to be converted, in which case they can be cloned
                     # into place instead of converted.  Therefore, only warn
                     # once.
                     msg = _('no ".hgsubstate" updates will be made for "%s"\n')
                     if len(revmap) == 0:
                         sub = self.repo.wvfs.reljoin(subpath, '.hg')
 
                         if self.repo.wvfs.exists(sub):
                             self.ui.warn(msg % subpath)
 
                 newid = revmap.get(revid)
                 if not newid:
                     if len(revmap) > 0:
                         self.ui.warn(_("%s is missing from %s/.hg/shamap\n") %
                                      (revid, subpath))
                 else:
                     revid = newid
 
             fp.write('%s %s\n' % (revid, subpath))
 
         return fp.getvalue()
 
     def _calculatemergedfiles(self, source, p1ctx, p2ctx):
         """Calculates the files from p2 that we need to pull in when merging p1
         and p2, given that the merge is coming from the given source.
 
         This prevents us from losing files that only exist in the target p2 and
         that don't come from the source repo (like if you're merging multiple
         repositories together).
         """
         anc = [p1ctx.ancestor(p2ctx)]
         # Calculate what files are coming from p2
         actions, diverge, rename = mergemod.calculateupdates(
             self.repo, p1ctx, p2ctx, anc,
             True,  # branchmerge
             True,  # force
             False, # acceptremote
             False, # followcopies
         )
 
         for file, (action, info, msg) in actions.iteritems():
             if source.targetfilebelongstosource(file):
                 # If the file belongs to the source repo, ignore the p2
                 # since it will be covered by the existing fileset.
                 continue
 
             # If the file requires actual merging, abort. We don't have enough
             # context to resolve merges correctly.
             if action in ['m', 'dm', 'cd', 'dc']:
                 raise error.Abort(_("unable to convert merge commit "
                     "since target parents do not merge cleanly (file "
                     "%s, parents %s and %s)") % (file, p1ctx,
                                                  p2ctx))
             elif action == 'k':
                 # 'keep' means nothing changed from p1
                 continue
             else:
                 # Any other change means we want to take the p2 version
                 yield file
 
     def putcommit(self, files, copies, parents, commit, source, revmap, full,
                   cleanp2):
         files = dict(files)
 
         def getfilectx(repo, memctx, f):
             if p2ctx and f in p2files and f not in copies:
                 self.ui.debug('reusing %s from p2\n' % f)
                 try:
                     return p2ctx[f]
                 except error.ManifestLookupError:
                     # If the file doesn't exist in p2, then we're syncing a
                     # delete, so just return None.
                     return None
             try:
                 v = files[f]
             except KeyError:
                 return None
             data, mode = source.getfile(f, v)
             if data is None:
                 return None
             if f == '.hgtags':
                 data = self._rewritetags(source, revmap, data)
             if f == '.hgsubstate':
                 data = self._rewritesubstate(source, data)
             return context.memfilectx(self.repo, memctx, f, data, 'l' in mode,
                                       'x' in mode, copies.get(f))
 
         pl = []
         for p in parents:
             if p not in pl:
                 pl.append(p)
         parents = pl
         nparents = len(parents)
         if self.filemapmode and nparents == 1:
             m1node = self.repo.changelog.read(nodemod.bin(parents[0]))[0]
             parent = parents[0]
 
         if len(parents) < 2:
             parents.append(nodemod.nullid)
         if len(parents) < 2:
             parents.append(nodemod.nullid)
         p2 = parents.pop(0)
 
         text = commit.desc
 
         sha1s = re.findall(sha1re, text)
         for sha1 in sha1s:
             oldrev = source.lookuprev(sha1)
             newrev = revmap.get(oldrev)
             if newrev is not None:
                 text = text.replace(sha1, newrev[:len(sha1)])
 
         extra = commit.extra.copy()
 
         sourcename = self.repo.ui.config('convert', 'hg.sourcename')
         if sourcename:
             extra['convert_source'] = sourcename
 
         for label in ('source', 'transplant_source', 'rebase_source',
                       'intermediate-source'):
             node = extra.get(label)
 
             if node is None:
                 continue
 
             # Only transplant stores its reference in binary
             if label == 'transplant_source':
                 node = nodemod.hex(node)
 
             newrev = revmap.get(node)
             if newrev is not None:
                 if label == 'transplant_source':
                     newrev = nodemod.bin(newrev)
 
                 extra[label] = newrev
 
         if self.branchnames and commit.branch:
             extra['branch'] = commit.branch
         if commit.rev and commit.saverev:
             extra['convert_revision'] = commit.rev
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
             p1ctx = self.repo[p1]
             p2ctx = None
             if p2 != nodemod.nullid:
                 p2ctx = self.repo[p2]
             fileset = set(files)
             if full:
                 fileset.update(self.repo[p1])
                 fileset.update(self.repo[p2])
 
             if p2ctx:
                 p2files = set(cleanp2)
                 for file in self._calculatemergedfiles(source, p1ctx, p2ctx):
                     p2files.add(file)
                     fileset.add(file)
 
             ctx = context.memctx(self.repo, (p1, p2), text, fileset,
                                  getfilectx, commit.author, commit.date, extra)
 
             # We won't know if the conversion changes the node until after the
             # commit, so copy the source's phase for now.
             self.repo.ui.setconfig('phases', 'new-commit',
                                    phases.phasenames[commit.phase], 'convert')
 
             with self.repo.transaction("convert") as tr:
                 node = nodemod.hex(self.repo.commitctx(ctx))
 
                 # If the node value has changed, but the phase is lower than
                 # draft, set it back to draft since it hasn't been exposed
                 # anywhere.
                 if commit.rev != node:
                     ctx = self.repo[node]
                     if ctx.phase() < phases.draft:
                         phases.registernew(self.repo, tr, phases.draft,
                                            [ctx.node()])
 
             text = "(octopus merge fixup)\n"
             p2 = node
 
         if self.filemapmode and nparents == 1:
             man = self.repo.manifestlog.getstorage(b'')
             mnode = self.repo.changelog.read(nodemod.bin(p2))[0]
             closed = 'close' in commit.extra
             if not closed and not man.cmp(m1node, man.revision(mnode)):
                 self.ui.status(_("filtering out empty revision\n"))
                 self.repo.rollback(force=True)
                 return parent
         return p2
 
     def puttags(self, tags):
         tagparent = self.repo.branchtip(self.tagsbranch, ignoremissing=True)
         tagparent = tagparent or nodemod.nullid
 
         oldlines = set()
         for branch, heads in self.repo.branchmap().iteritems():
             for h in heads:
                 if '.hgtags' in self.repo[h]:
                     oldlines.update(
                         set(self.repo[h]['.hgtags'].data().splitlines(True)))
         oldlines = sorted(list(oldlines))
 
         newlines = sorted([("%s %s\n" % (tags[tag], tag)) for tag in tags])
         if newlines == oldlines:
             return None, None
 
         # if the old and new tags match, then there is nothing to update
         oldtags = set()
         newtags = set()
         for line in oldlines:
             s = line.strip().split(' ', 1)
             if len(s) != 2:
                 continue
             oldtags.add(s[1])
         for line in newlines:
             s = line.strip().split(' ', 1)
             if len(s) != 2:
                 continue
             if s[1] not in oldtags:
                 newtags.add(s[1].strip())
 
         if not newtags:
             return None, None
 
         data = "".join(newlines)
         def getfilectx(repo, memctx, f):
             return context.memfilectx(repo, memctx, f, data, False, False, None)
 
         self.ui.status(_("updating tags\n"))
         date = "%d 0" % int(time.mktime(time.gmtime()))
         extra = {'branch': self.tagsbranch}
         ctx = context.memctx(self.repo, (tagparent, None), "update tags",
                              [".hgtags"], getfilectx, "convert-repo", date,
                              extra)
         node = self.repo.commitctx(ctx)
         return nodemod.hex(node), nodemod.hex(tagparent)
 
     def setfilemapmode(self, active):
         self.filemapmode = active
 
     def putbookmarks(self, updatedbookmark):
         if not len(updatedbookmark):
             return
         wlock = lock = tr = None
         try:
             wlock = self.repo.wlock()
             lock = self.repo.lock()
             tr = self.repo.transaction('bookmark')
             self.ui.status(_("updating bookmarks\n"))
             destmarks = self.repo._bookmarks
             changes = [(bookmark, nodemod.bin(updatedbookmark[bookmark]))
                        for bookmark in updatedbookmark]
             destmarks.applychanges(self.repo, tr, changes)
             tr.close()
         finally:
             lockmod.release(lock, wlock, tr)
 
     def hascommitfrommap(self, rev):
         # the exact semantics of clonebranches is unclear so we can't say no
         return rev in self.repo or self.clonebranches
 
     def hascommitforsplicemap(self, rev):
         if rev not in self.repo and self.clonebranches:
             raise error.Abort(_('revision %s not found in destination '
                                'repository (lookups with clonebranches=true '
                                'are not implemented)') % rev)
         return rev in self.repo
 
 class mercurial_source(common.converter_source):
     def __init__(self, ui, repotype, path, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         self.ignoreerrors = ui.configbool('convert', 'hg.ignoreerrors')
         self.ignored = set()
         self.saverev = ui.configbool('convert', 'hg.saverev')
         try:
             self.repo = hg.repository(self.ui, path)
             # try to provoke an exception if this isn't really a hg
             # repo, but some other bogus compatible-looking url
             if not self.repo.local():
                 raise error.RepoError
         except error.RepoError:
             ui.traceback()
             raise NoRepo(_("%s is not a local Mercurial repository") % path)
         self.lastrev = None
         self.lastctx = None
         self._changescache = None, None
         self.convertfp = None
         # Restrict converted revisions to startrev descendants
         startnode = ui.config('convert', 'hg.startrev')
         hgrevs = ui.config('convert', 'hg.revs')
         if hgrevs is None:
             if startnode is not None:
                 try:
                     startnode = self.repo.lookup(startnode)
                 except error.RepoError:
                     raise error.Abort(_('%s is not a valid start revision')
                                      % startnode)
                 startrev = self.repo.changelog.rev(startnode)
                 children = {startnode: 1}
                 for r in self.repo.changelog.descendants([startrev]):
                     children[self.repo.changelog.node(r)] = 1
                 self.keep = children.__contains__
             else:
                 self.keep = util.always
             if revs:
                 self._heads = [self.repo.lookup(r) for r in revs]
             else:
                 self._heads = self.repo.heads()
         else:
             if revs or startnode is not None:
                 raise error.Abort(_('hg.revs cannot be combined with '
                                    'hg.startrev or --rev'))
             nodes = set()
             parents = set()
             for r in scmutil.revrange(self.repo, [hgrevs]):
                 ctx = self.repo[r]
                 nodes.add(ctx.node())
                 parents.update(p.node() for p in ctx.parents())
             self.keep = nodes.__contains__
             self._heads = nodes - parents
 
     def _changectx(self, rev):
         if self.lastrev != rev:
             self.lastctx = self.repo[rev]
             self.lastrev = rev
         return self.lastctx
 
     def _parents(self, ctx):
         return [p for p in ctx.parents() if p and self.keep(p.node())]
 
     def getheads(self):
         return [nodemod.hex(h) for h in self._heads if self.keep(h)]
 
     def getfile(self, name, rev):
         try:
             fctx = self._changectx(rev)[name]
             return fctx.data(), fctx.flags()
         except error.LookupError:
             return None, None
 
     def _changedfiles(self, ctx1, ctx2):
         ma, r = [], []
         maappend = ma.append
         rappend = r.append
         d = ctx1.manifest().diff(ctx2.manifest())
         for f, ((node1, flag1), (node2, flag2)) in d.iteritems():
             if node2 is None:
                 rappend(f)
             else:
                 maappend(f)
         return ma, r
 
     def getchanges(self, rev, full):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if full or not parents:
             files = copyfiles = ctx.manifest()
         if parents:
             if self._changescache[0] == rev:
                 ma, r = self._changescache[1]
             else:
                 ma, r = self._changedfiles(parents[0], ctx)
             if not full:
                 files = ma + r
             copyfiles = ma
         # _getcopies() is also run for roots and before filtering so missing
         # revlogs are detected early
         copies = self._getcopies(ctx, parents, copyfiles)
         cleanp2 = set()
         if len(parents) == 2:
             d = parents[1].manifest().diff(ctx.manifest(), clean=True)
             for f, value in d.iteritems():
                 if value is None:
                     cleanp2.add(f)
         changes = [(f, rev) for f in files if f not in self.ignored]
         changes.sort()
         return changes, copies, cleanp2
 
     def _getcopies(self, ctx, parents, files):
         copies = {}
         for name in files:
             if name in self.ignored:
                 continue
             try:
                 copysource, _copynode = ctx.filectx(name).renamed()
                 if copysource in self.ignored:
                     continue
                 # Ignore copy sources not in parent revisions
                 if not any(copysource in p for p in parents):
                     continue
                 copies[name] = copysource
             except TypeError:
                 pass
             except error.LookupError as e:
                 if not self.ignoreerrors:
                     raise
                 self.ignored.add(name)
                 self.ui.warn(_('ignoring: %s\n') % e)
         return copies
 
     def getcommit(self, rev):
         ctx = self._changectx(rev)
         _parents = self._parents(ctx)
         parents = [p.hex() for p in _parents]
         optparents = [p.hex() for p in ctx.parents() if p and p not in _parents]
         crev = rev
 
         return common.commit(author=ctx.user(),
                              date=dateutil.datestr(ctx.date(),
                                                '%Y-%m-%d %H:%M:%S %1%2'),
                              desc=ctx.description(),
                              rev=crev,
                              parents=parents,
                              optparents=optparents,
                              branch=ctx.branch(),
                              extra=ctx.extra(),
                              sortkey=ctx.rev(),
                              saverev=self.saverev,
                              phase=ctx.phase())
 
+    def numcommits(self):
+        return len(self.repo)
+
     def gettags(self):
         # This will get written to .hgtags, filter non global tags out.
         tags = [t for t in self.repo.tagslist()
                 if self.repo.tagtype(t[0]) == 'global']
         return dict([(name, nodemod.hex(node)) for name, node in tags
                      if self.keep(node)])
 
     def getchangedfiles(self, rev, i):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if not parents and i is None:
             i = 0
             ma, r = ctx.manifest().keys(), []
         else:
             i = i or 0
             ma, r = self._changedfiles(parents[i], ctx)
         ma, r = [[f for f in l if f not in self.ignored] for l in (ma, r)]
 
         if i == 0:
             self._changescache = (rev, (ma, r))
 
         return ma + r
 
     def converted(self, rev, destrev):
         if self.convertfp is None:
             self.convertfp = open(self.repo.vfs.join('shamap'), 'ab')
         self.convertfp.write(util.tonativeeol('%s %s\n' % (destrev, rev)))
         self.convertfp.flush()
 
     def before(self):
         self.ui.debug('run hg source pre-conversion action\n')
 
     def after(self):
         self.ui.debug('run hg source post-conversion action\n')
 
     def hasnativeorder(self):
         return True
 
     def hasnativeclose(self):
         return True
 
     def lookuprev(self, rev):
         try:
             return nodemod.hex(self.repo.lookup(rev))
         except (error.RepoError, error.LookupError):
             return None
 
     def getbookmarks(self):
         return bookmarks.listbookmarks(self.repo)
 
     def checkrevformat(self, revstr, mapname='splicemap'):
         """ Mercurial, revision string is a 40 byte hex """
         self.checkhexformat(revstr, mapname)

hgext/extdiff.py

@@ -1,436 +1,446 b''
 # extdiff.py - external diff program support for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''command to allow external programs to compare revisions
 
 The extdiff Mercurial extension allows you to use external programs
 to compare revisions, or revision with working directory. The external
 diff programs are called with a configurable set of options and two
 non-option arguments: paths to directories containing snapshots of
 files to compare.
 
 If there is more than one file being compared and the "child" revision
 is the working directory, any modifications made in the external diff
 program will be copied back to the working directory from the temporary
 directory.
 
 The extdiff extension also allows you to configure new diff commands, so
 you do not need to type :hg:`extdiff -p kdiff3` always. ::
 
   [extdiff]
   # add new command that runs GNU diff(1) in 'context diff' mode
   cdiff = gdiff -Nprc5
   ## or the old way:
   #cmd.cdiff = gdiff
   #opts.cdiff = -Nprc5
 
   # add new command called meld, runs meld (no need to name twice).  If
   # the meld executable is not available, the meld tool in [merge-tools]
   # will be used, if available
   meld =
 
   # add new command called vimdiff, runs gvimdiff with DirDiff plugin
   # (see http://www.vim.org/scripts/script.php?script_id=102) Non
   # English user, be sure to put "let g:DirDiffDynamicDiffText = 1" in
   # your .vimrc
   vimdiff = gvim -f "+next" \\
             "+execute 'DirDiff' fnameescape(argv(0)) fnameescape(argv(1))"
 
 Tool arguments can include variables that are expanded at runtime::
 
   $parent1, $plabel1 - filename, descriptive label of first parent
   $child,   $clabel  - filename, descriptive label of child revision
   $parent2, $plabel2 - filename, descriptive label of second parent
   $root              - repository root
   $parent is an alias for $parent1.
 
 The extdiff extension will look in your [diff-tools] and [merge-tools]
 sections for diff tool arguments, when none are specified in [extdiff].
 
 ::
 
   [extdiff]
   kdiff3 =
 
   [diff-tools]
   kdiff3.diffargs=--L1 '$plabel1' --L2 '$clabel' $parent $child
 
 You can use -I/-X and list of file or directory names like normal
 :hg:`diff` command. The extdiff extension makes snapshots of only
 needed files, so running the external diff program will actually be
 pretty fast (at least faster than having to compare the entire tree).
 '''
 
 from __future__ import absolute_import
 
 import os
 import re
 import shutil
 import stat
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     short,
 )
 from mercurial import (
     archival,
     cmdutil,
     error,
     filemerge,
     formatter,
     pycompat,
     registrar,
     scmutil,
     util,
 )
 from mercurial.utils import (
     procutil,
     stringutil,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem('extdiff', br'opts\..*',
     default='',
     generic=True,
 )
 
 configitem('diff-tools', br'.*\.diffargs$',
     default=None,
     generic=True,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 def snapshot(ui, repo, files, node, tmproot, listsubrepos):
     '''snapshot files as of some revision
     if not using snapshot, -I/-X does not work and recursive diff
     in tools like kdiff3 and meld displays too many files.'''
     dirname = os.path.basename(repo.root)
     if dirname == "":
         dirname = "root"
     if node is not None:
         dirname = '%s.%s' % (dirname, short(node))
     base = os.path.join(tmproot, dirname)
     os.mkdir(base)
     fnsandstat = []
 
     if node is not None:
         ui.note(_('making snapshot of %d files from rev %s\n') %
                 (len(files), short(node)))
     else:
         ui.note(_('making snapshot of %d files from working directory\n') %
             (len(files)))
 
     if files:
         repo.ui.setconfig("ui", "archivemeta", False)
 
         archival.archive(repo, base, node, 'files',
-                         matchfn=scmutil.matchfiles(repo, files),
+                         match=scmutil.matchfiles(repo, files),
                          subrepos=listsubrepos)
 
         for fn in sorted(files):
             wfn = util.pconvert(fn)
             ui.note('  %s\n' % wfn)
 
             if node is None:
                 dest = os.path.join(base, wfn)
 
                 fnsandstat.append((dest, repo.wjoin(fn), os.lstat(dest)))
     return dirname, fnsandstat
 
+def formatcmdline(cmdline, repo_root, do3way,
+                  parent1, plabel1, parent2, plabel2, child, clabel):
+    # Function to quote file/dir names in the argument string.
+    # When not operating in 3-way mode, an empty string is
+    # returned for parent2
+    replace = {'parent': parent1, 'parent1': parent1, 'parent2': parent2,
+               'plabel1': plabel1, 'plabel2': plabel2,
+               'child': child, 'clabel': clabel,
+               'root': repo_root}
+    def quote(match):
+        pre = match.group(2)
+        key = match.group(3)
+        if not do3way and key == 'parent2':
+            return pre
+        return pre + procutil.shellquote(replace[key])
+
+    # Match parent2 first, so 'parent1?' will match both parent1 and parent
+    regex = (br'''(['"]?)([^\s'"$]*)'''
+             br'\$(parent2|parent1?|child|plabel1|plabel2|clabel|root)\1')
+    if not do3way and not re.search(regex, cmdline):
+        cmdline += ' $parent1 $child'
+    return re.sub(regex, quote, cmdline)
+
 def dodiff(ui, repo, cmdline, pats, opts):
     '''Do the actual diff:
 
     - copy to a temp structure if diffing 2 internal revisions
     - copy to a temp structure if diffing working revision with
       another one and more than 1 file is changed
     - just invoke the diff for a single file in the working dir
     '''
 
     revs = opts.get('rev')
     change = opts.get('change')
     do3way = '$parent2' in cmdline
 
     if revs and change:
         msg = _('cannot specify --rev and --change at the same time')
         raise error.Abort(msg)
     elif change:
         ctx2 = scmutil.revsingle(repo, change, None)
         ctx1a, ctx1b = ctx2.p1(), ctx2.p2()
     else:
         ctx1a, ctx2 = scmutil.revpair(repo, revs)
         if not revs:
             ctx1b = repo[None].p2()
         else:
             ctx1b = repo[nullid]
 
     node1a = ctx1a.node()
     node1b = ctx1b.node()
     node2 = ctx2.node()
 
     # Disable 3-way merge if there is only one parent
     if do3way:
         if node1b == nullid:
             do3way = False
 
     subrepos=opts.get('subrepos')
 
     matcher = scmutil.match(repo[node2], pats, opts)
 
     if opts.get('patch'):
         if subrepos:
             raise error.Abort(_('--patch cannot be used with --subrepos'))
         if node2 is None:
             raise error.Abort(_('--patch requires two revisions'))
     else:
         mod_a, add_a, rem_a = map(set, repo.status(node1a, node2, matcher,
                                                    listsubrepos=subrepos)[:3])
         if do3way:
             mod_b, add_b, rem_b = map(set,
                                       repo.status(node1b, node2, matcher,
                                                   listsubrepos=subrepos)[:3])
         else:
             mod_b, add_b, rem_b = set(), set(), set()
         modadd = mod_a | add_a | mod_b | add_b
         common = modadd | rem_a | rem_b
         if not common:
             return 0
 
     tmproot = pycompat.mkdtemp(prefix='extdiff.')
     try:
         if not opts.get('patch'):
             # Always make a copy of node1a (and node1b, if applicable)
             dir1a_files = mod_a | rem_a | ((mod_b | add_b) - add_a)
             dir1a = snapshot(ui, repo, dir1a_files, node1a, tmproot,
                              subrepos)[0]
             rev1a = '@%d' % repo[node1a].rev()
             if do3way:
                 dir1b_files = mod_b | rem_b | ((mod_a | add_a) - add_b)
                 dir1b = snapshot(ui, repo, dir1b_files, node1b, tmproot,
                                  subrepos)[0]
                 rev1b = '@%d' % repo[node1b].rev()
             else:
                 dir1b = None
                 rev1b = ''
 
             fnsandstat = []
 
             # If node2 in not the wc or there is >1 change, copy it
             dir2root = ''
             rev2 = ''
             if node2:
                 dir2 = snapshot(ui, repo, modadd, node2, tmproot, subrepos)[0]
                 rev2 = '@%d' % repo[node2].rev()
             elif len(common) > 1:
                 #we only actually need to get the files to copy back to
                 #the working dir in this case (because the other cases
                 #are: diffing 2 revisions or single file -- in which case
                 #the file is already directly passed to the diff tool).
                 dir2, fnsandstat = snapshot(ui, repo, modadd, None, tmproot,
                                             subrepos)
             else:
                 # This lets the diff tool open the changed file directly
                 dir2 = ''
                 dir2root = repo.root
 
             label1a = rev1a
             label1b = rev1b
             label2 = rev2
 
             # If only one change, diff the files instead of the directories
             # Handle bogus modifies correctly by checking if the files exist
             if len(common) == 1:
                 common_file = util.localpath(common.pop())
                 dir1a = os.path.join(tmproot, dir1a, common_file)
                 label1a = common_file + rev1a
                 if not os.path.isfile(dir1a):
                     dir1a = os.devnull
                 if do3way:
                     dir1b = os.path.join(tmproot, dir1b, common_file)
                     label1b = common_file + rev1b
                     if not os.path.isfile(dir1b):
                         dir1b = os.devnull
                 dir2 = os.path.join(dir2root, dir2, common_file)
                 label2 = common_file + rev2
         else:
             template = 'hg-%h.patch'
             with formatter.nullformatter(ui, 'extdiff', {}) as fm:
                 cmdutil.export(repo, [repo[node1a].rev(), repo[node2].rev()],
                                fm,
                                fntemplate=repo.vfs.reljoin(tmproot, template),
                                match=matcher)
             label1a = cmdutil.makefilename(repo[node1a], template)
             label2 = cmdutil.makefilename(repo[node2], template)
             dir1a = repo.vfs.reljoin(tmproot, label1a)
             dir2 = repo.vfs.reljoin(tmproot, label2)
             dir1b = None
             label1b = None
             fnsandstat = []
 
-        # Function to quote file/dir names in the argument string.
-        # When not operating in 3-way mode, an empty string is
-        # returned for parent2
-        replace = {'parent': dir1a, 'parent1': dir1a, 'parent2': dir1b,
-                   'plabel1': label1a, 'plabel2': label1b,
-                   'clabel': label2, 'child': dir2,
-                   'root': repo.root}
-        def quote(match):
-            pre = match.group(2)
-            key = match.group(3)
-            if not do3way and key == 'parent2':
-                return pre
-            return pre + procutil.shellquote(replace[key])
-
-        # Match parent2 first, so 'parent1?' will match both parent1 and parent
-        regex = (br'''(['"]?)([^\s'"$]*)'''
-                 br'\$(parent2|parent1?|child|plabel1|plabel2|clabel|root)\1')
-        if not do3way and not re.search(regex, cmdline):
-            cmdline += ' $parent1 $child'
-        cmdline = re.sub(regex, quote, cmdline)
-
-        ui.debug('running %r in %s\n' % (pycompat.bytestr(cmdline), tmproot))
+        # Run the external tool on the 2 temp directories or the patches
+        cmdline = formatcmdline(
+            cmdline, repo.root, do3way=do3way,
+            parent1=dir1a, plabel1=label1a,
+            parent2=dir1b, plabel2=label1b,
+            child=dir2, clabel=label2)
+        ui.debug('running %r in %s\n' % (pycompat.bytestr(cmdline),
+                                         tmproot))
         ui.system(cmdline, cwd=tmproot, blockedtag='extdiff')
 
         for copy_fn, working_fn, st in fnsandstat:
             cpstat = os.lstat(copy_fn)
             # Some tools copy the file and attributes, so mtime may not detect
             # all changes.  A size check will detect more cases, but not all.
             # The only certain way to detect every case is to diff all files,
             # which could be expensive.
             # copyfile() carries over the permission, so the mode check could
             # be in an 'elif' branch, but for the case where the file has
             # changed without affecting mtime or size.
             if (cpstat[stat.ST_MTIME] != st[stat.ST_MTIME]
                 or cpstat.st_size != st.st_size
                 or (cpstat.st_mode & 0o100) != (st.st_mode & 0o100)):
                 ui.debug('file changed while diffing. '
                          'Overwriting: %s (src: %s)\n' % (working_fn, copy_fn))
                 util.copyfile(copy_fn, working_fn)
 
         return 1
     finally:
         ui.note(_('cleaning up temp directory\n'))
         shutil.rmtree(tmproot)
 
 extdiffopts = [
     ('o', 'option', [],
      _('pass option to comparison program'), _('OPT')),
     ('r', 'rev', [], _('revision'), _('REV')),
     ('c', 'change', '', _('change made by revision'), _('REV')),
     ('', 'patch', None, _('compare patches for two revisions'))
     ] + cmdutil.walkopts + cmdutil.subrepoopts
 
 @command('extdiff',
     [('p', 'program', '', _('comparison program to run'), _('CMD')),
      ] + extdiffopts,
     _('hg extdiff [OPT]... [FILE]...'),
     helpcategory=command.CATEGORY_FILE_CONTENTS,
     inferrepo=True)
 def extdiff(ui, repo, *pats, **opts):
     '''use external program to diff repository (or selected files)
 
     Show differences between revisions for the specified files, using
     an external program. The default program used is diff, with
     default options "-Npru".
 
     To select a different program, use the -p/--program option. The
     program will be passed the names of two directories to compare. To
     pass additional options to the program, use -o/--option. These
     will be passed before the names of the directories to compare.
 
     When two revision arguments are given, then changes are shown
     between those revisions. If only one revision is specified then
     that revision is compared to the working directory, and, when no
     revisions are specified, the working directory files are compared
     to its parent.'''
     opts = pycompat.byteskwargs(opts)
     program = opts.get('program')
     option = opts.get('option')
     if not program:
         program = 'diff'
         option = option or ['-Npru']
     cmdline = ' '.join(map(procutil.shellquote, [program] + option))
     return dodiff(ui, repo, cmdline, pats, opts)
 
 class savedcmd(object):
     """use external program to diff repository (or selected files)
 
     Show differences between revisions for the specified files, using
     the following program::
 
         %(path)s
 
     When two revision arguments are given, then changes are shown
     between those revisions. If only one revision is specified then
     that revision is compared to the working directory, and, when no
     revisions are specified, the working directory files are compared
     to its parent.
     """
 
     def __init__(self, path, cmdline):
         # We can't pass non-ASCII through docstrings (and path is
-        # in an unknown encoding anyway)
-        docpath = stringutil.escapestr(path)
+        # in an unknown encoding anyway), but avoid double separators on
+        # Windows
+        docpath = stringutil.escapestr(path).replace(b'\\\\', b'\\')
         self.__doc__ %= {r'path': pycompat.sysstr(stringutil.uirepr(docpath))}
         self._cmdline = cmdline
 
     def __call__(self, ui, repo, *pats, **opts):
         opts = pycompat.byteskwargs(opts)
         options = ' '.join(map(procutil.shellquote, opts['option']))
         if options:
             options = ' ' + options
         return dodiff(ui, repo, self._cmdline + options, pats, opts)
 
 def uisetup(ui):
     for cmd, path in ui.configitems('extdiff'):
         path = util.expandpath(path)
         if cmd.startswith('cmd.'):
             cmd = cmd[4:]
             if not path:
                 path = procutil.findexe(cmd)
                 if path is None:
                     path = filemerge.findexternaltool(ui, cmd) or cmd
             diffopts = ui.config('extdiff', 'opts.' + cmd)
             cmdline = procutil.shellquote(path)
             if diffopts:
                 cmdline += ' ' + diffopts
         elif cmd.startswith('opts.'):
             continue
         else:
             if path:
                 # case "cmd = path opts"
                 cmdline = path
                 diffopts = len(pycompat.shlexsplit(cmdline)) > 1
             else:
                 # case "cmd ="
                 path = procutil.findexe(cmd)
                 if path is None:
                     path = filemerge.findexternaltool(ui, cmd) or cmd
                 cmdline = procutil.shellquote(path)
                 diffopts = False
         # look for diff arguments in [diff-tools] then [merge-tools]
         if not diffopts:
             args = ui.config('diff-tools', cmd+'.diffargs') or \
                    ui.config('merge-tools', cmd+'.diffargs')
             if args:
                 cmdline += ' ' + args
         command(cmd, extdiffopts[:], _('hg %s [OPTION]... [FILE]...') % cmd,
                 helpcategory=command.CATEGORY_FILE_CONTENTS,
                 inferrepo=True)(savedcmd(path, cmdline))
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = [savedcmd]

hgext/fastannotate/commands.py

@@ -1,285 +1,285 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # commands: fastannotate commands
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     commands,
     encoding,
     error,
     extensions,
     patch,
     pycompat,
     registrar,
     scmutil,
     util,
 )
 
 from . import (
     context as facontext,
     error as faerror,
     formatter as faformatter,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 def _matchpaths(repo, rev, pats, opts, aopts=facontext.defaultopts):
     """generate paths matching given patterns"""
     perfhack = repo.ui.configbool('fastannotate', 'perfhack')
 
     # disable perfhack if:
     # a) any walkopt is used
     # b) if we treat pats as plain file names, some of them do not have
     #    corresponding linelog files
     if perfhack:
         # cwd related to reporoot
         reporoot = os.path.dirname(repo.path)
         reldir = os.path.relpath(encoding.getcwd(), reporoot)
         if reldir == '.':
             reldir = ''
         if any(opts.get(o[1]) for o in commands.walkopts): # a)
             perfhack = False
         else: # b)
             relpats = [os.path.relpath(p, reporoot) if os.path.isabs(p) else p
                        for p in pats]
             # disable perfhack on '..' since it allows escaping from the repo
             if any(('..' in f or
                     not os.path.isfile(
                         facontext.pathhelper(repo, f, aopts).linelogpath))
                    for f in relpats):
                 perfhack = False
 
     # perfhack: emit paths directory without checking with manifest
     # this can be incorrect if the rev dos not have file.
     if perfhack:
         for p in relpats:
             yield os.path.join(reldir, p)
     else:
         def bad(x, y):
             raise error.Abort("%s: %s" % (x, y))
         ctx = scmutil.revsingle(repo, rev)
         m = scmutil.match(ctx, pats, opts, badfn=bad)
         for p in ctx.walk(m):
             yield p
 
 fastannotatecommandargs = {
     r'options': [
         ('r', 'rev', '.', _('annotate the specified revision'), _('REV')),
         ('u', 'user', None, _('list the author (long with -v)')),
         ('f', 'file', None, _('list the filename')),
         ('d', 'date', None, _('list the date (short with -q)')),
         ('n', 'number', None, _('list the revision number (default)')),
         ('c', 'changeset', None, _('list the changeset')),
         ('l', 'line-number', None, _('show line number at the first '
                                      'appearance')),
         ('e', 'deleted', None, _('show deleted lines (slow) (EXPERIMENTAL)')),
         ('', 'no-content', None, _('do not show file content (EXPERIMENTAL)')),
         ('', 'no-follow', None, _("don't follow copies and renames")),
         ('', 'linear', None, _('enforce linear history, ignore second parent '
                                'of merges (EXPERIMENTAL)')),
         ('', 'long-hash', None, _('show long changeset hash (EXPERIMENTAL)')),
         ('', 'rebuild', None, _('rebuild cache even if it exists '
                                 '(EXPERIMENTAL)')),
     ] + commands.diffwsopts + commands.walkopts + commands.formatteropts,
     r'synopsis': _('[-r REV] [-f] [-a] [-u] [-d] [-n] [-c] [-l] FILE...'),
     r'inferrepo': True,
 }
 
 def fastannotate(ui, repo, *pats, **opts):
     """show changeset information by line for each file
 
     List changes in files, showing the revision id responsible for each line.
 
     This command is useful for discovering when a change was made and by whom.
 
     By default this command prints revision numbers. If you include --file,
     --user, or --date, the revision number is suppressed unless you also
     include --number. The default format can also be customized by setting
     fastannotate.defaultformat.
 
     Returns 0 on success.
 
     .. container:: verbose
 
         This command uses an implementation different from the vanilla annotate
         command, which may produce slightly different (while still reasonable)
         outputs for some cases.
 
         Unlike the vanilla anootate, fastannotate follows rename regardless of
         the existence of --file.
 
         For the best performance when running on a full repo, use -c, -l,
         avoid -u, -d, -n. Use --linear and --no-content to make it even faster.
 
         For the best performance when running on a shallow (remotefilelog)
         repo, avoid --linear, --no-follow, or any diff options. As the server
         won't be able to populate annotate cache when non-default options
         affecting results are used.
     """
     if not pats:
         raise error.Abort(_('at least one filename or pattern is required'))
 
     # performance hack: filtered repo can be slow. unfilter by default.
     if ui.configbool('fastannotate', 'unfilteredrepo'):
         repo = repo.unfiltered()
 
     opts = pycompat.byteskwargs(opts)
 
     rev = opts.get('rev', '.')
     rebuild = opts.get('rebuild', False)
 
     diffopts = patch.difffeatureopts(ui, opts, section='annotate',
                                      whitespace=True)
     aopts = facontext.annotateopts(
         diffopts=diffopts,
         followmerge=not opts.get('linear', False),
         followrename=not opts.get('no_follow', False))
 
     if not any(opts.get(s)
                for s in ['user', 'date', 'file', 'number', 'changeset']):
         # default 'number' for compatibility. but fastannotate is more
         # efficient with "changeset", "line-number" and "no-content".
         for name in ui.configlist('fastannotate', 'defaultformat', ['number']):
             opts[name] = True
 
     ui.pager('fastannotate')
     template = opts.get('template')
     if template == 'json':
         formatter = faformatter.jsonformatter(ui, repo, opts)
     else:
         formatter = faformatter.defaultformatter(ui, repo, opts)
     showdeleted = opts.get('deleted', False)
     showlines = not bool(opts.get('no_content'))
     showpath = opts.get('file', False)
 
     # find the head of the main (master) branch
     master = ui.config('fastannotate', 'mainbranch') or rev
 
     # paths will be used for prefetching and the real annotating
     paths = list(_matchpaths(repo, rev, pats, opts, aopts))
 
     # for client, prefetch from the server
     if util.safehasattr(repo, 'prefetchfastannotate'):
         repo.prefetchfastannotate(paths)
 
     for path in paths:
         result = lines = existinglines = None
         while True:
             try:
                 with facontext.annotatecontext(repo, path, aopts, rebuild) as a:
                     result = a.annotate(rev, master=master, showpath=showpath,
                                         showlines=(showlines and
                                                    not showdeleted))
                     if showdeleted:
                         existinglines = set((l[0], l[1]) for l in result)
                         result = a.annotatealllines(
                             rev, showpath=showpath, showlines=showlines)
                 break
             except (faerror.CannotReuseError, faerror.CorruptedFileError):
                 # happens if master moves backwards, or the file was deleted
                 # and readded, or renamed to an existing name, or corrupted.
                 if rebuild: # give up since we have tried rebuild already
                     raise
                 else: # try a second time rebuilding the cache (slow)
                     rebuild = True
                     continue
 
         if showlines:
             result, lines = result
 
         formatter.write(result, lines, existinglines=existinglines)
     formatter.end()
 
 _newopts = set([])
 _knownopts = set([opt[1].replace('-', '_') for opt in
                   (fastannotatecommandargs[r'options'] + commands.globalopts)])
 
 def _annotatewrapper(orig, ui, repo, *pats, **opts):
     """used by wrapdefault"""
     # we need this hack until the obsstore has 0.0 seconds perf impact
     if ui.configbool('fastannotate', 'unfilteredrepo'):
         repo = repo.unfiltered()
 
     # treat the file as text (skip the isbinary check)
     if ui.configbool('fastannotate', 'forcetext'):
         opts[r'text'] = True
 
     # check if we need to do prefetch (client-side)
     rev = opts.get(r'rev')
     if util.safehasattr(repo, 'prefetchfastannotate') and rev is not None:
         paths = list(_matchpaths(repo, rev, pats, pycompat.byteskwargs(opts)))
         repo.prefetchfastannotate(paths)
 
     return orig(ui, repo, *pats, **opts)
 
 def registercommand():
     """register the fastannotate command"""
     name = 'fastannotate|fastblame|fa'
     command(name, helpbasic=True, **fastannotatecommandargs)(fastannotate)
 
 def wrapdefault():
     """wrap the default annotate command, to be aware of the protocol"""
     extensions.wrapcommand(commands.table, 'annotate', _annotatewrapper)
 
 @command('debugbuildannotatecache',
          [('r', 'rev', '', _('build up to the specific revision'), _('REV'))
          ] + commands.walkopts,
          _('[-r REV] FILE...'))
 def debugbuildannotatecache(ui, repo, *pats, **opts):
     """incrementally build fastannotate cache up to REV for specified files
 
     If REV is not specified, use the config 'fastannotate.mainbranch'.
 
     If fastannotate.client is True, download the annotate cache from the
     server. Otherwise, build the annotate cache locally.
 
     The annotate cache will be built using the default diff and follow
     options and lives in '.hg/fastannotate/default'.
     """
     opts = pycompat.byteskwargs(opts)
     rev = opts.get('REV') or ui.config('fastannotate', 'mainbranch')
     if not rev:
         raise error.Abort(_('you need to provide a revision'),
                           hint=_('set fastannotate.mainbranch or use --rev'))
     if ui.configbool('fastannotate', 'unfilteredrepo'):
         repo = repo.unfiltered()
     ctx = scmutil.revsingle(repo, rev)
     m = scmutil.match(ctx, pats, opts)
     paths = list(ctx.walk(m))
     if util.safehasattr(repo, 'prefetchfastannotate'):
         # client
         if opts.get('REV'):
             raise error.Abort(_('--rev cannot be used for client'))
         repo.prefetchfastannotate(paths)
     else:
         # server, or full repo
+        progress = ui.makeprogress(_('building'), total=len(paths))
         for i, path in enumerate(paths):
-            ui.progress(_('building'), i, total=len(paths))
+            progress.update(i)
             with facontext.annotatecontext(repo, path) as actx:
                 try:
                     if actx.isuptodate(rev):
                         continue
                     actx.annotate(rev, rev)
                 except (faerror.CannotReuseError, faerror.CorruptedFileError):
                     # the cache is broken (could happen with renaming so the
                     # file history gets invalidated). rebuild and try again.
                     ui.debug('fastannotate: %s: rebuilding broken cache\n'
                              % path)
                     actx.rebuild()
                     try:
                         actx.annotate(rev, rev)
                     except Exception as ex:
                         # possibly a bug, but should not stop us from building
                         # cache for other files.
                         ui.warn(_('fastannotate: %s: failed to '
                                   'build cache: %r\n') % (path, ex))
-        # clear the progress bar
-        ui.write()
+        progress.complete()

hgext/fastannotate/context.py

@@ -1,828 +1,826 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # context: context needed to annotate a file
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import hashlib
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     linelog as linelogmod,
     lock as lockmod,
     mdiff,
     node,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.utils import (
     stringutil,
 )
 
 from . import (
     error as faerror,
     revmap as revmapmod,
 )
 
 # given path, get filelog, cached
 @util.lrucachefunc
 def _getflog(repo, path):
     return repo.file(path)
 
 # extracted from mercurial.context.basefilectx.annotate
 def _parents(f, follow=True):
     # Cut _descendantrev here to mitigate the penalty of lazy linkrev
     # adjustment. Otherwise, p._adjustlinkrev() would walk changelog
     # from the topmost introrev (= srcrev) down to p.linkrev() if it
     # isn't an ancestor of the srcrev.
     f._changeid
     pl = f.parents()
 
     # Don't return renamed parents if we aren't following.
     if not follow:
         pl = [p for p in pl if p.path() == f.path()]
 
     # renamed filectx won't have a filelog yet, so set it
     # from the cache to save time
     for p in pl:
         if not '_filelog' in p.__dict__:
             p._filelog = _getflog(f._repo, p.path())
 
     return pl
 
 # extracted from mercurial.context.basefilectx.annotate. slightly modified
 # so it takes a fctx instead of a pair of text and fctx.
 def _decorate(fctx):
     text = fctx.data()
     linecount = text.count('\n')
     if text and not text.endswith('\n'):
         linecount += 1
     return ([(fctx, i) for i in pycompat.xrange(linecount)], text)
 
 # extracted from mercurial.context.basefilectx.annotate. slightly modified
 # so it takes an extra "blocks" parameter calculated elsewhere, instead of
 # calculating diff here.
 def _pair(parent, child, blocks):
     for (a1, a2, b1, b2), t in blocks:
         # Changed blocks ('!') or blocks made only of blank lines ('~')
         # belong to the child.
         if t == '=':
             child[0][b1:b2] = parent[0][a1:a2]
     return child
 
 # like scmutil.revsingle, but with lru cache, so their states (like manifests)
 # could be reused
 _revsingle = util.lrucachefunc(scmutil.revsingle)
 
 def resolvefctx(repo, rev, path, resolverev=False, adjustctx=None):
     """(repo, str, str) -> fctx
 
     get the filectx object from repo, rev, path, in an efficient way.
 
     if resolverev is True, "rev" is a revision specified by the revset
     language, otherwise "rev" is a nodeid, or a revision number that can
     be consumed by repo.__getitem__.
 
     if adjustctx is not None, the returned fctx will point to a changeset
     that introduces the change (last modified the file). if adjustctx
     is 'linkrev', trust the linkrev and do not adjust it. this is noticeably
     faster for big repos but is incorrect for some cases.
     """
     if resolverev and not isinstance(rev, int) and rev is not None:
         ctx = _revsingle(repo, rev)
     else:
         ctx = repo[rev]
 
     # If we don't need to adjust the linkrev, create the filectx using the
     # changectx instead of using ctx[path]. This means it already has the
     # changectx information, so blame -u will be able to look directly at the
     # commitctx object instead of having to resolve it by going through the
     # manifest. In a lazy-manifest world this can prevent us from downloading a
     # lot of data.
     if adjustctx is None:
         # ctx.rev() is None means it's the working copy, which is a special
         # case.
         if ctx.rev() is None:
             fctx = ctx[path]
         else:
             fctx = repo.filectx(path, changeid=ctx.rev())
     else:
         fctx = ctx[path]
         if adjustctx == 'linkrev':
             introrev = fctx.linkrev()
         else:
             introrev = fctx.introrev()
         if introrev != ctx.rev():
             fctx._changeid = introrev
             fctx._changectx = repo[introrev]
     return fctx
 
 # like mercurial.store.encodedir, but use linelog suffixes: .m, .l, .lock
 def encodedir(path):
     return (path
             .replace('.hg/', '.hg.hg/')
             .replace('.l/', '.l.hg/')
             .replace('.m/', '.m.hg/')
             .replace('.lock/', '.lock.hg/'))
 
 def hashdiffopts(diffopts):
     diffoptstr = stringutil.pprint(sorted(
         (k, getattr(diffopts, k))
         for k in mdiff.diffopts.defaults
     ))
-    return hashlib.sha1(diffoptstr).hexdigest()[:6]
+    return node.hex(hashlib.sha1(diffoptstr).digest())[:6]
 
 _defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
 
 class annotateopts(object):
     """like mercurial.mdiff.diffopts, but is for annotate
 
     followrename: follow renames, like "hg annotate -f"
     followmerge: follow p2 of a merge changeset, otherwise p2 is ignored
     """
 
     defaults = {
         'diffopts': None,
         'followrename': True,
         'followmerge': True,
     }
 
     def __init__(self, **opts):
+        opts = pycompat.byteskwargs(opts)
         for k, v in self.defaults.iteritems():
             setattr(self, k, opts.get(k, v))
 
     @util.propertycache
     def shortstr(self):
         """represent opts in a short string, suitable for a directory name"""
         result = ''
         if not self.followrename:
             result += 'r0'
         if not self.followmerge:
             result += 'm0'
         if self.diffopts is not None:
             assert isinstance(self.diffopts, mdiff.diffopts)
             diffopthash = hashdiffopts(self.diffopts)
             if diffopthash != _defaultdiffopthash:
                 result += 'i' + diffopthash
         return result or 'default'
 
 defaultopts = annotateopts()
 
 class _annotatecontext(object):
     """do not use this class directly as it does not use lock to protect
     writes. use "with annotatecontext(...)" instead.
     """
 
     def __init__(self, repo, path, linelogpath, revmappath, opts):
         self.repo = repo
         self.ui = repo.ui
         self.path = path
         self.opts = opts
         self.linelogpath = linelogpath
         self.revmappath = revmappath
         self._linelog = None
         self._revmap = None
         self._node2path = {} # {str: str}
 
     @property
     def linelog(self):
         if self._linelog is None:
             if os.path.exists(self.linelogpath):
                 with open(self.linelogpath, 'rb') as f:
                     try:
                         self._linelog = linelogmod.linelog.fromdata(f.read())
                     except linelogmod.LineLogError:
                         self._linelog = linelogmod.linelog()
             else:
                 self._linelog = linelogmod.linelog()
         return self._linelog
 
     @property
     def revmap(self):
         if self._revmap is None:
             self._revmap = revmapmod.revmap(self.revmappath)
         return self._revmap
 
     def close(self):
         if self._revmap is not None:
             self._revmap.flush()
             self._revmap = None
         if self._linelog is not None:
             with open(self.linelogpath, 'wb') as f:
                 f.write(self._linelog.encode())
             self._linelog = None
 
     __del__ = close
 
     def rebuild(self):
         """delete linelog and revmap, useful for rebuilding"""
         self.close()
         self._node2path.clear()
         _unlinkpaths([self.revmappath, self.linelogpath])
 
     @property
     def lastnode(self):
         """return last node in revmap, or None if revmap is empty"""
         if self._revmap is None:
             # fast path, read revmap without loading its full content
             return revmapmod.getlastnode(self.revmappath)
         else:
             return self._revmap.rev2hsh(self._revmap.maxrev)
 
     def isuptodate(self, master, strict=True):
         """return True if the revmap / linelog is up-to-date, or the file
         does not exist in the master revision. False otherwise.
 
         it tries to be fast and could return false negatives, because of the
         use of linkrev instead of introrev.
 
         useful for both server and client to decide whether to update
         fastannotate cache or not.
 
         if strict is True, even if fctx exists in the revmap, but is not the
         last node, isuptodate will return False. it's good for performance - no
         expensive check was done.
 
         if strict is False, if fctx exists in the revmap, this function may
         return True. this is useful for the client to skip downloading the
         cache if the client's master is behind the server's.
         """
         lastnode = self.lastnode
         try:
             f = self._resolvefctx(master, resolverev=True)
             # choose linkrev instead of introrev as the check is meant to be
             # *fast*.
             linknode = self.repo.changelog.node(f.linkrev())
             if not strict and lastnode and linknode != lastnode:
                 # check if f.node() is in the revmap. note: this loads the
                 # revmap and can be slow.
                 return self.revmap.hsh2rev(linknode) is not None
             # avoid resolving old manifest, or slow adjustlinkrev to be fast,
             # false negatives are acceptable in this case.
             return linknode == lastnode
         except LookupError:
             # master does not have the file, or the revmap is ahead
             return True
 
     def annotate(self, rev, master=None, showpath=False, showlines=False):
         """incrementally update the cache so it includes revisions in the main
         branch till 'master'. and run annotate on 'rev', which may or may not be
         included in the main branch.
 
         if master is None, do not update linelog.
 
         the first value returned is the annotate result, it is [(node, linenum)]
         by default. [(node, linenum, path)] if showpath is True.
 
         if showlines is True, a second value will be returned, it is a list of
         corresponding line contents.
         """
 
         # the fast path test requires commit hash, convert rev number to hash,
         # so it may hit the fast path. note: in the "fctx" mode, the "annotate"
         # command could give us a revision number even if the user passes a
         # commit hash.
         if isinstance(rev, int):
             rev = node.hex(self.repo.changelog.node(rev))
 
         # fast path: if rev is in the main branch already
         directly, revfctx = self.canannotatedirectly(rev)
         if directly:
             if self.ui.debugflag:
                 self.ui.debug('fastannotate: %s: using fast path '
                               '(resolved fctx: %s)\n'
                               % (self.path,
                                  stringutil.pprint(util.safehasattr(revfctx,
                                                                     'node'))))
             return self.annotatedirectly(revfctx, showpath, showlines)
 
         # resolve master
         masterfctx = None
         if master:
             try:
                 masterfctx = self._resolvefctx(master, resolverev=True,
                                                adjustctx=True)
             except LookupError: # master does not have the file
                 pass
             else:
                 if masterfctx in self.revmap: # no need to update linelog
                     masterfctx = None
 
         #                  ... - @ <- rev (can be an arbitrary changeset,
         #                 /                not necessarily a descendant
         #      master -> o                 of master)
         #                |
         #     a merge -> o         'o': new changesets in the main branch
         #                |\        '#': revisions in the main branch that
         #                o *            exist in linelog / revmap
         #                | .       '*': changesets in side branches, or
         # last master -> # .            descendants of master
         #                | .
         #                # *       joint: '#', and is a parent of a '*'
         #                |/
         #     a joint -> # ^^^^ --- side branches
         #                |
         #                ^ --- main branch (in linelog)
 
         # these DFSes are similar to the traditional annotate algorithm.
         # we cannot really reuse the code for perf reason.
 
         # 1st DFS calculates merges, joint points, and needed.
         # "needed" is a simple reference counting dict to free items in
         # "hist", reducing its memory usage otherwise could be huge.
         initvisit = [revfctx]
         if masterfctx:
             if masterfctx.rev() is None:
                 raise error.Abort(_('cannot update linelog to wdir()'),
                                   hint=_('set fastannotate.mainbranch'))
             initvisit.append(masterfctx)
         visit = initvisit[:]
         pcache = {}
         needed = {revfctx: 1}
         hist = {} # {fctx: ([(llrev or fctx, linenum)], text)}
         while visit:
             f = visit.pop()
             if f in pcache or f in hist:
                 continue
             if f in self.revmap: # in the old main branch, it's a joint
                 llrev = self.revmap.hsh2rev(f.node())
                 self.linelog.annotate(llrev)
                 result = self.linelog.annotateresult
                 hist[f] = (result, f.data())
                 continue
             pl = self._parentfunc(f)
             pcache[f] = pl
             for p in pl:
                 needed[p] = needed.get(p, 0) + 1
                 if p not in pcache:
                     visit.append(p)
 
         # 2nd (simple) DFS calculates new changesets in the main branch
         # ('o' nodes in # the above graph), so we know when to update linelog.
         newmainbranch = set()
         f = masterfctx
         while f and f not in self.revmap:
             newmainbranch.add(f)
             pl = pcache[f]
             if pl:
                 f = pl[0]
             else:
                 f = None
                 break
 
         # f, if present, is the position where the last build stopped at, and
         # should be the "master" last time. check to see if we can continue
         # building the linelog incrementally. (we cannot if diverged)
         if masterfctx is not None:
             self._checklastmasterhead(f)
 
         if self.ui.debugflag:
             if newmainbranch:
                 self.ui.debug('fastannotate: %s: %d new changesets in the main'
                               ' branch\n' % (self.path, len(newmainbranch)))
             elif not hist: # no joints, no updates
                 self.ui.debug('fastannotate: %s: linelog cannot help in '
                               'annotating this revision\n' % self.path)
 
         # prepare annotateresult so we can update linelog incrementally
         self.linelog.annotate(self.linelog.maxrev)
 
         # 3rd DFS does the actual annotate
         visit = initvisit[:]
-        progress = 0
+        progress = self.ui.makeprogress(('building cache'),
+                                        total=len(newmainbranch))
         while visit:
             f = visit[-1]
             if f in hist:
                 visit.pop()
                 continue
 
             ready = True
             pl = pcache[f]
             for p in pl:
                 if p not in hist:
                     ready = False
                     visit.append(p)
             if not ready:
                 continue
 
             visit.pop()
             blocks = None # mdiff blocks, used for appending linelog
             ismainbranch = (f in newmainbranch)
             # curr is the same as the traditional annotate algorithm,
             # if we only care about linear history (do not follow merge),
             # then curr is not actually used.
             assert f not in hist
             curr = _decorate(f)
             for i, p in enumerate(pl):
                 bs = list(self._diffblocks(hist[p][1], curr[1]))
                 if i == 0 and ismainbranch:
                     blocks = bs
                 curr = _pair(hist[p], curr, bs)
                 if needed[p] == 1:
                     del hist[p]
                     del needed[p]
                 else:
                     needed[p] -= 1
 
             hist[f] = curr
             del pcache[f]
 
             if ismainbranch: # need to write to linelog
-                if not self.ui.quiet:
-                    progress += 1
-                    self.ui.progress(_('building cache'), progress,
-                                     total=len(newmainbranch))
+                progress.increment()
                 bannotated = None
                 if len(pl) == 2 and self.opts.followmerge: # merge
                     bannotated = curr[0]
                 if blocks is None: # no parents, add an empty one
                     blocks = list(self._diffblocks('', curr[1]))
                 self._appendrev(f, blocks, bannotated)
             elif showpath: # not append linelog, but we need to record path
                 self._node2path[f.node()] = f.path()
 
-        if progress: # clean progress bar
-            self.ui.write()
+        progress.complete()
 
         result = [
             ((self.revmap.rev2hsh(fr) if isinstance(fr, int) else fr.node()), l)
             for fr, l in hist[revfctx][0]] # [(node, linenumber)]
         return self._refineannotateresult(result, revfctx, showpath, showlines)
 
     def canannotatedirectly(self, rev):
         """(str) -> bool, fctx or node.
         return (True, f) if we can annotate without updating the linelog, pass
         f to annotatedirectly.
         return (False, f) if we need extra calculation. f is the fctx resolved
         from rev.
         """
         result = True
         f = None
         if not isinstance(rev, int) and rev is not None:
             hsh = {20: bytes, 40: node.bin}.get(len(rev), lambda x: None)(rev)
             if hsh is not None and (hsh, self.path) in self.revmap:
                 f = hsh
         if f is None:
             adjustctx = 'linkrev' if self._perfhack else True
             f = self._resolvefctx(rev, adjustctx=adjustctx, resolverev=True)
             result = f in self.revmap
             if not result and self._perfhack:
                 # redo the resolution without perfhack - as we are going to
                 # do write operations, we need a correct fctx.
                 f = self._resolvefctx(rev, adjustctx=True, resolverev=True)
         return result, f
 
     def annotatealllines(self, rev, showpath=False, showlines=False):
         """(rev : str) -> [(node : str, linenum : int, path : str)]
 
         the result has the same format with annotate, but include all (including
         deleted) lines up to rev. call this after calling annotate(rev, ...) for
         better performance and accuracy.
         """
         revfctx = self._resolvefctx(rev, resolverev=True, adjustctx=True)
 
         # find a chain from rev to anything in the mainbranch
         if revfctx not in self.revmap:
             chain = [revfctx]
             a = ''
             while True:
                 f = chain[-1]
                 pl = self._parentfunc(f)
                 if not pl:
                     break
                 if pl[0] in self.revmap:
                     a = pl[0].data()
                     break
                 chain.append(pl[0])
 
             # both self.linelog and self.revmap is backed by filesystem. now
             # we want to modify them but do not want to write changes back to
             # files. so we create in-memory objects and copy them. it's like
             # a "fork".
             linelog = linelogmod.linelog()
             linelog.copyfrom(self.linelog)
             linelog.annotate(linelog.maxrev)
             revmap = revmapmod.revmap()
             revmap.copyfrom(self.revmap)
 
             for f in reversed(chain):
                 b = f.data()
                 blocks = list(self._diffblocks(a, b))
                 self._doappendrev(linelog, revmap, f, blocks)
                 a = b
         else:
             # fastpath: use existing linelog, revmap as we don't write to them
             linelog = self.linelog
             revmap = self.revmap
 
         lines = linelog.getalllines()
         hsh = revfctx.node()
         llrev = revmap.hsh2rev(hsh)
         result = [(revmap.rev2hsh(r), l) for r, l in lines if r <= llrev]
         # cannot use _refineannotateresult since we need custom logic for
         # resolving line contents
         if showpath:
             result = self._addpathtoresult(result, revmap)
         if showlines:
             linecontents = self._resolvelines(result, revmap, linelog)
             result = (result, linecontents)
         return result
 
     def _resolvelines(self, annotateresult, revmap, linelog):
         """(annotateresult) -> [line]. designed for annotatealllines.
         this is probably the most inefficient code in the whole fastannotate
         directory. but we have made a decision that the linelog does not
         store line contents. so getting them requires random accesses to
         the revlog data, since they can be many, it can be very slow.
         """
         # [llrev]
         revs = [revmap.hsh2rev(l[0]) for l in annotateresult]
         result = [None] * len(annotateresult)
         # {(rev, linenum): [lineindex]}
         key2idxs = collections.defaultdict(list)
         for i in pycompat.xrange(len(result)):
             key2idxs[(revs[i], annotateresult[i][1])].append(i)
         while key2idxs:
             # find an unresolved line and its linelog rev to annotate
             hsh = None
             try:
                 for (rev, _linenum), idxs in key2idxs.iteritems():
                     if revmap.rev2flag(rev) & revmapmod.sidebranchflag:
                         continue
                     hsh = annotateresult[idxs[0]][0]
                     break
             except StopIteration: # no more unresolved lines
                 return result
             if hsh is None:
                 # the remaining key2idxs are not in main branch, resolving them
                 # using the hard way...
                 revlines = {}
                 for (rev, linenum), idxs in key2idxs.iteritems():
                     if rev not in revlines:
                         hsh = annotateresult[idxs[0]][0]
                         if self.ui.debugflag:
                             self.ui.debug('fastannotate: reading %s line #%d '
                                           'to resolve lines %r\n'
                                           % (node.short(hsh), linenum, idxs))
                         fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
                         lines = mdiff.splitnewlines(fctx.data())
                         revlines[rev] = lines
                     for idx in idxs:
                         result[idx] = revlines[rev][linenum]
                 assert all(x is not None for x in result)
                 return result
 
             # run the annotate and the lines should match to the file content
             self.ui.debug('fastannotate: annotate %s to resolve lines\n'
                           % node.short(hsh))
             linelog.annotate(rev)
             fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
             annotated = linelog.annotateresult
             lines = mdiff.splitnewlines(fctx.data())
             if len(lines) != len(annotated):
                 raise faerror.CorruptedFileError('unexpected annotated lines')
             # resolve lines from the annotate result
             for i, line in enumerate(lines):
                 k = annotated[i]
                 if k in key2idxs:
                     for idx in key2idxs[k]:
                         result[idx] = line
                     del key2idxs[k]
         return result
 
     def annotatedirectly(self, f, showpath, showlines):
         """like annotate, but when we know that f is in linelog.
         f can be either a 20-char str (node) or a fctx. this is for perf - in
         the best case, the user provides a node and we don't need to read the
         filelog or construct any filecontext.
         """
-        if isinstance(f, str):
+        if isinstance(f, bytes):
             hsh = f
         else:
             hsh = f.node()
         llrev = self.revmap.hsh2rev(hsh)
         if not llrev:
             raise faerror.CorruptedFileError('%s is not in revmap'
                                              % node.hex(hsh))
         if (self.revmap.rev2flag(llrev) & revmapmod.sidebranchflag) != 0:
             raise faerror.CorruptedFileError('%s is not in revmap mainbranch'
                                              % node.hex(hsh))
         self.linelog.annotate(llrev)
         result = [(self.revmap.rev2hsh(r), l)
                   for r, l in self.linelog.annotateresult]
         return self._refineannotateresult(result, f, showpath, showlines)
 
     def _refineannotateresult(self, result, f, showpath, showlines):
         """add the missing path or line contents, they can be expensive.
         f could be either node or fctx.
         """
         if showpath:
             result = self._addpathtoresult(result)
         if showlines:
-            if isinstance(f, str): # f: node or fctx
+            if isinstance(f, bytes): # f: node or fctx
                 llrev = self.revmap.hsh2rev(f)
                 fctx = self._resolvefctx(f, self.revmap.rev2path(llrev))
             else:
                 fctx = f
             lines = mdiff.splitnewlines(fctx.data())
             if len(lines) != len(result): # linelog is probably corrupted
                 raise faerror.CorruptedFileError()
             result = (result, lines)
         return result
 
     def _appendrev(self, fctx, blocks, bannotated=None):
         self._doappendrev(self.linelog, self.revmap, fctx, blocks, bannotated)
 
     def _diffblocks(self, a, b):
         return mdiff.allblocks(a, b, self.opts.diffopts)
 
     @staticmethod
     def _doappendrev(linelog, revmap, fctx, blocks, bannotated=None):
         """append a revision to linelog and revmap"""
 
         def getllrev(f):
             """(fctx) -> int"""
             # f should not be a linelog revision
             if isinstance(f, int):
                 raise error.ProgrammingError('f should not be an int')
             # f is a fctx, allocate linelog rev on demand
             hsh = f.node()
             rev = revmap.hsh2rev(hsh)
             if rev is None:
                 rev = revmap.append(hsh, sidebranch=True, path=f.path())
             return rev
 
         # append sidebranch revisions to revmap
         siderevs = []
         siderevmap = {} # node: int
         if bannotated is not None:
             for (a1, a2, b1, b2), op in blocks:
                 if op != '=':
                     # f could be either linelong rev, or fctx.
                     siderevs += [f for f, l in bannotated[b1:b2]
                                  if not isinstance(f, int)]
         siderevs = set(siderevs)
         if fctx in siderevs: # mainnode must be appended seperately
             siderevs.remove(fctx)
         for f in siderevs:
             siderevmap[f] = getllrev(f)
 
         # the changeset in the main branch, could be a merge
         llrev = revmap.append(fctx.node(), path=fctx.path())
         siderevmap[fctx] = llrev
 
         for (a1, a2, b1, b2), op in reversed(blocks):
             if op == '=':
                 continue
             if bannotated is None:
                 linelog.replacelines(llrev, a1, a2, b1, b2)
             else:
                 blines = [((r if isinstance(r, int) else siderevmap[r]), l)
                           for r, l in bannotated[b1:b2]]
                 linelog.replacelines_vec(llrev, a1, a2, blines)
 
     def _addpathtoresult(self, annotateresult, revmap=None):
         """(revmap, [(node, linenum)]) -> [(node, linenum, path)]"""
         if revmap is None:
             revmap = self.revmap
 
         def _getpath(nodeid):
             path = self._node2path.get(nodeid)
             if path is None:
                 path = revmap.rev2path(revmap.hsh2rev(nodeid))
                 self._node2path[nodeid] = path
             return path
 
         return [(n, l, _getpath(n)) for n, l in annotateresult]
 
     def _checklastmasterhead(self, fctx):
         """check if fctx is the master's head last time, raise if not"""
         if fctx is None:
             llrev = 0
         else:
             llrev = self.revmap.hsh2rev(fctx.node())
             if not llrev:
                 raise faerror.CannotReuseError()
         if self.linelog.maxrev != llrev:
             raise faerror.CannotReuseError()
 
     @util.propertycache
     def _parentfunc(self):
         """-> (fctx) -> [fctx]"""
         followrename = self.opts.followrename
         followmerge = self.opts.followmerge
         def parents(f):
             pl = _parents(f, follow=followrename)
             if not followmerge:
                 pl = pl[:1]
             return pl
         return parents
 
     @util.propertycache
     def _perfhack(self):
         return self.ui.configbool('fastannotate', 'perfhack')
 
     def _resolvefctx(self, rev, path=None, **kwds):
         return resolvefctx(self.repo, rev, (path or self.path), **kwds)
 
 def _unlinkpaths(paths):
     """silent, best-effort unlink"""
     for path in paths:
         try:
             util.unlink(path)
         except OSError:
             pass
 
 class pathhelper(object):
     """helper for getting paths for lockfile, linelog and revmap"""
 
     def __init__(self, repo, path, opts=defaultopts):
         # different options use different directories
         self._vfspath = os.path.join('fastannotate',
                                      opts.shortstr, encodedir(path))
         self._repo = repo
 
     @property
     def dirname(self):
         return os.path.dirname(self._repo.vfs.join(self._vfspath))
 
     @property
     def linelogpath(self):
         return self._repo.vfs.join(self._vfspath + '.l')
 
     def lock(self):
         return lockmod.lock(self._repo.vfs, self._vfspath + '.lock')
 
     @contextlib.contextmanager
     def _lockflock(self):
         """the same as 'lock' but use flock instead of lockmod.lock, to avoid
         creating temporary symlinks."""
         import fcntl
         lockpath = self.linelogpath
         util.makedirs(os.path.dirname(lockpath))
         lockfd = os.open(lockpath, os.O_RDONLY | os.O_CREAT, 0o664)
         fcntl.flock(lockfd, fcntl.LOCK_EX)
         try:
             yield
         finally:
             fcntl.flock(lockfd, fcntl.LOCK_UN)
             os.close(lockfd)
 
     @property
     def revmappath(self):
         return self._repo.vfs.join(self._vfspath + '.m')
 
 @contextlib.contextmanager
 def annotatecontext(repo, path, opts=defaultopts, rebuild=False):
     """context needed to perform (fast) annotate on a file
 
     an annotatecontext of a single file consists of two structures: the
     linelog and the revmap. this function takes care of locking. only 1
     process is allowed to write that file's linelog and revmap at a time.
 
     when something goes wrong, this function will assume the linelog and the
     revmap are in a bad state, and remove them from disk.
 
     use this function in the following way:
 
         with annotatecontext(...) as actx:
             actx. ....
     """
     helper = pathhelper(repo, path, opts)
     util.makedirs(helper.dirname)
     revmappath = helper.revmappath
     linelogpath = helper.linelogpath
     actx = None
     try:
         with helper.lock():
             actx = _annotatecontext(repo, path, linelogpath, revmappath, opts)
             if rebuild:
                 actx.rebuild()
             yield actx
     except Exception:
         if actx is not None:
             actx.rebuild()
         repo.ui.debug('fastannotate: %s: cache broken and deleted\n' % path)
         raise
     finally:
         if actx is not None:
             actx.close()
 
 def fctxannotatecontext(fctx, follow=True, diffopts=None, rebuild=False):
     """like annotatecontext but get the context from a fctx. convenient when
     used in fctx.annotate
     """
     repo = fctx._repo
     path = fctx._path
     if repo.ui.configbool('fastannotate', 'forcefollow', True):
         follow = True
     aopts = annotateopts(diffopts=diffopts, followrename=follow)
     return annotatecontext(repo, path, aopts, rebuild)

hgext/fastannotate/formatter.py

@@ -1,161 +1,164 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # format: defines the format used to output annotate result
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 from mercurial import (
     encoding,
     node,
     pycompat,
     templatefilters,
     util,
 )
 from mercurial.utils import (
         dateutil,
 )
 
 # imitating mercurial.commands.annotate, not using the vanilla formatter since
 # the data structures are a bit different, and we have some fast paths.
 class defaultformatter(object):
     """the default formatter that does leftpad and support some common flags"""
 
     def __init__(self, ui, repo, opts):
         self.ui = ui
         self.opts = opts
 
         if ui.quiet:
             datefunc = dateutil.shortdate
         else:
             datefunc = dateutil.datestr
         datefunc = util.cachefunc(datefunc)
         getctx = util.cachefunc(lambda x: repo[x[0]])
         hexfunc = self._hexfunc
 
         # special handling working copy "changeset" and "rev" functions
         if self.opts.get('rev') == 'wdir()':
             orig = hexfunc
             hexfunc = lambda x: None if x is None else orig(x)
             wnode = hexfunc(repo[None].p1().node()) + '+'
-            wrev = str(repo[None].p1().rev())
+            wrev = '%d' % repo[None].p1().rev()
             wrevpad = ''
             if not opts.get('changeset'): # only show + if changeset is hidden
                 wrev += '+'
                 wrevpad = ' '
-            revenc = lambda x: wrev if x is None else str(x) + wrevpad
-            csetenc = lambda x: wnode if x is None else str(x) + ' '
+            revenc = lambda x: wrev if x is None else ('%d' % x) + wrevpad
+            def csetenc(x):
+                if x is None:
+                    return wnode
+                return pycompat.bytestr(x) + ' '
         else:
-            revenc = csetenc = str
+            revenc = csetenc = pycompat.bytestr
 
         # opt name, separator, raw value (for json/plain), encoder (for plain)
         opmap = [('user', ' ', lambda x: getctx(x).user(), ui.shortuser),
                  ('number', ' ', lambda x: getctx(x).rev(), revenc),
                  ('changeset', ' ', lambda x: hexfunc(x[0]), csetenc),
                  ('date', ' ', lambda x: getctx(x).date(), datefunc),
-                 ('file', ' ', lambda x: x[2], str),
-                 ('line_number', ':', lambda x: x[1] + 1, str)]
+                 ('file', ' ', lambda x: x[2], pycompat.bytestr),
+                 ('line_number', ':', lambda x: x[1] + 1, pycompat.bytestr)]
         fieldnamemap = {'number': 'rev', 'changeset': 'node'}
         funcmap = [(get, sep, fieldnamemap.get(op, op), enc)
                    for op, sep, get, enc in opmap
                    if opts.get(op)]
         # no separator for first column
         funcmap[0] = list(funcmap[0])
         funcmap[0][1] = ''
         self.funcmap = funcmap
 
     def write(self, annotatedresult, lines=None, existinglines=None):
         """(annotateresult, [str], set([rev, linenum])) -> None. write output.
         annotateresult can be [(node, linenum, path)], or [(node, linenum)]
         """
         pieces = [] # [[str]]
         maxwidths = [] # [int]
 
         # calculate padding
         for f, sep, name, enc in self.funcmap:
             l = [enc(f(x)) for x in annotatedresult]
             pieces.append(l)
             if name in ['node', 'date']: # node and date has fixed size
                 l = l[:1]
             widths = pycompat.maplist(encoding.colwidth, set(l))
             maxwidth = (max(widths) if widths else 0)
             maxwidths.append(maxwidth)
 
         # buffered output
         result = ''
         for i in pycompat.xrange(len(annotatedresult)):
             for j, p in enumerate(pieces):
                 sep = self.funcmap[j][1]
                 padding = ' ' * (maxwidths[j] - len(p[i]))
                 result += sep + padding + p[i]
             if lines:
                 if existinglines is None:
                     result += ': ' + lines[i]
                 else: # extra formatting showing whether a line exists
                     key = (annotatedresult[i][0], annotatedresult[i][1])
                     if key in existinglines:
                         result += ':  ' + lines[i]
                     else:
                         result += ': ' + self.ui.label('-' + lines[i],
                                                        'diff.deleted')
 
-            if result[-1] != '\n':
+            if result[-1:] != '\n':
                 result += '\n'
 
         self.ui.write(result)
 
     @util.propertycache
     def _hexfunc(self):
         if self.ui.debugflag or self.opts.get('long_hash'):
             return node.hex
         else:
             return node.short
 
     def end(self):
         pass
 
 class jsonformatter(defaultformatter):
     def __init__(self, ui, repo, opts):
         super(jsonformatter, self).__init__(ui, repo, opts)
         self.ui.write('[')
         self.needcomma = False
 
     def write(self, annotatedresult, lines=None, existinglines=None):
         if annotatedresult:
             self._writecomma()
 
-        pieces = [(name, map(f, annotatedresult))
+        pieces = [(name, pycompat.maplist(f, annotatedresult))
                   for f, sep, name, enc in self.funcmap]
         if lines is not None:
             pieces.append(('line', lines))
         pieces.sort()
 
         seps = [','] * len(pieces[:-1]) + ['']
 
         result = ''
         lasti = len(annotatedresult) - 1
         for i in pycompat.xrange(len(annotatedresult)):
             result += '\n {\n'
             for j, p in enumerate(pieces):
                 k, vs = p
                 result += ('  "%s": %s%s\n'
                            % (k, templatefilters.json(vs[i], paranoid=False),
                               seps[j]))
             result += ' }%s' % ('' if i == lasti else ',')
         if lasti >= 0:
             self.needcomma = True
 
         self.ui.write(result)
 
     def _writecomma(self):
         if self.needcomma:
             self.ui.write(',')
             self.needcomma = False
 
     @util.propertycache
     def _hexfunc(self):
         return node.hex
 
     def end(self):
         self.ui.write('\n]\n')

hgext/fastannotate/protocol.py

@@ -1,228 +1,228 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # protocol: logic for a server providing fastannotate support
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import contextlib
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     extensions,
     hg,
     util,
     wireprotov1peer,
     wireprotov1server,
 )
 from . import context
 
 # common
 
 def _getmaster(ui):
     """get the mainbranch, and enforce it is set"""
     master = ui.config('fastannotate', 'mainbranch')
     if not master:
         raise error.Abort(_('fastannotate.mainbranch is required '
                             'for both the client and the server'))
     return master
 
 # server-side
 
 def _capabilities(orig, repo, proto):
     result = orig(repo, proto)
     result.append('getannotate')
     return result
 
 def _getannotate(repo, proto, path, lastnode):
     # output:
     #   FILE := vfspath + '\0' + str(size) + '\0' + content
     #   OUTPUT := '' | FILE + OUTPUT
     result = ''
     buildondemand = repo.ui.configbool('fastannotate', 'serverbuildondemand',
                                        True)
     with context.annotatecontext(repo, path) as actx:
         if buildondemand:
             # update before responding to the client
             master = _getmaster(repo.ui)
             try:
                 if not actx.isuptodate(master):
                     actx.annotate(master, master)
             except Exception:
                 # non-fast-forward move or corrupted. rebuild automically.
                 actx.rebuild()
                 try:
                     actx.annotate(master, master)
                 except Exception:
                     actx.rebuild() # delete files
             finally:
                 # although the "with" context will also do a close/flush, we
                 # need to do it early so we can send the correct respond to
                 # client.
                 actx.close()
         # send back the full content of revmap and linelog, in the future we
         # may want to do some rsync-like fancy updating.
         # the lastnode check is not necessary if the client and the server
         # agree where the main branch is.
         if actx.lastnode != lastnode:
             for p in [actx.revmappath, actx.linelogpath]:
                 if not os.path.exists(p):
                     continue
                 content = ''
                 with open(p, 'rb') as f:
                     content = f.read()
                 vfsbaselen = len(repo.vfs.base + '/')
                 relpath = p[vfsbaselen:]
                 result += '%s\0%d\0%s' % (relpath, len(content), content)
     return result
 
 def _registerwireprotocommand():
     if 'getannotate' in wireprotov1server.commands:
         return
     wireprotov1server.wireprotocommand(
         'getannotate', 'path lastnode')(_getannotate)
 
 def serveruisetup(ui):
     _registerwireprotocommand()
     extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities)
 
 # client-side
 
 def _parseresponse(payload):
     result = {}
     i = 0
     l = len(payload) - 1
     state = 0 # 0: vfspath, 1: size
     vfspath = size = ''
     while i < l:
-        ch = payload[i]
+        ch = payload[i:i + 1]
         if ch == '\0':
             if state == 1:
-                result[vfspath] = buffer(payload, i + 1, int(size))
+                result[vfspath] = payload[i + 1:i + 1 + int(size)]
                 i += int(size)
                 state = 0
                 vfspath = size = ''
             elif state == 0:
                 state = 1
         else:
             if state == 1:
                 size += ch
             elif state == 0:
                 vfspath += ch
         i += 1
     return result
 
 def peersetup(ui, peer):
     class fastannotatepeer(peer.__class__):
         @wireprotov1peer.batchable
         def getannotate(self, path, lastnode=None):
             if not self.capable('getannotate'):
                 ui.warn(_('remote peer cannot provide annotate cache\n'))
                 yield None, None
             else:
                 args = {'path': path, 'lastnode': lastnode or ''}
                 f = wireprotov1peer.future()
                 yield args, f
                 yield _parseresponse(f.value)
     peer.__class__ = fastannotatepeer
 
 @contextlib.contextmanager
 def annotatepeer(repo):
     ui = repo.ui
 
     remotepath = ui.expandpath(
         ui.config('fastannotate', 'remotepath', 'default'))
     peer = hg.peer(ui, {}, remotepath)
 
     try:
         yield peer
     finally:
         peer.close()
 
 def clientfetch(repo, paths, lastnodemap=None, peer=None):
     """download annotate cache from the server for paths"""
     if not paths:
         return
 
     if peer is None:
         with annotatepeer(repo) as peer:
             return clientfetch(repo, paths, lastnodemap, peer)
 
     if lastnodemap is None:
         lastnodemap = {}
 
     ui = repo.ui
     results = []
     with peer.commandexecutor() as batcher:
         ui.debug('fastannotate: requesting %d files\n' % len(paths))
         for p in paths:
             results.append(batcher.callcommand(
                 'getannotate',
                 {'path': p, 'lastnode':lastnodemap.get(p)}))
 
         for result in results:
             r = result.result()
             # TODO: pconvert these paths on the server?
             r = {util.pconvert(p): v for p, v in r.iteritems()}
             for path in sorted(r):
                 # ignore malicious paths
                 if (not path.startswith('fastannotate/')
                     or '/../' in (path + '/')):
                     ui.debug('fastannotate: ignored malicious path %s\n' % path)
                     continue
                 content = r[path]
                 if ui.debugflag:
                     ui.debug('fastannotate: writing %d bytes to %s\n'
                              % (len(content), path))
                 repo.vfs.makedirs(os.path.dirname(path))
                 with repo.vfs(path, 'wb') as f:
                     f.write(content)
 
 def _filterfetchpaths(repo, paths):
     """return a subset of paths whose history is long and need to fetch linelog
     from the server. works with remotefilelog and non-remotefilelog repos.
     """
     threshold = repo.ui.configint('fastannotate', 'clientfetchthreshold', 10)
     if threshold <= 0:
         return paths
 
     result = []
     for path in paths:
         try:
             if len(repo.file(path)) >= threshold:
                 result.append(path)
         except Exception: # file not found etc.
             result.append(path)
 
     return result
 
 def localreposetup(ui, repo):
     class fastannotaterepo(repo.__class__):
         def prefetchfastannotate(self, paths, peer=None):
             master = _getmaster(self.ui)
             needupdatepaths = []
             lastnodemap = {}
             try:
                 for path in _filterfetchpaths(self, paths):
                     with context.annotatecontext(self, path) as actx:
                         if not actx.isuptodate(master, strict=False):
                             needupdatepaths.append(path)
                             lastnodemap[path] = actx.lastnode
                 if needupdatepaths:
                     clientfetch(self, needupdatepaths, lastnodemap, peer)
             except Exception as ex:
                 # could be directory not writable or so, not fatal
                 self.ui.debug('fastannotate: prefetch failed: %r\n' % ex)
     repo.__class__ = fastannotaterepo
 
 def clientreposetup(ui, repo):
     _registerwireprotocommand()
     if repo.local():
         localreposetup(ui, repo)
     # TODO: this mutates global state, but only if at least one repo
     # has the extension enabled. This is probably bad for hgweb.
     if peersetup not in hg.wirepeersetupfuncs:
         hg.wirepeersetupfuncs.append(peersetup)

hgext/fastannotate/revmap.py

@@ -1,254 +1,254 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # revmap: trivial hg hash - linelog rev bidirectional map
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import bisect
 import os
 import struct
 
 from mercurial.node import hex
 from mercurial import (
     error as hgerror,
     pycompat,
 )
 from . import error
 
 # the revmap file format is straightforward:
 #
 #    8 bytes: header
 #    1 byte : flag for linelog revision 1
 #    ? bytes: (optional) '\0'-terminated path string
 #             only exists if (flag & renameflag) != 0
 #   20 bytes: hg hash for linelog revision 1
 #    1 byte : flag for linelog revision 2
 #    ? bytes: (optional) '\0'-terminated path string
 #   20 bytes: hg hash for linelog revision 2
 #   ....
 #
 # the implementation is kinda stupid: __init__ loads the whole revmap.
 # no laziness. benchmark shows loading 10000 revisions is about 0.015
 # seconds, which looks enough for our use-case. if this implementation
 # becomes a bottleneck, we can change it to lazily read the file
 # from the end.
 
 # whether the changeset is in the side branch. i.e. not in the linear main
 # branch but only got referenced by lines in merge changesets.
 sidebranchflag = 1
 
 # whether the changeset changes the file path (ie. is a rename)
 renameflag = 2
 
 # len(mercurial.node.nullid)
 _hshlen = 20
 
 class revmap(object):
     """trivial hg bin hash - linelog rev bidirectional map
 
     also stores a flag (uint8) for each revision, and track renames.
     """
 
     HEADER = b'REVMAP1\0'
 
     def __init__(self, path=None):
         """create or load the revmap, optionally associate to a file
 
         if path is None, the revmap is entirely in-memory. the caller is
         responsible for locking. concurrent writes to a same file is unsafe.
         the caller needs to make sure one file is associated to at most one
         revmap object at a time."""
         self.path = path
         self._rev2hsh = [None]
         self._rev2flag = [None]
         self._hsh2rev = {}
         # since rename does not happen frequently, do not store path for every
         # revision. self._renamerevs can be used for bisecting.
         self._renamerevs = [0]
         self._renamepaths = ['']
         self._lastmaxrev = -1
         if path:
             if os.path.exists(path):
                 self._load()
             else:
                 # write the header so "append" can do incremental updates
                 self.flush()
 
     def copyfrom(self, rhs):
         """copy the map data from another revmap. do not affect self.path"""
         self._rev2hsh = rhs._rev2hsh[:]
         self._rev2flag = rhs._rev2flag[:]
         self._hsh2rev = rhs._hsh2rev.copy()
         self._renamerevs = rhs._renamerevs[:]
         self._renamepaths = rhs._renamepaths[:]
         self._lastmaxrev = -1
 
     @property
     def maxrev(self):
         """return max linelog revision number"""
         return len(self._rev2hsh) - 1
 
     def append(self, hsh, sidebranch=False, path=None, flush=False):
         """add a binary hg hash and return the mapped linelog revision.
         if flush is True, incrementally update the file.
         """
         if hsh in self._hsh2rev:
             raise error.CorruptedFileError('%r is in revmap already' % hex(hsh))
         if len(hsh) != _hshlen:
             raise hgerror.ProgrammingError('hsh must be %d-char long' % _hshlen)
         idx = len(self._rev2hsh)
         flag = 0
         if sidebranch:
             flag |= sidebranchflag
         if path is not None and path != self._renamepaths[-1]:
             flag |= renameflag
             self._renamerevs.append(idx)
             self._renamepaths.append(path)
         self._rev2hsh.append(hsh)
         self._rev2flag.append(flag)
         self._hsh2rev[hsh] = idx
         if flush:
             self.flush()
         return idx
 
     def rev2hsh(self, rev):
         """convert linelog revision to hg hash. return None if not found."""
         if rev > self.maxrev or rev < 0:
             return None
         return self._rev2hsh[rev]
 
     def rev2flag(self, rev):
         """get the flag (uint8) for a given linelog revision.
         return None if revision does not exist.
         """
         if rev > self.maxrev or rev < 0:
             return None
         return self._rev2flag[rev]
 
     def rev2path(self, rev):
         """get the path for a given linelog revision.
         return None if revision does not exist.
         """
         if rev > self.maxrev or rev < 0:
             return None
         idx = bisect.bisect_right(self._renamerevs, rev) - 1
         return self._renamepaths[idx]
 
     def hsh2rev(self, hsh):
         """convert hg hash to linelog revision. return None if not found."""
         return self._hsh2rev.get(hsh)
 
     def clear(self, flush=False):
         """make the map empty. if flush is True, write to disk"""
         # rev 0 is reserved, real rev starts from 1
         self._rev2hsh = [None]
         self._rev2flag = [None]
         self._hsh2rev = {}
         self._rev2path = ['']
         self._lastmaxrev = -1
         if flush:
             self.flush()
 
     def flush(self):
         """write the state down to the file"""
         if not self.path:
             return
         if self._lastmaxrev == -1: # write the entire file
             with open(self.path, 'wb') as f:
                 f.write(self.HEADER)
                 for i in pycompat.xrange(1, len(self._rev2hsh)):
                     self._writerev(i, f)
         else: # append incrementally
             with open(self.path, 'ab') as f:
                 for i in pycompat.xrange(self._lastmaxrev + 1,
                                          len(self._rev2hsh)):
                     self._writerev(i, f)
         self._lastmaxrev = self.maxrev
 
     def _load(self):
         """load state from file"""
         if not self.path:
             return
         # use local variables in a loop. CPython uses LOAD_FAST for them,
         # which is faster than both LOAD_CONST and LOAD_GLOBAL.
         flaglen = 1
         hshlen = _hshlen
         with open(self.path, 'rb') as f:
             if f.read(len(self.HEADER)) != self.HEADER:
                 raise error.CorruptedFileError()
             self.clear(flush=False)
             while True:
                 buf = f.read(flaglen)
                 if not buf:
                     break
                 flag = ord(buf)
                 rev = len(self._rev2hsh)
                 if flag & renameflag:
                     path = self._readcstr(f)
                     self._renamerevs.append(rev)
                     self._renamepaths.append(path)
                 hsh = f.read(hshlen)
                 if len(hsh) != hshlen:
                     raise error.CorruptedFileError()
                 self._hsh2rev[hsh] = rev
                 self._rev2flag.append(flag)
                 self._rev2hsh.append(hsh)
         self._lastmaxrev = self.maxrev
 
     def _writerev(self, rev, f):
         """append a revision data to file"""
         flag = self._rev2flag[rev]
         hsh = self._rev2hsh[rev]
         f.write(struct.pack('B', flag))
         if flag & renameflag:
             path = self.rev2path(rev)
             if path is None:
                 raise error.CorruptedFileError('cannot find path for %s' % rev)
-            f.write(path + '\0')
+            f.write(path + b'\0')
         f.write(hsh)
 
     @staticmethod
     def _readcstr(f):
         """read a C-language-like '\0'-terminated string"""
         buf = ''
         while True:
             ch = f.read(1)
             if not ch: # unexpected eof
                 raise error.CorruptedFileError()
             if ch == '\0':
                 break
             buf += ch
         return buf
 
     def __contains__(self, f):
         """(fctx or (node, path)) -> bool.
         test if (node, path) is in the map, and is not in a side branch.
         f can be either a tuple of (node, path), or a fctx.
         """
         if isinstance(f, tuple): # f: (node, path)
             hsh, path = f
         else: # f: fctx
             hsh, path = f.node(), f.path()
         rev = self.hsh2rev(hsh)
         if rev is None:
             return False
         if path is not None and path != self.rev2path(rev):
             return False
         return (self.rev2flag(rev) & sidebranchflag) == 0
 
 def getlastnode(path):
     """return the last hash in a revmap, without loading its full content.
     this is equivalent to `m = revmap(path); m.rev2hsh(m.maxrev)`, but faster.
     """
     hsh = None
     try:
         with open(path, 'rb') as f:
             f.seek(-_hshlen, 2)
             if f.tell() > len(revmap.HEADER):
                 hsh = f.read(_hshlen)
     except IOError:
         pass
     return hsh

hgext/fix.py

@@ -1,617 +1,691 b''
 # fix - rewrite file content in changesets and working copy
 #
 # Copyright 2018 Google LLC.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """rewrite file content in changesets or working copy (EXPERIMENTAL)
 
 Provides a command that runs configured tools on the contents of modified files,
 writing back any fixes to the working copy or replacing changesets.
 
 Here is an example configuration that causes :hg:`fix` to apply automatic
 formatting fixes to modified lines in C++ code::
 
   [fix]
   clang-format:command=clang-format --assume-filename={rootpath}
   clang-format:linerange=--lines={first}:{last}
-  clang-format:fileset=set:**.cpp or **.hpp
+  clang-format:pattern=set:**.cpp or **.hpp
 
 The :command suboption forms the first part of the shell command that will be
 used to fix a file. The content of the file is passed on standard input, and the
-fixed file content is expected on standard output. If there is any output on
-standard error, the file will not be affected. Some values may be substituted
-into the command::
+fixed file content is expected on standard output. Any output on standard error
+will be displayed as a warning. If the exit status is not zero, the file will
+not be affected. A placeholder warning is displayed if there is a non-zero exit
+status but no standard error output. Some values may be substituted into the
+command::
 
   {rootpath}  The path of the file being fixed, relative to the repo root
   {basename}  The name of the file being fixed, without the directory path
 
 If the :linerange suboption is set, the tool will only be run if there are
 changed lines in a file. The value of this suboption is appended to the shell
 command once for every range of changed lines in the file. Some values may be
 substituted into the command::
 
   {first}   The 1-based line number of the first line in the modified range
   {last}    The 1-based line number of the last line in the modified range
 
-The :fileset suboption determines which files will be passed through each
-configured tool. See :hg:`help fileset` for possible values. If there are file
-arguments to :hg:`fix`, the intersection of these filesets is used.
+The :pattern suboption determines which files will be passed through each
+configured tool. See :hg:`help patterns` for possible values. If there are file
+arguments to :hg:`fix`, the intersection of these patterns is used.
 
 There is also a configurable limit for the maximum size of file that will be
 processed by :hg:`fix`::
 
   [fix]
-  maxfilesize=2MB
+  maxfilesize = 2MB
+
+Normally, execution of configured tools will continue after a failure (indicated
+by a non-zero exit status). It can also be configured to abort after the first
+such failure, so that no files will be affected if any tool fails. This abort
+will also cause :hg:`fix` to exit with a non-zero status::
+
+  [fix]
+  failure = abort
 
+When multiple tools are configured to affect a file, they execute in an order
+defined by the :priority suboption. The priority suboption has a default value
+of zero for each tool. Tools are executed in order of descending priority. The
+execution order of tools with equal priority is unspecified. For example, you
+could use the 'sort' and 'head' utilities to keep only the 10 smallest numbers
+in a text file by ensuring that 'sort' runs before 'head'::
+
+  [fix]
+  sort:command = sort -n
+  head:command = head -n 10
+  sort:pattern = numbers.txt
+  head:pattern = numbers.txt
+  sort:priority = 2
+  head:priority = 1
+
+To account for changes made by each tool, the line numbers used for incremental
+formatting are recomputed before executing the next tool. So, each tool may see
+different values for the arguments added by the :linerange suboption.
 """
 
 from __future__ import absolute_import
 
 import collections
 import itertools
 import os
 import re
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.node import nullrev
 from mercurial.node import wdirrev
 
 from mercurial.utils import (
     procutil,
 )
 
 from mercurial import (
     cmdutil,
     context,
     copies,
     error,
     mdiff,
     merge,
     obsolete,
     pycompat,
     registrar,
     scmutil,
     util,
     worker,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # Register the suboptions allowed for each configured fixer.
-FIXER_ATTRS = ('command', 'linerange', 'fileset')
+FIXER_ATTRS = {
+    'command': None,
+    'linerange': None,
+    'fileset': None,
+    'pattern': None,
+    'priority': 0,
+}
 
-for key in FIXER_ATTRS:
-    configitem('fix', '.*(:%s)?' % key, default=None, generic=True)
+for key, default in FIXER_ATTRS.items():
+    configitem('fix', '.*(:%s)?' % key, default=default, generic=True)
 
 # A good default size allows most source code files to be fixed, but avoids
 # letting fixer tools choke on huge inputs, which could be surprising to the
 # user.
 configitem('fix', 'maxfilesize', default='2MB')
 
+# Allow fix commands to exit non-zero if an executed fixer tool exits non-zero.
+# This helps users do shell scripts that stop when a fixer tool signals a
+# problem.
+configitem('fix', 'failure', default='continue')
+
+def checktoolfailureaction(ui, message, hint=None):
+    """Abort with 'message' if fix.failure=abort"""
+    action = ui.config('fix', 'failure')
+    if action not in ('continue', 'abort'):
+        raise error.Abort(_('unknown fix.failure action: %s') % (action,),
+                          hint=_('use "continue" or "abort"'))
+    if action == 'abort':
+        raise error.Abort(message, hint=hint)
+
 allopt = ('', 'all', False, _('fix all non-public non-obsolete revisions'))
 baseopt = ('', 'base', [], _('revisions to diff against (overrides automatic '
                              'selection, and applies to every revision being '
                              'fixed)'), _('REV'))
 revopt = ('r', 'rev', [], _('revisions to fix'), _('REV'))
 wdiropt = ('w', 'working-dir', False, _('fix the working directory'))
 wholeopt = ('', 'whole', False, _('always fix every line of a file'))
 usage = _('[OPTION]... [FILE]...')
 
 @command('fix', [allopt, baseopt, revopt, wdiropt, wholeopt], usage,
         helpcategory=command.CATEGORY_FILE_CONTENTS)
 def fix(ui, repo, *pats, **opts):
     """rewrite file content in changesets or working directory
 
     Runs any configured tools to fix the content of files. Only affects files
     with changes, unless file arguments are provided. Only affects changed lines
     of files, unless the --whole flag is used. Some tools may always affect the
     whole file regardless of --whole.
 
     If revisions are specified with --rev, those revisions will be checked, and
     they may be replaced with new revisions that have fixed file content.  It is
     desirable to specify all descendants of each specified revision, so that the
     fixes propagate to the descendants. If all descendants are fixed at the same
     time, no merging, rebasing, or evolution will be required.
 
     If --working-dir is used, files with uncommitted changes in the working copy
     will be fixed. If the checked-out revision is also fixed, the working
     directory will update to the replacement revision.
 
     When determining what lines of each file to fix at each revision, the whole
     set of revisions being fixed is considered, so that fixes to earlier
     revisions are not forgotten in later ones. The --base flag can be used to
     override this default behavior, though it is not usually desirable to do so.
     """
     opts = pycompat.byteskwargs(opts)
     if opts['all']:
         if opts['rev']:
             raise error.Abort(_('cannot specify both "--rev" and "--all"'))
         opts['rev'] = ['not public() and not obsolete()']
         opts['working_dir'] = True
     with repo.wlock(), repo.lock(), repo.transaction('fix'):
         revstofix = getrevstofix(ui, repo, opts)
         basectxs = getbasectxs(repo, opts, revstofix)
         workqueue, numitems = getworkqueue(ui, repo, pats, opts, revstofix,
                                            basectxs)
         fixers = getfixers(ui)
 
         # There are no data dependencies between the workers fixing each file
         # revision, so we can use all available parallelism.
         def getfixes(items):
             for rev, path in items:
                 ctx = repo[rev]
                 olddata = ctx[path].data()
                 newdata = fixfile(ui, opts, fixers, ctx, path, basectxs[rev])
                 # Don't waste memory/time passing unchanged content back, but
                 # produce one result per item either way.
                 yield (rev, path, newdata if newdata != olddata else None)
         results = worker.worker(ui, 1.0, getfixes, tuple(), workqueue,
                                 threadsafe=False)
 
         # We have to hold on to the data for each successor revision in memory
         # until all its parents are committed. We ensure this by committing and
         # freeing memory for the revisions in some topological order. This
         # leaves a little bit of memory efficiency on the table, but also makes
         # the tests deterministic. It might also be considered a feature since
         # it makes the results more easily reproducible.
         filedata = collections.defaultdict(dict)
         replacements = {}
         wdirwritten = False
         commitorder = sorted(revstofix, reverse=True)
         with ui.makeprogress(topic=_('fixing'), unit=_('files'),
                              total=sum(numitems.values())) as progress:
             for rev, path, newdata in results:
                 progress.increment(item=path)
                 if newdata is not None:
                     filedata[rev][path] = newdata
                 numitems[rev] -= 1
                 # Apply the fixes for this and any other revisions that are
                 # ready and sitting at the front of the queue. Using a loop here
                 # prevents the queue from being blocked by the first revision to
                 # be ready out of order.
                 while commitorder and not numitems[commitorder[-1]]:
                     rev = commitorder.pop()
                     ctx = repo[rev]
                     if rev == wdirrev:
                         writeworkingdir(repo, ctx, filedata[rev], replacements)
                         wdirwritten = bool(filedata[rev])
                     else:
                         replacerev(ui, repo, ctx, filedata[rev], replacements)
                     del filedata[rev]
 
         cleanup(repo, replacements, wdirwritten)
 
 def cleanup(repo, replacements, wdirwritten):
     """Calls scmutil.cleanupnodes() with the given replacements.
 
     "replacements" is a dict from nodeid to nodeid, with one key and one value
     for every revision that was affected by fixing. This is slightly different
     from cleanupnodes().
 
     "wdirwritten" is a bool which tells whether the working copy was affected by
     fixing, since it has no entry in "replacements".
 
     Useful as a hook point for extending "hg fix" with output summarizing the
     effects of the command, though we choose not to output anything here.
     """
     replacements = {prec: [succ] for prec, succ in replacements.iteritems()}
     scmutil.cleanupnodes(repo, replacements, 'fix', fixphase=True)
 
 def getworkqueue(ui, repo, pats, opts, revstofix, basectxs):
     """"Constructs the list of files to be fixed at specific revisions
 
     It is up to the caller how to consume the work items, and the only
     dependence between them is that replacement revisions must be committed in
     topological order. Each work item represents a file in the working copy or
     in some revision that should be fixed and written back to the working copy
     or into a replacement revision.
 
     Work items for the same revision are grouped together, so that a worker
     pool starting with the first N items in parallel is likely to finish the
     first revision's work before other revisions. This can allow us to write
     the result to disk and reduce memory footprint. At time of writing, the
     partition strategy in worker.py seems favorable to this. We also sort the
     items by ascending revision number to match the order in which we commit
     the fixes later.
     """
     workqueue = []
     numitems = collections.defaultdict(int)
     maxfilesize = ui.configbytes('fix', 'maxfilesize')
     for rev in sorted(revstofix):
         fixctx = repo[rev]
         match = scmutil.match(fixctx, pats, opts)
         for path in pathstofix(ui, repo, pats, opts, match, basectxs[rev],
                                fixctx):
             if path not in fixctx:
                 continue
             fctx = fixctx[path]
             if fctx.islink():
                 continue
             if fctx.size() > maxfilesize:
                 ui.warn(_('ignoring file larger than %s: %s\n') %
                         (util.bytecount(maxfilesize), path))
                 continue
             workqueue.append((rev, path))
             numitems[rev] += 1
     return workqueue, numitems
 
 def getrevstofix(ui, repo, opts):
     """Returns the set of revision numbers that should be fixed"""
     revs = set(scmutil.revrange(repo, opts['rev']))
     for rev in revs:
         checkfixablectx(ui, repo, repo[rev])
     if revs:
         cmdutil.checkunfinished(repo)
         checknodescendants(repo, revs)
     if opts.get('working_dir'):
         revs.add(wdirrev)
         if list(merge.mergestate.read(repo).unresolved()):
             raise error.Abort('unresolved conflicts', hint="use 'hg resolve'")
     if not revs:
         raise error.Abort(
             'no changesets specified', hint='use --rev or --working-dir')
     return revs
 
 def checknodescendants(repo, revs):
     if (not obsolete.isenabled(repo, obsolete.allowunstableopt) and
         repo.revs('(%ld::) - (%ld)', revs, revs)):
         raise error.Abort(_('can only fix a changeset together '
                             'with all its descendants'))
 
 def checkfixablectx(ui, repo, ctx):
     """Aborts if the revision shouldn't be replaced with a fixed one."""
     if not ctx.mutable():
         raise error.Abort('can\'t fix immutable changeset %s' %
                           (scmutil.formatchangeid(ctx),))
     if ctx.obsolete():
         # It would be better to actually check if the revision has a successor.
         allowdivergence = ui.configbool('experimental',
                                         'evolution.allowdivergence')
         if not allowdivergence:
             raise error.Abort('fixing obsolete revision could cause divergence')
 
 def pathstofix(ui, repo, pats, opts, match, basectxs, fixctx):
     """Returns the set of files that should be fixed in a context
 
     The result depends on the base contexts; we include any file that has
     changed relative to any of the base contexts. Base contexts should be
     ancestors of the context being fixed.
     """
     files = set()
     for basectx in basectxs:
         stat = basectx.status(fixctx, match=match, listclean=bool(pats),
                               listunknown=bool(pats))
         files.update(
             set(itertools.chain(stat.added, stat.modified, stat.clean,
                                 stat.unknown)))
     return files
 
 def lineranges(opts, path, basectxs, fixctx, content2):
     """Returns the set of line ranges that should be fixed in a file
 
     Of the form [(10, 20), (30, 40)].
 
     This depends on the given base contexts; we must consider lines that have
     changed versus any of the base contexts, and whether the file has been
     renamed versus any of them.
 
     Another way to understand this is that we exclude line ranges that are
     common to the file in all base contexts.
     """
     if opts.get('whole'):
         # Return a range containing all lines. Rely on the diff implementation's
         # idea of how many lines are in the file, instead of reimplementing it.
         return difflineranges('', content2)
 
     rangeslist = []
     for basectx in basectxs:
         basepath = copies.pathcopies(basectx, fixctx).get(path, path)
         if basepath in basectx:
             content1 = basectx[basepath].data()
         else:
             content1 = ''
         rangeslist.extend(difflineranges(content1, content2))
     return unionranges(rangeslist)
 
 def unionranges(rangeslist):
     """Return the union of some closed intervals
 
     >>> unionranges([])
     []
     >>> unionranges([(1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (2, 100)])
     [(1, 100)]
     >>> unionranges([(1, 99), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (40, 60)])
     [(1, 100)]
     >>> unionranges([(1, 49), (50, 100)])
     [(1, 100)]
     >>> unionranges([(1, 48), (50, 100)])
     [(1, 48), (50, 100)]
     >>> unionranges([(1, 2), (3, 4), (5, 6)])
     [(1, 6)]
     """
     rangeslist = sorted(set(rangeslist))
     unioned = []
     if rangeslist:
         unioned, rangeslist = [rangeslist[0]], rangeslist[1:]
     for a, b in rangeslist:
         c, d = unioned[-1]
         if a > d + 1:
             unioned.append((a, b))
         else:
             unioned[-1] = (c, max(b, d))
     return unioned
 
 def difflineranges(content1, content2):
     """Return list of line number ranges in content2 that differ from content1.
 
     Line numbers are 1-based. The numbers are the first and last line contained
     in the range. Single-line ranges have the same line number for the first and
     last line. Excludes any empty ranges that result from lines that are only
     present in content1. Relies on mdiff's idea of where the line endings are in
     the string.
 
     >>> from mercurial import pycompat
     >>> lines = lambda s: b'\\n'.join([c for c in pycompat.iterbytestr(s)])
     >>> difflineranges2 = lambda a, b: difflineranges(lines(a), lines(b))
     >>> difflineranges2(b'', b'')
     []
     >>> difflineranges2(b'a', b'')
     []
     >>> difflineranges2(b'', b'A')
     [(1, 1)]
     >>> difflineranges2(b'a', b'a')
     []
     >>> difflineranges2(b'a', b'A')
     [(1, 1)]
     >>> difflineranges2(b'ab', b'')
     []
     >>> difflineranges2(b'', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abc', b'ac')
     []
     >>> difflineranges2(b'ab', b'aCb')
     [(2, 2)]
     >>> difflineranges2(b'abc', b'aBc')
     [(2, 2)]
     >>> difflineranges2(b'ab', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abcde', b'aBcDe')
     [(2, 2), (4, 4)]
     >>> difflineranges2(b'abcde', b'aBCDe')
     [(2, 4)]
     """
     ranges = []
     for lines, kind in mdiff.allblocks(content1, content2):
         firstline, lastline = lines[2:4]
         if kind == '!' and firstline != lastline:
             ranges.append((firstline + 1, lastline))
     return ranges
 
 def getbasectxs(repo, opts, revstofix):
     """Returns a map of the base contexts for each revision
 
     The base contexts determine which lines are considered modified when we
     attempt to fix just the modified lines in a file. It also determines which
     files we attempt to fix, so it is important to compute this even when
     --whole is used.
     """
     # The --base flag overrides the usual logic, and we give every revision
     # exactly the set of baserevs that the user specified.
     if opts.get('base'):
         baserevs = set(scmutil.revrange(repo, opts.get('base')))
         if not baserevs:
             baserevs = {nullrev}
         basectxs = {repo[rev] for rev in baserevs}
         return {rev: basectxs for rev in revstofix}
 
     # Proceed in topological order so that we can easily determine each
     # revision's baserevs by looking at its parents and their baserevs.
     basectxs = collections.defaultdict(set)
     for rev in sorted(revstofix):
         ctx = repo[rev]
         for pctx in ctx.parents():
             if pctx.rev() in basectxs:
                 basectxs[rev].update(basectxs[pctx.rev()])
             else:
                 basectxs[rev].add(pctx)
     return basectxs
 
 def fixfile(ui, opts, fixers, fixctx, path, basectxs):
     """Run any configured fixers that should affect the file in this context
 
     Returns the file content that results from applying the fixers in some order
     starting with the file's content in the fixctx. Fixers that support line
     ranges will affect lines that have changed relative to any of the basectxs
     (i.e. they will only avoid lines that are common to all basectxs).
 
     A fixer tool's stdout will become the file's new content if and only if it
     exits with code zero.
     """
     newdata = fixctx[path].data()
     for fixername, fixer in fixers.iteritems():
         if fixer.affects(opts, fixctx, path):
             rangesfn = lambda: lineranges(opts, path, basectxs, fixctx, newdata)
             command = fixer.command(ui, path, rangesfn)
             if command is None:
                 continue
             ui.debug('subprocess: %s\n' % (command,))
             proc = subprocess.Popen(
                 procutil.tonativestr(command),
                 shell=True,
                 cwd=procutil.tonativestr(b'/'),
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)
             newerdata, stderr = proc.communicate(newdata)
             if stderr:
                 showstderr(ui, fixctx.rev(), fixername, stderr)
             if proc.returncode == 0:
                 newdata = newerdata
-            elif not stderr:
-                showstderr(ui, fixctx.rev(), fixername,
-                           _('exited with status %d\n') % (proc.returncode,))
+            else:
+                if not stderr:
+                    message = _('exited with status %d\n') % (proc.returncode,)
+                    showstderr(ui, fixctx.rev(), fixername, message)
+                checktoolfailureaction(
+                    ui, _('no fixes will be applied'),
+                    hint=_('use --config fix.failure=continue to apply any '
+                           'successful fixes anyway'))
     return newdata
 
 def showstderr(ui, rev, fixername, stderr):
     """Writes the lines of the stderr string as warnings on the ui
 
     Uses the revision number and fixername to give more context to each line of
     the error message. Doesn't include file names, since those take up a lot of
     space and would tend to be included in the error message if they were
     relevant.
     """
     for line in re.split('[\r\n]+', stderr):
         if line:
             ui.warn(('['))
             if rev is None:
                 ui.warn(_('wdir'), label='evolve.rev')
             else:
                 ui.warn((str(rev)), label='evolve.rev')
             ui.warn(('] %s: %s\n') % (fixername, line))
 
 def writeworkingdir(repo, ctx, filedata, replacements):
     """Write new content to the working copy and check out the new p1 if any
 
     We check out a new revision if and only if we fixed something in both the
     working directory and its parent revision. This avoids the need for a full
     update/merge, and means that the working directory simply isn't affected
     unless the --working-dir flag is given.
 
     Directly updates the dirstate for the affected files.
     """
     for path, data in filedata.iteritems():
         fctx = ctx[path]
         fctx.write(data, fctx.flags())
         if repo.dirstate[path] == 'n':
             repo.dirstate.normallookup(path)
 
     oldparentnodes = repo.dirstate.parents()
     newparentnodes = [replacements.get(n, n) for n in oldparentnodes]
     if newparentnodes != oldparentnodes:
         repo.setparents(*newparentnodes)
 
 def replacerev(ui, repo, ctx, filedata, replacements):
     """Commit a new revision like the given one, but with file content changes
 
     "ctx" is the original revision to be replaced by a modified one.
 
     "filedata" is a dict that maps paths to their new file content. All other
     paths will be recreated from the original revision without changes.
     "filedata" may contain paths that didn't exist in the original revision;
     they will be added.
 
     "replacements" is a dict that maps a single node to a single node, and it is
     updated to indicate the original revision is replaced by the newly created
     one. No entry is added if the replacement's node already exists.
 
     The new revision has the same parents as the old one, unless those parents
     have already been replaced, in which case those replacements are the parents
     of this new revision. Thus, if revisions are replaced in topological order,
     there is no need to rebase them into the original topology later.
     """
 
     p1rev, p2rev = repo.changelog.parentrevs(ctx.rev())
     p1ctx, p2ctx = repo[p1rev], repo[p2rev]
     newp1node = replacements.get(p1ctx.node(), p1ctx.node())
     newp2node = replacements.get(p2ctx.node(), p2ctx.node())
 
+    # We don't want to create a revision that has no changes from the original,
+    # but we should if the original revision's parent has been replaced.
+    # Otherwise, we would produce an orphan that needs no actual human
+    # intervention to evolve. We can't rely on commit() to avoid creating the
+    # un-needed revision because the extra field added below produces a new hash
+    # regardless of file content changes.
+    if (not filedata and
+        p1ctx.node() not in replacements and
+        p2ctx.node() not in replacements):
+        return
+
     def filectxfn(repo, memctx, path):
         if path not in ctx:
             return None
         fctx = ctx[path]
         copied = fctx.renamed()
         if copied:
             copied = copied[0]
         return context.memfilectx(
             repo,
             memctx,
             path=fctx.path(),
             data=filedata.get(path, fctx.data()),
             islink=fctx.islink(),
             isexec=fctx.isexec(),
             copied=copied)
 
+    extra = ctx.extra().copy()
+    extra['fix_source'] = ctx.hex()
+
     memctx = context.memctx(
         repo,
         parents=(newp1node, newp2node),
         text=ctx.description(),
         files=set(ctx.files()) | set(filedata.keys()),
         filectxfn=filectxfn,
         user=ctx.user(),
         date=ctx.date(),
-        extra=ctx.extra(),
+        extra=extra,
         branch=ctx.branch(),
         editor=None)
     sucnode = memctx.commit()
     prenode = ctx.node()
     if prenode == sucnode:
         ui.debug('node %s already existed\n' % (ctx.hex()))
     else:
         replacements[ctx.node()] = sucnode
 
 def getfixers(ui):
     """Returns a map of configured fixer tools indexed by their names
 
     Each value is a Fixer object with methods that implement the behavior of the
     fixer's config suboptions. Does not validate the config values.
     """
-    result = {}
+    fixers = {}
     for name in fixernames(ui):
-        result[name] = Fixer()
+        fixers[name] = Fixer()
         attrs = ui.configsuboptions('fix', name)[1]
-        for key in FIXER_ATTRS:
-            setattr(result[name], pycompat.sysstr('_' + key),
-                    attrs.get(key, ''))
-    return result
+        if 'fileset' in attrs and 'pattern' not in attrs:
+            ui.warn(_('the fix.tool:fileset config name is deprecated; '
+                      'please rename it to fix.tool:pattern\n'))
+            attrs['pattern'] = attrs['fileset']
+        for key, default in FIXER_ATTRS.items():
+            setattr(fixers[name], pycompat.sysstr('_' + key),
+                    attrs.get(key, default))
+        fixers[name]._priority = int(fixers[name]._priority)
+    return collections.OrderedDict(
+        sorted(fixers.items(), key=lambda item: item[1]._priority,
+               reverse=True))
 
 def fixernames(ui):
     """Returns the names of [fix] config options that have suboptions"""
     names = set()
     for k, v in ui.configitems('fix'):
         if ':' in k:
             names.add(k.split(':', 1)[0])
     return names
 
 class Fixer(object):
     """Wraps the raw config values for a fixer with methods"""
 
     def affects(self, opts, fixctx, path):
         """Should this fixer run on the file at the given path and context?"""
-        return scmutil.match(fixctx, [self._fileset], opts)(path)
+        return scmutil.match(fixctx, [self._pattern], opts)(path)
 
     def command(self, ui, path, rangesfn):
         """A shell command to use to invoke this fixer on the given file/lines
 
         May return None if there is no appropriate command to run for the given
         parameters.
         """
         expand = cmdutil.rendercommandtemplate
         parts = [expand(ui, self._command,
                         {'rootpath': path, 'basename': os.path.basename(path)})]
         if self._linerange:
             ranges = rangesfn()
             if not ranges:
                 # No line ranges to fix, so don't run the fixer.
                 return None
             for first, last in ranges:
                 parts.append(expand(ui, self._linerange,
                                     {'first': first, 'last': last}))
         return ' '.join(parts)

hgext/highlight/__init__.py

@@ -1,96 +1,96 b''
 # highlight - syntax highlighting in hgweb, based on Pygments
 #
 #  Copyright 2008, 2009 Patrick Mezard <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # The original module was split in an interface and an implementation
 # file to defer pygments loading and speedup extension setup.
 
 """syntax highlighting for hgweb (requires Pygments)
 
 It depends on the Pygments syntax highlighting library:
 http://pygments.org/
 
 There are the following configuration options::
 
   [web]
   pygments_style = <style> (default: colorful)
   highlightfiles = <fileset> (default: size('<5M'))
   highlightonlymatchfilename = <bool> (default False)
 
 ``highlightonlymatchfilename`` will only highlight files if their type could
 be identified by their filename. When this is not enabled (the default),
 Pygments will try very hard to identify the file type from content and any
 match (even matches with a low confidence score) will be used.
 """
 
 from __future__ import absolute_import
 
 from . import highlight
 from mercurial.hgweb import (
     webcommands,
     webutil,
 )
 
 from mercurial import (
     extensions,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 def pygmentize(web, field, fctx, tmpl):
     style = web.config('web', 'pygments_style', 'colorful')
     expr = web.config('web', 'highlightfiles', "size('<5M')")
     filenameonly = web.configbool('web', 'highlightonlymatchfilename', False)
 
     ctx = fctx.changectx()
     m = ctx.matchfileset(expr)
     if m(fctx.path()):
         highlight.pygmentize(field, fctx, style, tmpl,
                 guessfilenameonly=filenameonly)
 
 def filerevision_highlight(orig, web, fctx):
     mt = web.res.headers['Content-Type']
     # only pygmentize for mimetype containing 'html' so we both match
     # 'text/html' and possibly 'application/xhtml+xml' in the future
     # so that we don't have to touch the extension when the mimetype
     # for a template changes; also hgweb optimizes the case that a
     # raw file is sent using rawfile() and doesn't call us, so we
     # can't clash with the file's content-type here in case we
     # pygmentize a html file
     if 'html' in mt:
         pygmentize(web, 'fileline', fctx, web.tmpl)
 
     return orig(web, fctx)
 
 def annotate_highlight(orig, web):
     mt = web.res.headers['Content-Type']
     if 'html' in mt:
         fctx = webutil.filectx(web.repo, web.req)
         pygmentize(web, 'annotateline', fctx, web.tmpl)
 
     return orig(web)
 
 def generate_css(web):
     pg_style = web.config('web', 'pygments_style', 'colorful')
     fmter = highlight.HtmlFormatter(style=pg_style)
     web.res.headers['Content-Type'] = 'text/css'
     web.res.setbodybytes(''.join([
         '/* pygments_style = %s */\n\n' % pg_style,
         fmter.get_style_defs(''),
     ]))
     return web.res.sendresponse()
 
-def extsetup():
+def extsetup(ui):
     # monkeypatch in the new version
     extensions.wrapfunction(webcommands, '_filerevision',
                             filerevision_highlight)
     extensions.wrapfunction(webcommands, 'annotate', annotate_highlight)
     webcommands.highlightcss = generate_css
     webcommands.__all__.append('highlightcss')