merge default into stable for 4.9 release

contrib/discovery-helper.sh

@@ -0,0 +1,64 b''
+#!/bin/bash
+#
+# produces two repositories with different common and missing subsets
+#
+#   $ discovery-helper.sh REPO NBHEADS DEPT
+#
+# The Goal is to produce two repositories with some common part and some
+# exclusive part on each side. Provide a source repository REPO, it will
+# produce two repositories REPO-left and REPO-right.
+#
+# Each repository will be missing some revisions exclusive to NBHEADS of the
+# repo topological heads. These heads and revisions exclusive to them (up to
+# DEPTH depth) are stripped.
+#
+# The "left" repository will use the NBHEADS first heads (sorted by
+# description). The "right" use the last NBHEADS one.
+#
+# To find out how many topological heads a repo has, use:
+#
+#   $ hg heads -t -T '{rev}\n' | wc -l
+#
+# Example:
+#
+#  The `pypy-2018-09-01` repository has 192 heads. To produce two repositories
+#  with 92 common heads and ~50 exclusive heads on each side.
+#
+#    $ ./discovery-helper.sh pypy-2018-08-01 50 10
+
+set -euo pipefail
+
+if [ $# -lt 3 ]; then
+     echo "usage: `basename $0` REPO NBHEADS DEPTH"
+     exit 64
+fi
+
+repo="$1"
+shift
+
+nbheads="$1"
+shift
+
+depth="$1"
+shift
+
+leftrepo="${repo}-left"
+rightrepo="${repo}-right"
+
+left="first(sort(heads(all()), 'desc'), $nbheads)"
+right="last(sort(heads(all()), 'desc'), $nbheads)"
+
+leftsubset="ancestors($left, $depth) and only($left, heads(all() - $left))"
+rightsubset="ancestors($right, $depth) and only($right, heads(all() - $right))"
+
+echo '### building left repository:' $left-repo
+echo '# cloning'
+hg clone --noupdate "${repo}" "${leftrepo}"
+echo '# stripping' '"'${leftsubset}'"'
+hg -R "${leftrepo}" --config extensions.strip= strip --rev "$leftsubset" --no-backup
+
+echo '### building right repository:' $right-repo
+echo '# cloning'
+hg clone --noupdate "${repo}" "${rightrepo}"
+echo '# stripping:' '"'${rightsubset}'"'
+hg -R "${rightrepo}" --config extensions.strip= strip --rev "$rightsubset" --no-backup

contrib/fuzz/dirstate.cc

@@ -0,0 +1,48 b''
+#include <Python.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "pyutil.h"
+
+extern "C" {
+
+static PyCodeObject *code;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+	contrib::initpy(*argv[0]);
+	code = (PyCodeObject *)Py_CompileString(R"py(
+from parsers import parse_dirstate
+try:
+    dmap = {}
+    copymap = {}
+    p = parse_dirstate(dmap, copymap, data)
+except Exception as e:
+    pass
+    # uncomment this print if you're editing this Python code
+    # to debug failures.
+    # print e
+)py",
+	                                        "fuzzer", Py_file_input);
+	return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+	PyObject *text =
+	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
+	PyObject *locals = PyDict_New();
+	PyDict_SetItemString(locals, "data", text);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
+	if (!res) {
+		PyErr_Print();
+	}
+	Py_XDECREF(res);
+	Py_DECREF(locals);
+	Py_DECREF(text);
+	return 0; // Non-zero return values are reserved for future use.
+}
+}

contrib/fuzz/dirstate_corpus.py

@@ -0,0 +1,18 b''
+from __future__ import absolute_import, print_function
+
+import argparse
+import os
+import zipfile
+
+ap = argparse.ArgumentParser()
+ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
+args = ap.parse_args()
+
+reporoot = os.path.normpath(os.path.join(os.path.dirname(__file__),
+                                         '..', '..'))
+dirstate = os.path.join(reporoot, '.hg', 'dirstate')
+
+with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
+    if os.path.exists(dirstate):
+        with open(dirstate) as f:
+            zf.writestr("dirstate", f.read())

contrib/fuzz/fm1readmarkers.cc

@@ -0,0 +1,60 b''
+#include <Python.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "pyutil.h"
+
+extern "C" {
+
+static PyCodeObject *code;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+	contrib::initpy(*argv[0]);
+	code = (PyCodeObject *)Py_CompileString(R"py(
+from parsers import fm1readmarkers
+def maybeint(s, default):
+    try:
+        return int(s)
+    except ValueError:
+        return default
+try:
+    parts = data.split('\0', 2)
+    if len(parts) == 3:
+        offset, stop, data = parts
+    elif len(parts) == 2:
+        stop, data = parts
+        offset = 0
+    else:
+        offset = stop = 0
+    offset, stop = maybeint(offset, 0), maybeint(stop, len(data))
+    fm1readmarkers(data, offset, stop)
+except Exception as e:
+    pass
+    # uncomment this print if you're editing this Python code
+    # to debug failures.
+    # print e
+)py",
+	                                        "fuzzer", Py_file_input);
+	return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+	PyObject *text =
+	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
+	PyObject *locals = PyDict_New();
+	PyDict_SetItemString(locals, "data", text);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
+	if (!res) {
+		PyErr_Print();
+	}
+	Py_XDECREF(res);
+	Py_DECREF(locals);
+	Py_DECREF(text);
+	return 0; // Non-zero return values are reserved for future use.
+}
+}

contrib/fuzz/fm1readmarkers_corpus.py

@@ -0,0 +1,36 b''
+from __future__ import absolute_import, print_function
+
+import argparse
+import zipfile
+
+ap = argparse.ArgumentParser()
+ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
+args = ap.parse_args()
+
+with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
+    zf.writestr(
+        'smallish_obsstore',
+        (
+            # header: fm1readmarkers should start at offset 1, and
+            # read until byte 597.
+            '1\x00597\x00'
+            # body of obsstore file
+            '\x01\x00\x00\x00vA\xd7\x02+C\x1a<)\x01,\x00\x00\x01\x03\x03\xe6'
+            '\x92\xde)x\x16\xd1Xph\xc7\xa7[\xe5\xe2\x1a\xab\x1e6e\xaf\xc2\xae'
+            '\xe7\xbc\x83\xe1\x88\xa5\xda\xce>O\xbd\x04\xe9\x03\xc4o\xeb\x03'
+            '\x01\t\x05\x04\x1fef18operationamenduserAugie Fackler <raf@duri'
+            'n42.com>\x00\x00\x00vA\xd7\x02-\x8aD\xaf-\x01,\x00\x00\x01\x03\x03'
+            '\x17*\xca\x8f\x9e}i\xe0i\xbb\xdf\x9fb\x03\xd2XG?\xd3h\x98\x89\x1a'
+            '=2\xeb\xc3\xc5<\xb3\x9e\xcc\x0e;#\xee\xc3\x10ux\x03\x01\t\x05\x04'
+            '\x1fef18operationamenduserAugie Fackler <[e-mail]>\x00\x00'
+            '\x00vA\xd7\x02Mn\xd9%\xea\x01,\x00\x00\x01\x03\x03\x98\x89\x1a='
+            '2\xeb\xc3\xc5<\xb3\x9e\xcc\x0e;#\xee\xc3\x10ux\xe0*\xcaT\x86Z8J'
+            '\x85)\x97\xff7\xcc)\xc1\x7f\x19\x0c\x01\x03\x01\t\x05\x04\x1fef'
+            '18operationamenduserAugie Fackler <[e-mail]>\x00\x00\x00'
+            'yA\xd7\x02MtA\xbfj\x01,\x00\x00\x01\x03\x03\xe0*\xcaT\x86Z8J\x85'
+            ')\x97\xff7\xcc)\xc1\x7f\x19\x0c\x01\x00\x94\x01\xa9\n\xf80\x92\xa3'
+            'j\xc5X\xb1\xc9:\xd51\xb8*\xa9\x03\x01\t\x08\x04\x1fef11operatio'
+            'nhistedituserAugie Fackler <[e-mail]>\x00\x00\x00yA\xd7\x02'
+            'MtA\xd4\xe1\x01,\x00\x00\x01\x03\x03"\xa5\xcb\x86\xb6\xf4\xbaO\xa0'
+            'sH\xe7?\xcb\x9b\xc2n\xcfI\x9e\x14\xf0D\xf0!\x18DN\xcd\x97\x016\xa5'
+            '\xef\xa06\xcb\x884\x8a\x03\x01\t\x08\x04\x1fef14operationhisted'))

contrib/fuzz/pyutil.cc

@@ -0,0 +1,49 b''
+#include "pyutil.h"
+
+#include <string>
+
+namespace contrib
+{
+
+static char cpypath[8192] = "\0";
+
+static PyObject *mainmod;
+static PyObject *globals;
+
+/* TODO: use Python 3 for this fuzzing? */
+PyMODINIT_FUNC initparsers(void);
+
+void initpy(const char *cselfpath)
+{
+	const std::string subdir = "/sanpy/lib/python2.7";
+	/* HACK ALERT: we need a full Python installation built without
+	   pymalloc and with ASAN, so we dump one in
+	   $OUT/sanpy/lib/python2.7. This helps us wire that up. */
+	std::string selfpath(cselfpath);
+	std::string pypath;
+	auto pos = selfpath.rfind("/");
+	if (pos == std::string::npos) {
+		char wd[8192];
+		getcwd(wd, 8192);
+		pypath = std::string(wd) + subdir;
+	} else {
+		pypath = selfpath.substr(0, pos) + subdir;
+	}
+	strncpy(cpypath, pypath.c_str(), pypath.size());
+	setenv("PYTHONPATH", cpypath, 1);
+	setenv("PYTHONNOUSERSITE", "1", 1);
+	/* prevent Python from looking up users in the fuzz environment */
+	setenv("PYTHONUSERBASE", cpypath, 1);
+	Py_SetPythonHome(cpypath);
+	Py_InitializeEx(0);
+	mainmod = PyImport_AddModule("__main__");
+	globals = PyModule_GetDict(mainmod);
+	initparsers();
+}
+
+PyObject *pyglobals()
+{
+	return globals;
+}
+
+} // namespace contrib

contrib/fuzz/pyutil.h

@@ -0,0 +1,9 b''
+#include <Python.h>
+
+namespace contrib
+{
+
+void initpy(const char *cselfpath);
+PyObject *pyglobals();
+
+} /* namespace contrib */

contrib/fuzz/revlog.cc

@@ -0,0 +1,47 b''
+#include <Python.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "pyutil.h"
+
+extern "C" {
+
+static PyCodeObject *code;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+	contrib::initpy(*argv[0]);
+	code = (PyCodeObject *)Py_CompileString(R"py(
+from parsers import parse_index2
+for inline in (True, False):
+    try:
+        index, cache = parse_index2(data, inline)
+    except Exception as e:
+        pass
+        # uncomment this print if you're editing this Python code
+        # to debug failures.
+        # print e
+)py",
+	                                        "fuzzer", Py_file_input);
+	return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+	PyObject *text =
+	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
+	PyObject *locals = PyDict_New();
+	PyDict_SetItemString(locals, "data", text);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
+	if (!res) {
+		PyErr_Print();
+	}
+	Py_XDECREF(res);
+	Py_DECREF(locals);
+	Py_DECREF(text);
+	return 0; // Non-zero return values are reserved for future use.
+}
+}

contrib/fuzz/revlog_corpus.py

@@ -0,0 +1,28 b''
+from __future__ import absolute_import, print_function
+
+import argparse
+import os
+import zipfile
+
+ap = argparse.ArgumentParser()
+ap.add_argument("out", metavar="some.zip", type=str, nargs=1)
+args = ap.parse_args()
+
+reporoot = os.path.normpath(os.path.join(os.path.dirname(__file__),
+                                         '..', '..'))
+# typically a standalone index
+changelog = os.path.join(reporoot, '.hg', 'store', '00changelog.i')
+# an inline revlog with only a few revisions
+contributing = os.path.join(
+    reporoot, '.hg', 'store', 'data', 'contrib', 'fuzz', 'mpatch.cc.i')
+
+print(changelog, os.path.exists(changelog))
+print(contributing, os.path.exists(contributing))
+
+with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
+    if os.path.exists(changelog):
+        with open(changelog) as f:
+            zf.writestr("00changelog.i", f.read())
+    if os.path.exists(contributing):
+        with open(contributing) as f:
+            zf.writestr("contributing.i", f.read())

contrib/perf-utils/perf-revlog-write-plot.py

@@ -0,0 +1,127 b''
+#!/usr/bin/env python
+#
+#  Copyright 2018 Paul Morelle <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+#
+# This script use the output of `hg perfrevlogwrite -T json --details` to draw
+# various plot related to write performance in a revlog
+#
+# usage: perf-revlog-write-plot.py details.json
+from __future__ import absolute_import, print_function
+import json
+import re
+
+import numpy as np
+import scipy.signal
+
+from matplotlib import (
+    pyplot as plt,
+    ticker as mticker,
+)
+
+
+def plot(data, title=None):
+    items = {}
+    re_title = re.compile(r'^revisions #\d+ of \d+, rev (\d+)$')
+    for item in data:
+        m = re_title.match(item['title'])
+        if m is None:
+            continue
+
+        rev = int(m.group(1))
+        items[rev] = item
+
+    min_rev = min(items.keys())
+    max_rev = max(items.keys())
+    ary = np.empty((2, max_rev - min_rev + 1))
+    for rev, item in items.items():
+        ary[0][rev - min_rev] = rev
+        ary[1][rev - min_rev] = item['wall']
+
+    fig = plt.figure()
+    comb_plt = fig.add_subplot(211)
+    other_plt = fig.add_subplot(212)
+
+    comb_plt.plot(ary[0],
+                  np.cumsum(ary[1]),
+                  color='red',
+                  linewidth=1,
+                  label='comb')
+
+    plots = []
+    p = other_plt.plot(ary[0],
+                       ary[1],
+                       color='red',
+                       linewidth=1,
+                       label='wall')
+    plots.append(p)
+
+    colors = {
+        10: ('green', 'xkcd:grass green'),
+        100: ('blue', 'xkcd:bright blue'),
+        1000: ('purple', 'xkcd:dark pink'),
+    }
+    for n, color in colors.items():
+        avg_n = np.convolve(ary[1], np.full(n, 1. / n), 'valid')
+        p = other_plt.plot(ary[0][n - 1:],
+                           avg_n,
+                           color=color[0],
+                           linewidth=1,
+                           label='avg time last %d' % n)
+        plots.append(p)
+
+        med_n = scipy.signal.medfilt(ary[1], n + 1)
+        p = other_plt.plot(ary[0],
+                           med_n,
+                           color=color[1],
+                           linewidth=1,
+                           label='median time last %d' % n)
+        plots.append(p)
+
+    formatter = mticker.ScalarFormatter()
+    formatter.set_scientific(False)
+    formatter.set_useOffset(False)
+
+    comb_plt.grid()
+    comb_plt.xaxis.set_major_formatter(formatter)
+    comb_plt.legend()
+
+    other_plt.grid()
+    other_plt.xaxis.set_major_formatter(formatter)
+    leg = other_plt.legend()
+    leg2plot = {}
+    for legline, plot in zip(leg.get_lines(), plots):
+        legline.set_picker(5)
+        leg2plot[legline] = plot
+
+    def onpick(event):
+        legline = event.artist
+        plot = leg2plot[legline]
+        visible = not plot[0].get_visible()
+        for l in plot:
+            l.set_visible(visible)
+
+        if visible:
+            legline.set_alpha(1.0)
+        else:
+            legline.set_alpha(0.2)
+        fig.canvas.draw()
+    if title is not None:
+        fig.canvas.set_window_title(title)
+    fig.canvas.mpl_connect('pick_event', onpick)
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    import sys
+
+    if len(sys.argv) > 1:
+        print('reading from %r' % sys.argv[1])
+        with open(sys.argv[1], 'r') as fp:
+            plot(json.load(fp), title=sys.argv[1])
+    else:
+        print('reading from stdin')
+        plot(json.load(sys.stdin))

hgext/bookflow.py

@@ -0,0 +1,104 b''
+"""implements bookmark-based branching (EXPERIMENTAL)
+
+ - Disables creation of new branches (config: enable_branches=False).
+ - Requires an active bookmark on commit (config: require_bookmark=True).
+ - Doesn't move the active bookmark on update, only on commit.
+ - Requires '--rev' for moving an existing bookmark.
+ - Protects special bookmarks (config: protect=@).
+
+ flow related commands
+
+    :hg book NAME: create a new bookmark
+    :hg book NAME -r REV: move bookmark to revision (fast-forward)
+    :hg up|co NAME: switch to bookmark
+    :hg push -B .: push active bookmark
+"""
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+from mercurial import (
+    bookmarks,
+    commands,
+    error,
+    extensions,
+    registrar,
+)
+
+MY_NAME = 'bookflow'
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem(MY_NAME, 'protect', ['@'])
+configitem(MY_NAME, 'require-bookmark', True)
+configitem(MY_NAME, 'enable-branches', False)
+
+cmdtable = {}
+command = registrar.command(cmdtable)
+
+def commit_hook(ui, repo, **kwargs):
+    active = repo._bookmarks.active
+    if active:
+        if active in ui.configlist(MY_NAME, 'protect'):
+            raise error.Abort(
+                _('cannot commit, bookmark %s is protected') % active)
+        if not cwd_at_bookmark(repo, active):
+            raise error.Abort(
+       _('cannot commit, working directory out of sync with active bookmark'),
+                hint=_("run 'hg up %s'") % active)
+    elif ui.configbool(MY_NAME, 'require-bookmark', True):
+        raise error.Abort(_('cannot commit without an active bookmark'))
+    return 0
+
+def bookmarks_update(orig, repo, parents, node):
+    if len(parents) == 2:
+        # called during commit
+        return orig(repo, parents, node)
+    else:
+        # called during update
+        return False
+
+def bookmarks_addbookmarks(
+        orig, repo, tr, names, rev=None, force=False, inactive=False):
+    if not rev:
+        marks = repo._bookmarks
+        for name in names:
+            if name in marks:
+                raise error.Abort(_(
+                    "bookmark %s already exists, to move use the --rev option"
+                    ) % name)
+    return orig(repo, tr, names, rev, force, inactive)
+
+def commands_commit(orig, ui, repo, *args, **opts):
+    commit_hook(ui, repo)
+    return orig(ui, repo, *args, **opts)
+
+def commands_pull(orig, ui, repo, *args, **opts):
+    rc = orig(ui, repo, *args, **opts)
+    active = repo._bookmarks.active
+    if active and not cwd_at_bookmark(repo, active):
+        ui.warn(_(
+            "working directory out of sync with active bookmark, run "
+            "'hg up %s'"
+        ) % active)
+    return rc
+
+def commands_branch(orig, ui, repo, label=None, **opts):
+    if label and not opts.get(r'clean') and not opts.get(r'rev'):
+        raise error.Abort(
+         _("creating named branches is disabled and you should use bookmarks"),
+            hint="see 'hg help bookflow'")
+    return orig(ui, repo, label, **opts)
+
+def cwd_at_bookmark(repo, mark):
+    mark_id = repo._bookmarks[mark]
+    cur_id = repo.lookup('.')
+    return cur_id == mark_id
+
+def uisetup(ui):
+    extensions.wrapfunction(bookmarks, 'update', bookmarks_update)
+    extensions.wrapfunction(bookmarks, 'addbookmarks', bookmarks_addbookmarks)
+    extensions.wrapcommand(commands.table, 'commit', commands_commit)
+    extensions.wrapcommand(commands.table, 'pull', commands_pull)
+    if not ui.configbool(MY_NAME, 'enable-branches'):
+        extensions.wrapcommand(commands.table, 'branch', commands_branch)

hgext/remotefilelog/README.md

@@ -0,0 +1,109 b''
+remotefilelog
+=============
+
+The remotefilelog extension allows Mercurial to clone shallow copies of a repository such that all file contents are left on the server and only downloaded on demand by the client. This greatly speeds up clone and pull performance for repositories that have long histories or that are growing quickly.
+
+In addition, the extension allows using a caching layer (such as memcache) to serve the file contents, thus providing better scalability and reducing server load.
+
+Installing
+==========
+
+**NOTE:** See the limitations section below to check if remotefilelog will work for your use case.
+
+remotefilelog can be installed like any other Mercurial extension. Download the source code and add the remotefilelog subdirectory to your `hgrc`:
+
+    :::ini
+    [extensions]
+    remotefilelog=path/to/remotefilelog/remotefilelog
+
+Configuring
+-----------
+
+**Server**
+
+* `server` (required) - Set to 'True' to indicate that the server can serve shallow clones.
+* `serverexpiration` - The server keeps a local cache of recently requested file revision blobs in .hg/remotefilelogcache. This setting specifies how many days they should be kept locally.  Defaults to 30.
+
+An example server configuration:
+
+    :::ini
+    [remotefilelog]
+    server = True
+    serverexpiration = 14
+
+**Client**
+
+* `cachepath` (required) - the location to store locally cached file revisions
+* `cachelimit` - the maximum size of the cachepath. By default it's 1000 GB.
+* `cachegroup` - the default unix group for the cachepath. Useful on shared systems so multiple users can read and write to the same cache.
+* `cacheprocess` - the external process that will handle the remote caching layer. If not set, all requests will go to the Mercurial server.
+* `fallbackpath` - the Mercurial repo path to fetch file revisions from. By default it uses the paths.default repo. This setting is useful for cloning from shallow clones and still talking to the central server for file revisions.
+* `includepattern` - a list of regex patterns matching files that should be kept remotely. Defaults to all files.
+* `excludepattern` - a list of regex patterns matching files that should not be kept remotely and should always be downloaded.
+* `pullprefetch` - a revset of commits whose file content should be prefetched after every pull. The most common value for this will be '(bookmark() + head()) & public()'. This is useful in environments where offline work is common, since it will enable offline updating to, rebasing to, and committing on every head and bookmark.
+
+An example client configuration:
+
+    :::ini
+    [remotefilelog]
+    cachepath = /dev/shm/hgcache
+    cachelimit = 2 GB
+
+Using as a largefiles replacement
+---------------------------------
+
+remotefilelog can theoretically be used as a replacement for the largefiles extension. You can use the `includepattern` setting to specify which directories or file types are considered large and they will be left on the server. Unlike the largefiles extension, this can be done without converting the server repository. Only the client configuration needs to specify the patterns.
+
+The include/exclude settings haven't been extensively tested, so this feature is still considered experimental.
+
+An example largefiles style client configuration:
+
+    :::ini
+    [remotefilelog]
+    cachepath = /dev/shm/hgcache
+    cachelimit = 2 GB
+    includepattern = *.sql3
+      bin/*
+
+Usage
+=====
+
+Once you have configured the server, you can get a shallow clone by doing:
+
+    :::bash
+    hg clone --shallow ssh://server//path/repo
+
+After that, all normal mercurial commands should work.
+
+Occasionly the client or server caches may grow too big. Run `hg gc` to clean up the cache. It will remove cached files that appear to no longer be necessary, or any files that exceed the configured maximum size. This does not improve performance; it just frees up space.
+
+Limitations
+===========
+
+1. The extension must be used with Mercurial 3.3 (commit d7d08337b3f6) or higher (earlier versions of the extension work with earlier versions of Mercurial though, up to Mercurial 2.7).
+
+2. remotefilelog has only been tested on linux with case-sensitive filesystems. It should work on other unix systems but may have problems on case-insensitive filesystems.
+
+3. remotefilelog only works with ssh based Mercurial repos. http based repos are currently not supported, though it shouldn't be too difficult for some motivated individual to implement.
+
+4. Tags are not supported in completely shallow repos. If you use tags in your repo you will have to specify `excludepattern=.hgtags` in your client configuration to ensure that file is downloaded. The include/excludepattern settings are experimental at the moment and have yet to be deployed in a production environment.
+
+5. A few commands will be slower. `hg log <filename>` will be much slower since it has to walk the entire commit history instead of just the filelog. Use `hg log -f <filename>` instead, which remains very fast.
+
+Contributing
+============
+
+Patches are welcome as pull requests, though they will be collapsed and rebased to maintain a linear history.  Tests can be run via:
+
+    :::bash
+    cd tests
+    ./run-tests --with-hg=path/to/hgrepo/hg
+
+We (Facebook) have to ask for a "Contributor License Agreement" from someone who sends in a patch or code that we want to include in the codebase. This is a legal requirement; a similar situation applies to Apache and other ASF projects.
+
+If we ask you to fill out a CLA we'll direct you to our [online CLA page](https://developers.facebook.com/opensource/cla) where you can complete it easily. We use the same form as the Apache CLA so that friction is minimal.
+
+License
+=======
+
+remotefilelog is made available under the terms of the GNU General Public License version 2, or any later version. See the COPYING file that accompanies this distribution for the full text of the license.

hgext/remotefilelog/__init__.py

@@ -0,0 +1,1143 b''
+# __init__.py - remotefilelog extension
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+"""remotefilelog causes Mercurial to lazilly fetch file contents (EXPERIMENTAL)
+
+This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY
+GUARANTEES. This means that repositories created with this extension may
+only be usable with the exact version of this extension/Mercurial that was
+used. The extension attempts to enforce this in order to prevent repository
+corruption.
+
+remotefilelog works by fetching file contents lazily and storing them
+in a cache on the client rather than in revlogs. This allows enormous
+histories to be transferred only partially, making them easier to
+operate on.
+
+Configs:
+
+    ``packs.maxchainlen`` specifies the maximum delta chain length in pack files
+
+    ``packs.maxpacksize`` specifies the maximum pack file size
+
+    ``packs.maxpackfilecount`` specifies the maximum number of packs in the
+      shared cache (trees only for now)
+
+    ``remotefilelog.backgroundprefetch`` runs prefetch in background when True
+
+    ``remotefilelog.bgprefetchrevs`` specifies revisions to fetch on commit and
+      update, and on other commands that use them. Different from pullprefetch.
+
+    ``remotefilelog.gcrepack`` does garbage collection during repack when True
+
+    ``remotefilelog.nodettl`` specifies maximum TTL of a node in seconds before
+      it is garbage collected
+
+    ``remotefilelog.repackonhggc`` runs repack on hg gc when True
+
+    ``remotefilelog.prefetchdays`` specifies the maximum age of a commit in
+      days after which it is no longer prefetched.
+
+    ``remotefilelog.prefetchdelay`` specifies delay between background
+      prefetches in seconds after operations that change the working copy parent
+
+    ``remotefilelog.data.gencountlimit`` constraints the minimum number of data
+      pack files required to be considered part of a generation. In particular,
+      minimum number of packs files > gencountlimit.
+
+    ``remotefilelog.data.generations`` list for specifying the lower bound of
+      each generation of the data pack files. For example, list ['100MB','1MB']
+      or ['1MB', '100MB'] will lead to three generations: [0, 1MB), [
+      1MB, 100MB) and [100MB, infinity).
+
+    ``remotefilelog.data.maxrepackpacks`` the maximum number of pack files to
+      include in an incremental data repack.
+
+    ``remotefilelog.data.repackmaxpacksize`` the maximum size of a pack file for
+      it to be considered for an incremental data repack.
+
+    ``remotefilelog.data.repacksizelimit`` the maximum total size of pack files
+      to include in an incremental data repack.
+
+    ``remotefilelog.history.gencountlimit`` constraints the minimum number of
+      history pack files required to be considered part of a generation. In
+      particular, minimum number of packs files > gencountlimit.
+
+    ``remotefilelog.history.generations`` list for specifying the lower bound of
+      each generation of the history pack files. For example, list [
+      '100MB', '1MB'] or ['1MB', '100MB'] will lead to three generations: [
+      0, 1MB), [1MB, 100MB) and [100MB, infinity).
+
+    ``remotefilelog.history.maxrepackpacks`` the maximum number of pack files to
+      include in an incremental history repack.
+
+    ``remotefilelog.history.repackmaxpacksize`` the maximum size of a pack file
+      for it to be considered for an incremental history repack.
+
+    ``remotefilelog.history.repacksizelimit`` the maximum total size of pack
+      files to include in an incremental history repack.
+
+    ``remotefilelog.backgroundrepack`` automatically consolidate packs in the
+      background
+
+    ``remotefilelog.cachepath`` path to cache
+
+    ``remotefilelog.cachegroup`` if set, make cache directory sgid to this
+      group
+
+    ``remotefilelog.cacheprocess`` binary to invoke for fetching file data
+
+    ``remotefilelog.debug`` turn on remotefilelog-specific debug output
+
+    ``remotefilelog.excludepattern`` pattern of files to exclude from pulls
+
+    ``remotefilelog.includepattern`` pattern of files to include in pulls
+
+    ``remotefilelog.fetchwarning``: message to print when too many
+      single-file fetches occur
+
+    ``remotefilelog.getfilesstep`` number of files to request in a single RPC
+
+    ``remotefilelog.getfilestype`` if set to 'threaded' use threads to fetch
+      files, otherwise use optimistic fetching
+
+    ``remotefilelog.pullprefetch`` revset for selecting files that should be
+      eagerly downloaded rather than lazily
+
+    ``remotefilelog.reponame`` name of the repo. If set, used to partition
+      data from other repos in a shared store.
+
+    ``remotefilelog.server`` if true, enable server-side functionality
+
+    ``remotefilelog.servercachepath`` path for caching blobs on the server
+
+    ``remotefilelog.serverexpiration`` number of days to keep cached server
+      blobs
+
+    ``remotefilelog.validatecache`` if set, check cache entries for corruption
+      before returning blobs
+
+    ``remotefilelog.validatecachelog`` if set, check cache entries for
+      corruption before returning metadata
+
+"""
+from __future__ import absolute_import
+
+import os
+import time
+import traceback
+
+from mercurial.node import hex
+from mercurial.i18n import _
+from mercurial import (
+    changegroup,
+    changelog,
+    cmdutil,
+    commands,
+    configitems,
+    context,
+    copies,
+    debugcommands as hgdebugcommands,
+    dispatch,
+    error,
+    exchange,
+    extensions,
+    hg,
+    localrepo,
+    match,
+    merge,
+    node as nodemod,
+    patch,
+    pycompat,
+    registrar,
+    repair,
+    repoview,
+    revset,
+    scmutil,
+    smartset,
+    streamclone,
+    templatekw,
+    util,
+)
+from . import (
+    constants,
+    debugcommands,
+    fileserverclient,
+    remotefilectx,
+    remotefilelog,
+    remotefilelogserver,
+    repack as repackmod,
+    shallowbundle,
+    shallowrepo,
+    shallowstore,
+    shallowutil,
+    shallowverifier,
+)
+
+# ensures debug commands are registered
+hgdebugcommands.command
+
+cmdtable = {}
+command = registrar.command(cmdtable)
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem('remotefilelog', 'debug', default=False)
+
+configitem('remotefilelog', 'reponame', default='')
+configitem('remotefilelog', 'cachepath', default=None)
+configitem('remotefilelog', 'cachegroup', default=None)
+configitem('remotefilelog', 'cacheprocess', default=None)
+configitem('remotefilelog', 'cacheprocess.includepath', default=None)
+configitem("remotefilelog", "cachelimit", default="1000 GB")
+
+configitem('remotefilelog', 'fallbackpath', default=configitems.dynamicdefault,
+           alias=[('remotefilelog', 'fallbackrepo')])
+
+configitem('remotefilelog', 'validatecachelog', default=None)
+configitem('remotefilelog', 'validatecache', default='on')
+configitem('remotefilelog', 'server', default=None)
+configitem('remotefilelog', 'servercachepath', default=None)
+configitem("remotefilelog", "serverexpiration", default=30)
+configitem('remotefilelog', 'backgroundrepack', default=False)
+configitem('remotefilelog', 'bgprefetchrevs', default=None)
+configitem('remotefilelog', 'pullprefetch', default=None)
+configitem('remotefilelog', 'backgroundprefetch', default=False)
+configitem('remotefilelog', 'prefetchdelay', default=120)
+configitem('remotefilelog', 'prefetchdays', default=14)
+
+configitem('remotefilelog', 'getfilesstep', default=10000)
+configitem('remotefilelog', 'getfilestype', default='optimistic')
+configitem('remotefilelog', 'batchsize', configitems.dynamicdefault)
+configitem('remotefilelog', 'fetchwarning', default='')
+
+configitem('remotefilelog', 'includepattern', default=None)
+configitem('remotefilelog', 'excludepattern', default=None)
+
+configitem('remotefilelog', 'gcrepack', default=False)
+configitem('remotefilelog', 'repackonhggc', default=False)
+configitem('repack', 'chainorphansbysize', default=True)
+
+configitem('packs', 'maxpacksize', default=0)
+configitem('packs', 'maxchainlen', default=1000)
+
+#  default TTL limit is 30 days
+_defaultlimit = 60 * 60 * 24 * 30
+configitem('remotefilelog', 'nodettl', default=_defaultlimit)
+
+configitem('remotefilelog', 'data.gencountlimit', default=2),
+configitem('remotefilelog', 'data.generations',
+           default=['1GB', '100MB', '1MB'])
+configitem('remotefilelog', 'data.maxrepackpacks', default=50)
+configitem('remotefilelog', 'data.repackmaxpacksize', default='4GB')
+configitem('remotefilelog', 'data.repacksizelimit', default='100MB')
+
+configitem('remotefilelog', 'history.gencountlimit', default=2),
+configitem('remotefilelog', 'history.generations', default=['100MB'])
+configitem('remotefilelog', 'history.maxrepackpacks', default=50)
+configitem('remotefilelog', 'history.repackmaxpacksize', default='400MB')
+configitem('remotefilelog', 'history.repacksizelimit', default='100MB')
+
+# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
+# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
+# be specifying the version(s) of Mercurial they are tested with, or
+# leave the attribute unspecified.
+testedwith = 'ships-with-hg-core'
+
+repoclass = localrepo.localrepository
+repoclass._basesupported.add(constants.SHALLOWREPO_REQUIREMENT)
+
+isenabled = shallowutil.isenabled
+
+def uisetup(ui):
+    """Wraps user facing Mercurial commands to swap them out with shallow
+    versions.
+    """
+    hg.wirepeersetupfuncs.append(fileserverclient.peersetup)
+
+    entry = extensions.wrapcommand(commands.table, 'clone', cloneshallow)
+    entry[1].append(('', 'shallow', None,
+                     _("create a shallow clone which uses remote file "
+                       "history")))
+
+    extensions.wrapcommand(commands.table, 'debugindex',
+        debugcommands.debugindex)
+    extensions.wrapcommand(commands.table, 'debugindexdot',
+        debugcommands.debugindexdot)
+    extensions.wrapcommand(commands.table, 'log', log)
+    extensions.wrapcommand(commands.table, 'pull', pull)
+
+    # Prevent 'hg manifest --all'
+    def _manifest(orig, ui, repo, *args, **opts):
+        if (isenabled(repo) and opts.get(r'all')):
+            raise error.Abort(_("--all is not supported in a shallow repo"))
+
+        return orig(ui, repo, *args, **opts)
+    extensions.wrapcommand(commands.table, "manifest", _manifest)
+
+    # Wrap remotefilelog with lfs code
+    def _lfsloaded(loaded=False):
+        lfsmod = None
+        try:
+            lfsmod = extensions.find('lfs')
+        except KeyError:
+            pass
+        if lfsmod:
+            lfsmod.wrapfilelog(remotefilelog.remotefilelog)
+            fileserverclient._lfsmod = lfsmod
+    extensions.afterloaded('lfs', _lfsloaded)
+
+    # debugdata needs remotefilelog.len to work
+    extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow)
+
+def cloneshallow(orig, ui, repo, *args, **opts):
+    if opts.get(r'shallow'):
+        repos = []
+        def pull_shallow(orig, self, *args, **kwargs):
+            if not isenabled(self):
+                repos.append(self.unfiltered())
+                # set up the client hooks so the post-clone update works
+                setupclient(self.ui, self.unfiltered())
+
+                # setupclient fixed the class on the repo itself
+                # but we also need to fix it on the repoview
+                if isinstance(self, repoview.repoview):
+                    self.__class__.__bases__ = (self.__class__.__bases__[0],
+                                                self.unfiltered().__class__)
+                self.requirements.add(constants.SHALLOWREPO_REQUIREMENT)
+                self._writerequirements()
+
+                # Since setupclient hadn't been called, exchange.pull was not
+                # wrapped. So we need to manually invoke our version of it.
+                return exchangepull(orig, self, *args, **kwargs)
+            else:
+                return orig(self, *args, **kwargs)
+        extensions.wrapfunction(exchange, 'pull', pull_shallow)
+
+        # Wrap the stream logic to add requirements and to pass include/exclude
+        # patterns around.
+        def setup_streamout(repo, remote):
+            # Replace remote.stream_out with a version that sends file
+            # patterns.
+            def stream_out_shallow(orig):
+                caps = remote.capabilities()
+                if constants.NETWORK_CAP_LEGACY_SSH_GETFILES in caps:
+                    opts = {}
+                    if repo.includepattern:
+                        opts[r'includepattern'] = '\0'.join(repo.includepattern)
+                    if repo.excludepattern:
+                        opts[r'excludepattern'] = '\0'.join(repo.excludepattern)
+                    return remote._callstream('stream_out_shallow', **opts)
+                else:
+                    return orig()
+            extensions.wrapfunction(remote, 'stream_out', stream_out_shallow)
+        def stream_wrap(orig, op):
+            setup_streamout(op.repo, op.remote)
+            return orig(op)
+        extensions.wrapfunction(
+            streamclone, 'maybeperformlegacystreamclone', stream_wrap)
+
+        def canperformstreamclone(orig, pullop, bundle2=False):
+            # remotefilelog is currently incompatible with the
+            # bundle2 flavor of streamclones, so force us to use
+            # v1 instead.
+            if 'v2' in pullop.remotebundle2caps.get('stream', []):
+                pullop.remotebundle2caps['stream'] = [
+                    c for c in pullop.remotebundle2caps['stream']
+                    if c != 'v2']
+            if bundle2:
+                return False, None
+            supported, requirements = orig(pullop, bundle2=bundle2)
+            if requirements is not None:
+                requirements.add(constants.SHALLOWREPO_REQUIREMENT)
+            return supported, requirements
+        extensions.wrapfunction(
+            streamclone, 'canperformstreamclone', canperformstreamclone)
+
+    try:
+        orig(ui, repo, *args, **opts)
+    finally:
+        if opts.get(r'shallow'):
+            for r in repos:
+                if util.safehasattr(r, 'fileservice'):
+                    r.fileservice.close()
+
+def debugdatashallow(orig, *args, **kwds):
+    oldlen = remotefilelog.remotefilelog.__len__
+    try:
+        remotefilelog.remotefilelog.__len__ = lambda x: 1
+        return orig(*args, **kwds)
+    finally:
+        remotefilelog.remotefilelog.__len__ = oldlen
+
+def reposetup(ui, repo):
+    if not repo.local():
+        return
+
+    # put here intentionally bc doesnt work in uisetup
+    ui.setconfig('hooks', 'update.prefetch', wcpprefetch)
+    ui.setconfig('hooks', 'commit.prefetch', wcpprefetch)
+
+    isserverenabled = ui.configbool('remotefilelog', 'server')
+    isshallowclient = isenabled(repo)
+
+    if isserverenabled and isshallowclient:
+        raise RuntimeError("Cannot be both a server and shallow client.")
+
+    if isshallowclient:
+        setupclient(ui, repo)
+
+    if isserverenabled:
+        remotefilelogserver.setupserver(ui, repo)
+
+def setupclient(ui, repo):
+    if not isinstance(repo, localrepo.localrepository):
+        return
+
+    # Even clients get the server setup since they need to have the
+    # wireprotocol endpoints registered.
+    remotefilelogserver.onetimesetup(ui)
+    onetimeclientsetup(ui)
+
+    shallowrepo.wraprepo(repo)
+    repo.store = shallowstore.wrapstore(repo.store)
+
+clientonetime = False
+def onetimeclientsetup(ui):
+    global clientonetime
+    if clientonetime:
+        return
+    clientonetime = True
+
+    changegroup.cgpacker = shallowbundle.shallowcg1packer
+
+    extensions.wrapfunction(changegroup, '_addchangegroupfiles',
+                            shallowbundle.addchangegroupfiles)
+    extensions.wrapfunction(
+        changegroup, 'makechangegroup', shallowbundle.makechangegroup)
+
+    def storewrapper(orig, requirements, path, vfstype):
+        s = orig(requirements, path, vfstype)
+        if constants.SHALLOWREPO_REQUIREMENT in requirements:
+            s = shallowstore.wrapstore(s)
+
+        return s
+    extensions.wrapfunction(localrepo, 'makestore', storewrapper)
+
+    extensions.wrapfunction(exchange, 'pull', exchangepull)
+
+    # prefetch files before update
+    def applyupdates(orig, repo, actions, wctx, mctx, overwrite, labels=None):
+        if isenabled(repo):
+            manifest = mctx.manifest()
+            files = []
+            for f, args, msg in actions['g']:
+                files.append((f, hex(manifest[f])))
+            # batch fetch the needed files from the server
+            repo.fileservice.prefetch(files)
+        return orig(repo, actions, wctx, mctx, overwrite, labels=labels)
+    extensions.wrapfunction(merge, 'applyupdates', applyupdates)
+
+    # Prefetch merge checkunknownfiles
+    def checkunknownfiles(orig, repo, wctx, mctx, force, actions,
+                          *args, **kwargs):
+        if isenabled(repo):
+            files = []
+            sparsematch = repo.maybesparsematch(mctx.rev())
+            for f, (m, actionargs, msg) in actions.iteritems():
+                if sparsematch and not sparsematch(f):
+                    continue
+                if m in ('c', 'dc', 'cm'):
+                    files.append((f, hex(mctx.filenode(f))))
+                elif m == 'dg':
+                    f2 = actionargs[0]
+                    files.append((f2, hex(mctx.filenode(f2))))
+            # batch fetch the needed files from the server
+            repo.fileservice.prefetch(files)
+        return orig(repo, wctx, mctx, force, actions, *args, **kwargs)
+    extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles)
+
+    # Prefetch files before status attempts to look at their size and contents
+    def checklookup(orig, self, files):
+        repo = self._repo
+        if isenabled(repo):
+            prefetchfiles = []
+            for parent in self._parents:
+                for f in files:
+                    if f in parent:
+                        prefetchfiles.append((f, hex(parent.filenode(f))))
+            # batch fetch the needed files from the server
+            repo.fileservice.prefetch(prefetchfiles)
+        return orig(self, files)
+    extensions.wrapfunction(context.workingctx, '_checklookup', checklookup)
+
+    # Prefetch the logic that compares added and removed files for renames
+    def findrenames(orig, repo, matcher, added, removed, *args, **kwargs):
+        if isenabled(repo):
+            files = []
+            parentctx = repo['.']
+            for f in removed:
+                files.append((f, hex(parentctx.filenode(f))))
+            # batch fetch the needed files from the server
+            repo.fileservice.prefetch(files)
+        return orig(repo, matcher, added, removed, *args, **kwargs)
+    extensions.wrapfunction(scmutil, '_findrenames', findrenames)
+
+    # prefetch files before mergecopies check
+    def computenonoverlap(orig, repo, c1, c2, *args, **kwargs):
+        u1, u2 = orig(repo, c1, c2, *args, **kwargs)
+        if isenabled(repo):
+            m1 = c1.manifest()
+            m2 = c2.manifest()
+            files = []
+
+            sparsematch1 = repo.maybesparsematch(c1.rev())
+            if sparsematch1:
+                sparseu1 = []
+                for f in u1:
+                    if sparsematch1(f):
+                        files.append((f, hex(m1[f])))
+                        sparseu1.append(f)
+                u1 = sparseu1
+
+            sparsematch2 = repo.maybesparsematch(c2.rev())
+            if sparsematch2:
+                sparseu2 = []
+                for f in u2:
+                    if sparsematch2(f):
+                        files.append((f, hex(m2[f])))
+                        sparseu2.append(f)
+                u2 = sparseu2
+
+            # batch fetch the needed files from the server
+            repo.fileservice.prefetch(files)
+        return u1, u2
+    extensions.wrapfunction(copies, '_computenonoverlap', computenonoverlap)
+
+    # prefetch files before pathcopies check
+    def computeforwardmissing(orig, a, b, match=None):
+        missing = list(orig(a, b, match=match))
+        repo = a._repo
+        if isenabled(repo):
+            mb = b.manifest()
+
+            files = []
+            sparsematch = repo.maybesparsematch(b.rev())
+            if sparsematch:
+                sparsemissing = []
+                for f in missing:
+                    if sparsematch(f):
+                        files.append((f, hex(mb[f])))
+                        sparsemissing.append(f)
+                missing = sparsemissing
+
+            # batch fetch the needed files from the server
+            repo.fileservice.prefetch(files)
+        return missing
+    extensions.wrapfunction(copies, '_computeforwardmissing',
+                            computeforwardmissing)
+
+    # close cache miss server connection after the command has finished
+    def runcommand(orig, lui, repo, *args, **kwargs):
+        fileservice = None
+        # repo can be None when running in chg:
+        # - at startup, reposetup was called because serve is not norepo
+        # - a norepo command like "help" is called
+        if repo and isenabled(repo):
+            fileservice = repo.fileservice
+        try:
+            return orig(lui, repo, *args, **kwargs)
+        finally:
+            if fileservice:
+                fileservice.close()
+    extensions.wrapfunction(dispatch, 'runcommand', runcommand)
+
+    # disappointing hacks below
+    templatekw.getrenamedfn = getrenamedfn
+    extensions.wrapfunction(revset, 'filelog', filelogrevset)
+    revset.symbols['filelog'] = revset.filelog
+    extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs)
+
+    # prevent strip from stripping remotefilelogs
+    def _collectbrokencsets(orig, repo, files, striprev):
+        if isenabled(repo):
+            files = list([f for f in files if not repo.shallowmatch(f)])
+        return orig(repo, files, striprev)
+    extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets)
+
+    # Don't commit filelogs until we know the commit hash, since the hash
+    # is present in the filelog blob.
+    # This violates Mercurial's filelog->manifest->changelog write order,
+    # but is generally fine for client repos.
+    pendingfilecommits = []
+    def addrawrevision(orig, self, rawtext, transaction, link, p1, p2, node,
+                       flags, cachedelta=None, _metatuple=None):
+        if isinstance(link, int):
+            pendingfilecommits.append(
+                (self, rawtext, transaction, link, p1, p2, node, flags,
+                 cachedelta, _metatuple))
+            return node
+        else:
+            return orig(self, rawtext, transaction, link, p1, p2, node, flags,
+                        cachedelta, _metatuple=_metatuple)
+    extensions.wrapfunction(
+        remotefilelog.remotefilelog, 'addrawrevision', addrawrevision)
+
+    def changelogadd(orig, self, *args):
+        oldlen = len(self)
+        node = orig(self, *args)
+        newlen = len(self)
+        if oldlen != newlen:
+            for oldargs in pendingfilecommits:
+                log, rt, tr, link, p1, p2, n, fl, c, m = oldargs
+                linknode = self.node(link)
+                if linknode == node:
+                    log.addrawrevision(rt, tr, linknode, p1, p2, n, fl, c, m)
+                else:
+                    raise error.ProgrammingError(
+                        'pending multiple integer revisions are not supported')
+        else:
+            # "link" is actually wrong here (it is set to len(changelog))
+            # if changelog remains unchanged, skip writing file revisions
+            # but still do a sanity check about pending multiple revisions
+            if len(set(x[3] for x in pendingfilecommits)) > 1:
+                raise error.ProgrammingError(
+                    'pending multiple integer revisions are not supported')
+        del pendingfilecommits[:]
+        return node
+    extensions.wrapfunction(changelog.changelog, 'add', changelogadd)
+
+    # changectx wrappers
+    def filectx(orig, self, path, fileid=None, filelog=None):
+        if fileid is None:
+            fileid = self.filenode(path)
+        if (isenabled(self._repo) and self._repo.shallowmatch(path)):
+            return remotefilectx.remotefilectx(self._repo, path,
+                fileid=fileid, changectx=self, filelog=filelog)
+        return orig(self, path, fileid=fileid, filelog=filelog)
+    extensions.wrapfunction(context.changectx, 'filectx', filectx)
+
+    def workingfilectx(orig, self, path, filelog=None):
+        if (isenabled(self._repo) and self._repo.shallowmatch(path)):
+            return remotefilectx.remoteworkingfilectx(self._repo,
+                path, workingctx=self, filelog=filelog)
+        return orig(self, path, filelog=filelog)
+    extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx)
+
+    # prefetch required revisions before a diff
+    def trydiff(orig, repo, revs, ctx1, ctx2, modified, added, removed,
+                copy, getfilectx, *args, **kwargs):
+        if isenabled(repo):
+            prefetch = []
+            mf1 = ctx1.manifest()
+            for fname in modified + added + removed:
+                if fname in mf1:
+                    fnode = getfilectx(fname, ctx1).filenode()
+                    # fnode can be None if it's a edited working ctx file
+                    if fnode:
+                        prefetch.append((fname, hex(fnode)))
+                if fname not in removed:
+                    fnode = getfilectx(fname, ctx2).filenode()
+                    if fnode:
+                        prefetch.append((fname, hex(fnode)))
+
+            repo.fileservice.prefetch(prefetch)
+
+        return orig(repo, revs, ctx1, ctx2, modified, added, removed,
+            copy, getfilectx, *args, **kwargs)
+    extensions.wrapfunction(patch, 'trydiff', trydiff)
+
+    # Prevent verify from processing files
+    # a stub for mercurial.hg.verify()
+    def _verify(orig, repo):
+        lock = repo.lock()
+        try:
+            return shallowverifier.shallowverifier(repo).verify()
+        finally:
+            lock.release()
+
+    extensions.wrapfunction(hg, 'verify', _verify)
+
+    scmutil.fileprefetchhooks.add('remotefilelog', _fileprefetchhook)
+
+def getrenamedfn(repo, endrev=None):
+    rcache = {}
+
+    def getrenamed(fn, rev):
+        '''looks up all renames for a file (up to endrev) the first
+        time the file is given. It indexes on the changerev and only
+        parses the manifest if linkrev != changerev.
+        Returns rename info for fn at changerev rev.'''
+        if rev in rcache.setdefault(fn, {}):
+            return rcache[fn][rev]
+
+        try:
+            fctx = repo[rev].filectx(fn)
+            for ancestor in fctx.ancestors():
+                if ancestor.path() == fn:
+                    renamed = ancestor.renamed()
+                    rcache[fn][ancestor.rev()] = renamed and renamed[0]
+
+            renamed = fctx.renamed()
+            return renamed and renamed[0]
+        except error.LookupError:
+            return None
+
+    return getrenamed
+
+def walkfilerevs(orig, repo, match, follow, revs, fncache):
+    if not isenabled(repo):
+        return orig(repo, match, follow, revs, fncache)
+
+    # remotefilelog's can't be walked in rev order, so throw.
+    # The caller will see the exception and walk the commit tree instead.
+    if not follow:
+        raise cmdutil.FileWalkError("Cannot walk via filelog")
+
+    wanted = set()
+    minrev, maxrev = min(revs), max(revs)
+
+    pctx = repo['.']
+    for filename in match.files():
+        if filename not in pctx:
+            raise error.Abort(_('cannot follow file not in parent '
+                               'revision: "%s"') % filename)
+        fctx = pctx[filename]
+
+        linkrev = fctx.linkrev()
+        if linkrev >= minrev and linkrev <= maxrev:
+            fncache.setdefault(linkrev, []).append(filename)
+            wanted.add(linkrev)
+
+        for ancestor in fctx.ancestors():
+            linkrev = ancestor.linkrev()
+            if linkrev >= minrev and linkrev <= maxrev:
+                fncache.setdefault(linkrev, []).append(ancestor.path())
+                wanted.add(linkrev)
+
+    return wanted
+
+def filelogrevset(orig, repo, subset, x):
+    """``filelog(pattern)``
+    Changesets connected to the specified filelog.
+
+    For performance reasons, ``filelog()`` does not show every changeset
+    that affects the requested file(s). See :hg:`help log` for details. For
+    a slower, more accurate result, use ``file()``.
+    """
+
+    if not isenabled(repo):
+        return orig(repo, subset, x)
+
+    # i18n: "filelog" is a keyword
+    pat = revset.getstring(x, _("filelog requires a pattern"))
+    m = match.match(repo.root, repo.getcwd(), [pat], default='relpath',
+                       ctx=repo[None])
+    s = set()
+
+    if not match.patkind(pat):
+        # slow
+        for r in subset:
+            ctx = repo[r]
+            cfiles = ctx.files()
+            for f in m.files():
+                if f in cfiles:
+                    s.add(ctx.rev())
+                    break
+    else:
+        # partial
+        files = (f for f in repo[None] if m(f))
+        for f in files:
+            fctx = repo[None].filectx(f)
+            s.add(fctx.linkrev())
+            for actx in fctx.ancestors():
+                s.add(actx.linkrev())
+
+    return smartset.baseset([r for r in subset if r in s])
+
+@command('gc', [], _('hg gc [REPO...]'), norepo=True)
+def gc(ui, *args, **opts):
+    '''garbage collect the client and server filelog caches
+    '''
+    cachepaths = set()
+
+    # get the system client cache
+    systemcache = shallowutil.getcachepath(ui, allowempty=True)
+    if systemcache:
+        cachepaths.add(systemcache)
+
+    # get repo client and server cache
+    repopaths = []
+    pwd = ui.environ.get('PWD')
+    if pwd:
+        repopaths.append(pwd)
+
+    repopaths.extend(args)
+    repos = []
+    for repopath in repopaths:
+        try:
+            repo = hg.peer(ui, {}, repopath)
+            repos.append(repo)
+
+            repocache = shallowutil.getcachepath(repo.ui, allowempty=True)
+            if repocache:
+                cachepaths.add(repocache)
+        except error.RepoError:
+            pass
+
+    # gc client cache
+    for cachepath in cachepaths:
+        gcclient(ui, cachepath)
+
+    # gc server cache
+    for repo in repos:
+        remotefilelogserver.gcserver(ui, repo._repo)
+
+def gcclient(ui, cachepath):
+    # get list of repos that use this cache
+    repospath = os.path.join(cachepath, 'repos')
+    if not os.path.exists(repospath):
+        ui.warn(_("no known cache at %s\n") % cachepath)
+        return
+
+    reposfile = open(repospath, 'rb')
+    repos = set([r[:-1] for r in reposfile.readlines()])
+    reposfile.close()
+
+    # build list of useful files
+    validrepos = []
+    keepkeys = set()
+
+    sharedcache = None
+    filesrepacked = False
+
+    count = 0
+    progress = ui.makeprogress(_("analyzing repositories"), unit="repos",
+                               total=len(repos))
+    for path in repos:
+        progress.update(count)
+        count += 1
+        try:
+            path = ui.expandpath(os.path.normpath(path))
+        except TypeError as e:
+            ui.warn(_("warning: malformed path: %r:%s\n") % (path, e))
+            traceback.print_exc()
+            continue
+        try:
+            peer = hg.peer(ui, {}, path)
+            repo = peer._repo
+        except error.RepoError:
+            continue
+
+        validrepos.append(path)
+
+        # Protect against any repo or config changes that have happened since
+        # this repo was added to the repos file. We'd rather this loop succeed
+        # and too much be deleted, than the loop fail and nothing gets deleted.
+        if not isenabled(repo):
+            continue
+
+        if not util.safehasattr(repo, 'name'):
+            ui.warn(_("repo %s is a misconfigured remotefilelog repo\n") % path)
+            continue
+
+        # If garbage collection on repack and repack on hg gc are enabled
+        # then loose files are repacked and garbage collected.
+        # Otherwise regular garbage collection is performed.
+        repackonhggc = repo.ui.configbool('remotefilelog', 'repackonhggc')
+        gcrepack = repo.ui.configbool('remotefilelog', 'gcrepack')
+        if repackonhggc and gcrepack:
+            try:
+                repackmod.incrementalrepack(repo)
+                filesrepacked = True
+                continue
+            except (IOError, repackmod.RepackAlreadyRunning):
+                # If repack cannot be performed due to not enough disk space
+                # continue doing garbage collection of loose files w/o repack
+                pass
+
+        reponame = repo.name
+        if not sharedcache:
+            sharedcache = repo.sharedstore
+
+        # Compute a keepset which is not garbage collected
+        def keyfn(fname, fnode):
+            return fileserverclient.getcachekey(reponame, fname, hex(fnode))
+        keepkeys = repackmod.keepset(repo, keyfn=keyfn, lastkeepkeys=keepkeys)
+
+    progress.complete()
+
+    # write list of valid repos back
+    oldumask = os.umask(0o002)
+    try:
+        reposfile = open(repospath, 'wb')
+        reposfile.writelines([("%s\n" % r) for r in validrepos])
+        reposfile.close()
+    finally:
+        os.umask(oldumask)
+
+    # prune cache
+    if sharedcache is not None:
+        sharedcache.gc(keepkeys)
+    elif not filesrepacked:
+        ui.warn(_("warning: no valid repos in repofile\n"))
+
+def log(orig, ui, repo, *pats, **opts):
+    if not isenabled(repo):
+        return orig(ui, repo, *pats, **opts)
+
+    follow = opts.get(r'follow')
+    revs = opts.get(r'rev')
+    if pats:
+        # Force slowpath for non-follow patterns and follows that start from
+        # non-working-copy-parent revs.
+        if not follow or revs:
+            # This forces the slowpath
+            opts[r'removed'] = True
+
+        # If this is a non-follow log without any revs specified, recommend that
+        # the user add -f to speed it up.
+        if not follow and not revs:
+            match, pats = scmutil.matchandpats(repo['.'], pats,
+                                               pycompat.byteskwargs(opts))
+            isfile = not match.anypats()
+            if isfile:
+                for file in match.files():
+                    if not os.path.isfile(repo.wjoin(file)):
+                        isfile = False
+                        break
+
+            if isfile:
+                ui.warn(_("warning: file log can be slow on large repos - " +
+                          "use -f to speed it up\n"))
+
+    return orig(ui, repo, *pats, **opts)
+
+def revdatelimit(ui, revset):
+    """Update revset so that only changesets no older than 'prefetchdays' days
+    are included. The default value is set to 14 days. If 'prefetchdays' is set
+    to zero or negative value then date restriction is not applied.
+    """
+    days = ui.configint('remotefilelog', 'prefetchdays')
+    if days > 0:
+        revset = '(%s) & date(-%s)' % (revset, days)
+    return revset
+
+def readytofetch(repo):
+    """Check that enough time has passed since the last background prefetch.
+    This only relates to prefetches after operations that change the working
+    copy parent. Default delay between background prefetches is 2 minutes.
+    """
+    timeout = repo.ui.configint('remotefilelog', 'prefetchdelay')
+    fname = repo.vfs.join('lastprefetch')
+
+    ready = False
+    with open(fname, 'a'):
+        # the with construct above is used to avoid race conditions
+        modtime = os.path.getmtime(fname)
+        if (time.time() - modtime) > timeout:
+            os.utime(fname, None)
+            ready = True
+
+    return ready
+
+def wcpprefetch(ui, repo, **kwargs):
+    """Prefetches in background revisions specified by bgprefetchrevs revset.
+    Does background repack if backgroundrepack flag is set in config.
+    """
+    shallow = isenabled(repo)
+    bgprefetchrevs = ui.config('remotefilelog', 'bgprefetchrevs')
+    isready = readytofetch(repo)
+
+    if not (shallow and bgprefetchrevs and isready):
+        return
+
+    bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
+    # update a revset with a date limit
+    bgprefetchrevs = revdatelimit(ui, bgprefetchrevs)
+
+    def anon():
+        if util.safehasattr(repo, 'ranprefetch') and repo.ranprefetch:
+            return
+        repo.ranprefetch = True
+        repo.backgroundprefetch(bgprefetchrevs, repack=bgrepack)
+
+    repo._afterlock(anon)
+
+def pull(orig, ui, repo, *pats, **opts):
+    result = orig(ui, repo, *pats, **opts)
+
+    if isenabled(repo):
+        # prefetch if it's configured
+        prefetchrevset = ui.config('remotefilelog', 'pullprefetch')
+        bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
+        bgprefetch = repo.ui.configbool('remotefilelog', 'backgroundprefetch')
+
+        if prefetchrevset:
+            ui.status(_("prefetching file contents\n"))
+            revs = scmutil.revrange(repo, [prefetchrevset])
+            base = repo['.'].rev()
+            if bgprefetch:
+                repo.backgroundprefetch(prefetchrevset, repack=bgrepack)
+            else:
+                repo.prefetch(revs, base=base)
+                if bgrepack:
+                    repackmod.backgroundrepack(repo, incremental=True)
+        elif bgrepack:
+            repackmod.backgroundrepack(repo, incremental=True)
+
+    return result
+
+def exchangepull(orig, repo, remote, *args, **kwargs):
+    # Hook into the callstream/getbundle to insert bundle capabilities
+    # during a pull.
+    def localgetbundle(orig, source, heads=None, common=None, bundlecaps=None,
+                       **kwargs):
+        if not bundlecaps:
+            bundlecaps = set()
+        bundlecaps.add(constants.BUNDLE2_CAPABLITY)
+        return orig(source, heads=heads, common=common, bundlecaps=bundlecaps,
+                    **kwargs)
+
+    if util.safehasattr(remote, '_callstream'):
+        remote._localrepo = repo
+    elif util.safehasattr(remote, 'getbundle'):
+        extensions.wrapfunction(remote, 'getbundle', localgetbundle)
+
+    return orig(repo, remote, *args, **kwargs)
+
+def _fileprefetchhook(repo, revs, match):
+    if isenabled(repo):
+        allfiles = []
+        for rev in revs:
+            if rev == nodemod.wdirrev or rev is None:
+                continue
+            ctx = repo[rev]
+            mf = ctx.manifest()
+            sparsematch = repo.maybesparsematch(ctx.rev())
+            for path in ctx.walk(match):
+                if path.endswith('/'):
+                    # Tree manifest that's being excluded as part of narrow
+                    continue
+                if (not sparsematch or sparsematch(path)) and path in mf:
+                    allfiles.append((path, hex(mf[path])))
+        repo.fileservice.prefetch(allfiles)
+
+@command('debugremotefilelog', [
+    ('d', 'decompress', None, _('decompress the filelog first')),
+    ], _('hg debugremotefilelog <path>'), norepo=True)
+def debugremotefilelog(ui, path, **opts):
+    return debugcommands.debugremotefilelog(ui, path, **opts)
+
+@command('verifyremotefilelog', [
+    ('d', 'decompress', None, _('decompress the filelogs first')),
+    ], _('hg verifyremotefilelogs <directory>'), norepo=True)
+def verifyremotefilelog(ui, path, **opts):
+    return debugcommands.verifyremotefilelog(ui, path, **opts)
+
+@command('debugdatapack', [
+    ('', 'long', None, _('print the long hashes')),
+    ('', 'node', '', _('dump the contents of node'), 'NODE'),
+    ], _('hg debugdatapack <paths>'), norepo=True)
+def debugdatapack(ui, *paths, **opts):
+    return debugcommands.debugdatapack(ui, *paths, **opts)
+
+@command('debughistorypack', [
+    ], _('hg debughistorypack <path>'), norepo=True)
+def debughistorypack(ui, path, **opts):
+    return debugcommands.debughistorypack(ui, path)
+
+@command('debugkeepset', [
+    ], _('hg debugkeepset'))
+def debugkeepset(ui, repo, **opts):
+    # The command is used to measure keepset computation time
+    def keyfn(fname, fnode):
+        return fileserverclient.getcachekey(repo.name, fname, hex(fnode))
+    repackmod.keepset(repo, keyfn)
+    return
+
+@command('debugwaitonrepack', [
+    ], _('hg debugwaitonrepack'))
+def debugwaitonrepack(ui, repo, **opts):
+    return debugcommands.debugwaitonrepack(repo)
+
+@command('debugwaitonprefetch', [
+    ], _('hg debugwaitonprefetch'))
+def debugwaitonprefetch(ui, repo, **opts):
+    return debugcommands.debugwaitonprefetch(repo)
+
+def resolveprefetchopts(ui, opts):
+    if not opts.get('rev'):
+        revset = ['.', 'draft()']
+
+        prefetchrevset = ui.config('remotefilelog', 'pullprefetch', None)
+        if prefetchrevset:
+            revset.append('(%s)' % prefetchrevset)
+        bgprefetchrevs = ui.config('remotefilelog', 'bgprefetchrevs', None)
+        if bgprefetchrevs:
+            revset.append('(%s)' % bgprefetchrevs)
+        revset = '+'.join(revset)
+
+        # update a revset with a date limit
+        revset = revdatelimit(ui, revset)
+
+        opts['rev'] = [revset]
+
+    if not opts.get('base'):
+        opts['base'] = None
+
+    return opts
+
+@command('prefetch', [
+    ('r', 'rev', [], _('prefetch the specified revisions'), _('REV')),
+    ('', 'repack', False, _('run repack after prefetch')),
+    ('b', 'base', '', _("rev that is assumed to already be local")),
+    ] + commands.walkopts, _('hg prefetch [OPTIONS] [FILE...]'))
+def prefetch(ui, repo, *pats, **opts):
+    """prefetch file revisions from the server
+
+    Prefetchs file revisions for the specified revs and stores them in the
+    local remotefilelog cache.  If no rev is specified, the default rev is
+    used which is the union of dot, draft, pullprefetch and bgprefetchrev.
+    File names or patterns can be used to limit which files are downloaded.
+
+    Return 0 on success.
+    """
+    opts = pycompat.byteskwargs(opts)
+    if not isenabled(repo):
+        raise error.Abort(_("repo is not shallow"))
+
+    opts = resolveprefetchopts(ui, opts)
+    revs = scmutil.revrange(repo, opts.get('rev'))
+    repo.prefetch(revs, opts.get('base'), pats, opts)
+
+    # Run repack in background
+    if opts.get('repack'):
+        repackmod.backgroundrepack(repo, incremental=True)
+
+@command('repack', [
+     ('', 'background', None, _('run in a background process'), None),
+     ('', 'incremental', None, _('do an incremental repack'), None),
+     ('', 'packsonly', None, _('only repack packs (skip loose objects)'), None),
+    ], _('hg repack [OPTIONS]'))
+def repack_(ui, repo, *pats, **opts):
+    if opts.get(r'background'):
+        repackmod.backgroundrepack(repo, incremental=opts.get(r'incremental'),
+                                   packsonly=opts.get(r'packsonly', False))
+        return
+
+    options = {'packsonly': opts.get(r'packsonly')}
+
+    try:
+        if opts.get(r'incremental'):
+            repackmod.incrementalrepack(repo, options=options)
+        else:
+            repackmod.fullrepack(repo, options=options)
+    except repackmod.RepackAlreadyRunning as ex:
+        # Don't propogate the exception if the repack is already in
+        # progress, since we want the command to exit 0.
+        repo.ui.warn('%s\n' % ex)

hgext/remotefilelog/basepack.py

@@ -0,0 +1,541 b''
+from __future__ import absolute_import
+
+import collections
+import errno
+import hashlib
+import mmap
+import os
+import struct
+import time
+
+from mercurial.i18n import _
+from mercurial import (
+    node as nodemod,
+    policy,
+    pycompat,
+    util,
+    vfs as vfsmod,
+)
+from . import shallowutil
+
+osutil = policy.importmod(r'osutil')
+
+# The pack version supported by this implementation. This will need to be
+# rev'd whenever the byte format changes. Ex: changing the fanout prefix,
+# changing any of the int sizes, changing the delta algorithm, etc.
+PACKVERSIONSIZE = 1
+INDEXVERSIONSIZE = 2
+
+FANOUTSTART = INDEXVERSIONSIZE
+
+# Constant that indicates a fanout table entry hasn't been filled in. (This does
+# not get serialized)
+EMPTYFANOUT = -1
+
+# The fanout prefix is the number of bytes that can be addressed by the fanout
+# table. Example: a fanout prefix of 1 means we use the first byte of a hash to
+# look in the fanout table (which will be 2^8 entries long).
+SMALLFANOUTPREFIX = 1
+LARGEFANOUTPREFIX = 2
+
+# The number of entries in the index at which point we switch to a large fanout.
+# It is chosen to balance the linear scan through a sparse fanout, with the
+# size of the bisect in actual index.
+# 2^16 / 8 was chosen because it trades off (1 step fanout scan + 5 step
+# bisect) with (8 step fanout scan + 1 step bisect)
+# 5 step bisect = log(2^16 / 8 / 255)  # fanout
+# 10 step fanout scan = 2^16 / (2^16 / 8)  # fanout space divided by entries
+SMALLFANOUTCUTOFF = 2**16 // 8
+
+# The amount of time to wait between checking for new packs. This prevents an
+# exception when data is moved to a new pack after the process has already
+# loaded the pack list.
+REFRESHRATE = 0.1
+
+if pycompat.isposix and not pycompat.ispy3:
+    # With glibc 2.7+ the 'e' flag uses O_CLOEXEC when opening.
+    # The 'e' flag will be ignored on older versions of glibc.
+    # Python 3 can't handle the 'e' flag.
+    PACKOPENMODE = 'rbe'
+else:
+    PACKOPENMODE = 'rb'
+
+class _cachebackedpacks(object):
+    def __init__(self, packs, cachesize):
+        self._packs = set(packs)
+        self._lrucache = util.lrucachedict(cachesize)
+        self._lastpack = None
+
+        # Avoid cold start of the cache by populating the most recent packs
+        # in the cache.
+        for i in reversed(range(min(cachesize, len(packs)))):
+            self._movetofront(packs[i])
+
+    def _movetofront(self, pack):
+        # This effectively makes pack the first entry in the cache.
+        self._lrucache[pack] = True
+
+    def _registerlastpackusage(self):
+        if self._lastpack is not None:
+            self._movetofront(self._lastpack)
+            self._lastpack = None
+
+    def add(self, pack):
+        self._registerlastpackusage()
+
+        # This method will mostly be called when packs are not in cache.
+        # Therefore, adding pack to the cache.
+        self._movetofront(pack)
+        self._packs.add(pack)
+
+    def __iter__(self):
+        self._registerlastpackusage()
+
+        # Cache iteration is based on LRU.
+        for pack in self._lrucache:
+            self._lastpack = pack
+            yield pack
+
+        cachedpacks = set(pack for pack in self._lrucache)
+        # Yield for paths not in the cache.
+        for pack in self._packs - cachedpacks:
+            self._lastpack = pack
+            yield pack
+
+        # Data not found in any pack.
+        self._lastpack = None
+
+class basepackstore(object):
+    # Default cache size limit for the pack files.
+    DEFAULTCACHESIZE = 100
+
+    def __init__(self, ui, path):
+        self.ui = ui
+        self.path = path
+
+        # lastrefesh is 0 so we'll immediately check for new packs on the first
+        # failure.
+        self.lastrefresh = 0
+
+        packs = []
+        for filepath, __, __ in self._getavailablepackfilessorted():
+            try:
+                pack = self.getpack(filepath)
+            except Exception as ex:
+                # An exception may be thrown if the pack file is corrupted
+                # somehow.  Log a warning but keep going in this case, just
+                # skipping this pack file.
+                #
+                # If this is an ENOENT error then don't even bother logging.
+                # Someone could have removed the file since we retrieved the
+                # list of paths.
+                if getattr(ex, 'errno', None) != errno.ENOENT:
+                    ui.warn(_('unable to load pack %s: %s\n') % (filepath, ex))
+                continue
+            packs.append(pack)
+
+        self.packs = _cachebackedpacks(packs, self.DEFAULTCACHESIZE)
+
+    def _getavailablepackfiles(self):
+        """For each pack file (a index/data file combo), yields:
+          (full path without extension, mtime, size)
+
+        mtime will be the mtime of the index/data file (whichever is newer)
+        size is the combined size of index/data file
+        """
+        indexsuffixlen = len(self.INDEXSUFFIX)
+        packsuffixlen = len(self.PACKSUFFIX)
+
+        ids = set()
+        sizes = collections.defaultdict(lambda: 0)
+        mtimes = collections.defaultdict(lambda: [])
+        try:
+            for filename, type, stat in osutil.listdir(self.path, stat=True):
+                id = None
+                if filename[-indexsuffixlen:] == self.INDEXSUFFIX:
+                    id = filename[:-indexsuffixlen]
+                elif filename[-packsuffixlen:] == self.PACKSUFFIX:
+                    id = filename[:-packsuffixlen]
+
+                # Since we expect to have two files corresponding to each ID
+                # (the index file and the pack file), we can yield once we see
+                # it twice.
+                if id:
+                    sizes[id] += stat.st_size # Sum both files' sizes together
+                    mtimes[id].append(stat.st_mtime)
+                    if id in ids:
+                        yield (os.path.join(self.path, id), max(mtimes[id]),
+                            sizes[id])
+                    else:
+                        ids.add(id)
+        except OSError as ex:
+            if ex.errno != errno.ENOENT:
+                raise
+
+    def _getavailablepackfilessorted(self):
+        """Like `_getavailablepackfiles`, but also sorts the files by mtime,
+        yielding newest files first.
+
+        This is desirable, since it is more likely newer packfiles have more
+        desirable data.
+        """
+        files = []
+        for path, mtime, size in self._getavailablepackfiles():
+            files.append((mtime, size, path))
+        files = sorted(files, reverse=True)
+        for mtime, size, path in files:
+            yield path, mtime, size
+
+    def gettotalsizeandcount(self):
+        """Returns the total disk size (in bytes) of all the pack files in
+        this store, and the count of pack files.
+
+        (This might be smaller than the total size of the ``self.path``
+        directory, since this only considers fuly-writen pack files, and not
+        temporary files or other detritus on the directory.)
+        """
+        totalsize = 0
+        count = 0
+        for __, __, size in self._getavailablepackfiles():
+            totalsize += size
+            count += 1
+        return totalsize, count
+
+    def getmetrics(self):
+        """Returns metrics on the state of this store."""
+        size, count = self.gettotalsizeandcount()
+        return {
+            'numpacks': count,
+            'totalpacksize': size,
+        }
+
+    def getpack(self, path):
+        raise NotImplementedError()
+
+    def getmissing(self, keys):
+        missing = keys
+        for pack in self.packs:
+            missing = pack.getmissing(missing)
+
+            # Ensures better performance of the cache by keeping the most
+            # recently accessed pack at the beginning in subsequent iterations.
+            if not missing:
+                return missing
+
+        if missing:
+            for pack in self.refresh():
+                missing = pack.getmissing(missing)
+
+        return missing
+
+    def markledger(self, ledger, options=None):
+        for pack in self.packs:
+            pack.markledger(ledger)
+
+    def markforrefresh(self):
+        """Tells the store that there may be new pack files, so the next time it
+        has a lookup miss it should check for new files."""
+        self.lastrefresh = 0
+
+    def refresh(self):
+        """Checks for any new packs on disk, adds them to the main pack list,
+        and returns a list of just the new packs."""
+        now = time.time()
+
+        # If we experience a lot of misses (like in the case of getmissing() on
+        # new objects), let's only actually check disk for new stuff every once
+        # in a while. Generally this code path should only ever matter when a
+        # repack is going on in the background, and that should be pretty rare
+        # to have that happen twice in quick succession.
+        newpacks = []
+        if now > self.lastrefresh + REFRESHRATE:
+            self.lastrefresh = now
+            previous = set(p.path for p in self.packs)
+            for filepath, __, __ in self._getavailablepackfilessorted():
+                if filepath not in previous:
+                    newpack = self.getpack(filepath)
+                    newpacks.append(newpack)
+                    self.packs.add(newpack)
+
+        return newpacks
+
+class versionmixin(object):
+    # Mix-in for classes with multiple supported versions
+    VERSION = None
+    SUPPORTED_VERSIONS = [2]
+
+    def _checkversion(self, version):
+        if version in self.SUPPORTED_VERSIONS:
+            if self.VERSION is None:
+                # only affect this instance
+                self.VERSION = version
+            elif self.VERSION != version:
+                raise RuntimeError('inconsistent version: %s' % version)
+        else:
+            raise RuntimeError('unsupported version: %s' % version)
+
+class basepack(versionmixin):
+    # The maximum amount we should read via mmap before remmaping so the old
+    # pages can be released (100MB)
+    MAXPAGEDIN = 100 * 1024**2
+
+    SUPPORTED_VERSIONS = [2]
+
+    def __init__(self, path):
+        self.path = path
+        self.packpath = path + self.PACKSUFFIX
+        self.indexpath = path + self.INDEXSUFFIX
+
+        self.indexsize = os.stat(self.indexpath).st_size
+        self.datasize = os.stat(self.packpath).st_size
+
+        self._index = None
+        self._data = None
+        self.freememory() # initialize the mmap
+
+        version = struct.unpack('!B', self._data[:PACKVERSIONSIZE])[0]
+        self._checkversion(version)
+
+        version, config = struct.unpack('!BB', self._index[:INDEXVERSIONSIZE])
+        self._checkversion(version)
+
+        if 0b10000000 & config:
+            self.params = indexparams(LARGEFANOUTPREFIX, version)
+        else:
+            self.params = indexparams(SMALLFANOUTPREFIX, version)
+
+    @util.propertycache
+    def _fanouttable(self):
+        params = self.params
+        rawfanout = self._index[FANOUTSTART:FANOUTSTART + params.fanoutsize]
+        fanouttable = []
+        for i in pycompat.xrange(0, params.fanoutcount):
+            loc = i * 4
+            fanoutentry = struct.unpack('!I', rawfanout[loc:loc + 4])[0]
+            fanouttable.append(fanoutentry)
+        return fanouttable
+
+    @util.propertycache
+    def _indexend(self):
+        nodecount = struct.unpack_from('!Q', self._index,
+                                       self.params.indexstart - 8)[0]
+        return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
+
+    def freememory(self):
+        """Unmap and remap the memory to free it up after known expensive
+        operations. Return True if self._data and self._index were reloaded.
+        """
+        if self._index:
+            if self._pagedin < self.MAXPAGEDIN:
+                return False
+
+            self._index.close()
+            self._data.close()
+
+        # TODO: use an opener/vfs to access these paths
+        with open(self.indexpath, PACKOPENMODE) as indexfp:
+            # memory-map the file, size 0 means whole file
+            self._index = mmap.mmap(indexfp.fileno(), 0,
+                                    access=mmap.ACCESS_READ)
+        with open(self.packpath, PACKOPENMODE) as datafp:
+            self._data = mmap.mmap(datafp.fileno(), 0, access=mmap.ACCESS_READ)
+
+        self._pagedin = 0
+        return True
+
+    def getmissing(self, keys):
+        raise NotImplementedError()
+
+    def markledger(self, ledger, options=None):
+        raise NotImplementedError()
+
+    def cleanup(self, ledger):
+        raise NotImplementedError()
+
+    def __iter__(self):
+        raise NotImplementedError()
+
+    def iterentries(self):
+        raise NotImplementedError()
+
+class mutablebasepack(versionmixin):
+
+    def __init__(self, ui, packdir, version=2):
+        self._checkversion(version)
+        # TODO(augie): make this configurable
+        self._compressor = 'GZ'
+        opener = vfsmod.vfs(packdir)
+        opener.createmode = 0o444
+        self.opener = opener
+
+        self.entries = {}
+
+        shallowutil.mkstickygroupdir(ui, packdir)
+        self.packfp, self.packpath = opener.mkstemp(
+            suffix=self.PACKSUFFIX + '-tmp')
+        self.idxfp, self.idxpath = opener.mkstemp(
+            suffix=self.INDEXSUFFIX + '-tmp')
+        self.packfp = os.fdopen(self.packfp, r'wb+')
+        self.idxfp = os.fdopen(self.idxfp, r'wb+')
+        self.sha = hashlib.sha1()
+        self._closed = False
+
+        # The opener provides no way of doing permission fixup on files created
+        # via mkstemp, so we must fix it ourselves. We can probably fix this
+        # upstream in vfs.mkstemp so we don't need to use the private method.
+        opener._fixfilemode(opener.join(self.packpath))
+        opener._fixfilemode(opener.join(self.idxpath))
+
+        # Write header
+        # TODO: make it extensible (ex: allow specifying compression algorithm,
+        # a flexible key/value header, delta algorithm, fanout size, etc)
+        versionbuf = struct.pack('!B', self.VERSION) # unsigned 1 byte int
+        self.writeraw(versionbuf)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type is None:
+            self.close()
+        else:
+            self.abort()
+
+    def abort(self):
+        # Unclean exit
+        self._cleantemppacks()
+
+    def writeraw(self, data):
+        self.packfp.write(data)
+        self.sha.update(data)
+
+    def close(self, ledger=None):
+        if self._closed:
+            return
+
+        try:
+            sha = nodemod.hex(self.sha.digest())
+            self.packfp.close()
+            self.writeindex()
+
+            if len(self.entries) == 0:
+                # Empty pack
+                self._cleantemppacks()
+                self._closed = True
+                return None
+
+            self.opener.rename(self.packpath, sha + self.PACKSUFFIX)
+            try:
+                self.opener.rename(self.idxpath, sha + self.INDEXSUFFIX)
+            except Exception as ex:
+                try:
+                    self.opener.unlink(sha + self.PACKSUFFIX)
+                except Exception:
+                    pass
+                # Throw exception 'ex' explicitly since a normal 'raise' would
+                # potentially throw an exception from the unlink cleanup.
+                raise ex
+        except Exception:
+            # Clean up temp packs in all exception cases
+            self._cleantemppacks()
+            raise
+
+        self._closed = True
+        result = self.opener.join(sha)
+        if ledger:
+            ledger.addcreated(result)
+        return result
+
+    def _cleantemppacks(self):
+        try:
+            self.opener.unlink(self.packpath)
+        except Exception:
+            pass
+        try:
+            self.opener.unlink(self.idxpath)
+        except Exception:
+            pass
+
+    def writeindex(self):
+        rawindex = ''
+
+        largefanout = len(self.entries) > SMALLFANOUTCUTOFF
+        if largefanout:
+            params = indexparams(LARGEFANOUTPREFIX, self.VERSION)
+        else:
+            params = indexparams(SMALLFANOUTPREFIX, self.VERSION)
+
+        fanouttable = [EMPTYFANOUT] * params.fanoutcount
+
+        # Precompute the location of each entry
+        locations = {}
+        count = 0
+        for node in sorted(self.entries):
+            location = count * self.INDEXENTRYLENGTH
+            locations[node] = location
+            count += 1
+
+            # Must use [0] on the unpack result since it's always a tuple.
+            fanoutkey = struct.unpack(params.fanoutstruct,
+                                      node[:params.fanoutprefix])[0]
+            if fanouttable[fanoutkey] == EMPTYFANOUT:
+                fanouttable[fanoutkey] = location
+
+        rawfanouttable = ''
+        last = 0
+        for offset in fanouttable:
+            offset = offset if offset != EMPTYFANOUT else last
+            last = offset
+            rawfanouttable += struct.pack('!I', offset)
+
+        rawentrieslength = struct.pack('!Q', len(self.entries))
+
+        # The index offset is the it's location in the file. So after the 2 byte
+        # header and the fanouttable.
+        rawindex = self.createindex(locations, 2 + len(rawfanouttable))
+
+        self._writeheader(params)
+        self.idxfp.write(rawfanouttable)
+        self.idxfp.write(rawentrieslength)
+        self.idxfp.write(rawindex)
+        self.idxfp.close()
+
+    def createindex(self, nodelocations):
+        raise NotImplementedError()
+
+    def _writeheader(self, indexparams):
+        # Index header
+        #    <version: 1 byte>
+        #    <large fanout: 1 bit> # 1 means 2^16, 0 means 2^8
+        #    <unused: 7 bit> # future use (compression, delta format, etc)
+        config = 0
+        if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
+            config = 0b10000000
+        self.idxfp.write(struct.pack('!BB', self.VERSION, config))
+
+class indexparams(object):
+    __slots__ = (r'fanoutprefix', r'fanoutstruct', r'fanoutcount',
+                 r'fanoutsize', r'indexstart')
+
+    def __init__(self, prefixsize, version):
+        self.fanoutprefix = prefixsize
+
+        # The struct pack format for fanout table location (i.e. the format that
+        # converts the node prefix into an integer location in the fanout
+        # table).
+        if prefixsize == SMALLFANOUTPREFIX:
+            self.fanoutstruct = '!B'
+        elif prefixsize == LARGEFANOUTPREFIX:
+            self.fanoutstruct = '!H'
+        else:
+            raise ValueError("invalid fanout prefix size: %s" % prefixsize)
+
+        # The number of fanout table entries
+        self.fanoutcount = 2**(prefixsize * 8)
+
+        # The total bytes used by the fanout table
+        self.fanoutsize = self.fanoutcount * 4
+
+        self.indexstart = FANOUTSTART + self.fanoutsize
+        # Skip the index length
+        self.indexstart += 8

hgext/remotefilelog/basestore.py

@@ -0,0 +1,425 b''
+from __future__ import absolute_import
+
+import errno
+import hashlib
+import os
+import shutil
+import stat
+import time
+
+from mercurial.i18n import _
+from mercurial.node import bin, hex
+from mercurial import (
+    error,
+    pycompat,
+    util,
+)
+from . import (
+    constants,
+    shallowutil,
+)
+
+class basestore(object):
+    def __init__(self, repo, path, reponame, shared=False):
+        """Creates a remotefilelog store object for the given repo name.
+
+        `path` - The file path where this store keeps its data
+        `reponame` - The name of the repo. This is used to partition data from
+        many repos.
+        `shared` - True if this store is a shared cache of data from the central
+        server, for many repos on this machine. False means this store is for
+        the local data for one repo.
+        """
+        self.repo = repo
+        self.ui = repo.ui
+        self._path = path
+        self._reponame = reponame
+        self._shared = shared
+        self._uid = os.getuid() if not pycompat.iswindows else None
+
+        self._validatecachelog = self.ui.config("remotefilelog",
+                                                "validatecachelog")
+        self._validatecache = self.ui.config("remotefilelog", "validatecache",
+                                             'on')
+        if self._validatecache not in ('on', 'strict', 'off'):
+            self._validatecache = 'on'
+        if self._validatecache == 'off':
+            self._validatecache = False
+
+        if shared:
+            shallowutil.mkstickygroupdir(self.ui, path)
+
+    def getmissing(self, keys):
+        missing = []
+        for name, node in keys:
+            filepath = self._getfilepath(name, node)
+            exists = os.path.exists(filepath)
+            if (exists and self._validatecache == 'strict' and
+                not self._validatekey(filepath, 'contains')):
+                exists = False
+            if not exists:
+                missing.append((name, node))
+
+        return missing
+
+    # BELOW THIS ARE IMPLEMENTATIONS OF REPACK SOURCE
+
+    def markledger(self, ledger, options=None):
+        if options and options.get(constants.OPTION_PACKSONLY):
+            return
+        if self._shared:
+            for filename, nodes in self._getfiles():
+                for node in nodes:
+                    ledger.markdataentry(self, filename, node)
+                    ledger.markhistoryentry(self, filename, node)
+
+    def cleanup(self, ledger):
+        ui = self.ui
+        entries = ledger.sources.get(self, [])
+        count = 0
+        progress = ui.makeprogress(_("cleaning up"), unit="files",
+                                   total=len(entries))
+        for entry in entries:
+            if entry.gced or (entry.datarepacked and entry.historyrepacked):
+                progress.update(count)
+                path = self._getfilepath(entry.filename, entry.node)
+                util.tryunlink(path)
+            count += 1
+        progress.complete()
+
+        # Clean up the repo cache directory.
+        self._cleanupdirectory(self._getrepocachepath())
+
+    # BELOW THIS ARE NON-STANDARD APIS
+
+    def _cleanupdirectory(self, rootdir):
+        """Removes the empty directories and unnecessary files within the root
+        directory recursively. Note that this method does not remove the root
+        directory itself. """
+
+        oldfiles = set()
+        otherfiles = set()
+        # osutil.listdir returns stat information which saves some rmdir/listdir
+        # syscalls.
+        for name, mode in util.osutil.listdir(rootdir):
+            if stat.S_ISDIR(mode):
+                dirpath = os.path.join(rootdir, name)
+                self._cleanupdirectory(dirpath)
+
+                # Now that the directory specified by dirpath is potentially
+                # empty, try and remove it.
+                try:
+                    os.rmdir(dirpath)
+                except OSError:
+                    pass
+
+            elif stat.S_ISREG(mode):
+                if name.endswith('_old'):
+                    oldfiles.add(name[:-4])
+                else:
+                    otherfiles.add(name)
+
+        # Remove the files which end with suffix '_old' and have no
+        # corresponding file without the suffix '_old'. See addremotefilelognode
+        # method for the generation/purpose of files with '_old' suffix.
+        for filename in oldfiles - otherfiles:
+            filepath = os.path.join(rootdir, filename + '_old')
+            util.tryunlink(filepath)
+
+    def _getfiles(self):
+        """Return a list of (filename, [node,...]) for all the revisions that
+        exist in the store.
+
+        This is useful for obtaining a list of all the contents of the store
+        when performing a repack to another store, since the store API requires
+        name+node keys and not namehash+node keys.
+        """
+        existing = {}
+        for filenamehash, node in self._listkeys():
+            existing.setdefault(filenamehash, []).append(node)
+
+        filenamemap = self._resolvefilenames(existing.keys())
+
+        for filename, sha in filenamemap.iteritems():
+            yield (filename, existing[sha])
+
+    def _resolvefilenames(self, hashes):
+        """Given a list of filename hashes that are present in the
+        remotefilelog store, return a mapping from filename->hash.
+
+        This is useful when converting remotefilelog blobs into other storage
+        formats.
+        """
+        if not hashes:
+            return {}
+
+        filenames = {}
+        missingfilename = set(hashes)
+
+        # Start with a full manifest, since it'll cover the majority of files
+        for filename in self.repo['tip'].manifest():
+            sha = hashlib.sha1(filename).digest()
+            if sha in missingfilename:
+                filenames[filename] = sha
+                missingfilename.discard(sha)
+
+        # Scan the changelog until we've found every file name
+        cl = self.repo.unfiltered().changelog
+        for rev in pycompat.xrange(len(cl) - 1, -1, -1):
+            if not missingfilename:
+                break
+            files = cl.readfiles(cl.node(rev))
+            for filename in files:
+                sha = hashlib.sha1(filename).digest()
+                if sha in missingfilename:
+                    filenames[filename] = sha
+                    missingfilename.discard(sha)
+
+        return filenames
+
+    def _getrepocachepath(self):
+        return os.path.join(
+            self._path, self._reponame) if self._shared else self._path
+
+    def _listkeys(self):
+        """List all the remotefilelog keys that exist in the store.
+
+        Returns a iterator of (filename hash, filecontent hash) tuples.
+        """
+
+        for root, dirs, files in os.walk(self._getrepocachepath()):
+            for filename in files:
+                if len(filename) != 40:
+                    continue
+                node = filename
+                if self._shared:
+                    # .../1a/85ffda..be21
+                    filenamehash = root[-41:-39] + root[-38:]
+                else:
+                    filenamehash = root[-40:]
+                yield (bin(filenamehash), bin(node))
+
+    def _getfilepath(self, name, node):
+        node = hex(node)
+        if self._shared:
+            key = shallowutil.getcachekey(self._reponame, name, node)
+        else:
+            key = shallowutil.getlocalkey(name, node)
+
+        return os.path.join(self._path, key)
+
+    def _getdata(self, name, node):
+        filepath = self._getfilepath(name, node)
+        try:
+            data = shallowutil.readfile(filepath)
+            if self._validatecache and not self._validatedata(data, filepath):
+                if self._validatecachelog:
+                    with open(self._validatecachelog, 'a+') as f:
+                        f.write("corrupt %s during read\n" % filepath)
+                os.rename(filepath, filepath + ".corrupt")
+                raise KeyError("corrupt local cache file %s" % filepath)
+        except IOError:
+            raise KeyError("no file found at %s for %s:%s" % (filepath, name,
+                                                              hex(node)))
+
+        return data
+
+    def addremotefilelognode(self, name, node, data):
+        filepath = self._getfilepath(name, node)
+
+        oldumask = os.umask(0o002)
+        try:
+            # if this node already exists, save the old version for
+            # recovery/debugging purposes.
+            if os.path.exists(filepath):
+                newfilename = filepath + '_old'
+                # newfilename can be read-only and shutil.copy will fail.
+                # Delete newfilename to avoid it
+                if os.path.exists(newfilename):
+                    shallowutil.unlinkfile(newfilename)
+                shutil.copy(filepath, newfilename)
+
+            shallowutil.mkstickygroupdir(self.ui, os.path.dirname(filepath))
+            shallowutil.writefile(filepath, data, readonly=True)
+
+            if self._validatecache:
+                if not self._validatekey(filepath, 'write'):
+                    raise error.Abort(_("local cache write was corrupted %s") %
+                                      filepath)
+        finally:
+            os.umask(oldumask)
+
+    def markrepo(self, path):
+        """Call this to add the given repo path to the store's list of
+        repositories that are using it. This is useful later when doing garbage
+        collection, since it allows us to insecpt the repos to see what nodes
+        they want to be kept alive in the store.
+        """
+        repospath = os.path.join(self._path, "repos")
+        with open(repospath, 'ab') as reposfile:
+            reposfile.write(os.path.dirname(path) + "\n")
+
+        repospathstat = os.stat(repospath)
+        if repospathstat.st_uid == self._uid:
+            os.chmod(repospath, 0o0664)
+
+    def _validatekey(self, path, action):
+        with open(path, 'rb') as f:
+            data = f.read()
+
+        if self._validatedata(data, path):
+            return True
+
+        if self._validatecachelog:
+            with open(self._validatecachelog, 'ab+') as f:
+                f.write("corrupt %s during %s\n" % (path, action))
+
+        os.rename(path, path + ".corrupt")
+        return False
+
+    def _validatedata(self, data, path):
+        try:
+            if len(data) > 0:
+                # see remotefilelogserver.createfileblob for the format
+                offset, size, flags = shallowutil.parsesizeflags(data)
+                if len(data) <= size:
+                    # it is truncated
+                    return False
+
+                # extract the node from the metadata
+                offset += size
+                datanode = data[offset:offset + 20]
+
+                # and compare against the path
+                if os.path.basename(path) == hex(datanode):
+                    # Content matches the intended path
+                    return True
+                return False
+        except (ValueError, RuntimeError):
+            pass
+
+        return False
+
+    def gc(self, keepkeys):
+        ui = self.ui
+        cachepath = self._path
+
+        # prune cache
+        queue = pycompat.queue.PriorityQueue()
+        originalsize = 0
+        size = 0
+        count = 0
+        removed = 0
+
+        # keep files newer than a day even if they aren't needed
+        limit = time.time() - (60 * 60 * 24)
+
+        progress = ui.makeprogress(_("removing unnecessary files"),
+                                   unit="files")
+        progress.update(0)
+        for root, dirs, files in os.walk(cachepath):
+            for file in files:
+                if file == 'repos':
+                    continue
+
+                # Don't delete pack files
+                if '/packs/' in root:
+                    continue
+
+                progress.update(count)
+                path = os.path.join(root, file)
+                key = os.path.relpath(path, cachepath)
+                count += 1
+                try:
+                    pathstat = os.stat(path)
+                except OSError as e:
+                    # errno.ENOENT = no such file or directory
+                    if e.errno != errno.ENOENT:
+                        raise
+                    msg = _("warning: file %s was removed by another process\n")
+                    ui.warn(msg % path)
+                    continue
+
+                originalsize += pathstat.st_size
+
+                if key in keepkeys or pathstat.st_atime > limit:
+                    queue.put((pathstat.st_atime, path, pathstat))
+                    size += pathstat.st_size
+                else:
+                    try:
+                        shallowutil.unlinkfile(path)
+                    except OSError as e:
+                        # errno.ENOENT = no such file or directory
+                        if e.errno != errno.ENOENT:
+                            raise
+                        msg = _("warning: file %s was removed by another "
+                                "process\n")
+                        ui.warn(msg % path)
+                        continue
+                    removed += 1
+        progress.complete()
+
+        # remove oldest files until under limit
+        limit = ui.configbytes("remotefilelog", "cachelimit")
+        if size > limit:
+            excess = size - limit
+            progress = ui.makeprogress(_("enforcing cache limit"), unit="bytes",
+                                       total=excess)
+            removedexcess = 0
+            while queue and size > limit and size > 0:
+                progress.update(removedexcess)
+                atime, oldpath, oldpathstat = queue.get()
+                try:
+                    shallowutil.unlinkfile(oldpath)
+                except OSError as e:
+                    # errno.ENOENT = no such file or directory
+                    if e.errno != errno.ENOENT:
+                        raise
+                    msg = _("warning: file %s was removed by another process\n")
+                    ui.warn(msg % oldpath)
+                size -= oldpathstat.st_size
+                removed += 1
+                removedexcess += oldpathstat.st_size
+            progress.complete()
+
+        ui.status(_("finished: removed %d of %d files (%0.2f GB to %0.2f GB)\n")
+                  % (removed, count,
+                     float(originalsize) / 1024.0 / 1024.0 / 1024.0,
+                     float(size) / 1024.0 / 1024.0 / 1024.0))
+
+class baseunionstore(object):
+    def __init__(self, *args, **kwargs):
+        # If one of the functions that iterates all of the stores is about to
+        # throw a KeyError, try this many times with a full refresh between
+        # attempts. A repack operation may have moved data from one store to
+        # another while we were running.
+        self.numattempts = kwargs.get(r'numretries', 0) + 1
+        # If not-None, call this function on every retry and if the attempts are
+        # exhausted.
+        self.retrylog = kwargs.get(r'retrylog', None)
+
+    def markforrefresh(self):
+        for store in self.stores:
+            if util.safehasattr(store, 'markforrefresh'):
+                store.markforrefresh()
+
+    @staticmethod
+    def retriable(fn):
+        def noop(*args):
+            pass
+        def wrapped(self, *args, **kwargs):
+            retrylog = self.retrylog or noop
+            funcname = fn.__name__
+            for i in pycompat.xrange(self.numattempts):
+                if i > 0:
+                    retrylog('re-attempting (n=%d) %s\n' % (i, funcname))
+                    self.markforrefresh()
+                try:
+                    return fn(self, *args, **kwargs)
+                except KeyError:
+                    pass
+            # retries exhausted
+            retrylog('retries exhausted in %s, raising KeyError\n' %
+                     pycompat.sysbytes(funcname))
+            raise
+        return wrapped

hgext/remotefilelog/connectionpool.py

@@ -0,0 +1,84 b''
+# connectionpool.py - class for pooling peer connections for reuse
+#
+# Copyright 2017 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+from mercurial import (
+    extensions,
+    hg,
+    sshpeer,
+    util,
+)
+
+_sshv1peer = sshpeer.sshv1peer
+
+class connectionpool(object):
+    def __init__(self, repo):
+        self._repo = repo
+        self._pool = dict()
+
+    def get(self, path):
+        pathpool = self._pool.get(path)
+        if pathpool is None:
+            pathpool = list()
+            self._pool[path] = pathpool
+
+        conn = None
+        if len(pathpool) > 0:
+            try:
+                conn = pathpool.pop()
+                peer = conn.peer
+                # If the connection has died, drop it
+                if isinstance(peer, _sshv1peer):
+                    if peer._subprocess.poll() is not None:
+                        conn = None
+            except IndexError:
+                pass
+
+        if conn is None:
+            def _cleanup(orig):
+                # close pipee first so peer.cleanup reading it won't deadlock,
+                # if there are other processes with pipeo open (i.e. us).
+                peer = orig.im_self
+                if util.safehasattr(peer, 'pipee'):
+                    peer.pipee.close()
+                return orig()
+
+            peer = hg.peer(self._repo.ui, {}, path)
+            if util.safehasattr(peer, 'cleanup'):
+                extensions.wrapfunction(peer, 'cleanup', _cleanup)
+
+            conn = connection(pathpool, peer)
+
+        return conn
+
+    def close(self):
+        for pathpool in self._pool.itervalues():
+            for conn in pathpool:
+                conn.close()
+            del pathpool[:]
+
+class connection(object):
+    def __init__(self, pool, peer):
+        self._pool = pool
+        self.peer = peer
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        # Only add the connection back to the pool if there was no exception,
+        # since an exception could mean the connection is not in a reusable
+        # state.
+        if type is None:
+            self._pool.append(self)
+        else:
+            self.close()
+
+    def close(self):
+        if util.safehasattr(self.peer, 'cleanup'):
+            self.peer.cleanup()

hgext/remotefilelog/constants.py

@@ -0,0 +1,41 b''
+from __future__ import absolute_import
+
+import struct
+
+from mercurial.i18n import _
+
+NETWORK_CAP_LEGACY_SSH_GETFILES = 'exp-remotefilelog-ssh-getfiles-1'
+
+SHALLOWREPO_REQUIREMENT = "exp-remotefilelog-repo-req-1"
+
+BUNDLE2_CAPABLITY = "exp-remotefilelog-b2cap-1"
+
+FILENAMESTRUCT = '!H'
+FILENAMESIZE = struct.calcsize(FILENAMESTRUCT)
+
+NODESIZE = 20
+PACKREQUESTCOUNTSTRUCT = '!I'
+
+NODECOUNTSTRUCT = '!I'
+NODECOUNTSIZE = struct.calcsize(NODECOUNTSTRUCT)
+
+PATHCOUNTSTRUCT = '!I'
+PATHCOUNTSIZE = struct.calcsize(PATHCOUNTSTRUCT)
+
+FILEPACK_CATEGORY=""
+TREEPACK_CATEGORY="manifests"
+
+ALL_CATEGORIES = [FILEPACK_CATEGORY, TREEPACK_CATEGORY]
+
+# revision metadata keys. must be a single character.
+METAKEYFLAG = 'f'  # revlog flag
+METAKEYSIZE = 's'  # full rawtext size
+
+def getunits(category):
+    if category == FILEPACK_CATEGORY:
+        return _("files")
+    if category == TREEPACK_CATEGORY:
+        return _("trees")
+
+# Repack options passed to ``markledger``.
+OPTION_PACKSONLY = 'packsonly'

hgext/remotefilelog/contentstore.py

@@ -0,0 +1,376 b''
+from __future__ import absolute_import
+
+import threading
+
+from mercurial.node import hex, nullid
+from mercurial import (
+    mdiff,
+    pycompat,
+    revlog,
+)
+from . import (
+    basestore,
+    constants,
+    shallowutil,
+)
+
+class ChainIndicies(object):
+    """A static class for easy reference to the delta chain indicies.
+    """
+    # The filename of this revision delta
+    NAME = 0
+    # The mercurial file node for this revision delta
+    NODE = 1
+    # The filename of the delta base's revision. This is useful when delta
+    # between different files (like in the case of a move or copy, we can delta
+    # against the original file content).
+    BASENAME = 2
+    # The mercurial file node for the delta base revision. This is the nullid if
+    # this delta is a full text.
+    BASENODE = 3
+    # The actual delta or full text data.
+    DATA = 4
+
+class unioncontentstore(basestore.baseunionstore):
+    def __init__(self, *args, **kwargs):
+        super(unioncontentstore, self).__init__(*args, **kwargs)
+
+        self.stores = args
+        self.writestore = kwargs.get(r'writestore')
+
+        # If allowincomplete==True then the union store can return partial
+        # delta chains, otherwise it will throw a KeyError if a full
+        # deltachain can't be found.
+        self.allowincomplete = kwargs.get(r'allowincomplete', False)
+
+    def get(self, name, node):
+        """Fetches the full text revision contents of the given name+node pair.
+        If the full text doesn't exist, throws a KeyError.
+
+        Under the hood, this uses getdeltachain() across all the stores to build
+        up a full chain to produce the full text.
+        """
+        chain = self.getdeltachain(name, node)
+
+        if chain[-1][ChainIndicies.BASENODE] != nullid:
+            # If we didn't receive a full chain, throw
+            raise KeyError((name, hex(node)))
+
+        # The last entry in the chain is a full text, so we start our delta
+        # applies with that.
+        fulltext = chain.pop()[ChainIndicies.DATA]
+
+        text = fulltext
+        while chain:
+            delta = chain.pop()[ChainIndicies.DATA]
+            text = mdiff.patches(text, [delta])
+
+        return text
+
+    @basestore.baseunionstore.retriable
+    def getdelta(self, name, node):
+        """Return the single delta entry for the given name/node pair.
+        """
+        for store in self.stores:
+            try:
+                return store.getdelta(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    def getdeltachain(self, name, node):
+        """Returns the deltachain for the given name/node pair.
+
+        Returns an ordered list of:
+
+          [(name, node, deltabasename, deltabasenode, deltacontent),...]
+
+        where the chain is terminated by a full text entry with a nullid
+        deltabasenode.
+        """
+        chain = self._getpartialchain(name, node)
+        while chain[-1][ChainIndicies.BASENODE] != nullid:
+            x, x, deltabasename, deltabasenode, x = chain[-1]
+            try:
+                morechain = self._getpartialchain(deltabasename, deltabasenode)
+                chain.extend(morechain)
+            except KeyError:
+                # If we allow incomplete chains, don't throw.
+                if not self.allowincomplete:
+                    raise
+                break
+
+        return chain
+
+    @basestore.baseunionstore.retriable
+    def getmeta(self, name, node):
+        """Returns the metadata dict for given node."""
+        for store in self.stores:
+            try:
+                return store.getmeta(name, node)
+            except KeyError:
+                pass
+        raise KeyError((name, hex(node)))
+
+    def getmetrics(self):
+        metrics = [s.getmetrics() for s in self.stores]
+        return shallowutil.sumdicts(*metrics)
+
+    @basestore.baseunionstore.retriable
+    def _getpartialchain(self, name, node):
+        """Returns a partial delta chain for the given name/node pair.
+
+        A partial chain is a chain that may not be terminated in a full-text.
+        """
+        for store in self.stores:
+            try:
+                return store.getdeltachain(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add content only to remotefilelog "
+                           "contentstore")
+
+    def getmissing(self, keys):
+        missing = keys
+        for store in self.stores:
+            if missing:
+                missing = store.getmissing(missing)
+        return missing
+
+    def addremotefilelognode(self, name, node, data):
+        if self.writestore:
+            self.writestore.addremotefilelognode(name, node, data)
+        else:
+            raise RuntimeError("no writable store configured")
+
+    def markledger(self, ledger, options=None):
+        for store in self.stores:
+            store.markledger(ledger, options)
+
+class remotefilelogcontentstore(basestore.basestore):
+    def __init__(self, *args, **kwargs):
+        super(remotefilelogcontentstore, self).__init__(*args, **kwargs)
+        self._threaddata = threading.local()
+
+    def get(self, name, node):
+        # return raw revision text
+        data = self._getdata(name, node)
+
+        offset, size, flags = shallowutil.parsesizeflags(data)
+        content = data[offset:offset + size]
+
+        ancestormap = shallowutil.ancestormap(data)
+        p1, p2, linknode, copyfrom = ancestormap[node]
+        copyrev = None
+        if copyfrom:
+            copyrev = hex(p1)
+
+        self._updatemetacache(node, size, flags)
+
+        # lfs tracks renames in its own metadata, remove hg copy metadata,
+        # because copy metadata will be re-added by lfs flag processor.
+        if flags & revlog.REVIDX_EXTSTORED:
+            copyrev = copyfrom = None
+        revision = shallowutil.createrevlogtext(content, copyfrom, copyrev)
+        return revision
+
+    def getdelta(self, name, node):
+        # Since remotefilelog content stores only contain full texts, just
+        # return that.
+        revision = self.get(name, node)
+        return revision, name, nullid, self.getmeta(name, node)
+
+    def getdeltachain(self, name, node):
+        # Since remotefilelog content stores just contain full texts, we return
+        # a fake delta chain that just consists of a single full text revision.
+        # The nullid in the deltabasenode slot indicates that the revision is a
+        # fulltext.
+        revision = self.get(name, node)
+        return [(name, node, None, nullid, revision)]
+
+    def getmeta(self, name, node):
+        self._sanitizemetacache()
+        if node != self._threaddata.metacache[0]:
+            data = self._getdata(name, node)
+            offset, size, flags = shallowutil.parsesizeflags(data)
+            self._updatemetacache(node, size, flags)
+        return self._threaddata.metacache[1]
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add content only to remotefilelog "
+                           "contentstore")
+
+    def _sanitizemetacache(self):
+        metacache = getattr(self._threaddata, 'metacache', None)
+        if metacache is None:
+            self._threaddata.metacache = (None, None) # (node, meta)
+
+    def _updatemetacache(self, node, size, flags):
+        self._sanitizemetacache()
+        if node == self._threaddata.metacache[0]:
+            return
+        meta = {constants.METAKEYFLAG: flags,
+                constants.METAKEYSIZE: size}
+        self._threaddata.metacache = (node, meta)
+
+class remotecontentstore(object):
+    def __init__(self, ui, fileservice, shared):
+        self._fileservice = fileservice
+        # type(shared) is usually remotefilelogcontentstore
+        self._shared = shared
+
+    def get(self, name, node):
+        self._fileservice.prefetch([(name, hex(node))], force=True,
+                                   fetchdata=True)
+        return self._shared.get(name, node)
+
+    def getdelta(self, name, node):
+        revision = self.get(name, node)
+        return revision, name, nullid, self._shared.getmeta(name, node)
+
+    def getdeltachain(self, name, node):
+        # Since our remote content stores just contain full texts, we return a
+        # fake delta chain that just consists of a single full text revision.
+        # The nullid in the deltabasenode slot indicates that the revision is a
+        # fulltext.
+        revision = self.get(name, node)
+        return [(name, node, None, nullid, revision)]
+
+    def getmeta(self, name, node):
+        self._fileservice.prefetch([(name, hex(node))], force=True,
+                                   fetchdata=True)
+        return self._shared.getmeta(name, node)
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add to a remote store")
+
+    def getmissing(self, keys):
+        return keys
+
+    def markledger(self, ledger, options=None):
+        pass
+
+class manifestrevlogstore(object):
+    def __init__(self, repo):
+        self._store = repo.store
+        self._svfs = repo.svfs
+        self._revlogs = dict()
+        self._cl = revlog.revlog(self._svfs, '00changelog.i')
+        self._repackstartlinkrev = 0
+
+    def get(self, name, node):
+        return self._revlog(name).revision(node, raw=True)
+
+    def getdelta(self, name, node):
+        revision = self.get(name, node)
+        return revision, name, nullid, self.getmeta(name, node)
+
+    def getdeltachain(self, name, node):
+        revision = self.get(name, node)
+        return [(name, node, None, nullid, revision)]
+
+    def getmeta(self, name, node):
+        rl = self._revlog(name)
+        rev = rl.rev(node)
+        return {constants.METAKEYFLAG: rl.flags(rev),
+                constants.METAKEYSIZE: rl.rawsize(rev)}
+
+    def getancestors(self, name, node, known=None):
+        if known is None:
+            known = set()
+        if node in known:
+            return []
+
+        rl = self._revlog(name)
+        ancestors = {}
+        missing = set((node,))
+        for ancrev in rl.ancestors([rl.rev(node)], inclusive=True):
+            ancnode = rl.node(ancrev)
+            missing.discard(ancnode)
+
+            p1, p2 = rl.parents(ancnode)
+            if p1 != nullid and p1 not in known:
+                missing.add(p1)
+            if p2 != nullid and p2 not in known:
+                missing.add(p2)
+
+            linknode = self._cl.node(rl.linkrev(ancrev))
+            ancestors[rl.node(ancrev)] = (p1, p2, linknode, '')
+            if not missing:
+                break
+        return ancestors
+
+    def getnodeinfo(self, name, node):
+        cl = self._cl
+        rl = self._revlog(name)
+        parents = rl.parents(node)
+        linkrev = rl.linkrev(rl.rev(node))
+        return (parents[0], parents[1], cl.node(linkrev), None)
+
+    def add(self, *args):
+        raise RuntimeError("cannot add to a revlog store")
+
+    def _revlog(self, name):
+        rl = self._revlogs.get(name)
+        if rl is None:
+            revlogname = '00manifesttree.i'
+            if name != '':
+                revlogname = 'meta/%s/00manifest.i' % name
+            rl = revlog.revlog(self._svfs, revlogname)
+            self._revlogs[name] = rl
+        return rl
+
+    def getmissing(self, keys):
+        missing = []
+        for name, node in keys:
+            mfrevlog = self._revlog(name)
+            if node not in mfrevlog.nodemap:
+                missing.append((name, node))
+
+        return missing
+
+    def setrepacklinkrevrange(self, startrev, endrev):
+        self._repackstartlinkrev = startrev
+        self._repackendlinkrev = endrev
+
+    def markledger(self, ledger, options=None):
+        if options and options.get(constants.OPTION_PACKSONLY):
+            return
+        treename = ''
+        rl = revlog.revlog(self._svfs, '00manifesttree.i')
+        startlinkrev = self._repackstartlinkrev
+        endlinkrev = self._repackendlinkrev
+        for rev in pycompat.xrange(len(rl) - 1, -1, -1):
+            linkrev = rl.linkrev(rev)
+            if linkrev < startlinkrev:
+                break
+            if linkrev > endlinkrev:
+                continue
+            node = rl.node(rev)
+            ledger.markdataentry(self, treename, node)
+            ledger.markhistoryentry(self, treename, node)
+
+        for path, encoded, size in self._store.datafiles():
+            if path[:5] != 'meta/' or path[-2:] != '.i':
+                continue
+
+            treename = path[5:-len('/00manifest.i')]
+
+            rl = revlog.revlog(self._svfs, path)
+            for rev in pycompat.xrange(len(rl) - 1, -1, -1):
+                linkrev = rl.linkrev(rev)
+                if linkrev < startlinkrev:
+                    break
+                if linkrev > endlinkrev:
+                    continue
+                node = rl.node(rev)
+                ledger.markdataentry(self, treename, node)
+                ledger.markhistoryentry(self, treename, node)
+
+    def cleanup(self, ledger):
+        pass

hgext/remotefilelog/datapack.py

@@ -0,0 +1,460 b''
+from __future__ import absolute_import
+
+import struct
+import zlib
+
+from mercurial.node import hex, nullid
+from mercurial.i18n import _
+from mercurial import (
+    pycompat,
+    util,
+)
+from . import (
+    basepack,
+    constants,
+    shallowutil,
+)
+
+NODELENGTH = 20
+
+# The indicator value in the index for a fulltext entry.
+FULLTEXTINDEXMARK = -1
+NOBASEINDEXMARK = -2
+
+INDEXSUFFIX = '.dataidx'
+PACKSUFFIX = '.datapack'
+
+class datapackstore(basepack.basepackstore):
+    INDEXSUFFIX = INDEXSUFFIX
+    PACKSUFFIX = PACKSUFFIX
+
+    def __init__(self, ui, path):
+        super(datapackstore, self).__init__(ui, path)
+
+    def getpack(self, path):
+        return datapack(path)
+
+    def get(self, name, node):
+        raise RuntimeError("must use getdeltachain with datapackstore")
+
+    def getmeta(self, name, node):
+        for pack in self.packs:
+            try:
+                return pack.getmeta(name, node)
+            except KeyError:
+                pass
+
+        for pack in self.refresh():
+            try:
+                return pack.getmeta(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    def getdelta(self, name, node):
+        for pack in self.packs:
+            try:
+                return pack.getdelta(name, node)
+            except KeyError:
+                pass
+
+        for pack in self.refresh():
+            try:
+                return pack.getdelta(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    def getdeltachain(self, name, node):
+        for pack in self.packs:
+            try:
+                return pack.getdeltachain(name, node)
+            except KeyError:
+                pass
+
+        for pack in self.refresh():
+            try:
+                return pack.getdeltachain(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add to datapackstore")
+
+class datapack(basepack.basepack):
+    INDEXSUFFIX = INDEXSUFFIX
+    PACKSUFFIX = PACKSUFFIX
+
+    # Format is <node><delta offset><pack data offset><pack data size>
+    # See the mutabledatapack doccomment for more details.
+    INDEXFORMAT = '!20siQQ'
+    INDEXENTRYLENGTH = 40
+
+    SUPPORTED_VERSIONS = [2]
+
+    def getmissing(self, keys):
+        missing = []
+        for name, node in keys:
+            value = self._find(node)
+            if not value:
+                missing.append((name, node))
+
+        return missing
+
+    def get(self, name, node):
+        raise RuntimeError("must use getdeltachain with datapack (%s:%s)"
+                           % (name, hex(node)))
+
+    def getmeta(self, name, node):
+        value = self._find(node)
+        if value is None:
+            raise KeyError((name, hex(node)))
+
+        node, deltabaseoffset, offset, size = value
+        rawentry = self._data[offset:offset + size]
+
+        # see docstring of mutabledatapack for the format
+        offset = 0
+        offset += struct.unpack_from('!H', rawentry, offset)[0] + 2 # filename
+        offset += 40 # node, deltabase node
+        offset += struct.unpack_from('!Q', rawentry, offset)[0] + 8 # delta
+
+        metalen = struct.unpack_from('!I', rawentry, offset)[0]
+        offset += 4
+
+        meta = shallowutil.parsepackmeta(rawentry[offset:offset + metalen])
+
+        return meta
+
+    def getdelta(self, name, node):
+        value = self._find(node)
+        if value is None:
+            raise KeyError((name, hex(node)))
+
+        node, deltabaseoffset, offset, size = value
+        entry = self._readentry(offset, size, getmeta=True)
+        filename, node, deltabasenode, delta, meta = entry
+
+        # If we've read a lot of data from the mmap, free some memory.
+        self.freememory()
+
+        return delta, filename, deltabasenode, meta
+
+    def getdeltachain(self, name, node):
+        value = self._find(node)
+        if value is None:
+            raise KeyError((name, hex(node)))
+
+        params = self.params
+
+        # Precompute chains
+        chain = [value]
+        deltabaseoffset = value[1]
+        entrylen = self.INDEXENTRYLENGTH
+        while (deltabaseoffset != FULLTEXTINDEXMARK
+               and deltabaseoffset != NOBASEINDEXMARK):
+            loc = params.indexstart + deltabaseoffset
+            value = struct.unpack(self.INDEXFORMAT,
+                                  self._index[loc:loc + entrylen])
+            deltabaseoffset = value[1]
+            chain.append(value)
+
+        # Read chain data
+        deltachain = []
+        for node, deltabaseoffset, offset, size in chain:
+            filename, node, deltabasenode, delta = self._readentry(offset, size)
+            deltachain.append((filename, node, filename, deltabasenode, delta))
+
+        # If we've read a lot of data from the mmap, free some memory.
+        self.freememory()
+
+        return deltachain
+
+    def _readentry(self, offset, size, getmeta=False):
+        rawentry = self._data[offset:offset + size]
+        self._pagedin += len(rawentry)
+
+        # <2 byte len> + <filename>
+        lengthsize = 2
+        filenamelen = struct.unpack('!H', rawentry[:2])[0]
+        filename = rawentry[lengthsize:lengthsize + filenamelen]
+
+        # <20 byte node> + <20 byte deltabase>
+        nodestart = lengthsize + filenamelen
+        deltabasestart = nodestart + NODELENGTH
+        node = rawentry[nodestart:deltabasestart]
+        deltabasenode = rawentry[deltabasestart:deltabasestart + NODELENGTH]
+
+        # <8 byte len> + <delta>
+        deltastart = deltabasestart + NODELENGTH
+        rawdeltalen = rawentry[deltastart:deltastart + 8]
+        deltalen = struct.unpack('!Q', rawdeltalen)[0]
+
+        delta = rawentry[deltastart + 8:deltastart + 8 + deltalen]
+        delta = self._decompress(delta)
+
+        if getmeta:
+            metastart = deltastart + 8 + deltalen
+            metalen = struct.unpack_from('!I', rawentry, metastart)[0]
+
+            rawmeta = rawentry[metastart + 4:metastart + 4 + metalen]
+            meta = shallowutil.parsepackmeta(rawmeta)
+            return filename, node, deltabasenode, delta, meta
+        else:
+            return filename, node, deltabasenode, delta
+
+    def _decompress(self, data):
+        return zlib.decompress(data)
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add to datapack (%s:%s)" % (name, node))
+
+    def _find(self, node):
+        params = self.params
+        fanoutkey = struct.unpack(params.fanoutstruct,
+                                  node[:params.fanoutprefix])[0]
+        fanout = self._fanouttable
+
+        start = fanout[fanoutkey] + params.indexstart
+        indexend = self._indexend
+
+        # Scan forward to find the first non-same entry, which is the upper
+        # bound.
+        for i in pycompat.xrange(fanoutkey + 1, params.fanoutcount):
+            end = fanout[i] + params.indexstart
+            if end != start:
+                break
+        else:
+            end = indexend
+
+        # Bisect between start and end to find node
+        index = self._index
+        startnode = index[start:start + NODELENGTH]
+        endnode = index[end:end + NODELENGTH]
+        entrylen = self.INDEXENTRYLENGTH
+        if startnode == node:
+            entry = index[start:start + entrylen]
+        elif endnode == node:
+            entry = index[end:end + entrylen]
+        else:
+            while start < end - entrylen:
+                mid = start  + (end - start) / 2
+                mid = mid - ((mid - params.indexstart) % entrylen)
+                midnode = index[mid:mid + NODELENGTH]
+                if midnode == node:
+                    entry = index[mid:mid + entrylen]
+                    break
+                if node > midnode:
+                    start = mid
+                    startnode = midnode
+                elif node < midnode:
+                    end = mid
+                    endnode = midnode
+            else:
+                return None
+
+        return struct.unpack(self.INDEXFORMAT, entry)
+
+    def markledger(self, ledger, options=None):
+        for filename, node in self:
+            ledger.markdataentry(self, filename, node)
+
+    def cleanup(self, ledger):
+        entries = ledger.sources.get(self, [])
+        allkeys = set(self)
+        repackedkeys = set((e.filename, e.node) for e in entries if
+                           e.datarepacked or e.gced)
+
+        if len(allkeys - repackedkeys) == 0:
+            if self.path not in ledger.created:
+                util.unlinkpath(self.indexpath, ignoremissing=True)
+                util.unlinkpath(self.packpath, ignoremissing=True)
+
+    def __iter__(self):
+        for f, n, deltabase, deltalen in self.iterentries():
+            yield f, n
+
+    def iterentries(self):
+        # Start at 1 to skip the header
+        offset = 1
+        data = self._data
+        while offset < self.datasize:
+            oldoffset = offset
+
+            # <2 byte len> + <filename>
+            filenamelen = struct.unpack('!H', data[offset:offset + 2])[0]
+            offset += 2
+            filename = data[offset:offset + filenamelen]
+            offset += filenamelen
+
+            # <20 byte node>
+            node = data[offset:offset + constants.NODESIZE]
+            offset += constants.NODESIZE
+            # <20 byte deltabase>
+            deltabase = data[offset:offset + constants.NODESIZE]
+            offset += constants.NODESIZE
+
+            # <8 byte len> + <delta>
+            rawdeltalen = data[offset:offset + 8]
+            deltalen = struct.unpack('!Q', rawdeltalen)[0]
+            offset += 8
+
+            # TODO(augie): we should store a header that is the
+            # uncompressed size.
+            uncompressedlen = len(self._decompress(
+                data[offset:offset + deltalen]))
+            offset += deltalen
+
+            # <4 byte len> + <metadata-list>
+            metalen = struct.unpack_from('!I', data, offset)[0]
+            offset += 4 + metalen
+
+            yield (filename, node, deltabase, uncompressedlen)
+
+            # If we've read a lot of data from the mmap, free some memory.
+            self._pagedin += offset - oldoffset
+            if self.freememory():
+                data = self._data
+
+class mutabledatapack(basepack.mutablebasepack):
+    """A class for constructing and serializing a datapack file and index.
+
+    A datapack is a pair of files that contain the revision contents for various
+    file revisions in Mercurial. It contains only revision contents (like file
+    contents), not any history information.
+
+    It consists of two files, with the following format. All bytes are in
+    network byte order (big endian).
+
+    .datapack
+        The pack itself is a series of revision deltas with some basic header
+        information on each. A revision delta may be a fulltext, represented by
+        a deltabasenode equal to the nullid.
+
+        datapack = <version: 1 byte>
+                   [<revision>,...]
+        revision = <filename len: 2 byte unsigned int>
+                   <filename>
+                   <node: 20 byte>
+                   <deltabasenode: 20 byte>
+                   <delta len: 8 byte unsigned int>
+                   <delta>
+                   <metadata-list len: 4 byte unsigned int> [1]
+                   <metadata-list>                          [1]
+        metadata-list = [<metadata-item>, ...]
+        metadata-item = <metadata-key: 1 byte>
+                        <metadata-value len: 2 byte unsigned>
+                        <metadata-value>
+
+        metadata-key could be METAKEYFLAG or METAKEYSIZE or other single byte
+        value in the future.
+
+    .dataidx
+        The index file consists of two parts, the fanout and the index.
+
+        The index is a list of index entries, sorted by node (one per revision
+        in the pack). Each entry has:
+
+        - node (The 20 byte node of the entry; i.e. the commit hash, file node
+                hash, etc)
+        - deltabase index offset (The location in the index of the deltabase for
+                                  this entry. The deltabase is the next delta in
+                                  the chain, with the chain eventually
+                                  terminating in a full-text, represented by a
+                                  deltabase offset of -1. This lets us compute
+                                  delta chains from the index, then do
+                                  sequential reads from the pack if the revision
+                                  are nearby on disk.)
+        - pack entry offset (The location of this entry in the datapack)
+        - pack content size (The on-disk length of this entry's pack data)
+
+        The fanout is a quick lookup table to reduce the number of steps for
+        bisecting the index. It is a series of 4 byte pointers to positions
+        within the index. It has 2^16 entries, which corresponds to hash
+        prefixes [0000, 0001,..., FFFE, FFFF]. Example: the pointer in slot
+        4F0A points to the index position of the first revision whose node
+        starts with 4F0A. This saves log(2^16)=16 bisect steps.
+
+        dataidx = <fanouttable>
+                  <index>
+        fanouttable = [<index offset: 4 byte unsigned int>,...] (2^16 entries)
+        index = [<index entry>,...]
+        indexentry = <node: 20 byte>
+                     <deltabase location: 4 byte signed int>
+                     <pack entry offset: 8 byte unsigned int>
+                     <pack entry size: 8 byte unsigned int>
+
+    [1]: new in version 1.
+    """
+    INDEXSUFFIX = INDEXSUFFIX
+    PACKSUFFIX = PACKSUFFIX
+
+    # v[01] index format: <node><delta offset><pack data offset><pack data size>
+    INDEXFORMAT = datapack.INDEXFORMAT
+    INDEXENTRYLENGTH = datapack.INDEXENTRYLENGTH
+
+    # v1 has metadata support
+    SUPPORTED_VERSIONS = [2]
+
+    def _compress(self, data):
+        return zlib.compress(data)
+
+    def add(self, name, node, deltabasenode, delta, metadata=None):
+        # metadata is a dict, ex. {METAKEYFLAG: flag}
+        if len(name) > 2**16:
+            raise RuntimeError(_("name too long %s") % name)
+        if len(node) != 20:
+            raise RuntimeError(_("node should be 20 bytes %s") % node)
+
+        if node in self.entries:
+            # The revision has already been added
+            return
+
+        # TODO: allow configurable compression
+        delta = self._compress(delta)
+
+        rawdata = ''.join((
+            struct.pack('!H', len(name)), # unsigned 2 byte int
+            name,
+            node,
+            deltabasenode,
+            struct.pack('!Q', len(delta)), # unsigned 8 byte int
+            delta,
+        ))
+
+        # v1 support metadata
+        rawmeta = shallowutil.buildpackmeta(metadata)
+        rawdata += struct.pack('!I', len(rawmeta)) # unsigned 4 byte
+        rawdata += rawmeta
+
+        offset = self.packfp.tell()
+
+        size = len(rawdata)
+
+        self.entries[node] = (deltabasenode, offset, size)
+
+        self.writeraw(rawdata)
+
+    def createindex(self, nodelocations, indexoffset):
+        entries = sorted((n, db, o, s) for n, (db, o, s)
+                         in self.entries.iteritems())
+
+        rawindex = ''
+        fmt = self.INDEXFORMAT
+        for node, deltabase, offset, size in entries:
+            if deltabase == nullid:
+                deltabaselocation = FULLTEXTINDEXMARK
+            else:
+                # Instead of storing the deltabase node in the index, let's
+                # store a pointer directly to the index entry for the deltabase.
+                deltabaselocation = nodelocations.get(deltabase,
+                                                      NOBASEINDEXMARK)
+
+            entry = struct.pack(fmt, node, deltabaselocation, offset, size)
+            rawindex += entry
+
+        return rawindex

hgext/remotefilelog/debugcommands.py

@@ -0,0 +1,378 b''
+# debugcommands.py - debug logic for remotefilelog
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
+
+import hashlib
+import os
+import zlib
+
+from mercurial.node import bin, hex, nullid, short
+from mercurial.i18n import _
+from mercurial import (
+    error,
+    filelog,
+    node as nodemod,
+    revlog,
+)
+from . import (
+    constants,
+    datapack,
+    extutil,
+    fileserverclient,
+    historypack,
+    repack,
+    shallowutil,
+)
+
+def debugremotefilelog(ui, path, **opts):
+    decompress = opts.get(r'decompress')
+
+    size, firstnode, mapping = parsefileblob(path, decompress)
+
+    ui.status(_("size: %d bytes\n") % (size))
+    ui.status(_("path: %s \n") % (path))
+    ui.status(_("key: %s \n") % (short(firstnode)))
+    ui.status(_("\n"))
+    ui.status(_("%12s => %12s %13s %13s %12s\n") %
+              ("node", "p1", "p2", "linknode", "copyfrom"))
+
+    queue = [firstnode]
+    while queue:
+        node = queue.pop(0)
+        p1, p2, linknode, copyfrom = mapping[node]
+        ui.status(_("%s => %s  %s  %s  %s\n") %
+            (short(node), short(p1), short(p2), short(linknode), copyfrom))
+        if p1 != nullid:
+            queue.append(p1)
+        if p2 != nullid:
+            queue.append(p2)
+
+def buildtemprevlog(repo, file):
+    # get filename key
+    filekey = nodemod.hex(hashlib.sha1(file).digest())
+    filedir = os.path.join(repo.path, 'store/data', filekey)
+
+    # sort all entries based on linkrev
+    fctxs = []
+    for filenode in os.listdir(filedir):
+        if '_old' not in filenode:
+            fctxs.append(repo.filectx(file, fileid=bin(filenode)))
+
+    fctxs = sorted(fctxs, key=lambda x: x.linkrev())
+
+    # add to revlog
+    temppath = repo.sjoin('data/temprevlog.i')
+    if os.path.exists(temppath):
+        os.remove(temppath)
+    r = filelog.filelog(repo.svfs, 'temprevlog')
+
+    class faket(object):
+        def add(self, a, b, c):
+            pass
+    t = faket()
+    for fctx in fctxs:
+        if fctx.node() not in repo:
+            continue
+
+        p = fctx.filelog().parents(fctx.filenode())
+        meta = {}
+        if fctx.renamed():
+            meta['copy'] = fctx.renamed()[0]
+            meta['copyrev'] = hex(fctx.renamed()[1])
+
+        r.add(fctx.data(), meta, t, fctx.linkrev(), p[0], p[1])
+
+    return r
+
+def debugindex(orig, ui, repo, file_=None, **opts):
+    """dump the contents of an index file"""
+    if (opts.get(r'changelog') or
+        opts.get(r'manifest') or
+        opts.get(r'dir') or
+        not shallowutil.isenabled(repo) or
+        not repo.shallowmatch(file_)):
+        return orig(ui, repo, file_, **opts)
+
+    r = buildtemprevlog(repo, file_)
+
+    # debugindex like normal
+    format = opts.get('format', 0)
+    if format not in (0, 1):
+        raise error.Abort(_("unknown format %d") % format)
+
+    generaldelta = r.version & revlog.FLAG_GENERALDELTA
+    if generaldelta:
+        basehdr = ' delta'
+    else:
+        basehdr = '  base'
+
+    if format == 0:
+        ui.write(("   rev    offset  length " + basehdr + " linkrev"
+                  " nodeid       p1           p2\n"))
+    elif format == 1:
+        ui.write(("   rev flag   offset   length"
+                  "     size " + basehdr + "   link     p1     p2"
+                  "       nodeid\n"))
+
+    for i in r:
+        node = r.node(i)
+        if generaldelta:
+            base = r.deltaparent(i)
+        else:
+            base = r.chainbase(i)
+        if format == 0:
+            try:
+                pp = r.parents(node)
+            except Exception:
+                pp = [nullid, nullid]
+            ui.write("% 6d % 9d % 7d % 6d % 7d %s %s %s\n" % (
+                    i, r.start(i), r.length(i), base, r.linkrev(i),
+                    short(node), short(pp[0]), short(pp[1])))
+        elif format == 1:
+            pr = r.parentrevs(i)
+            ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % (
+                    i, r.flags(i), r.start(i), r.length(i), r.rawsize(i),
+                    base, r.linkrev(i), pr[0], pr[1], short(node)))
+
+def debugindexdot(orig, ui, repo, file_):
+    """dump an index DAG as a graphviz dot file"""
+    if not shallowutil.isenabled(repo):
+        return orig(ui, repo, file_)
+
+    r = buildtemprevlog(repo, os.path.basename(file_)[:-2])
+
+    ui.write(("digraph G {\n"))
+    for i in r:
+        node = r.node(i)
+        pp = r.parents(node)
+        ui.write("\t%d -> %d\n" % (r.rev(pp[0]), i))
+        if pp[1] != nullid:
+            ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i))
+    ui.write("}\n")
+
+def verifyremotefilelog(ui, path, **opts):
+    decompress = opts.get(r'decompress')
+
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            if file == "repos":
+                continue
+            filepath = os.path.join(root, file)
+            size, firstnode, mapping = parsefileblob(filepath, decompress)
+            for p1, p2, linknode, copyfrom in mapping.itervalues():
+                if linknode == nullid:
+                    actualpath = os.path.relpath(root, path)
+                    key = fileserverclient.getcachekey("reponame", actualpath,
+                                                       file)
+                    ui.status("%s %s\n" % (key, os.path.relpath(filepath,
+                                                                path)))
+
+def _decompressblob(raw):
+    return zlib.decompress(raw)
+
+def parsefileblob(path, decompress):
+    raw = None
+    f = open(path, "rb")
+    try:
+        raw = f.read()
+    finally:
+        f.close()
+
+    if decompress:
+        raw = _decompressblob(raw)
+
+    offset, size, flags = shallowutil.parsesizeflags(raw)
+    start = offset + size
+
+    firstnode = None
+
+    mapping = {}
+    while start < len(raw):
+        divider = raw.index('\0', start + 80)
+
+        currentnode = raw[start:(start + 20)]
+        if not firstnode:
+            firstnode = currentnode
+
+        p1 = raw[(start + 20):(start + 40)]
+        p2 = raw[(start + 40):(start + 60)]
+        linknode = raw[(start + 60):(start + 80)]
+        copyfrom = raw[(start + 80):divider]
+
+        mapping[currentnode] = (p1, p2, linknode, copyfrom)
+        start = divider + 1
+
+    return size, firstnode, mapping
+
+def debugdatapack(ui, *paths, **opts):
+    for path in paths:
+        if '.data' in path:
+            path = path[:path.index('.data')]
+        ui.write("%s:\n" % path)
+        dpack = datapack.datapack(path)
+        node = opts.get(r'node')
+        if node:
+            deltachain = dpack.getdeltachain('', bin(node))
+            dumpdeltachain(ui, deltachain, **opts)
+            return
+
+        if opts.get(r'long'):
+            hashformatter = hex
+            hashlen = 42
+        else:
+            hashformatter = short
+            hashlen = 14
+
+        lastfilename = None
+        totaldeltasize = 0
+        totalblobsize = 0
+        def printtotals():
+            if lastfilename is not None:
+                ui.write("\n")
+            if not totaldeltasize or not totalblobsize:
+                return
+            difference = totalblobsize - totaldeltasize
+            deltastr = "%0.1f%% %s" % (
+                (100.0 * abs(difference) / totalblobsize),
+                ("smaller" if difference > 0 else "bigger"))
+
+            ui.write(("Total:%s%s  %s (%s)\n") % (
+                "".ljust(2 * hashlen - len("Total:")),
+                ('%d' % totaldeltasize).ljust(12),
+                ('%d' % totalblobsize).ljust(9),
+                deltastr
+            ))
+
+        bases = {}
+        nodes = set()
+        failures = 0
+        for filename, node, deltabase, deltalen in dpack.iterentries():
+            bases[node] = deltabase
+            if node in nodes:
+                ui.write(("Bad entry: %s appears twice\n" % short(node)))
+                failures += 1
+            nodes.add(node)
+            if filename != lastfilename:
+                printtotals()
+                name = '(empty name)' if filename == '' else filename
+                ui.write("%s:\n" % name)
+                ui.write("%s%s%s%s\n" % (
+                    "Node".ljust(hashlen),
+                    "Delta Base".ljust(hashlen),
+                    "Delta Length".ljust(14),
+                    "Blob Size".ljust(9)))
+                lastfilename = filename
+                totalblobsize = 0
+                totaldeltasize = 0
+
+            # Metadata could be missing, in which case it will be an empty dict.
+            meta = dpack.getmeta(filename, node)
+            if constants.METAKEYSIZE in meta:
+                blobsize = meta[constants.METAKEYSIZE]
+                totaldeltasize += deltalen
+                totalblobsize += blobsize
+            else:
+                blobsize = "(missing)"
+            ui.write("%s  %s  %s%d\n" % (
+                hashformatter(node),
+                hashformatter(deltabase),
+                ('%d' % deltalen).ljust(14),
+                blobsize))
+
+        if filename is not None:
+            printtotals()
+
+        failures += _sanitycheck(ui, set(nodes), bases)
+        if failures > 1:
+            ui.warn(("%d failures\n" % failures))
+            return 1
+
+def _sanitycheck(ui, nodes, bases):
+    """
+    Does some basic sanity checking on a packfiles with ``nodes`` ``bases`` (a
+    mapping of node->base):
+
+    - Each deltabase must itself be a node elsewhere in the pack
+    - There must be no cycles
+    """
+    failures = 0
+    for node in nodes:
+        seen = set()
+        current = node
+        deltabase = bases[current]
+
+        while deltabase != nullid:
+            if deltabase not in nodes:
+                ui.warn(("Bad entry: %s has an unknown deltabase (%s)\n" %
+                        (short(node), short(deltabase))))
+                failures += 1
+                break
+
+            if deltabase in seen:
+                ui.warn(("Bad entry: %s has a cycle (at %s)\n" %
+                        (short(node), short(deltabase))))
+                failures += 1
+                break
+
+            current = deltabase
+            seen.add(current)
+            deltabase = bases[current]
+        # Since ``node`` begins a valid chain, reset/memoize its base to nullid
+        # so we don't traverse it again.
+        bases[node] = nullid
+    return failures
+
+def dumpdeltachain(ui, deltachain, **opts):
+    hashformatter = hex
+    hashlen = 40
+
+    lastfilename = None
+    for filename, node, filename, deltabasenode, delta in deltachain:
+        if filename != lastfilename:
+            ui.write("\n%s\n" % filename)
+            lastfilename = filename
+        ui.write("%s  %s  %s  %s\n" % (
+            "Node".ljust(hashlen),
+            "Delta Base".ljust(hashlen),
+            "Delta SHA1".ljust(hashlen),
+            "Delta Length".ljust(6),
+        ))
+
+        ui.write("%s  %s  %s  %d\n" % (
+            hashformatter(node),
+            hashformatter(deltabasenode),
+            nodemod.hex(hashlib.sha1(delta).digest()),
+            len(delta)))
+
+def debughistorypack(ui, path):
+    if '.hist' in path:
+        path = path[:path.index('.hist')]
+    hpack = historypack.historypack(path)
+
+    lastfilename = None
+    for entry in hpack.iterentries():
+        filename, node, p1node, p2node, linknode, copyfrom = entry
+        if filename != lastfilename:
+            ui.write("\n%s\n" % filename)
+            ui.write("%s%s%s%s%s\n" % (
+                "Node".ljust(14),
+                "P1 Node".ljust(14),
+                "P2 Node".ljust(14),
+                "Link Node".ljust(14),
+                "Copy From"))
+            lastfilename = filename
+        ui.write("%s  %s  %s  %s  %s\n" % (short(node), short(p1node),
+            short(p2node), short(linknode), copyfrom))
+
+def debugwaitonrepack(repo):
+    with extutil.flock(repack.repacklockvfs(repo).join('repacklock'), ''):
+        return
+
+def debugwaitonprefetch(repo):
+    with repo._lock(repo.svfs, "prefetchlock", True, None,
+                         None, _('prefetching in %s') % repo.origroot):
+        pass

hgext/remotefilelog/extutil.py

@@ -0,0 +1,66 b''
+# extutil.py - useful utility methods for extensions
+#
+# Copyright 2016 Facebook
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import os
+import time
+
+from mercurial import (
+    error,
+    lock as lockmod,
+    util,
+    vfs as vfsmod,
+)
+
[e-mail]
+def flock(lockpath, description, timeout=-1):
+    """A flock based lock object. Currently it is always non-blocking.
+
+    Note that since it is flock based, you can accidentally take it multiple
+    times within one process and the first one to be released will release all
+    of them. So the caller needs to be careful to not create more than one
+    instance per lock.
+    """
+
+    # best effort lightweight lock
+    try:
+        import fcntl
+        fcntl.flock
+    except ImportError:
+        # fallback to Mercurial lock
+        vfs = vfsmod.vfs(os.path.dirname(lockpath))
+        with lockmod.lock(vfs, os.path.basename(lockpath), timeout=timeout):
+            yield
+        return
+    # make sure lock file exists
+    util.makedirs(os.path.dirname(lockpath))
+    with open(lockpath, 'a'):
+        pass
+    lockfd = os.open(lockpath, os.O_RDONLY, 0o664)
+    start = time.time()
+    while True:
+        try:
+            fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            break
+        except IOError as ex:
+            if ex.errno == errno.EAGAIN:
+                if timeout != -1 and time.time() - start > timeout:
+                    raise error.LockHeld(errno.EAGAIN, lockpath, description,
+                                         '')
+                else:
+                    time.sleep(0.05)
+                    continue
+            raise
+
+    try:
+        yield
+    finally:
+        fcntl.flock(lockfd, fcntl.LOCK_UN)
+        os.close(lockfd)

hgext/remotefilelog/fileserverclient.py

@@ -0,0 +1,581 b''
+# fileserverclient.py - client for communicating with the cache process
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import hashlib
+import io
+import os
+import threading
+import time
+import zlib
+
+from mercurial.i18n import _
+from mercurial.node import bin, hex, nullid
+from mercurial import (
+    error,
+    node,
+    pycompat,
+    revlog,
+    sshpeer,
+    util,
+    wireprotov1peer,
+)
+from mercurial.utils import procutil
+
+from . import (
+    constants,
+    contentstore,
+    metadatastore,
+)
+
+_sshv1peer = sshpeer.sshv1peer
+
+# Statistics for debugging
+fetchcost = 0
+fetches = 0
+fetched = 0
+fetchmisses = 0
+
+_lfsmod = None
+
+def getcachekey(reponame, file, id):
+    pathhash = node.hex(hashlib.sha1(file).digest())
+    return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
+
+def getlocalkey(file, id):
+    pathhash = node.hex(hashlib.sha1(file).digest())
+    return os.path.join(pathhash, id)
+
+def peersetup(ui, peer):
+
+    class remotefilepeer(peer.__class__):
+        @wireprotov1peer.batchable
+        def x_rfl_getfile(self, file, node):
+            if not self.capable('x_rfl_getfile'):
+                raise error.Abort(
+                    'configured remotefile server does not support getfile')
+            f = wireprotov1peer.future()
+            yield {'file': file, 'node': node}, f
+            code, data = f.value.split('\0', 1)
+            if int(code):
+                raise error.LookupError(file, node, data)
+            yield data
+
+        @wireprotov1peer.batchable
+        def x_rfl_getflogheads(self, path):
+            if not self.capable('x_rfl_getflogheads'):
+                raise error.Abort('configured remotefile server does not '
+                                  'support getflogheads')
+            f = wireprotov1peer.future()
+            yield {'path': path}, f
+            heads = f.value.split('\n') if f.value else []
+            yield heads
+
+        def _updatecallstreamopts(self, command, opts):
+            if command != 'getbundle':
+                return
+            if (constants.NETWORK_CAP_LEGACY_SSH_GETFILES
+                not in self.capabilities()):
+                return
+            if not util.safehasattr(self, '_localrepo'):
+                return
+            if (constants.SHALLOWREPO_REQUIREMENT
+                not in self._localrepo.requirements):
+                return
+
+            bundlecaps = opts.get('bundlecaps')
+            if bundlecaps:
+                bundlecaps = [bundlecaps]
+            else:
+                bundlecaps = []
+
+            # shallow, includepattern, and excludepattern are a hacky way of
+            # carrying over data from the local repo to this getbundle
+            # command. We need to do it this way because bundle1 getbundle
+            # doesn't provide any other place we can hook in to manipulate
+            # getbundle args before it goes across the wire. Once we get rid
+            # of bundle1, we can use bundle2's _pullbundle2extraprepare to
+            # do this more cleanly.
+            bundlecaps.append(constants.BUNDLE2_CAPABLITY)
+            if self._localrepo.includepattern:
+                patterns = '\0'.join(self._localrepo.includepattern)
+                includecap = "includepattern=" + patterns
+                bundlecaps.append(includecap)
+            if self._localrepo.excludepattern:
+                patterns = '\0'.join(self._localrepo.excludepattern)
+                excludecap = "excludepattern=" + patterns
+                bundlecaps.append(excludecap)
+            opts['bundlecaps'] = ','.join(bundlecaps)
+
+        def _sendrequest(self, command, args, **opts):
+            self._updatecallstreamopts(command, args)
+            return super(remotefilepeer, self)._sendrequest(command, args,
+                                                            **opts)
+
+        def _callstream(self, command, **opts):
+            supertype = super(remotefilepeer, self)
+            if not util.safehasattr(supertype, '_sendrequest'):
+                self._updatecallstreamopts(command, pycompat.byteskwargs(opts))
+            return super(remotefilepeer, self)._callstream(command, **opts)
+
+    peer.__class__ = remotefilepeer
+
+class cacheconnection(object):
+    """The connection for communicating with the remote cache. Performs
+    gets and sets by communicating with an external process that has the
+    cache-specific implementation.
+    """
+    def __init__(self):
+        self.pipeo = self.pipei = self.pipee = None
+        self.subprocess = None
+        self.connected = False
+
+    def connect(self, cachecommand):
+        if self.pipeo:
+            raise error.Abort(_("cache connection already open"))
+        self.pipei, self.pipeo, self.pipee, self.subprocess = \
+            procutil.popen4(cachecommand)
+        self.connected = True
+
+    def close(self):
+        def tryclose(pipe):
+            try:
+                pipe.close()
+            except Exception:
+                pass
+        if self.connected:
+            try:
+                self.pipei.write("exit\n")
+            except Exception:
+                pass
+            tryclose(self.pipei)
+            self.pipei = None
+            tryclose(self.pipeo)
+            self.pipeo = None
+            tryclose(self.pipee)
+            self.pipee = None
+            try:
+                # Wait for process to terminate, making sure to avoid deadlock.
+                # See https://docs.python.org/2/library/subprocess.html for
+                # warnings about wait() and deadlocking.
+                self.subprocess.communicate()
+            except Exception:
+                pass
+            self.subprocess = None
+        self.connected = False
+
+    def request(self, request, flush=True):
+        if self.connected:
+            try:
+                self.pipei.write(request)
+                if flush:
+                    self.pipei.flush()
+            except IOError:
+                self.close()
+
+    def receiveline(self):
+        if not self.connected:
+            return None
+        try:
+            result = self.pipeo.readline()[:-1]
+            if not result:
+                self.close()
+        except IOError:
+            self.close()
+
+        return result
+
+def _getfilesbatch(
+        remote, receivemissing, progresstick, missed, idmap, batchsize):
+    # Over http(s), iterbatch is a streamy method and we can start
+    # looking at results early. This means we send one (potentially
+    # large) request, but then we show nice progress as we process
+    # file results, rather than showing chunks of $batchsize in
+    # progress.
+    #
+    # Over ssh, iterbatch isn't streamy because batch() wasn't
+    # explicitly designed as a streaming method. In the future we
+    # should probably introduce a streambatch() method upstream and
+    # use that for this.
+    with remote.commandexecutor() as e:
+        futures = []
+        for m in missed:
+            futures.append(e.callcommand('x_rfl_getfile', {
+                'file': idmap[m],
+                'node': m[-40:]
+            }))
+
+        for i, m in enumerate(missed):
+            r = futures[i].result()
+            futures[i] = None  # release memory
+            file_ = idmap[m]
+            node = m[-40:]
+            receivemissing(io.BytesIO('%d\n%s' % (len(r), r)), file_, node)
+            progresstick()
+
+def _getfiles_optimistic(
+    remote, receivemissing, progresstick, missed, idmap, step):
+    remote._callstream("x_rfl_getfiles")
+    i = 0
+    pipeo = remote._pipeo
+    pipei = remote._pipei
+    while i < len(missed):
+        # issue a batch of requests
+        start = i
+        end = min(len(missed), start + step)
+        i = end
+        for missingid in missed[start:end]:
+            # issue new request
+            versionid = missingid[-40:]
+            file = idmap[missingid]
+            sshrequest = "%s%s\n" % (versionid, file)
+            pipeo.write(sshrequest)
+        pipeo.flush()
+
+        # receive batch results
+        for missingid in missed[start:end]:
+            versionid = missingid[-40:]
+            file = idmap[missingid]
+            receivemissing(pipei, file, versionid)
+            progresstick()
+
+    # End the command
+    pipeo.write('\n')
+    pipeo.flush()
+
+def _getfiles_threaded(
+    remote, receivemissing, progresstick, missed, idmap, step):
+    remote._callstream("getfiles")
+    pipeo = remote._pipeo
+    pipei = remote._pipei
+
+    def writer():
+        for missingid in missed:
+            versionid = missingid[-40:]
+            file = idmap[missingid]
+            sshrequest = "%s%s\n" % (versionid, file)
+            pipeo.write(sshrequest)
+        pipeo.flush()
+    writerthread = threading.Thread(target=writer)
+    writerthread.daemon = True
+    writerthread.start()
+
+    for missingid in missed:
+        versionid = missingid[-40:]
+        file = idmap[missingid]
+        receivemissing(pipei, file, versionid)
+        progresstick()
+
+    writerthread.join()
+    # End the command
+    pipeo.write('\n')
+    pipeo.flush()
+
+class fileserverclient(object):
+    """A client for requesting files from the remote file server.
+    """
+    def __init__(self, repo):
+        ui = repo.ui
+        self.repo = repo
+        self.ui = ui
+        self.cacheprocess = ui.config("remotefilelog", "cacheprocess")
+        if self.cacheprocess:
+            self.cacheprocess = util.expandpath(self.cacheprocess)
+
+        # This option causes remotefilelog to pass the full file path to the
+        # cacheprocess instead of a hashed key.
+        self.cacheprocesspasspath = ui.configbool(
+            "remotefilelog", "cacheprocess.includepath")
+
+        self.debugoutput = ui.configbool("remotefilelog", "debug")
+
+        self.remotecache = cacheconnection()
+
+    def setstore(self, datastore, historystore, writedata, writehistory):
+        self.datastore = datastore
+        self.historystore = historystore
+        self.writedata = writedata
+        self.writehistory = writehistory
+
+    def _connect(self):
+        return self.repo.connectionpool.get(self.repo.fallbackpath)
+
+    def request(self, fileids):
+        """Takes a list of filename/node pairs and fetches them from the
+        server. Files are stored in the local cache.
+        A list of nodes that the server couldn't find is returned.
+        If the connection fails, an exception is raised.
+        """
+        if not self.remotecache.connected:
+            self.connect()
+        cache = self.remotecache
+        writedata = self.writedata
+
+        repo = self.repo
+        total = len(fileids)
+        request = "get\n%d\n" % total
+        idmap = {}
+        reponame = repo.name
+        for file, id in fileids:
+            fullid = getcachekey(reponame, file, id)
+            if self.cacheprocesspasspath:
+                request += file + '\0'
+            request += fullid + "\n"
+            idmap[fullid] = file
+
+        cache.request(request)
+
+        progress = self.ui.makeprogress(_('downloading'), total=total)
+        progress.update(0)
+
+        missed = []
+        while True:
+            missingid = cache.receiveline()
+            if not missingid:
+                missedset = set(missed)
+                for missingid in idmap:
+                    if not missingid in missedset:
+                        missed.append(missingid)
+                self.ui.warn(_("warning: cache connection closed early - " +
+                    "falling back to server\n"))
+                break
+            if missingid == "0":
+                break
+            if missingid.startswith("_hits_"):
+                # receive progress reports
+                parts = missingid.split("_")
+                progress.increment(int(parts[2]))
+                continue
+
+            missed.append(missingid)
+
+        global fetchmisses
+        fetchmisses += len(missed)
+
+        fromcache = total - len(missed)
+        progress.update(fromcache, total=total)
+        self.ui.log("remotefilelog", "remote cache hit rate is %r of %r\n",
+                    fromcache, total, hit=fromcache, total=total)
+
+        oldumask = os.umask(0o002)
+        try:
+            # receive cache misses from master
+            if missed:
+                # When verbose is true, sshpeer prints 'running ssh...'
+                # to stdout, which can interfere with some command
+                # outputs
+                verbose = self.ui.verbose
+                self.ui.verbose = False
+                try:
+                    with self._connect() as conn:
+                        remote = conn.peer
+                        if remote.capable(
+                                constants.NETWORK_CAP_LEGACY_SSH_GETFILES):
+                            if not isinstance(remote, _sshv1peer):
+                                raise error.Abort('remotefilelog requires ssh '
+                                                  'servers')
+                            step = self.ui.configint('remotefilelog',
+                                                     'getfilesstep')
+                            getfilestype = self.ui.config('remotefilelog',
+                                                          'getfilestype')
+                            if getfilestype == 'threaded':
+                                _getfiles = _getfiles_threaded
+                            else:
+                                _getfiles = _getfiles_optimistic
+                            _getfiles(remote, self.receivemissing,
+                                      progress.increment, missed, idmap, step)
+                        elif remote.capable("x_rfl_getfile"):
+                            if remote.capable('batch'):
+                                batchdefault = 100
+                            else:
+                                batchdefault = 10
+                            batchsize = self.ui.configint(
+                                'remotefilelog', 'batchsize', batchdefault)
+                            _getfilesbatch(
+                                remote, self.receivemissing, progress.increment,
+                                missed, idmap, batchsize)
+                        else:
+                            raise error.Abort("configured remotefilelog server"
+                                             " does not support remotefilelog")
+
+                    self.ui.log("remotefilefetchlog",
+                                "Success\n",
+                                fetched_files = progress.pos - fromcache,
+                                total_to_fetch = total - fromcache)
+                except Exception:
+                    self.ui.log("remotefilefetchlog",
+                                "Fail\n",
+                                fetched_files = progress.pos - fromcache,
+                                total_to_fetch = total - fromcache)
+                    raise
+                finally:
+                    self.ui.verbose = verbose
+                # send to memcache
+                request = "set\n%d\n%s\n" % (len(missed), "\n".join(missed))
+                cache.request(request)
+
+            progress.complete()
+
+            # mark ourselves as a user of this cache
+            writedata.markrepo(self.repo.path)
+        finally:
+            os.umask(oldumask)
+
+    def receivemissing(self, pipe, filename, node):
+        line = pipe.readline()[:-1]
+        if not line:
+            raise error.ResponseError(_("error downloading file contents:"),
+                                      _("connection closed early"))
+        size = int(line)
+        data = pipe.read(size)
+        if len(data) != size:
+            raise error.ResponseError(_("error downloading file contents:"),
+                                      _("only received %s of %s bytes")
+                                      % (len(data), size))
+
+        self.writedata.addremotefilelognode(filename, bin(node),
+                                             zlib.decompress(data))
+
+    def connect(self):
+        if self.cacheprocess:
+            cmd = "%s %s" % (self.cacheprocess, self.writedata._path)
+            self.remotecache.connect(cmd)
+        else:
+            # If no cache process is specified, we fake one that always
+            # returns cache misses.  This enables tests to run easily
+            # and may eventually allow us to be a drop in replacement
+            # for the largefiles extension.
+            class simplecache(object):
+                def __init__(self):
+                    self.missingids = []
+                    self.connected = True
+
+                def close(self):
+                    pass
+
+                def request(self, value, flush=True):
+                    lines = value.split("\n")
+                    if lines[0] != "get":
+                        return
+                    self.missingids = lines[2:-1]
+                    self.missingids.append('0')
+
+                def receiveline(self):
+                    if len(self.missingids) > 0:
+                        return self.missingids.pop(0)
+                    return None
+
+            self.remotecache = simplecache()
+
+    def close(self):
+        if fetches:
+            msg = ("%d files fetched over %d fetches - " +
+                   "(%d misses, %0.2f%% hit ratio) over %0.2fs\n") % (
+                       fetched,
+                       fetches,
+                       fetchmisses,
+                       float(fetched - fetchmisses) / float(fetched) * 100.0,
+                       fetchcost)
+            if self.debugoutput:
+                self.ui.warn(msg)
+            self.ui.log("remotefilelog.prefetch", msg.replace("%", "%%"),
+                remotefilelogfetched=fetched,
+                remotefilelogfetches=fetches,
+                remotefilelogfetchmisses=fetchmisses,
+                remotefilelogfetchtime=fetchcost * 1000)
+
+        if self.remotecache.connected:
+            self.remotecache.close()
+
+    def prefetch(self, fileids, force=False, fetchdata=True,
+                 fetchhistory=False):
+        """downloads the given file versions to the cache
+        """
+        repo = self.repo
+        idstocheck = []
+        for file, id in fileids:
+            # hack
+            # - we don't use .hgtags
+            # - workingctx produces ids with length 42,
+            #   which we skip since they aren't in any cache
+            if (file == '.hgtags' or len(id) == 42
+                or not repo.shallowmatch(file)):
+                continue
+
+            idstocheck.append((file, bin(id)))
+
+        datastore = self.datastore
+        historystore = self.historystore
+        if force:
+            datastore = contentstore.unioncontentstore(*repo.shareddatastores)
+            historystore = metadatastore.unionmetadatastore(
+                *repo.sharedhistorystores)
+
+        missingids = set()
+        if fetchdata:
+            missingids.update(datastore.getmissing(idstocheck))
+        if fetchhistory:
+            missingids.update(historystore.getmissing(idstocheck))
+
+        # partition missing nodes into nullid and not-nullid so we can
+        # warn about this filtering potentially shadowing bugs.
+        nullids = len([None for unused, id in missingids if id == nullid])
+        if nullids:
+            missingids = [(f, id) for f, id in missingids if id != nullid]
+            repo.ui.develwarn(
+                ('remotefilelog not fetching %d null revs'
+                 ' - this is likely hiding bugs' % nullids),
+                config='remotefilelog-ext')
+        if missingids:
+            global fetches, fetched, fetchcost
+            fetches += 1
+
+            # We want to be able to detect excess individual file downloads, so
+            # let's log that information for debugging.
+            if fetches >= 15 and fetches < 18:
+                if fetches == 15:
+                    fetchwarning = self.ui.config('remotefilelog',
+                                                  'fetchwarning')
+                    if fetchwarning:
+                        self.ui.warn(fetchwarning + '\n')
+                self.logstacktrace()
+            missingids = [(file, hex(id)) for file, id in missingids]
+            fetched += len(missingids)
+            start = time.time()
+            missingids = self.request(missingids)
+            if missingids:
+                raise error.Abort(_("unable to download %d files") %
+                                  len(missingids))
+            fetchcost += time.time() - start
+            self._lfsprefetch(fileids)
+
+    def _lfsprefetch(self, fileids):
+        if not _lfsmod or not util.safehasattr(
+                self.repo.svfs, 'lfslocalblobstore'):
+            return
+        if not _lfsmod.wrapper.candownload(self.repo):
+            return
+        pointers = []
+        store = self.repo.svfs.lfslocalblobstore
+        for file, id in fileids:
+            node = bin(id)
+            rlog = self.repo.file(file)
+            if rlog.flags(node) & revlog.REVIDX_EXTSTORED:
+                text = rlog.revision(node, raw=True)
+                p = _lfsmod.pointer.deserialize(text)
+                oid = p.oid()
+                if not store.has(oid):
+                    pointers.append(p)
+        if len(pointers) > 0:
+            self.repo.svfs.lfsremoteblobstore.readbatch(pointers, store)
+            assert all(store.has(p.oid()) for p in pointers)
+
+    def logstacktrace(self):
+        import traceback
+        self.ui.log('remotefilelog', 'excess remotefilelog fetching:\n%s\n',
+                    ''.join(traceback.format_stack()))

hgext/remotefilelog/historypack.py

@@ -0,0 +1,520 b''
+from __future__ import absolute_import
+
+import hashlib
+import struct
+
+from mercurial.node import hex, nullid
+from mercurial import (
+    pycompat,
+    util,
+)
+from . import (
+    basepack,
+    constants,
+    shallowutil,
+)
+
+# (filename hash, offset, size)
+INDEXFORMAT2 = '!20sQQII'
+INDEXENTRYLENGTH2 = struct.calcsize(INDEXFORMAT2)
+NODELENGTH = 20
+
+NODEINDEXFORMAT = '!20sQ'
+NODEINDEXENTRYLENGTH = struct.calcsize(NODEINDEXFORMAT)
+
+# (node, p1, p2, linknode)
+PACKFORMAT = "!20s20s20s20sH"
+PACKENTRYLENGTH = 82
+
+ENTRYCOUNTSIZE = 4
+
+INDEXSUFFIX = '.histidx'
+PACKSUFFIX = '.histpack'
+
+ANC_NODE = 0
+ANC_P1NODE = 1
+ANC_P2NODE = 2
+ANC_LINKNODE = 3
+ANC_COPYFROM = 4
+
+class historypackstore(basepack.basepackstore):
+    INDEXSUFFIX = INDEXSUFFIX
+    PACKSUFFIX = PACKSUFFIX
+
+    def getpack(self, path):
+        return historypack(path)
+
+    def getancestors(self, name, node, known=None):
+        for pack in self.packs:
+            try:
+                return pack.getancestors(name, node, known=known)
+            except KeyError:
+                pass
+
+        for pack in self.refresh():
+            try:
+                return pack.getancestors(name, node, known=known)
+            except KeyError:
+                pass
+
+        raise KeyError((name, node))
+
+    def getnodeinfo(self, name, node):
+        for pack in self.packs:
+            try:
+                return pack.getnodeinfo(name, node)
+            except KeyError:
+                pass
+
+        for pack in self.refresh():
+            try:
+                return pack.getnodeinfo(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, node))
+
+    def add(self, filename, node, p1, p2, linknode, copyfrom):
+        raise RuntimeError("cannot add to historypackstore (%s:%s)"
+                           % (filename, hex(node)))
+
+class historypack(basepack.basepack):
+    INDEXSUFFIX = INDEXSUFFIX
+    PACKSUFFIX = PACKSUFFIX
+
+    SUPPORTED_VERSIONS = [2]
+
+    def __init__(self, path):
+        super(historypack, self).__init__(path)
+        self.INDEXFORMAT = INDEXFORMAT2
+        self.INDEXENTRYLENGTH = INDEXENTRYLENGTH2
+
+    def getmissing(self, keys):
+        missing = []
+        for name, node in keys:
+            try:
+                self._findnode(name, node)
+            except KeyError:
+                missing.append((name, node))
+
+        return missing
+
+    def getancestors(self, name, node, known=None):
+        """Returns as many ancestors as we're aware of.
+
+        return value: {
+           node: (p1, p2, linknode, copyfrom),
+           ...
+        }
+        """
+        if known and node in known:
+            return []
+
+        ancestors = self._getancestors(name, node, known=known)
+        results = {}
+        for ancnode, p1, p2, linknode, copyfrom in ancestors:
+            results[ancnode] = (p1, p2, linknode, copyfrom)
+
+        if not results:
+            raise KeyError((name, node))
+        return results
+
+    def getnodeinfo(self, name, node):
+        # Drop the node from the tuple before returning, since the result should
+        # just be (p1, p2, linknode, copyfrom)
+        return self._findnode(name, node)[1:]
+
+    def _getancestors(self, name, node, known=None):
+        if known is None:
+            known = set()
+        section = self._findsection(name)
+        filename, offset, size, nodeindexoffset, nodeindexsize = section
+        pending = set((node,))
+        o = 0
+        while o < size:
+            if not pending:
+                break
+            entry, copyfrom = self._readentry(offset + o)
+            o += PACKENTRYLENGTH
+            if copyfrom:
+                o += len(copyfrom)
+
+            ancnode = entry[ANC_NODE]
+            if ancnode in pending:
+                pending.remove(ancnode)
+                p1node = entry[ANC_P1NODE]
+                p2node = entry[ANC_P2NODE]
+                if p1node != nullid and p1node not in known:
+                    pending.add(p1node)
+                if p2node != nullid and p2node not in known:
+                    pending.add(p2node)
+
+                yield (ancnode, p1node, p2node, entry[ANC_LINKNODE], copyfrom)
+
+    def _readentry(self, offset):
+        data = self._data
+        entry = struct.unpack(PACKFORMAT, data[offset:offset + PACKENTRYLENGTH])
+        copyfrom = None
+        copyfromlen = entry[ANC_COPYFROM]
+        if copyfromlen != 0:
+            offset += PACKENTRYLENGTH
+            copyfrom = data[offset:offset + copyfromlen]
+        return entry, copyfrom
+
+    def add(self, filename, node, p1, p2, linknode, copyfrom):
+        raise RuntimeError("cannot add to historypack (%s:%s)" %
+                           (filename, hex(node)))
+
+    def _findnode(self, name, node):
+        if self.VERSION == 0:
+            ancestors = self._getancestors(name, node)
+            for ancnode, p1node, p2node, linknode, copyfrom in ancestors:
+                if ancnode == node:
+                    return (ancnode, p1node, p2node, linknode, copyfrom)
+        else:
+            section = self._findsection(name)
+            nodeindexoffset, nodeindexsize = section[3:]
+            entry = self._bisect(node, nodeindexoffset,
+                                 nodeindexoffset + nodeindexsize,
+                                 NODEINDEXENTRYLENGTH)
+            if entry is not None:
+                node, offset = struct.unpack(NODEINDEXFORMAT, entry)
+                entry, copyfrom = self._readentry(offset)
+                # Drop the copyfromlen from the end of entry, and replace it
+                # with the copyfrom string.
+                return entry[:4] + (copyfrom,)
+
+        raise KeyError("unable to find history for %s:%s" % (name, hex(node)))
+
+    def _findsection(self, name):
+        params = self.params
+        namehash = hashlib.sha1(name).digest()
+        fanoutkey = struct.unpack(params.fanoutstruct,
+                                  namehash[:params.fanoutprefix])[0]
+        fanout = self._fanouttable
+
+        start = fanout[fanoutkey] + params.indexstart
+        indexend = self._indexend
+
+        for i in pycompat.xrange(fanoutkey + 1, params.fanoutcount):
+            end = fanout[i] + params.indexstart
+            if end != start:
+                break
+        else:
+            end = indexend
+
+        entry = self._bisect(namehash, start, end, self.INDEXENTRYLENGTH)
+        if not entry:
+            raise KeyError(name)
+
+        rawentry = struct.unpack(self.INDEXFORMAT, entry)
+        x, offset, size, nodeindexoffset, nodeindexsize = rawentry
+        rawnamelen = self._index[nodeindexoffset:nodeindexoffset +
+                                                 constants.FILENAMESIZE]
+        actualnamelen = struct.unpack('!H', rawnamelen)[0]
+        nodeindexoffset += constants.FILENAMESIZE
+        actualname = self._index[nodeindexoffset:nodeindexoffset +
+                                                 actualnamelen]
+        if actualname != name:
+            raise KeyError("found file name %s when looking for %s" %
+                           (actualname, name))
+        nodeindexoffset += actualnamelen
+
+        filenamelength = struct.unpack('!H', self._data[offset:offset +
+                                                    constants.FILENAMESIZE])[0]
+        offset += constants.FILENAMESIZE
+
+        actualname = self._data[offset:offset + filenamelength]
+        offset += filenamelength
+
+        if name != actualname:
+            raise KeyError("found file name %s when looking for %s" %
+                           (actualname, name))
+
+        # Skip entry list size
+        offset += ENTRYCOUNTSIZE
+
+        nodelistoffset = offset
+        nodelistsize = (size - constants.FILENAMESIZE - filenamelength -
+                        ENTRYCOUNTSIZE)
+        return (name, nodelistoffset, nodelistsize,
+                nodeindexoffset, nodeindexsize)
+
+    def _bisect(self, node, start, end, entrylen):
+        # Bisect between start and end to find node
+        origstart = start
+        startnode = self._index[start:start + NODELENGTH]
+        endnode = self._index[end:end + NODELENGTH]
+
+        if startnode == node:
+            return self._index[start:start + entrylen]
+        elif endnode == node:
+            return self._index[end:end + entrylen]
+        else:
+            while start < end - entrylen:
+                mid = start + (end - start) // 2
+                mid = mid - ((mid - origstart) % entrylen)
+                midnode = self._index[mid:mid + NODELENGTH]
+                if midnode == node:
+                    return self._index[mid:mid + entrylen]
+                if node > midnode:
+                    start = mid
+                    startnode = midnode
+                elif node < midnode:
+                    end = mid
+                    endnode = midnode
+        return None
+
+    def markledger(self, ledger, options=None):
+        for filename, node in self:
+            ledger.markhistoryentry(self, filename, node)
+
+    def cleanup(self, ledger):
+        entries = ledger.sources.get(self, [])
+        allkeys = set(self)
+        repackedkeys = set((e.filename, e.node) for e in entries if
+                           e.historyrepacked)
+
+        if len(allkeys - repackedkeys) == 0:
+            if self.path not in ledger.created:
+                util.unlinkpath(self.indexpath, ignoremissing=True)
+                util.unlinkpath(self.packpath, ignoremissing=True)
+
+    def __iter__(self):
+        for f, n, x, x, x, x in self.iterentries():
+            yield f, n
+
+    def iterentries(self):
+        # Start at 1 to skip the header
+        offset = 1
+        while offset < self.datasize:
+            data = self._data
+            # <2 byte len> + <filename>
+            filenamelen = struct.unpack('!H', data[offset:offset +
+                                                   constants.FILENAMESIZE])[0]
+            offset += constants.FILENAMESIZE
+            filename = data[offset:offset + filenamelen]
+            offset += filenamelen
+
+            revcount = struct.unpack('!I', data[offset:offset +
+                                                ENTRYCOUNTSIZE])[0]
+            offset += ENTRYCOUNTSIZE
+
+            for i in pycompat.xrange(revcount):
+                entry = struct.unpack(PACKFORMAT, data[offset:offset +
+                                                              PACKENTRYLENGTH])
+                offset += PACKENTRYLENGTH
+
+                copyfrom = data[offset:offset + entry[ANC_COPYFROM]]
+                offset += entry[ANC_COPYFROM]
+
+                yield (filename, entry[ANC_NODE], entry[ANC_P1NODE],
+                        entry[ANC_P2NODE], entry[ANC_LINKNODE], copyfrom)
+
+                self._pagedin += PACKENTRYLENGTH
+
+            # If we've read a lot of data from the mmap, free some memory.
+            self.freememory()
+
+class mutablehistorypack(basepack.mutablebasepack):
+    """A class for constructing and serializing a histpack file and index.
+
+    A history pack is a pair of files that contain the revision history for
+    various file revisions in Mercurial. It contains only revision history (like
+    parent pointers and linknodes), not any revision content information.
+
+    It consists of two files, with the following format:
+
+    .histpack
+        The pack itself is a series of file revisions with some basic header
+        information on each.
+
+        datapack = <version: 1 byte>
+                   [<filesection>,...]
+        filesection = <filename len: 2 byte unsigned int>
+                      <filename>
+                      <revision count: 4 byte unsigned int>
+                      [<revision>,...]
+        revision = <node: 20 byte>
+                   <p1node: 20 byte>
+                   <p2node: 20 byte>
+                   <linknode: 20 byte>
+                   <copyfromlen: 2 byte>
+                   <copyfrom>
+
+        The revisions within each filesection are stored in topological order
+        (newest first). If a given entry has a parent from another file (a copy)
+        then p1node is the node from the other file, and copyfrom is the
+        filepath of the other file.
+
+    .histidx
+        The index file provides a mapping from filename to the file section in
+        the histpack. In V1 it also contains sub-indexes for specific nodes
+        within each file. It consists of three parts, the fanout, the file index
+        and the node indexes.
+
+        The file index is a list of index entries, sorted by filename hash (one
+        per file section in the pack). Each entry has:
+
+        - node (The 20 byte hash of the filename)
+        - pack entry offset (The location of this file section in the histpack)
+        - pack content size (The on-disk length of this file section's pack
+                             data)
+        - node index offset (The location of the file's node index in the index
+                             file) [1]
+        - node index size (the on-disk length of this file's node index) [1]
+
+        The fanout is a quick lookup table to reduce the number of steps for
+        bisecting the index. It is a series of 4 byte pointers to positions
+        within the index. It has 2^16 entries, which corresponds to hash
+        prefixes [00, 01, 02,..., FD, FE, FF]. Example: the pointer in slot 4F
+        points to the index position of the first revision whose node starts
+        with 4F. This saves log(2^16) bisect steps.
+
+        dataidx = <fanouttable>
+                  <file count: 8 byte unsigned> [1]
+                  <fileindex>
+                  <node count: 8 byte unsigned> [1]
+                  [<nodeindex>,...] [1]
+        fanouttable = [<index offset: 4 byte unsigned int>,...] (2^16 entries)
+
+        fileindex = [<file index entry>,...]
+        fileindexentry = <node: 20 byte>
+                         <pack file section offset: 8 byte unsigned int>
+                         <pack file section size: 8 byte unsigned int>
+                         <node index offset: 4 byte unsigned int> [1]
+                         <node index size: 4 byte unsigned int>   [1]
+        nodeindex = <filename>[<node index entry>,...] [1]
+        filename = <filename len : 2 byte unsigned int><filename value> [1]
+        nodeindexentry = <node: 20 byte> [1]
+                         <pack file node offset: 8 byte unsigned int> [1]
+
+    [1]: new in version 1.
+    """
+    INDEXSUFFIX = INDEXSUFFIX
+    PACKSUFFIX = PACKSUFFIX
+
+    SUPPORTED_VERSIONS = [2]
+
+    def __init__(self, ui, packpath, version=2):
+        super(mutablehistorypack, self).__init__(ui, packpath, version=version)
+        self.files = {}
+        self.entrylocations = {}
+        self.fileentries = {}
+
+        self.INDEXFORMAT = INDEXFORMAT2
+        self.INDEXENTRYLENGTH = INDEXENTRYLENGTH2
+
+        self.NODEINDEXFORMAT = NODEINDEXFORMAT
+        self.NODEINDEXENTRYLENGTH = NODEINDEXENTRYLENGTH
+
+    def add(self, filename, node, p1, p2, linknode, copyfrom):
+        copyfrom = copyfrom or ''
+        copyfromlen = struct.pack('!H', len(copyfrom))
+        self.fileentries.setdefault(filename, []).append((node, p1, p2,
+                                                          linknode,
+                                                          copyfromlen,
+                                                          copyfrom))
+
+    def _write(self):
+        for filename in sorted(self.fileentries):
+            entries = self.fileentries[filename]
+            sectionstart = self.packfp.tell()
+
+            # Write the file section content
+            entrymap = dict((e[0], e) for e in entries)
+            def parentfunc(node):
+                x, p1, p2, x, x, x = entrymap[node]
+                parents = []
+                if p1 != nullid:
+                    parents.append(p1)
+                if p2 != nullid:
+                    parents.append(p2)
+                return parents
+
+            sortednodes = list(reversed(shallowutil.sortnodes(
+                (e[0] for e in entries),
+                parentfunc)))
+
+            # Write the file section header
+            self.writeraw("%s%s%s" % (
+                struct.pack('!H', len(filename)),
+                filename,
+                struct.pack('!I', len(sortednodes)),
+            ))
+
+            sectionlen = constants.FILENAMESIZE + len(filename) + 4
+
+            rawstrings = []
+
+            # Record the node locations for the index
+            locations = self.entrylocations.setdefault(filename, {})
+            offset = sectionstart + sectionlen
+            for node in sortednodes:
+                locations[node] = offset
+                raw = '%s%s%s%s%s%s' % entrymap[node]
+                rawstrings.append(raw)
+                offset += len(raw)
+
+            rawdata = ''.join(rawstrings)
+            sectionlen += len(rawdata)
+
+            self.writeraw(rawdata)
+
+            # Record metadata for the index
+            self.files[filename] = (sectionstart, sectionlen)
+            node = hashlib.sha1(filename).digest()
+            self.entries[node] = node
+
+    def close(self, ledger=None):
+        if self._closed:
+            return
+
+        self._write()
+
+        return super(mutablehistorypack, self).close(ledger=ledger)
+
+    def createindex(self, nodelocations, indexoffset):
+        fileindexformat = self.INDEXFORMAT
+        fileindexlength = self.INDEXENTRYLENGTH
+        nodeindexformat = self.NODEINDEXFORMAT
+        nodeindexlength = self.NODEINDEXENTRYLENGTH
+
+        files = ((hashlib.sha1(filename).digest(), filename, offset, size)
+                for filename, (offset, size) in self.files.iteritems())
+        files = sorted(files)
+
+        # node index is after file index size, file index, and node index size
+        indexlensize = struct.calcsize('!Q')
+        nodeindexoffset = (indexoffset + indexlensize +
+                           (len(files) * fileindexlength) + indexlensize)
+
+        fileindexentries = []
+        nodeindexentries = []
+        nodecount = 0
+        for namehash, filename, offset, size in files:
+            # File section index
+            nodelocations = self.entrylocations[filename]
+
+            nodeindexsize = len(nodelocations) * nodeindexlength
+
+            rawentry = struct.pack(fileindexformat, namehash, offset, size,
+                                   nodeindexoffset, nodeindexsize)
+            # Node index
+            nodeindexentries.append(struct.pack(constants.FILENAMESTRUCT,
+                                                len(filename)) + filename)
+            nodeindexoffset += constants.FILENAMESIZE + len(filename)
+
+            for node, location in sorted(nodelocations.iteritems()):
+                nodeindexentries.append(struct.pack(nodeindexformat, node,
+                                                    location))
+                nodecount += 1
+
+            nodeindexoffset += len(nodelocations) * nodeindexlength
+
+            fileindexentries.append(rawentry)
+
+        nodecountraw = ''
+        nodecountraw = struct.pack('!Q', nodecount)
+        return (''.join(fileindexentries) + nodecountraw +
+                ''.join(nodeindexentries))

hgext/remotefilelog/metadatastore.py

@@ -0,0 +1,156 b''
+from __future__ import absolute_import
+
+from mercurial.node import hex, nullid
+from . import (
+    basestore,
+    shallowutil,
+)
+
+class unionmetadatastore(basestore.baseunionstore):
+    def __init__(self, *args, **kwargs):
+        super(unionmetadatastore, self).__init__(*args, **kwargs)
+
+        self.stores = args
+        self.writestore = kwargs.get(r'writestore')
+
+        # If allowincomplete==True then the union store can return partial
+        # ancestor lists, otherwise it will throw a KeyError if a full
+        # history can't be found.
+        self.allowincomplete = kwargs.get(r'allowincomplete', False)
+
+    def getancestors(self, name, node, known=None):
+        """Returns as many ancestors as we're aware of.
+
+        return value: {
+           node: (p1, p2, linknode, copyfrom),
+           ...
+        }
+        """
+        if known is None:
+            known = set()
+        if node in known:
+            return []
+
+        ancestors = {}
+        def traverse(curname, curnode):
+            # TODO: this algorithm has the potential to traverse parts of
+            # history twice. Ex: with A->B->C->F and A->B->D->F, both D and C
+            # may be queued as missing, then B and A are traversed for both.
+            queue = [(curname, curnode)]
+            missing = []
+            seen = set()
+            while queue:
+                name, node = queue.pop()
+                if (name, node) in seen:
+                    continue
+                seen.add((name, node))
+                value = ancestors.get(node)
+                if not value:
+                    missing.append((name, node))
+                    continue
+                p1, p2, linknode, copyfrom = value
+                if p1 != nullid and p1 not in known:
+                    queue.append((copyfrom or curname, p1))
+                if p2 != nullid and p2 not in known:
+                    queue.append((curname, p2))
+            return missing
+
+        missing = [(name, node)]
+        while missing:
+            curname, curnode = missing.pop()
+            try:
+                ancestors.update(self._getpartialancestors(curname, curnode,
+                                                           known=known))
+                newmissing = traverse(curname, curnode)
+                missing.extend(newmissing)
+            except KeyError:
+                # If we allow incomplete histories, don't throw.
+                if not self.allowincomplete:
+                    raise
+                # If the requested name+node doesn't exist, always throw.
+                if (curname, curnode) == (name, node):
+                    raise
+
+        # TODO: ancestors should probably be (name, node) -> (value)
+        return ancestors
+
+    @basestore.baseunionstore.retriable
+    def _getpartialancestors(self, name, node, known=None):
+        for store in self.stores:
+            try:
+                return store.getancestors(name, node, known=known)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    @basestore.baseunionstore.retriable
+    def getnodeinfo(self, name, node):
+        for store in self.stores:
+            try:
+                return store.getnodeinfo(name, node)
+            except KeyError:
+                pass
+
+        raise KeyError((name, hex(node)))
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add content only to remotefilelog "
+                           "contentstore")
+
+    def getmissing(self, keys):
+        missing = keys
+        for store in self.stores:
+            if missing:
+                missing = store.getmissing(missing)
+        return missing
+
+    def markledger(self, ledger, options=None):
+        for store in self.stores:
+            store.markledger(ledger, options)
+
+    def getmetrics(self):
+        metrics = [s.getmetrics() for s in self.stores]
+        return shallowutil.sumdicts(*metrics)
+
+class remotefilelogmetadatastore(basestore.basestore):
+    def getancestors(self, name, node, known=None):
+        """Returns as many ancestors as we're aware of.
+
+        return value: {
+           node: (p1, p2, linknode, copyfrom),
+           ...
+        }
+        """
+        data = self._getdata(name, node)
+        ancestors = shallowutil.ancestormap(data)
+        return ancestors
+
+    def getnodeinfo(self, name, node):
+        return self.getancestors(name, node)[node]
+
+    def add(self, name, node, parents, linknode):
+        raise RuntimeError("cannot add metadata only to remotefilelog "
+                           "metadatastore")
+
+class remotemetadatastore(object):
+    def __init__(self, ui, fileservice, shared):
+        self._fileservice = fileservice
+        self._shared = shared
+
+    def getancestors(self, name, node, known=None):
+        self._fileservice.prefetch([(name, hex(node))], force=True,
+                                   fetchdata=False, fetchhistory=True)
+        return self._shared.getancestors(name, node, known=known)
+
+    def getnodeinfo(self, name, node):
+        return self.getancestors(name, node)[node]
+
+    def add(self, name, node, data):
+        raise RuntimeError("cannot add to a remote store")
+
+    def getmissing(self, keys):
+        return keys
+
+    def markledger(self, ledger, options=None):
+        pass

hgext/remotefilelog/remotefilectx.py

@@ -0,0 +1,491 b''
+# remotefilectx.py - filectx/workingfilectx implementations for remotefilelog
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
+
+import collections
+import time
+
+from mercurial.node import bin, hex, nullid, nullrev
+from mercurial import (
+    ancestor,
+    context,
+    error,
+    phases,
+    pycompat,
+    util,
+)
+from . import shallowutil
+
+propertycache = util.propertycache
+FASTLOG_TIMEOUT_IN_SECS = 0.5
+
+class remotefilectx(context.filectx):
+    def __init__(self, repo, path, changeid=None, fileid=None,
+                 filelog=None, changectx=None, ancestormap=None):
+        if fileid == nullrev:
+            fileid = nullid
+        if fileid and len(fileid) == 40:
+            fileid = bin(fileid)
+        super(remotefilectx, self).__init__(repo, path, changeid,
+            fileid, filelog, changectx)
+        self._ancestormap = ancestormap
+
+    def size(self):
+        return self._filelog.size(self._filenode)
+
+    @propertycache
+    def _changeid(self):
+        if '_changeid' in self.__dict__:
+            return self._changeid
+        elif '_changectx' in self.__dict__:
+            return self._changectx.rev()
+        elif '_descendantrev' in self.__dict__:
+            # this file context was created from a revision with a known
+            # descendant, we can (lazily) correct for linkrev aliases
+            linknode = self._adjustlinknode(self._path, self._filelog,
+                                            self._filenode, self._descendantrev)
+            return self._repo.unfiltered().changelog.rev(linknode)
+        else:
+            return self.linkrev()
+
+    def filectx(self, fileid, changeid=None):
+        '''opens an arbitrary revision of the file without
+        opening a new filelog'''
+        return remotefilectx(self._repo, self._path, fileid=fileid,
+                             filelog=self._filelog, changeid=changeid)
+
+    def linkrev(self):
+        return self._linkrev
+
+    @propertycache
+    def _linkrev(self):
+        if self._filenode == nullid:
+            return nullrev
+
+        ancestormap = self.ancestormap()
+        p1, p2, linknode, copyfrom = ancestormap[self._filenode]
+        rev = self._repo.changelog.nodemap.get(linknode)
+        if rev is not None:
+            return rev
+
+        # Search all commits for the appropriate linkrev (slow, but uncommon)
+        path = self._path
+        fileid = self._filenode
+        cl = self._repo.unfiltered().changelog
+        mfl = self._repo.manifestlog
+
+        for rev in range(len(cl) - 1, 0, -1):
+            node = cl.node(rev)
+            data = cl.read(node) # get changeset data (we avoid object creation)
+            if path in data[3]: # checking the 'files' field.
+                # The file has been touched, check if the hash is what we're
+                # looking for.
+                if fileid == mfl[data[0]].readfast().get(path):
+                    return rev
+
+        # Couldn't find the linkrev. This should generally not happen, and will
+        # likely cause a crash.
+        return None
+
+    def introrev(self):
+        """return the rev of the changeset which introduced this file revision
+
+        This method is different from linkrev because it take into account the
+        changeset the filectx was created from. It ensures the returned
+        revision is one of its ancestors. This prevents bugs from
+        'linkrev-shadowing' when a file revision is used by multiple
+        changesets.
+        """
+        lkr = self.linkrev()
+        attrs = vars(self)
+        noctx = not ('_changeid' in attrs or '_changectx' in attrs)
+        if noctx or self.rev() == lkr:
+            return lkr
+        linknode = self._adjustlinknode(self._path, self._filelog,
+                                        self._filenode, self.rev(),
+                                        inclusive=True)
+        return self._repo.changelog.rev(linknode)
+
+    def renamed(self):
+        """check if file was actually renamed in this changeset revision
+
+        If rename logged in file revision, we report copy for changeset only
+        if file revisions linkrev points back to the changeset in question
+        or both changeset parents contain different file revisions.
+        """
+        ancestormap = self.ancestormap()
+
+        p1, p2, linknode, copyfrom = ancestormap[self._filenode]
+        if not copyfrom:
+            return None
+
+        renamed = (copyfrom, p1)
+        if self.rev() == self.linkrev():
+            return renamed
+
+        name = self.path()
+        fnode = self._filenode
+        for p in self._changectx.parents():
+            try:
+                if fnode == p.filenode(name):
+                    return None
+            except error.LookupError:
+                pass
+        return renamed
+
+    def ancestormap(self):
+        if not self._ancestormap:
+            self._ancestormap = self.filelog().ancestormap(self._filenode)
+
+        return self._ancestormap
+
+    def parents(self):
+        repo = self._repo
+        ancestormap = self.ancestormap()
+
+        p1, p2, linknode, copyfrom = ancestormap[self._filenode]
+        results = []
+        if p1 != nullid:
+            path = copyfrom or self._path
+            flog = repo.file(path)
+            p1ctx = remotefilectx(repo, path, fileid=p1, filelog=flog,
+                                  ancestormap=ancestormap)
+            p1ctx._descendantrev = self.rev()
+            results.append(p1ctx)
+
+        if p2 != nullid:
+            path = self._path
+            flog = repo.file(path)
+            p2ctx = remotefilectx(repo, path, fileid=p2, filelog=flog,
+                                  ancestormap=ancestormap)
+            p2ctx._descendantrev = self.rev()
+            results.append(p2ctx)
+
+        return results
+
+    def _nodefromancrev(self, ancrev, cl, mfl, path, fnode):
+        """returns the node for <path> in <ancrev> if content matches <fnode>"""
+        ancctx = cl.read(ancrev) # This avoids object creation.
+        manifestnode, files = ancctx[0], ancctx[3]
+        # If the file was touched in this ancestor, and the content is similar
+        # to the one we are searching for.
+        if path in files and fnode == mfl[manifestnode].readfast().get(path):
+            return cl.node(ancrev)
+        return None
+
+    def _adjustlinknode(self, path, filelog, fnode, srcrev, inclusive=False):
+        """return the first ancestor of <srcrev> introducing <fnode>
+
+        If the linkrev of the file revision does not point to an ancestor of
+        srcrev, we'll walk down the ancestors until we find one introducing
+        this file revision.
+
+        :repo: a localrepository object (used to access changelog and manifest)
+        :path: the file path
+        :fnode: the nodeid of the file revision
+        :filelog: the filelog of this path
+        :srcrev: the changeset revision we search ancestors from
+        :inclusive: if true, the src revision will also be checked
+
+        Note: This is based on adjustlinkrev in core, but it's quite different.
+
+        adjustlinkrev depends on the fact that the linkrev is the bottom most
+        node, and uses that as a stopping point for the ancestor traversal. We
+        can't do that here because the linknode is not guaranteed to be the
+        bottom most one.
+
+        In our code here, we actually know what a bunch of potential ancestor
+        linknodes are, so instead of stopping the cheap-ancestor-traversal when
+        we get to a linkrev, we stop when we see any of the known linknodes.
+        """
+        repo = self._repo
+        cl = repo.unfiltered().changelog
+        mfl = repo.manifestlog
+        ancestormap = self.ancestormap()
+        linknode = ancestormap[fnode][2]
+
+        if srcrev is None:
+            # wctx case, used by workingfilectx during mergecopy
+            revs = [p.rev() for p in self._repo[None].parents()]
+            inclusive = True # we skipped the real (revless) source
+        else:
+            revs = [srcrev]
+
+        if self._verifylinknode(revs, linknode):
+            return linknode
+
+        commonlogkwargs = {
+            r'revs': ' '.join([hex(cl.node(rev)) for rev in revs]),
+            r'fnode': hex(fnode),
+            r'filepath': path,
+            r'user': shallowutil.getusername(repo.ui),
+            r'reponame': shallowutil.getreponame(repo.ui),
+        }
+
+        repo.ui.log('linkrevfixup', 'adjusting linknode\n', **commonlogkwargs)
+
+        pc = repo._phasecache
+        seenpublic = False
+        iteranc = cl.ancestors(revs, inclusive=inclusive)
+        for ancrev in iteranc:
+            # First, check locally-available history.
+            lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode)
+            if lnode is not None:
+                return lnode
+
+            # adjusting linknode can be super-slow. To mitigate the issue
+            # we use two heuristics: calling fastlog and forcing remotefilelog
+            # prefetch
+            if not seenpublic and pc.phase(repo, ancrev) == phases.public:
+                # TODO: there used to be a codepath to fetch linknodes
+                # from a server as a fast path, but it appeared to
+                # depend on an API FB added to their phabricator.
+                lnode = self._forceprefetch(repo, path, fnode, revs,
+                                            commonlogkwargs)
+                if lnode:
+                    return lnode
+                seenpublic = True
+
+        return linknode
+
+    def _forceprefetch(self, repo, path, fnode, revs,
+                       commonlogkwargs):
+        # This next part is super non-obvious, so big comment block time!
+        #
+        # It is possible to get extremely bad performance here when a fairly
+        # common set of circumstances occur when this extension is combined
+        # with a server-side commit rewriting extension like pushrebase.
+        #
+        # First, an engineer creates Commit A and pushes it to the server.
+        # While the server's data structure will have the correct linkrev
+        # for the files touched in Commit A, the client will have the
+        # linkrev of the local commit, which is "invalid" because it's not
+        # an ancestor of the main line of development.
+        #
+        # The client will never download the remotefilelog with the correct
+        # linkrev as long as nobody else touches that file, since the file
+        # data and history hasn't changed since Commit A.
+        #
+        # After a long time (or a short time in a heavily used repo), if the
+        # same engineer returns to change the same file, some commands --
+        # such as amends of commits with file moves, logs, diffs, etc  --
+        # can trigger this _adjustlinknode code. In those cases, finding
+        # the correct rev can become quite expensive, as the correct
+        # revision is far back in history and we need to walk back through
+        # history to find it.
+        #
+        # In order to improve this situation, we force a prefetch of the
+        # remotefilelog data blob for the file we were called on. We do this
+        # at most once, when we first see a public commit in the history we
+        # are traversing.
+        #
+        # Forcing the prefetch means we will download the remote blob even
+        # if we have the "correct" blob in the local store. Since the union
+        # store checks the remote store first, this means we are much more
+        # likely to get the correct linkrev at this point.
+        #
+        # In rare circumstances (such as the server having a suboptimal
+        # linkrev for our use case), we will fall back to the old slow path.
+        #
+        # We may want to add additional heuristics here in the future if
+        # the slow path is used too much. One promising possibility is using
+        # obsolescence markers to find a more-likely-correct linkrev.
+
+        logmsg = ''
+        start = time.time()
+        try:
+            repo.fileservice.prefetch([(path, hex(fnode))], force=True)
+
+            # Now that we've downloaded a new blob from the server,
+            # we need to rebuild the ancestor map to recompute the
+            # linknodes.
+            self._ancestormap = None
+            linknode = self.ancestormap()[fnode][2] # 2 is linknode
+            if self._verifylinknode(revs, linknode):
+                logmsg = 'remotefilelog prefetching succeeded'
+                return linknode
+            logmsg = 'remotefilelog prefetching not found'
+            return None
+        except Exception as e:
+            logmsg = 'remotefilelog prefetching failed (%s)' % e
+            return None
+        finally:
+            elapsed = time.time() - start
+            repo.ui.log('linkrevfixup', logmsg + '\n', elapsed=elapsed * 1000,
+                        **pycompat.strkwargs(commonlogkwargs))
+
+    def _verifylinknode(self, revs, linknode):
+        """
+        Check if a linknode is correct one for the current history.
+
+        That is, return True if the linkrev is the ancestor of any of the
+        passed in revs, otherwise return False.
+
+        `revs` is a list that usually has one element -- usually the wdir parent
+        or the user-passed rev we're looking back from. It may contain two revs
+        when there is a merge going on, or zero revs when a root node with no
+        parents is being created.
+        """
+        if not revs:
+            return False
+        try:
+            # Use the C fastpath to check if the given linknode is correct.
+            cl = self._repo.unfiltered().changelog
+            return any(cl.isancestor(linknode, cl.node(r)) for r in revs)
+        except error.LookupError:
+            # The linknode read from the blob may have been stripped or
+            # otherwise not present in the repository anymore. Do not fail hard
+            # in this case. Instead, return false and continue the search for
+            # the correct linknode.
+            return False
+
+    def ancestors(self, followfirst=False):
+        ancestors = []
+        queue = collections.deque((self,))
+        seen = set()
+        while queue:
+            current = queue.pop()
+            if current.filenode() in seen:
+                continue
+            seen.add(current.filenode())
+
+            ancestors.append(current)
+
+            parents = current.parents()
+            first = True
+            for p in parents:
+                if first or not followfirst:
+                    queue.append(p)
+                first = False
+
+        # Remove self
+        ancestors.pop(0)
+
+        # Sort by linkrev
+        # The copy tracing algorithm depends on these coming out in order
+        ancestors = sorted(ancestors, reverse=True, key=lambda x:x.linkrev())
+
+        for ancestor in ancestors:
+            yield ancestor
+
+    def ancestor(self, fc2, actx):
+        # the easy case: no (relevant) renames
+        if fc2.path() == self.path() and self.path() in actx:
+            return actx[self.path()]
+
+        # the next easiest cases: unambiguous predecessor (name trumps
+        # history)
+        if self.path() in actx and fc2.path() not in actx:
+            return actx[self.path()]
+        if fc2.path() in actx and self.path() not in actx:
+            return actx[fc2.path()]
+
+        # do a full traversal
+        amap = self.ancestormap()
+        bmap = fc2.ancestormap()
+
+        def parents(x):
+            f, n = x
+            p = amap.get(n) or bmap.get(n)
+            if not p:
+                return []
+
+            return [(p[3] or f, p[0]), (f, p[1])]
+
+        a = (self.path(), self.filenode())
+        b = (fc2.path(), fc2.filenode())
+        result = ancestor.genericancestor(a, b, parents)
+        if result:
+            f, n = result
+            r = remotefilectx(self._repo, f, fileid=n,
+                                 ancestormap=amap)
+            return r
+
+        return None
+
+    def annotate(self, *args, **kwargs):
+        introctx = self
+        prefetchskip = kwargs.pop(r'prefetchskip', None)
+        if prefetchskip:
+            # use introrev so prefetchskip can be accurately tested
+            introrev = self.introrev()
+            if self.rev() != introrev:
+                introctx = remotefilectx(self._repo, self._path,
+                                         changeid=introrev,
+                                         fileid=self._filenode,
+                                         filelog=self._filelog,
+                                         ancestormap=self._ancestormap)
+
+        # like self.ancestors, but append to "fetch" and skip visiting parents
+        # of nodes in "prefetchskip".
+        fetch = []
+        seen = set()
+        queue = collections.deque((introctx,))
+        seen.add(introctx.node())
+        while queue:
+            current = queue.pop()
+            if current.filenode() != self.filenode():
+                # this is a "joint point". fastannotate needs contents of
+                # "joint point"s to calculate diffs for side branches.
+                fetch.append((current.path(), hex(current.filenode())))
+            if prefetchskip and current in prefetchskip:
+                continue
+            for parent in current.parents():
+                if parent.node() not in seen:
+                    seen.add(parent.node())
+                    queue.append(parent)
+
+        self._repo.ui.debug('remotefilelog: prefetching %d files '
+                            'for annotate\n' % len(fetch))
+        if fetch:
+            self._repo.fileservice.prefetch(fetch)
+        return super(remotefilectx, self).annotate(*args, **kwargs)
+
+    # Return empty set so that the hg serve and thg don't stack trace
+    def children(self):
+        return []
+
+class remoteworkingfilectx(context.workingfilectx, remotefilectx):
+    def __init__(self, repo, path, filelog=None, workingctx=None):
+        self._ancestormap = None
+        return super(remoteworkingfilectx, self).__init__(repo, path,
+            filelog, workingctx)
+
+    def parents(self):
+        return remotefilectx.parents(self)
+
+    def ancestormap(self):
+        if not self._ancestormap:
+            path = self._path
+            pcl = self._changectx._parents
+            renamed = self.renamed()
+
+            if renamed:
+                p1 = renamed
+            else:
+                p1 = (path, pcl[0]._manifest.get(path, nullid))
+
+            p2 = (path, nullid)
+            if len(pcl) > 1:
+                p2 = (path, pcl[1]._manifest.get(path, nullid))
+
+            m = {}
+            if p1[1] != nullid:
+                p1ctx = self._repo.filectx(p1[0], fileid=p1[1])
+                m.update(p1ctx.filelog().ancestormap(p1[1]))
+
+            if p2[1] != nullid:
+                p2ctx = self._repo.filectx(p2[0], fileid=p2[1])
+                m.update(p2ctx.filelog().ancestormap(p2[1]))
+
+            copyfrom = ''
+            if renamed:
+                copyfrom = renamed[0]
+            m[None] = (p1[1], p2[1], nullid, copyfrom)
+            self._ancestormap = m
+
+        return self._ancestormap

hgext/remotefilelog/remotefilelog.py

@@ -0,0 +1,454 b''
+# remotefilelog.py - filelog implementation where filelog history is stored
+#                    remotely
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
+
+import collections
+import os
+
+from mercurial.node import bin, nullid
+from mercurial.i18n import _
+from mercurial import (
+    ancestor,
+    error,
+    mdiff,
+    revlog,
+)
+from mercurial.utils import storageutil
+
+from . import (
+    constants,
+    fileserverclient,
+    shallowutil,
+)
+
+class remotefilelognodemap(object):
+    def __init__(self, filename, store):
+        self._filename = filename
+        self._store = store
+
+    def __contains__(self, node):
+        missing = self._store.getmissing([(self._filename, node)])
+        return not bool(missing)
+
+    def __get__(self, node):
+        if node not in self:
+            raise KeyError(node)
+        return node
+
+class remotefilelog(object):
+
+    _generaldelta = True
+
+    def __init__(self, opener, path, repo):
+        self.opener = opener
+        self.filename = path
+        self.repo = repo
+        self.nodemap = remotefilelognodemap(self.filename, repo.contentstore)
+
+        self.version = 1
+
+    def read(self, node):
+        """returns the file contents at this node"""
+        t = self.revision(node)
+        if not t.startswith('\1\n'):
+            return t
+        s = t.index('\1\n', 2)
+        return t[s + 2:]
+
+    def add(self, text, meta, transaction, linknode, p1=None, p2=None):
+        hashtext = text
+
+        # hash with the metadata, like in vanilla filelogs
+        hashtext = shallowutil.createrevlogtext(text, meta.get('copy'),
+                                                meta.get('copyrev'))
+        node = storageutil.hashrevisionsha1(hashtext, p1, p2)
+        return self.addrevision(hashtext, transaction, linknode, p1, p2,
+                                node=node)
+
+    def _createfileblob(self, text, meta, flags, p1, p2, node, linknode):
+        # text passed to "_createfileblob" does not include filelog metadata
+        header = shallowutil.buildfileblobheader(len(text), flags)
+        data = "%s\0%s" % (header, text)
+
+        realp1 = p1
+        copyfrom = ""
+        if meta and 'copy' in meta:
+            copyfrom = meta['copy']
+            realp1 = bin(meta['copyrev'])
+
+        data += "%s%s%s%s%s\0" % (node, realp1, p2, linknode, copyfrom)
+
+        visited = set()
+
+        pancestors = {}
+        queue = []
+        if realp1 != nullid:
+            p1flog = self
+            if copyfrom:
+                p1flog = remotefilelog(self.opener, copyfrom, self.repo)
+
+            pancestors.update(p1flog.ancestormap(realp1))
+            queue.append(realp1)
+            visited.add(realp1)
+        if p2 != nullid:
+            pancestors.update(self.ancestormap(p2))
+            queue.append(p2)
+            visited.add(p2)
+
+        ancestortext = ""
+
+        # add the ancestors in topological order
+        while queue:
+            c = queue.pop(0)
+            pa1, pa2, ancestorlinknode, pacopyfrom = pancestors[c]
+
+            pacopyfrom = pacopyfrom or ''
+            ancestortext += "%s%s%s%s%s\0" % (
+                c, pa1, pa2, ancestorlinknode, pacopyfrom)
+
+            if pa1 != nullid and pa1 not in visited:
+                queue.append(pa1)
+                visited.add(pa1)
+            if pa2 != nullid and pa2 not in visited:
+                queue.append(pa2)
+                visited.add(pa2)
+
+        data += ancestortext
+
+        return data
+
+    def addrevision(self, text, transaction, linknode, p1, p2, cachedelta=None,
+                    node=None, flags=revlog.REVIDX_DEFAULT_FLAGS):
+        # text passed to "addrevision" includes hg filelog metadata header
+        if node is None:
+            node = storageutil.hashrevisionsha1(text, p1, p2)
+
+        meta, metaoffset = storageutil.parsemeta(text)
+        rawtext, validatehash = self._processflags(text, flags, 'write')
+        return self.addrawrevision(rawtext, transaction, linknode, p1, p2,
+                                   node, flags, cachedelta,
+                                   _metatuple=(meta, metaoffset))
+
+    def addrawrevision(self, rawtext, transaction, linknode, p1, p2, node,
+                       flags, cachedelta=None, _metatuple=None):
+        if _metatuple:
+            # _metatuple: used by "addrevision" internally by remotefilelog
+            # meta was parsed confidently
+            meta, metaoffset = _metatuple
+        else:
+            # not from self.addrevision, but something else (repo._filecommit)
+            # calls addrawrevision directly. remotefilelog needs to get and
+            # strip filelog metadata.
+            # we don't have confidence about whether rawtext contains filelog
+            # metadata or not (flag processor could replace it), so we just
+            # parse it as best-effort.
+            # in LFS (flags != 0)'s case, the best way is to call LFS code to
+            # get the meta information, instead of storageutil.parsemeta.
+            meta, metaoffset = storageutil.parsemeta(rawtext)
+        if flags != 0:
+            # when flags != 0, be conservative and do not mangle rawtext, since
+            # a read flag processor expects the text not being mangled at all.
+            metaoffset = 0
+        if metaoffset:
+            # remotefilelog fileblob stores copy metadata in its ancestortext,
+            # not its main blob. so we need to remove filelog metadata
+            # (containing copy information) from text.
+            blobtext = rawtext[metaoffset:]
+        else:
+            blobtext = rawtext
+        data = self._createfileblob(blobtext, meta, flags, p1, p2, node,
+                                    linknode)
+        self.repo.contentstore.addremotefilelognode(self.filename, node, data)
+
+        return node
+
+    def renamed(self, node):
+        ancestors = self.repo.metadatastore.getancestors(self.filename, node)
+        p1, p2, linknode, copyfrom = ancestors[node]
+        if copyfrom:
+            return (copyfrom, p1)
+
+        return False
+
+    def size(self, node):
+        """return the size of a given revision"""
+        return len(self.read(node))
+
+    rawsize = size
+
+    def cmp(self, node, text):
+        """compare text with a given file revision
+
+        returns True if text is different than what is stored.
+        """
+
+        if node == nullid:
+            return True
+
+        nodetext = self.read(node)
+        return nodetext != text
+
+    def __nonzero__(self):
+        return True
+
+    __bool__ = __nonzero__
+
+    def __len__(self):
+        if self.filename == '.hgtags':
+            # The length of .hgtags is used to fast path tag checking.
+            # remotefilelog doesn't support .hgtags since the entire .hgtags
+            # history is needed.  Use the excludepattern setting to make
+            # .hgtags a normal filelog.
+            return 0
+
+        raise RuntimeError("len not supported")
+
+    def empty(self):
+        return False
+
+    def flags(self, node):
+        if isinstance(node, int):
+            raise error.ProgrammingError(
+                'remotefilelog does not accept integer rev for flags')
+        store = self.repo.contentstore
+        return store.getmeta(self.filename, node).get(constants.METAKEYFLAG, 0)
+
+    def parents(self, node):
+        if node == nullid:
+            return nullid, nullid
+
+        ancestormap = self.repo.metadatastore.getancestors(self.filename, node)
+        p1, p2, linknode, copyfrom = ancestormap[node]
+        if copyfrom:
+            p1 = nullid
+
+        return p1, p2
+
+    def parentrevs(self, rev):
+        # TODO(augie): this is a node and should be a rev, but for now
+        # nothing in core seems to actually break.
+        return self.parents(rev)
+
+    def linknode(self, node):
+        ancestormap = self.repo.metadatastore.getancestors(self.filename, node)
+        p1, p2, linknode, copyfrom = ancestormap[node]
+        return linknode
+
+    def linkrev(self, node):
+        return self.repo.unfiltered().changelog.rev(self.linknode(node))
+
+    def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
+                      assumehaveparentrevisions=False, deltaprevious=False,
+                      deltamode=None):
+        # we don't use any of these parameters here
+        del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
+        del deltamode
+        prevnode = None
+        for node in nodes:
+            p1, p2 = self.parents(node)
+            if prevnode is None:
+                basenode = prevnode = p1
+            if basenode == node:
+                basenode = nullid
+            if basenode != nullid:
+                revision = None
+                delta = self.revdiff(basenode, node)
+            else:
+                revision = self.revision(node, raw=True)
+                delta = None
+            yield revlog.revlogrevisiondelta(
+                node=node,
+                p1node=p1,
+                p2node=p2,
+                linknode=self.linknode(node),
+                basenode=basenode,
+                flags=self.flags(node),
+                baserevisionsize=None,
+                revision=revision,
+                delta=delta,
+                )
+
+    def revdiff(self, node1, node2):
+        return mdiff.textdiff(self.revision(node1, raw=True),
+                              self.revision(node2, raw=True))
+
+    def lookup(self, node):
+        if len(node) == 40:
+            node = bin(node)
+        if len(node) != 20:
+            raise error.LookupError(node, self.filename,
+                                    _('invalid lookup input'))
+
+        return node
+
+    def rev(self, node):
+        # This is a hack to make TortoiseHG work.
+        return node
+
+    def node(self, rev):
+        # This is a hack.
+        if isinstance(rev, int):
+            raise error.ProgrammingError(
+                'remotefilelog does not convert integer rev to node')
+        return rev
+
+    def revision(self, node, raw=False):
+        """returns the revlog contents at this node.
+        this includes the meta data traditionally included in file revlogs.
+        this is generally only used for bundling and communicating with vanilla
+        hg clients.
+        """
+        if node == nullid:
+            return ""
+        if len(node) != 20:
+            raise error.LookupError(node, self.filename,
+                                    _('invalid revision input'))
+
+        store = self.repo.contentstore
+        rawtext = store.get(self.filename, node)
+        if raw:
+            return rawtext
+        flags = store.getmeta(self.filename, node).get(constants.METAKEYFLAG, 0)
+        if flags == 0:
+            return rawtext
+        text, verifyhash = self._processflags(rawtext, flags, 'read')
+        return text
+
+    def _processflags(self, text, flags, operation, raw=False):
+        # mostly copied from hg/mercurial/revlog.py
+        validatehash = True
+        orderedflags = revlog.REVIDX_FLAGS_ORDER
+        if operation == 'write':
+            orderedflags = reversed(orderedflags)
+        for flag in orderedflags:
+            if flag & flags:
+                vhash = True
+                if flag not in revlog._flagprocessors:
+                    message = _("missing processor for flag '%#x'") % (flag)
+                    raise revlog.RevlogError(message)
+                readfunc, writefunc, rawfunc = revlog._flagprocessors[flag]
+                if raw:
+                    vhash = rawfunc(self, text)
+                elif operation == 'read':
+                    text, vhash = readfunc(self, text)
+                elif operation == 'write':
+                    text, vhash = writefunc(self, text)
+                validatehash = validatehash and vhash
+        return text, validatehash
+
+    def _read(self, id):
+        """reads the raw file blob from disk, cache, or server"""
+        fileservice = self.repo.fileservice
+        localcache = fileservice.localcache
+        cachekey = fileserverclient.getcachekey(self.repo.name, self.filename,
+                                                id)
+        try:
+            return localcache.read(cachekey)
+        except KeyError:
+            pass
+
+        localkey = fileserverclient.getlocalkey(self.filename, id)
+        localpath = os.path.join(self.localpath, localkey)
+        try:
+            return shallowutil.readfile(localpath)
+        except IOError:
+            pass
+
+        fileservice.prefetch([(self.filename, id)])
+        try:
+            return localcache.read(cachekey)
+        except KeyError:
+            pass
+
+        raise error.LookupError(id, self.filename, _('no node'))
+
+    def ancestormap(self, node):
+        return self.repo.metadatastore.getancestors(self.filename, node)
+
+    def ancestor(self, a, b):
+        if a == nullid or b == nullid:
+            return nullid
+
+        revmap, parentfunc = self._buildrevgraph(a, b)
+        nodemap = dict(((v, k) for (k, v) in revmap.iteritems()))
+
+        ancs = ancestor.ancestors(parentfunc, revmap[a], revmap[b])
+        if ancs:
+            # choose a consistent winner when there's a tie
+            return min(map(nodemap.__getitem__, ancs))
+        return nullid
+
+    def commonancestorsheads(self, a, b):
+        """calculate all the heads of the common ancestors of nodes a and b"""
+
+        if a == nullid or b == nullid:
+            return nullid
+
+        revmap, parentfunc = self._buildrevgraph(a, b)
+        nodemap = dict(((v, k) for (k, v) in revmap.iteritems()))
+
+        ancs = ancestor.commonancestorsheads(parentfunc, revmap[a], revmap[b])
+        return map(nodemap.__getitem__, ancs)
+
+    def _buildrevgraph(self, a, b):
+        """Builds a numeric revision graph for the given two nodes.
+        Returns a node->rev map and a rev->[revs] parent function.
+        """
+        amap = self.ancestormap(a)
+        bmap = self.ancestormap(b)
+
+        # Union the two maps
+        parentsmap = collections.defaultdict(list)
+        allparents = set()
+        for mapping in (amap, bmap):
+            for node, pdata in mapping.iteritems():
+                parents = parentsmap[node]
+                p1, p2, linknode, copyfrom = pdata
+                # Don't follow renames (copyfrom).
+                # remotefilectx.ancestor does that.
+                if p1 != nullid and not copyfrom:
+                    parents.append(p1)
+                    allparents.add(p1)
+                if p2 != nullid:
+                    parents.append(p2)
+                    allparents.add(p2)
+
+        # Breadth first traversal to build linkrev graph
+        parentrevs = collections.defaultdict(list)
+        revmap = {}
+        queue = collections.deque(((None, n) for n in parentsmap
+                 if n not in allparents))
+        while queue:
+            prevrev, current = queue.pop()
+            if current in revmap:
+                if prevrev:
+                    parentrevs[prevrev].append(revmap[current])
+                continue
+
+            # Assign linkrevs in reverse order, so start at
+            # len(parentsmap) and work backwards.
+            currentrev = len(parentsmap) - len(revmap) - 1
+            revmap[current] = currentrev
+
+            if prevrev:
+                parentrevs[prevrev].append(currentrev)
+
+            for parent in parentsmap.get(current):
+                queue.appendleft((currentrev, parent))
+
+        return revmap, parentrevs.__getitem__
+
+    def strip(self, minlink, transaction):
+        pass
+
+    # misc unused things
+    def files(self):
+        return []
+
+    def checksize(self):
+        return 0, 0

hgext/remotefilelog/remotefilelogserver.py

@@ -0,0 +1,404 b''
+# remotefilelogserver.py - server logic for a remotefilelog server
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
+
+import errno
+import os
+import stat
+import time
+import zlib
+
+from mercurial.i18n import _
+from mercurial.node import bin, hex, nullid
+from mercurial import (
+    changegroup,
+    changelog,
+    context,
+    error,
+    extensions,
+    match,
+    store,
+    streamclone,
+    util,
+    wireprotoserver,
+    wireprototypes,
+    wireprotov1server,
+)
+from .  import (
+    constants,
+    shallowutil,
+)
+
+_sshv1server = wireprotoserver.sshv1protocolhandler
+
+def setupserver(ui, repo):
+    """Sets up a normal Mercurial repo so it can serve files to shallow repos.
+    """
+    onetimesetup(ui)
+
+    # don't send files to shallow clients during pulls
+    def generatefiles(orig, self, changedfiles, linknodes, commonrevs, source,
+                      *args, **kwargs):
+        caps = self._bundlecaps or []
+        if constants.BUNDLE2_CAPABLITY in caps:
+            # only send files that don't match the specified patterns
+            includepattern = None
+            excludepattern = None
+            for cap in (self._bundlecaps or []):
+                if cap.startswith("includepattern="):
+                    includepattern = cap[len("includepattern="):].split('\0')
+                elif cap.startswith("excludepattern="):
+                    excludepattern = cap[len("excludepattern="):].split('\0')
+
+            m = match.always(repo.root, '')
+            if includepattern or excludepattern:
+                m = match.match(repo.root, '', None,
+                    includepattern, excludepattern)
+
+            changedfiles = list([f for f in changedfiles if not m(f)])
+        return orig(self, changedfiles, linknodes, commonrevs, source,
+                    *args, **kwargs)
+
+    extensions.wrapfunction(
+        changegroup.cgpacker, 'generatefiles', generatefiles)
+
+onetime = False
+def onetimesetup(ui):
+    """Configures the wireprotocol for both clients and servers.
+    """
+    global onetime
+    if onetime:
+        return
+    onetime = True
+
+    # support file content requests
+    wireprotov1server.wireprotocommand(
+        'x_rfl_getflogheads', 'path', permission='pull')(getflogheads)
+    wireprotov1server.wireprotocommand(
+        'x_rfl_getfiles', '', permission='pull')(getfiles)
+    wireprotov1server.wireprotocommand(
+        'x_rfl_getfile', 'file node', permission='pull')(getfile)
+
+    class streamstate(object):
+        match = None
+        shallowremote = False
+        noflatmf = False
+    state = streamstate()
+
+    def stream_out_shallow(repo, proto, other):
+        includepattern = None
+        excludepattern = None
+        raw = other.get('includepattern')
+        if raw:
+            includepattern = raw.split('\0')
+        raw = other.get('excludepattern')
+        if raw:
+            excludepattern = raw.split('\0')
+
+        oldshallow = state.shallowremote
+        oldmatch = state.match
+        oldnoflatmf = state.noflatmf
+        try:
+            state.shallowremote = True
+            state.match = match.always(repo.root, '')
+            state.noflatmf = other.get('noflatmanifest') == 'True'
+            if includepattern or excludepattern:
+                state.match = match.match(repo.root, '', None,
+                    includepattern, excludepattern)
+            streamres = wireprotov1server.stream(repo, proto)
+
+            # Force the first value to execute, so the file list is computed
+            # within the try/finally scope
+            first = next(streamres.gen)
+            second = next(streamres.gen)
+            def gen():
+                yield first
+                yield second
+                for value in streamres.gen:
+                    yield value
+            return wireprototypes.streamres(gen())
+        finally:
+            state.shallowremote = oldshallow
+            state.match = oldmatch
+            state.noflatmf = oldnoflatmf
+
+    wireprotov1server.commands['stream_out_shallow'] = (stream_out_shallow, '*')
+
+    # don't clone filelogs to shallow clients
+    def _walkstreamfiles(orig, repo, matcher=None):
+        if state.shallowremote:
+            # if we are shallow ourselves, stream our local commits
+            if shallowutil.isenabled(repo):
+                striplen = len(repo.store.path) + 1
+                readdir = repo.store.rawvfs.readdir
+                visit = [os.path.join(repo.store.path, 'data')]
+                while visit:
+                    p = visit.pop()
+                    for f, kind, st in readdir(p, stat=True):
+                        fp = p + '/' + f
+                        if kind == stat.S_IFREG:
+                            if not fp.endswith('.i') and not fp.endswith('.d'):
+                                n = util.pconvert(fp[striplen:])
+                                yield (store.decodedir(n), n, st.st_size)
+                        if kind == stat.S_IFDIR:
+                            visit.append(fp)
+
+            if 'treemanifest' in repo.requirements:
+                for (u, e, s) in repo.store.datafiles():
+                    if (u.startswith('meta/') and
+                        (u.endswith('.i') or u.endswith('.d'))):
+                        yield (u, e, s)
+
+            # Return .d and .i files that do not match the shallow pattern
+            match = state.match
+            if match and not match.always():
+                for (u, e, s) in repo.store.datafiles():
+                    f = u[5:-2]  # trim data/...  and .i/.d
+                    if not state.match(f):
+                        yield (u, e, s)
+
+            for x in repo.store.topfiles():
+                if state.noflatmf and x[0][:11] == '00manifest.':
+                    continue
+                yield x
+
+        elif shallowutil.isenabled(repo):
+            # don't allow cloning from a shallow repo to a full repo
+            # since it would require fetching every version of every
+            # file in order to create the revlogs.
+            raise error.Abort(_("Cannot clone from a shallow repo "
+                                "to a full repo."))
+        else:
+            for x in orig(repo, matcher):
+                yield x
+
+    extensions.wrapfunction(streamclone, '_walkstreamfiles', _walkstreamfiles)
+
+    # expose remotefilelog capabilities
+    def _capabilities(orig, repo, proto):
+        caps = orig(repo, proto)
+        if (shallowutil.isenabled(repo) or ui.configbool('remotefilelog',
+                                                         'server')):
+            if isinstance(proto, _sshv1server):
+                # legacy getfiles method which only works over ssh
+                caps.append(constants.NETWORK_CAP_LEGACY_SSH_GETFILES)
+            caps.append('x_rfl_getflogheads')
+            caps.append('x_rfl_getfile')
+        return caps
+    extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities)
+
+    def _adjustlinkrev(orig, self, *args, **kwargs):
+        # When generating file blobs, taking the real path is too slow on large
+        # repos, so force it to just return the linkrev directly.
+        repo = self._repo
+        if util.safehasattr(repo, 'forcelinkrev') and repo.forcelinkrev:
+            return self._filelog.linkrev(self._filelog.rev(self._filenode))
+        return orig(self, *args, **kwargs)
+
+    extensions.wrapfunction(
+        context.basefilectx, '_adjustlinkrev', _adjustlinkrev)
+
+    def _iscmd(orig, cmd):
+        if cmd == 'x_rfl_getfiles':
+            return False
+        return orig(cmd)
+
+    extensions.wrapfunction(wireprotoserver, 'iscmd', _iscmd)
+
+def _loadfileblob(repo, cachepath, path, node):
+    filecachepath = os.path.join(cachepath, path, hex(node))
+    if not os.path.exists(filecachepath) or os.path.getsize(filecachepath) == 0:
+        filectx = repo.filectx(path, fileid=node)
+        if filectx.node() == nullid:
+            repo.changelog = changelog.changelog(repo.svfs)
+            filectx = repo.filectx(path, fileid=node)
+
+        text = createfileblob(filectx)
+        # TODO configurable compression engines
+        text = zlib.compress(text)
+
+        # everything should be user & group read/writable
+        oldumask = os.umask(0o002)
+        try:
+            dirname = os.path.dirname(filecachepath)
+            if not os.path.exists(dirname):
+                try:
+                    os.makedirs(dirname)
+                except OSError as ex:
+                    if ex.errno != errno.EEXIST:
+                        raise
+
+            f = None
+            try:
+                f = util.atomictempfile(filecachepath, "wb")
+                f.write(text)
+            except (IOError, OSError):
+                # Don't abort if the user only has permission to read,
+                # and not write.
+                pass
+            finally:
+                if f:
+                    f.close()
+        finally:
+            os.umask(oldumask)
+    else:
+        with open(filecachepath, "rb") as f:
+            text = f.read()
+    return text
+
+def getflogheads(repo, proto, path):
+    """A server api for requesting a filelog's heads
+    """
+    flog = repo.file(path)
+    heads = flog.heads()
+    return '\n'.join((hex(head) for head in heads if head != nullid))
+
+def getfile(repo, proto, file, node):
+    """A server api for requesting a particular version of a file. Can be used
+    in batches to request many files at once. The return protocol is:
+    <errorcode>\0<data/errormsg> where <errorcode> is 0 for success or
+    non-zero for an error.
+
+    data is a compressed blob with revlog flag and ancestors information. See
+    createfileblob for its content.
+    """
+    if shallowutil.isenabled(repo):
+        return '1\0' + _('cannot fetch remote files from shallow repo')
+    cachepath = repo.ui.config("remotefilelog", "servercachepath")
+    if not cachepath:
+        cachepath = os.path.join(repo.path, "remotefilelogcache")
+    node = bin(node.strip())
+    if node == nullid:
+        return '0\0'
+    return '0\0' + _loadfileblob(repo, cachepath, file, node)
+
+def getfiles(repo, proto):
+    """A server api for requesting particular versions of particular files.
+    """
+    if shallowutil.isenabled(repo):
+        raise error.Abort(_('cannot fetch remote files from shallow repo'))
+    if not isinstance(proto, _sshv1server):
+        raise error.Abort(_('cannot fetch remote files over non-ssh protocol'))
+
+    def streamer():
+        fin = proto._fin
+
+        cachepath = repo.ui.config("remotefilelog", "servercachepath")
+        if not cachepath:
+            cachepath = os.path.join(repo.path, "remotefilelogcache")
+
+        while True:
+            request = fin.readline()[:-1]
+            if not request:
+                break
+
+            node = bin(request[:40])
+            if node == nullid:
+                yield '0\n'
+                continue
+
+            path = request[40:]
+
+            text = _loadfileblob(repo, cachepath, path, node)
+
+            yield '%d\n%s' % (len(text), text)
+
+            # it would be better to only flush after processing a whole batch
+            # but currently we don't know if there are more requests coming
+            proto._fout.flush()
+    return wireprototypes.streamres(streamer())
+
+def createfileblob(filectx):
+    """
+    format:
+        v0:
+            str(len(rawtext)) + '\0' + rawtext + ancestortext
+        v1:
+            'v1' + '\n' + metalist + '\0' + rawtext + ancestortext
+            metalist := metalist + '\n' + meta | meta
+            meta := sizemeta | flagmeta
+            sizemeta := METAKEYSIZE + str(len(rawtext))
+            flagmeta := METAKEYFLAG + str(flag)
+
+            note: sizemeta must exist. METAKEYFLAG and METAKEYSIZE must have a
+            length of 1.
+    """
+    flog = filectx.filelog()
+    frev = filectx.filerev()
+    revlogflags = flog._revlog.flags(frev)
+    if revlogflags == 0:
+        # normal files
+        text = filectx.data()
+    else:
+        # lfs, read raw revision data
+        text = flog.revision(frev, raw=True)
+
+    repo = filectx._repo
+
+    ancestors = [filectx]
+
+    try:
+        repo.forcelinkrev = True
+        ancestors.extend([f for f in filectx.ancestors()])
+
+        ancestortext = ""
+        for ancestorctx in ancestors:
+            parents = ancestorctx.parents()
+            p1 = nullid
+            p2 = nullid
+            if len(parents) > 0:
+                p1 = parents[0].filenode()
+            if len(parents) > 1:
+                p2 = parents[1].filenode()
+
+            copyname = ""
+            rename = ancestorctx.renamed()
+            if rename:
+                copyname = rename[0]
+            linknode = ancestorctx.node()
+            ancestortext += "%s%s%s%s%s\0" % (
+                ancestorctx.filenode(), p1, p2, linknode,
+                copyname)
+    finally:
+        repo.forcelinkrev = False
+
+    header = shallowutil.buildfileblobheader(len(text), revlogflags)
+
+    return "%s\0%s%s" % (header, text, ancestortext)
+
+def gcserver(ui, repo):
+    if not repo.ui.configbool("remotefilelog", "server"):
+        return
+
+    neededfiles = set()
+    heads = repo.revs("heads(tip~25000:) - null")
+
+    cachepath = repo.vfs.join("remotefilelogcache")
+    for head in heads:
+        mf = repo[head].manifest()
+        for filename, filenode in mf.iteritems():
+            filecachepath = os.path.join(cachepath, filename, hex(filenode))
+            neededfiles.add(filecachepath)
+
+    # delete unneeded older files
+    days = repo.ui.configint("remotefilelog", "serverexpiration")
+    expiration = time.time() - (days * 24 * 60 * 60)
+
+    progress = ui.makeprogress(_("removing old server cache"), unit="files")
+    progress.update(0)
+    for root, dirs, files in os.walk(cachepath):
+        for file in files:
+            filepath = os.path.join(root, file)
+            progress.increment()
+            if filepath in neededfiles:
+                continue
+
+            stat = os.stat(filepath)
+            if stat.st_mtime < expiration:
+                os.remove(filepath)
+
+    progress.complete()

hgext/remotefilelog/repack.py

@@ -0,0 +1,778 b''
+from __future__ import absolute_import
+
+import os
+import time
+
+from mercurial.i18n import _
+from mercurial.node import (
+    nullid,
+    short,
+)
+from mercurial import (
+    encoding,
+    error,
+    mdiff,
+    policy,
+    pycompat,
+    scmutil,
+    util,
+    vfs,
+)
+from mercurial.utils import procutil
+from . import (
+    constants,
+    contentstore,
+    datapack,
+    extutil,
+    historypack,
+    metadatastore,
+    shallowutil,
+)
+
+osutil = policy.importmod(r'osutil')
+
+class RepackAlreadyRunning(error.Abort):
+    pass
+
+def backgroundrepack(repo, incremental=True, packsonly=False):
+    cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'repack']
+    msg = _("(running background repack)\n")
+    if incremental:
+        cmd.append('--incremental')
+        msg = _("(running background incremental repack)\n")
+    if packsonly:
+        cmd.append('--packsonly')
+    repo.ui.warn(msg)
+    procutil.runbgcommand(cmd, encoding.environ)
+
+def fullrepack(repo, options=None):
+    """If ``packsonly`` is True, stores creating only loose objects are skipped.
+    """
+    if util.safehasattr(repo, 'shareddatastores'):
+        datasource = contentstore.unioncontentstore(
+            *repo.shareddatastores)
+        historysource = metadatastore.unionmetadatastore(
+            *repo.sharedhistorystores,
+            allowincomplete=True)
+
+        packpath = shallowutil.getcachepackpath(
+            repo,
+            constants.FILEPACK_CATEGORY)
+        _runrepack(repo, datasource, historysource, packpath,
+                   constants.FILEPACK_CATEGORY, options=options)
+
+    if util.safehasattr(repo.manifestlog, 'datastore'):
+        localdata, shareddata = _getmanifeststores(repo)
+        lpackpath, ldstores, lhstores = localdata
+        spackpath, sdstores, shstores = shareddata
+
+        # Repack the shared manifest store
+        datasource = contentstore.unioncontentstore(*sdstores)
+        historysource = metadatastore.unionmetadatastore(
+                        *shstores,
+                        allowincomplete=True)
+        _runrepack(repo, datasource, historysource, spackpath,
+                   constants.TREEPACK_CATEGORY, options=options)
+
+        # Repack the local manifest store
+        datasource = contentstore.unioncontentstore(
+                        *ldstores,
+                        allowincomplete=True)
+        historysource = metadatastore.unionmetadatastore(
+                        *lhstores,
+                        allowincomplete=True)
+        _runrepack(repo, datasource, historysource, lpackpath,
+                   constants.TREEPACK_CATEGORY, options=options)
+
+def incrementalrepack(repo, options=None):
+    """This repacks the repo by looking at the distribution of pack files in the
+    repo and performing the most minimal repack to keep the repo in good shape.
+    """
+    if util.safehasattr(repo, 'shareddatastores'):
+        packpath = shallowutil.getcachepackpath(
+            repo,
+            constants.FILEPACK_CATEGORY)
+        _incrementalrepack(repo,
+                           repo.shareddatastores,
+                           repo.sharedhistorystores,
+                           packpath,
+                           constants.FILEPACK_CATEGORY,
+                           options=options)
+
+    if util.safehasattr(repo.manifestlog, 'datastore'):
+        localdata, shareddata = _getmanifeststores(repo)
+        lpackpath, ldstores, lhstores = localdata
+        spackpath, sdstores, shstores = shareddata
+
+        # Repack the shared manifest store
+        _incrementalrepack(repo,
+                           sdstores,
+                           shstores,
+                           spackpath,
+                           constants.TREEPACK_CATEGORY,
+                           options=options)
+
+        # Repack the local manifest store
+        _incrementalrepack(repo,
+                           ldstores,
+                           lhstores,
+                           lpackpath,
+                           constants.TREEPACK_CATEGORY,
+                           allowincompletedata=True,
+                           options=options)
+
+def _getmanifeststores(repo):
+    shareddatastores = repo.manifestlog.shareddatastores
+    localdatastores = repo.manifestlog.localdatastores
+    sharedhistorystores = repo.manifestlog.sharedhistorystores
+    localhistorystores = repo.manifestlog.localhistorystores
+
+    sharedpackpath = shallowutil.getcachepackpath(repo,
+                                            constants.TREEPACK_CATEGORY)
+    localpackpath = shallowutil.getlocalpackpath(repo.svfs.vfs.base,
+                                            constants.TREEPACK_CATEGORY)
+
+    return ((localpackpath, localdatastores, localhistorystores),
+            (sharedpackpath, shareddatastores, sharedhistorystores))
+
+def _topacks(packpath, files, constructor):
+    paths = list(os.path.join(packpath, p) for p in files)
+    packs = list(constructor(p) for p in paths)
+    return packs
+
+def _deletebigpacks(repo, folder, files):
+    """Deletes packfiles that are bigger than ``packs.maxpacksize``.
+
+    Returns ``files` with the removed files omitted."""
+    maxsize = repo.ui.configbytes("packs", "maxpacksize")
+    if maxsize <= 0:
+        return files
+
+    # This only considers datapacks today, but we could broaden it to include
+    # historypacks.
+    VALIDEXTS = [".datapack", ".dataidx"]
+
+    # Either an oversize index or datapack will trigger cleanup of the whole
+    # pack:
+    oversized = set([os.path.splitext(path)[0] for path, ftype, stat in files
+        if (stat.st_size > maxsize and (os.path.splitext(path)[1]
+                                        in VALIDEXTS))])
+
+    for rootfname in oversized:
+        rootpath = os.path.join(folder, rootfname)
+        for ext in VALIDEXTS:
+            path = rootpath + ext
+            repo.ui.debug('removing oversize packfile %s (%s)\n' %
+                          (path, util.bytecount(os.stat(path).st_size)))
+            os.unlink(path)
+    return [row for row in files if os.path.basename(row[0]) not in oversized]
+
+def _incrementalrepack(repo, datastore, historystore, packpath, category,
+        allowincompletedata=False, options=None):
+    shallowutil.mkstickygroupdir(repo.ui, packpath)
+
+    files = osutil.listdir(packpath, stat=True)
+    files = _deletebigpacks(repo, packpath, files)
+    datapacks = _topacks(packpath,
+        _computeincrementaldatapack(repo.ui, files),
+        datapack.datapack)
+    datapacks.extend(s for s in datastore
+                     if not isinstance(s, datapack.datapackstore))
+
+    historypacks = _topacks(packpath,
+        _computeincrementalhistorypack(repo.ui, files),
+        historypack.historypack)
+    historypacks.extend(s for s in historystore
+                        if not isinstance(s, historypack.historypackstore))
+
+    # ``allhistory{files,packs}`` contains all known history packs, even ones we
+    # don't plan to repack. They are used during the datapack repack to ensure
+    # good ordering of nodes.
+    allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX,
+                            historypack.INDEXSUFFIX)
+    allhistorypacks = _topacks(packpath,
+        (f for f, mode, stat in allhistoryfiles),
+        historypack.historypack)
+    allhistorypacks.extend(s for s in historystore
+                        if not isinstance(s, historypack.historypackstore))
+    _runrepack(repo,
+               contentstore.unioncontentstore(
+                   *datapacks,
+                   allowincomplete=allowincompletedata),
+               metadatastore.unionmetadatastore(
+                   *historypacks,
+                   allowincomplete=True),
+               packpath, category,
+               fullhistory=metadatastore.unionmetadatastore(
+                   *allhistorypacks,
+                   allowincomplete=True),
+                options=options)
+
+def _computeincrementaldatapack(ui, files):
+    opts = {
+        'gencountlimit' : ui.configint(
+            'remotefilelog', 'data.gencountlimit'),
+        'generations' : ui.configlist(
+            'remotefilelog', 'data.generations'),
+        'maxrepackpacks' : ui.configint(
+            'remotefilelog', 'data.maxrepackpacks'),
+        'repackmaxpacksize' : ui.configbytes(
+            'remotefilelog', 'data.repackmaxpacksize'),
+        'repacksizelimit' : ui.configbytes(
+            'remotefilelog', 'data.repacksizelimit'),
+    }
+
+    packfiles = _allpackfileswithsuffix(
+        files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX)
+    return _computeincrementalpack(packfiles, opts)
+
+def _computeincrementalhistorypack(ui, files):
+    opts = {
+        'gencountlimit' : ui.configint(
+            'remotefilelog', 'history.gencountlimit'),
+        'generations' : ui.configlist(
+            'remotefilelog', 'history.generations', ['100MB']),
+        'maxrepackpacks' : ui.configint(
+            'remotefilelog', 'history.maxrepackpacks'),
+        'repackmaxpacksize' : ui.configbytes(
+            'remotefilelog', 'history.repackmaxpacksize', '400MB'),
+        'repacksizelimit' : ui.configbytes(
+            'remotefilelog', 'history.repacksizelimit'),
+    }
+
+    packfiles = _allpackfileswithsuffix(
+        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX)
+    return _computeincrementalpack(packfiles, opts)
+
+def _allpackfileswithsuffix(files, packsuffix, indexsuffix):
+    result = []
+    fileset = set(fn for fn, mode, stat in files)
+    for filename, mode, stat in files:
+        if not filename.endswith(packsuffix):
+            continue
+
+        prefix = filename[:-len(packsuffix)]
+
+        # Don't process a pack if it doesn't have an index.
+        if (prefix + indexsuffix) not in fileset:
+            continue
+        result.append((prefix, mode, stat))
+
+    return result
+
+def _computeincrementalpack(files, opts):
+    """Given a set of pack files along with the configuration options, this
+    function computes the list of files that should be packed as part of an
+    incremental repack.
+
+    It tries to strike a balance between keeping incremental repacks cheap (i.e.
+    packing small things when possible, and rolling the packs up to the big ones
+    over time).
+    """
+
+    limits = list(sorted((util.sizetoint(s) for s in opts['generations']),
+                                reverse=True))
+    limits.append(0)
+
+    # Group the packs by generation (i.e. by size)
+    generations = []
+    for i in pycompat.xrange(len(limits)):
+        generations.append([])
+
+    sizes = {}
+    for prefix, mode, stat in files:
+        size = stat.st_size
+        if size > opts['repackmaxpacksize']:
+            continue
+
+        sizes[prefix] = size
+        for i, limit in enumerate(limits):
+            if size > limit:
+                generations[i].append(prefix)
+                break
+
+    # Steps for picking what packs to repack:
+    # 1. Pick the largest generation with > gencountlimit pack files.
+    # 2. Take the smallest three packs.
+    # 3. While total-size-of-packs < repacksizelimit: add another pack
+
+    # Find the largest generation with more than gencountlimit packs
+    genpacks = []
+    for i, limit in enumerate(limits):
+        if len(generations[i]) > opts['gencountlimit']:
+            # Sort to be smallest last, for easy popping later
+            genpacks.extend(sorted(generations[i], reverse=True,
+                                   key=lambda x: sizes[x]))
+            break
+
+    # Take as many packs from the generation as we can
+    chosenpacks = genpacks[-3:]
+    genpacks = genpacks[:-3]
+    repacksize = sum(sizes[n] for n in chosenpacks)
+    while (repacksize < opts['repacksizelimit'] and genpacks and
+           len(chosenpacks) < opts['maxrepackpacks']):
+        chosenpacks.append(genpacks.pop())
+        repacksize += sizes[chosenpacks[-1]]
+
+    return chosenpacks
+
+def _runrepack(repo, data, history, packpath, category, fullhistory=None,
+               options=None):
+    shallowutil.mkstickygroupdir(repo.ui, packpath)
+
+    def isold(repo, filename, node):
+        """Check if the file node is older than a limit.
+        Unless a limit is specified in the config the default limit is taken.
+        """
+        filectx = repo.filectx(filename, fileid=node)
+        filetime = repo[filectx.linkrev()].date()
+
+        ttl = repo.ui.configint('remotefilelog', 'nodettl')
+
+        limit = time.time() - ttl
+        return filetime[0] < limit
+
+    garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')
+    if not fullhistory:
+        fullhistory = history
+    packer = repacker(repo, data, history, fullhistory, category,
+                      gc=garbagecollect, isold=isold, options=options)
+
+    with datapack.mutabledatapack(repo.ui, packpath, version=2) as dpack:
+        with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
+            try:
+                packer.run(dpack, hpack)
+            except error.LockHeld:
+                raise RepackAlreadyRunning(_("skipping repack - another repack "
+                                             "is already running"))
+
+def keepset(repo, keyfn, lastkeepkeys=None):
+    """Computes a keepset which is not garbage collected.
+    'keyfn' is a function that maps filename, node to a unique key.
+    'lastkeepkeys' is an optional argument and if provided the keepset
+    function updates lastkeepkeys with more keys and returns the result.
+    """
+    if not lastkeepkeys:
+        keepkeys = set()
+    else:
+        keepkeys = lastkeepkeys
+
+    # We want to keep:
+    # 1. Working copy parent
+    # 2. Draft commits
+    # 3. Parents of draft commits
+    # 4. Pullprefetch and bgprefetchrevs revsets if specified
+    revs = ['.', 'draft()', 'parents(draft())']
+    prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)
+    if prefetchrevs:
+        revs.append('(%s)' % prefetchrevs)
+    prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)
+    if prefetchrevs:
+        revs.append('(%s)' % prefetchrevs)
+    revs = '+'.join(revs)
+
+    revs = ['sort((%s), "topo")' % revs]
+    keep = scmutil.revrange(repo, revs)
+
+    processed = set()
+    lastmanifest = None
+
+    # process the commits in toposorted order starting from the oldest
+    for r in reversed(keep._list):
+        if repo[r].p1().rev() in processed:
+            # if the direct parent has already been processed
+            # then we only need to process the delta
+            m = repo[r].manifestctx().readdelta()
+        else:
+            # otherwise take the manifest and diff it
+            # with the previous manifest if one exists
+            if lastmanifest:
+                m = repo[r].manifest().diff(lastmanifest)
+            else:
+                m = repo[r].manifest()
+        lastmanifest = repo[r].manifest()
+        processed.add(r)
+
+        # populate keepkeys with keys from the current manifest
+        if type(m) is dict:
+            # m is a result of diff of two manifests and is a dictionary that
+            # maps filename to ((newnode, newflag), (oldnode, oldflag)) tuple
+            for filename, diff in m.iteritems():
+                if diff[0][0] is not None:
+                    keepkeys.add(keyfn(filename, diff[0][0]))
+        else:
+            # m is a manifest object
+            for filename, filenode in m.iteritems():
+                keepkeys.add(keyfn(filename, filenode))
+
+    return keepkeys
+
+class repacker(object):
+    """Class for orchestrating the repack of data and history information into a
+    new format.
+    """
+    def __init__(self, repo, data, history, fullhistory, category, gc=False,
+                 isold=None, options=None):
+        self.repo = repo
+        self.data = data
+        self.history = history
+        self.fullhistory = fullhistory
+        self.unit = constants.getunits(category)
+        self.garbagecollect = gc
+        self.options = options
+        if self.garbagecollect:
+            if not isold:
+                raise ValueError("Function 'isold' is not properly specified")
+            # use (filename, node) tuple as a keepset key
+            self.keepkeys = keepset(repo, lambda f, n : (f, n))
+            self.isold = isold
+
+    def run(self, targetdata, targethistory):
+        ledger = repackledger()
+
+        with extutil.flock(repacklockvfs(self.repo).join("repacklock"),
+                           _('repacking %s') % self.repo.origroot, timeout=0):
+            self.repo.hook('prerepack')
+
+            # Populate ledger from source
+            self.data.markledger(ledger, options=self.options)
+            self.history.markledger(ledger, options=self.options)
+
+            # Run repack
+            self.repackdata(ledger, targetdata)
+            self.repackhistory(ledger, targethistory)
+
+            # Call cleanup on each source
+            for source in ledger.sources:
+                source.cleanup(ledger)
+
+    def _chainorphans(self, ui, filename, nodes, orphans, deltabases):
+        """Reorderes ``orphans`` into a single chain inside ``nodes`` and
+        ``deltabases``.
+
+        We often have orphan entries (nodes without a base that aren't
+        referenced by other nodes -- i.e., part of a chain) due to gaps in
+        history. Rather than store them as individual fulltexts, we prefer to
+        insert them as one chain sorted by size.
+        """
+        if not orphans:
+            return nodes
+
+        def getsize(node, default=0):
+            meta = self.data.getmeta(filename, node)
+            if constants.METAKEYSIZE in meta:
+                return meta[constants.METAKEYSIZE]
+            else:
+                return default
+
+        # Sort orphans by size; biggest first is preferred, since it's more
+        # likely to be the newest version assuming files grow over time.
+        # (Sort by node first to ensure the sort is stable.)
+        orphans = sorted(orphans)
+        orphans = list(sorted(orphans, key=getsize, reverse=True))
+        if ui.debugflag:
+            ui.debug("%s: orphan chain: %s\n" % (filename,
+                ", ".join([short(s) for s in orphans])))
+
+        # Create one contiguous chain and reassign deltabases.
+        for i, node in enumerate(orphans):
+            if i == 0:
+                deltabases[node] = (nullid, 0)
+            else:
+                parent = orphans[i - 1]
+                deltabases[node] = (parent, deltabases[parent][1] + 1)
+        nodes = [n for n in nodes if n not in orphans]
+        nodes += orphans
+        return nodes
+
+    def repackdata(self, ledger, target):
+        ui = self.repo.ui
+        maxchainlen = ui.configint('packs', 'maxchainlen', 1000)
+
+        byfile = {}
+        for entry in ledger.entries.itervalues():
+            if entry.datasource:
+                byfile.setdefault(entry.filename, {})[entry.node] = entry
+
+        count = 0
+        repackprogress = ui.makeprogress(_("repacking data"), unit=self.unit,
+                                            total=len(byfile))
+        for filename, entries in sorted(byfile.iteritems()):
+            repackprogress.update(count)
+
+            ancestors = {}
+            nodes = list(node for node in entries)
+            nohistory = []
+            buildprogress = ui.makeprogress(_("building history"), unit='nodes',
+                                            total=len(nodes))
+            for i, node in enumerate(nodes):
+                if node in ancestors:
+                    continue
+                buildprogress.update(i)
+                try:
+                    ancestors.update(self.fullhistory.getancestors(filename,
+                        node, known=ancestors))
+                except KeyError:
+                    # Since we're packing data entries, we may not have the
+                    # corresponding history entries for them. It's not a big
+                    # deal, but the entries won't be delta'd perfectly.
+                    nohistory.append(node)
+            buildprogress.complete()
+
+            # Order the nodes children first, so we can produce reverse deltas
+            orderednodes = list(reversed(self._toposort(ancestors)))
+            if len(nohistory) > 0:
+                ui.debug('repackdata: %d nodes without history\n' %
+                         len(nohistory))
+            orderednodes.extend(sorted(nohistory))
+
+            # Filter orderednodes to just the nodes we want to serialize (it
+            # currently also has the edge nodes' ancestors).
+            orderednodes = list(filter(lambda node: node in nodes,
+                                orderednodes))
+
+            # Garbage collect old nodes:
+            if self.garbagecollect:
+                neworderednodes = []
+                for node in orderednodes:
+                    # If the node is old and is not in the keepset, we skip it,
+                    # and mark as garbage collected
+                    if ((filename, node) not in self.keepkeys and
+                        self.isold(self.repo, filename, node)):
+                        entries[node].gced = True
+                        continue
+                    neworderednodes.append(node)
+                orderednodes = neworderednodes
+
+            # Compute delta bases for nodes:
+            deltabases = {}
+            nobase = set()
+            referenced = set()
+            nodes = set(nodes)
+            processprogress = ui.makeprogress(_("processing nodes"),
+                                              unit='nodes',
+                                              total=len(orderednodes))
+            for i, node in enumerate(orderednodes):
+                processprogress.update(i)
+                # Find delta base
+                # TODO: allow delta'ing against most recent descendant instead
+                # of immediate child
+                deltatuple = deltabases.get(node, None)
+                if deltatuple is None:
+                    deltabase, chainlen = nullid, 0
+                    deltabases[node] = (nullid, 0)
+                    nobase.add(node)
+                else:
+                    deltabase, chainlen = deltatuple
+                    referenced.add(deltabase)
+
+                # Use available ancestor information to inform our delta choices
+                ancestorinfo = ancestors.get(node)
+                if ancestorinfo:
+                    p1, p2, linknode, copyfrom = ancestorinfo
+
+                    # The presence of copyfrom means we're at a point where the
+                    # file was copied from elsewhere. So don't attempt to do any
+                    # deltas with the other file.
+                    if copyfrom:
+                        p1 = nullid
+
+                    if chainlen < maxchainlen:
+                        # Record this child as the delta base for its parents.
+                        # This may be non optimal, since the parents may have
+                        # many children, and this will only choose the last one.
+                        # TODO: record all children and try all deltas to find
+                        # best
+                        if p1 != nullid:
+                            deltabases[p1] = (node, chainlen + 1)
+                        if p2 != nullid:
+                            deltabases[p2] = (node, chainlen + 1)
+
+            # experimental config: repack.chainorphansbysize
+            if ui.configbool('repack', 'chainorphansbysize'):
+                orphans = nobase - referenced
+                orderednodes = self._chainorphans(ui, filename, orderednodes,
+                    orphans, deltabases)
+
+            # Compute deltas and write to the pack
+            for i, node in enumerate(orderednodes):
+                deltabase, chainlen = deltabases[node]
+                # Compute delta
+                # TODO: Optimize the deltachain fetching. Since we're
+                # iterating over the different version of the file, we may
+                # be fetching the same deltachain over and over again.
+                meta = None
+                if deltabase != nullid:
+                    deltaentry = self.data.getdelta(filename, node)
+                    delta, deltabasename, origdeltabase, meta = deltaentry
+                    size = meta.get(constants.METAKEYSIZE)
+                    if (deltabasename != filename or origdeltabase != deltabase
+                        or size is None):
+                        deltabasetext = self.data.get(filename, deltabase)
+                        original = self.data.get(filename, node)
+                        size = len(original)
+                        delta = mdiff.textdiff(deltabasetext, original)
+                else:
+                    delta = self.data.get(filename, node)
+                    size = len(delta)
+                    meta = self.data.getmeta(filename, node)
+
+                # TODO: don't use the delta if it's larger than the fulltext
+                if constants.METAKEYSIZE not in meta:
+                    meta[constants.METAKEYSIZE] = size
+                target.add(filename, node, deltabase, delta, meta)
+
+                entries[node].datarepacked = True
+
+            processprogress.complete()
+            count += 1
+
+        repackprogress.complete()
+        target.close(ledger=ledger)
+
+    def repackhistory(self, ledger, target):
+        ui = self.repo.ui
+
+        byfile = {}
+        for entry in ledger.entries.itervalues():
+            if entry.historysource:
+                byfile.setdefault(entry.filename, {})[entry.node] = entry
+
+        progress = ui.makeprogress(_("repacking history"), unit=self.unit,
+                                   total=len(byfile))
+        for filename, entries in sorted(byfile.iteritems()):
+            ancestors = {}
+            nodes = list(node for node in entries)
+
+            for node in nodes:
+                if node in ancestors:
+                    continue
+                ancestors.update(self.history.getancestors(filename, node,
+                                                           known=ancestors))
+
+            # Order the nodes children first
+            orderednodes = reversed(self._toposort(ancestors))
+
+            # Write to the pack
+            dontprocess = set()
+            for node in orderednodes:
+                p1, p2, linknode, copyfrom = ancestors[node]
+
+                # If the node is marked dontprocess, but it's also in the
+                # explicit entries set, that means the node exists both in this
+                # file and in another file that was copied to this file.
+                # Usually this happens if the file was copied to another file,
+                # then the copy was deleted, then reintroduced without copy
+                # metadata. The original add and the new add have the same hash
+                # since the content is identical and the parents are null.
+                if node in dontprocess and node not in entries:
+                    # If copyfrom == filename, it means the copy history
+                    # went to come other file, then came back to this one, so we
+                    # should continue processing it.
+                    if p1 != nullid and copyfrom != filename:
+                        dontprocess.add(p1)
+                    if p2 != nullid:
+                        dontprocess.add(p2)
+                    continue
+
+                if copyfrom:
+                    dontprocess.add(p1)
+
+                target.add(filename, node, p1, p2, linknode, copyfrom)
+
+                if node in entries:
+                    entries[node].historyrepacked = True
+
+            progress.increment()
+
+        progress.complete()
+        target.close(ledger=ledger)
+
+    def _toposort(self, ancestors):
+        def parentfunc(node):
+            p1, p2, linknode, copyfrom = ancestors[node]
+            parents = []
+            if p1 != nullid:
+                parents.append(p1)
+            if p2 != nullid:
+                parents.append(p2)
+            return parents
+
+        sortednodes = shallowutil.sortnodes(ancestors.keys(), parentfunc)
+        return sortednodes
+
+class repackledger(object):
+    """Storage for all the bookkeeping that happens during a repack. It contains
+    the list of revisions being repacked, what happened to each revision, and
+    which source store contained which revision originally (for later cleanup).
+    """
+    def __init__(self):
+        self.entries = {}
+        self.sources = {}
+        self.created = set()
+
+    def markdataentry(self, source, filename, node):
+        """Mark the given filename+node revision as having a data rev in the
+        given source.
+        """
+        entry = self._getorcreateentry(filename, node)
+        entry.datasource = True
+        entries = self.sources.get(source)
+        if not entries:
+            entries = set()
+            self.sources[source] = entries
+        entries.add(entry)
+
+    def markhistoryentry(self, source, filename, node):
+        """Mark the given filename+node revision as having a history rev in the
+        given source.
+        """
+        entry = self._getorcreateentry(filename, node)
+        entry.historysource = True
+        entries = self.sources.get(source)
+        if not entries:
+            entries = set()
+            self.sources[source] = entries
+        entries.add(entry)
+
+    def _getorcreateentry(self, filename, node):
+        key = (filename, node)
+        value = self.entries.get(key)
+        if not value:
+            value = repackentry(filename, node)
+            self.entries[key] = value
+
+        return value
+
+    def addcreated(self, value):
+        self.created.add(value)
+
+class repackentry(object):
+    """Simple class representing a single revision entry in the repackledger.
+    """
+    __slots__ = (r'filename', r'node', r'datasource', r'historysource',
+                 r'datarepacked', r'historyrepacked', r'gced')
+    def __init__(self, filename, node):
+        self.filename = filename
+        self.node = node
+        # If the revision has a data entry in the source
+        self.datasource = False
+        # If the revision has a history entry in the source
+        self.historysource = False
+        # If the revision's data entry was repacked into the repack target
+        self.datarepacked = False
+        # If the revision's history entry was repacked into the repack target
+        self.historyrepacked = False
+        # If garbage collected
+        self.gced = False
+
+def repacklockvfs(repo):
+    if util.safehasattr(repo, 'name'):
+        # Lock in the shared cache so repacks across multiple copies of the same
+        # repo are coordinated.
+        sharedcachepath = shallowutil.getcachepackpath(
+            repo,
+            constants.FILEPACK_CATEGORY)
+        return vfs.vfs(sharedcachepath)
+    else:
+        return repo.svfs

hgext/remotefilelog/shallowbundle.py

@@ -0,0 +1,293 b''
+# shallowbundle.py - bundle10 implementation for use with shallow repositories
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+from mercurial.node import bin, hex, nullid
+from mercurial import (
+    bundlerepo,
+    changegroup,
+    error,
+    match,
+    mdiff,
+    pycompat,
+)
+from . import (
+    constants,
+    remotefilelog,
+    shallowutil,
+)
+
+NoFiles = 0
+LocalFiles = 1
+AllFiles = 2
+
+def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None):
+    if not isinstance(rlog, remotefilelog.remotefilelog):
+        for c in super(cls, self).group(nodelist, rlog, lookup,
+                                        units=units):
+            yield c
+        return
+
+    if len(nodelist) == 0:
+        yield self.close()
+        return
+
+    nodelist = shallowutil.sortnodes(nodelist, rlog.parents)
+
+    # add the parent of the first rev
+    p = rlog.parents(nodelist[0])[0]
+    nodelist.insert(0, p)
+
+    # build deltas
+    for i in pycompat.xrange(len(nodelist) - 1):
+        prev, curr = nodelist[i], nodelist[i + 1]
+        linknode = lookup(curr)
+        for c in self.nodechunk(rlog, curr, prev, linknode):
+            yield c
+
+    yield self.close()
+
+class shallowcg1packer(changegroup.cgpacker):
+    def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
+        if shallowutil.isenabled(self._repo):
+            fastpathlinkrev = False
+
+        return super(shallowcg1packer, self).generate(commonrevs, clnodes,
+            fastpathlinkrev, source)
+
+    def group(self, nodelist, rlog, lookup, units=None, reorder=None):
+        return shallowgroup(shallowcg1packer, self, nodelist, rlog, lookup,
+                            units=units)
+
+    def generatefiles(self, changedfiles, *args):
+        try:
+            linknodes, commonrevs, source = args
+        except ValueError:
+            commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args
+        if shallowutil.isenabled(self._repo):
+            repo = self._repo
+            if isinstance(repo, bundlerepo.bundlerepository):
+                # If the bundle contains filelogs, we can't pull from it, since
+                # bundlerepo is heavily tied to revlogs. Instead require that
+                # the user use unbundle instead.
+                # Force load the filelog data.
+                bundlerepo.bundlerepository.file(repo, 'foo')
+                if repo._cgfilespos:
+                    raise error.Abort("cannot pull from full bundles",
+                                      hint="use `hg unbundle` instead")
+                return []
+            filestosend = self.shouldaddfilegroups(source)
+            if filestosend == NoFiles:
+                changedfiles = list([f for f in changedfiles
+                                     if not repo.shallowmatch(f)])
+
+        return super(shallowcg1packer, self).generatefiles(
+            changedfiles, *args)
+
+    def shouldaddfilegroups(self, source):
+        repo = self._repo
+        if not shallowutil.isenabled(repo):
+            return AllFiles
+
+        if source == "push" or source == "bundle":
+            return AllFiles
+
+        caps = self._bundlecaps or []
+        if source == "serve" or source == "pull":
+            if constants.BUNDLE2_CAPABLITY in caps:
+                return LocalFiles
+            else:
+                # Serving to a full repo requires us to serve everything
+                repo.ui.warn(_("pulling from a shallow repo\n"))
+                return AllFiles
+
+        return NoFiles
+
+    def prune(self, rlog, missing, commonrevs):
+        if not isinstance(rlog, remotefilelog.remotefilelog):
+            return super(shallowcg1packer, self).prune(rlog, missing,
+                commonrevs)
+
+        repo = self._repo
+        results = []
+        for fnode in missing:
+            fctx = repo.filectx(rlog.filename, fileid=fnode)
+            if fctx.linkrev() not in commonrevs:
+                results.append(fnode)
+        return results
+
+    def nodechunk(self, revlog, node, prevnode, linknode):
+        prefix = ''
+        if prevnode == nullid:
+            delta = revlog.revision(node, raw=True)
+            prefix = mdiff.trivialdiffheader(len(delta))
+        else:
+            # Actually uses remotefilelog.revdiff which works on nodes, not revs
+            delta = revlog.revdiff(prevnode, node)
+        p1, p2 = revlog.parents(node)
+        flags = revlog.flags(node)
+        meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags)
+        meta += prefix
+        l = len(meta) + len(delta)
+        yield changegroup.chunkheader(l)
+        yield meta
+        yield delta
+
+def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs):
+    if not shallowutil.isenabled(repo):
+        return orig(repo, outgoing, version, source, *args, **kwargs)
+
+    original = repo.shallowmatch
+    try:
+        # if serving, only send files the clients has patterns for
+        if source == 'serve':
+            bundlecaps = kwargs.get(r'bundlecaps')
+            includepattern = None
+            excludepattern = None
+            for cap in (bundlecaps or []):
+                if cap.startswith("includepattern="):
+                    raw = cap[len("includepattern="):]
+                    if raw:
+                        includepattern = raw.split('\0')
+                elif cap.startswith("excludepattern="):
+                    raw = cap[len("excludepattern="):]
+                    if raw:
+                        excludepattern = raw.split('\0')
+            if includepattern or excludepattern:
+                repo.shallowmatch = match.match(repo.root, '', None,
+                    includepattern, excludepattern)
+            else:
+                repo.shallowmatch = match.always(repo.root, '')
+        return orig(repo, outgoing, version, source, *args, **kwargs)
+    finally:
+        repo.shallowmatch = original
+
+def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
+    if not shallowutil.isenabled(repo):
+        return orig(repo, source, revmap, trp, expectedfiles, *args)
+
+    newfiles = 0
+    visited = set()
+    revisiondatas = {}
+    queue = []
+
+    # Normal Mercurial processes each file one at a time, adding all
+    # the new revisions for that file at once. In remotefilelog a file
+    # revision may depend on a different file's revision (in the case
+    # of a rename/copy), so we must lay all revisions down across all
+    # files in topological order.
+
+    # read all the file chunks but don't add them
+    progress = repo.ui.makeprogress(_('files'), total=expectedfiles)
+    while True:
+        chunkdata = source.filelogheader()
+        if not chunkdata:
+            break
+        f = chunkdata["filename"]
+        repo.ui.debug("adding %s revisions\n" % f)
+        progress.increment()
+
+        if not repo.shallowmatch(f):
+            fl = repo.file(f)
+            deltas = source.deltaiter()
+            fl.addgroup(deltas, revmap, trp)
+            continue
+
+        chain = None
+        while True:
+            # returns: (node, p1, p2, cs, deltabase, delta, flags) or None
+            revisiondata = source.deltachunk(chain)
+            if not revisiondata:
+                break
+
+            chain = revisiondata[0]
+
+            revisiondatas[(f, chain)] = revisiondata
+            queue.append((f, chain))
+
+            if f not in visited:
+                newfiles += 1
+                visited.add(f)
+
+        if chain is None:
+            raise error.Abort(_("received file revlog group is empty"))
+
+    processed = set()
+    def available(f, node, depf, depnode):
+        if depnode != nullid and (depf, depnode) not in processed:
+            if not (depf, depnode) in revisiondatas:
+                # It's not in the changegroup, assume it's already
+                # in the repo
+                return True
+            # re-add self to queue
+            queue.insert(0, (f, node))
+            # add dependency in front
+            queue.insert(0, (depf, depnode))
+            return False
+        return True
+
+    skipcount = 0
+
+    # Prefetch the non-bundled revisions that we will need
+    prefetchfiles = []
+    for f, node in queue:
+        revisiondata = revisiondatas[(f, node)]
+        # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
+        dependents = [revisiondata[1], revisiondata[2], revisiondata[4]]
+
+        for dependent in dependents:
+            if dependent == nullid or (f, dependent) in revisiondatas:
+                continue
+            prefetchfiles.append((f, hex(dependent)))
+
+    repo.fileservice.prefetch(prefetchfiles)
+
+    # Apply the revisions in topological order such that a revision
+    # is only written once it's deltabase and parents have been written.
+    while queue:
+        f, node = queue.pop(0)
+        if (f, node) in processed:
+            continue
+
+        skipcount += 1
+        if skipcount > len(queue) + 1:
+            raise error.Abort(_("circular node dependency"))
+
+        fl = repo.file(f)
+
+        revisiondata = revisiondatas[(f, node)]
+        # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
+        node, p1, p2, linknode, deltabase, delta, flags = revisiondata
+
+        if not available(f, node, f, deltabase):
+            continue
+
+        base = fl.revision(deltabase, raw=True)
+        text = mdiff.patch(base, delta)
+        if not isinstance(text, bytes):
+            text = bytes(text)
+
+        meta, text = shallowutil.parsemeta(text)
+        if 'copy' in meta:
+            copyfrom = meta['copy']
+            copynode = bin(meta['copyrev'])
+            if not available(f, node, copyfrom, copynode):
+                continue
+
+        for p in [p1, p2]:
+            if p != nullid:
+                if not available(f, node, f, p):
+                    continue
+
+        fl.add(text, meta, trp, linknode, p1, p2)
+        processed.add((f, node))
+        skipcount = 0
+
+    progress.complete()
+
+    return len(revisiondatas), newfiles

Makefile

@@ -72,6 +72,8 b' cleanbutpackages:'
 	rm -rf build mercurial/locale
 	$(MAKE) -C doc clean
 	$(MAKE) -C contrib/chg distclean
+	rm -rf rust/target
+	rm -f mercurial/rustext.so
 
 clean: cleanbutpackages
 	rm -rf packages
@@ -178,6 +180,7 b' packaging_targets := \\'
   docker-fedora20 \
   docker-fedora21 \
   docker-fedora28 \
+  docker-fedora29 \
   docker-ubuntu-trusty \
   docker-ubuntu-trusty-ppa \
   docker-ubuntu-xenial \
@@ -189,6 +192,7 b' packaging_targets := \\'
   fedora20 \
   fedora21 \
   fedora28 \
+  fedora29 \
   linux-wheels \
   linux-wheels-x86_64 \
   linux-wheels-i686 \

contrib/all-revsets.txt

@@ -139,3 +139,18 b' secret()'
 # test finding common ancestors
 heads(commonancestors(last(head(), 2)))
 heads(commonancestors(head()))
+
+# more heads testing
+heads(all())
+heads(-10000:-1)
+(-5000:-1000) and heads(-10000:-1)
+heads(matching(tip, "author"))
+heads(matching(tip, "author")) and -10000:-1
+(-10000:-1) and heads(matching(tip, "author"))
+# more roots testing
+roots(all())
+roots(-10000:-1)
+(-5000:-1000) and roots(-10000:-1)
+roots(matching(tip, "author"))
+roots(matching(tip, "author")) and -10000:-1
+(-10000:-1) and roots(matching(tip, "author"))

contrib/catapipe.py

@@ -6,6 +6,14 b''
 # GNU General Public License version 2 or any later version.
 """Tool read primitive events from a pipe to produce a catapult trace.
 
+Usage:
+    Terminal 1: $ catapipe.py /tmp/mypipe /tmp/trace.json
+    Terminal 2: $ HGCATAPULTSERVERPIPE=/tmp/mypipe hg root
+    <ctrl-c catapipe.py in Terminal 1>
+    $ catapult/tracing/bin/trace2html /tmp/trace.json  # produce /tmp/trace.html
+    <open trace.html in your browser of choice; the WASD keys are very useful>
+    (catapult is located at https://github.com/catapult-project/catapult)
+
 For now the event stream supports
 
   START $SESSIONID ...
@@ -24,7 +32,7 b' run-tests.py.'
 
 Typically you'll want to place the path to the named pipe in the
 HGCATAPULTSERVERPIPE environment variable, which both run-tests and hg
-understand.
+understand. To trace *only* run-tests, use HGTESTCATAPULTSERVERPIPE instead.
 """
 from __future__ import absolute_import, print_function
 

contrib/check-commit

@@ -34,7 +34,7 b' errors = ['
     (commitheader + r"(?!merge with )[^#]\S+[^:] ",
      "summary line doesn't start with 'topic: '"),
     (afterheader + r"[A-Z][a-z]\S+", "don't capitalize summary lines"),
-    (afterheader + r"[^\n]*: *[A-Z][a-z]\S+", "don't capitalize summary lines"),
+    (afterheader + r"^\S+: *[A-Z][a-z]\S+", "don't capitalize summary lines"),
     (afterheader + r"\S*[^A-Za-z0-9-_]\S*: ",
      "summary keyword should be most user-relevant one-word command or topic"),
     (afterheader + r".*\.\s*\n", "don't add trailing period on summary line"),

contrib/clang-format-ignorelist

@@ -3,7 +3,6 b''
 mercurial/cext/dirs.c
 mercurial/cext/manifest.c
 mercurial/cext/osutil.c
-mercurial/cext/revlog.c
 # Vendored code that we should never format:
 contrib/python-zstandard/c-ext/bufferutil.c
 contrib/python-zstandard/c-ext/compressionchunker.c

contrib/fuzz/Makefile

@@ -4,7 +4,7 b' CXX = clang++'
 all: bdiff mpatch xdiff
 
 fuzzutil.o: fuzzutil.cc fuzzutil.h
-	$(CXX) $(CXXFLAGS) -g -O1 -fsanitize=fuzzer-no-link,address \
+	$(CXX) $(CXXFLAGS) -g -O1 \
 	  -std=c++17 \
 	  -I../../mercurial -c -o fuzzutil.o fuzzutil.cc
 
@@ -12,6 +12,11 b' fuzzutil-oss-fuzz.o: fuzzutil.cc fuzzuti'
 	$(CXX) $(CXXFLAGS) -std=c++17 \
 	  -I../../mercurial -c -o fuzzutil-oss-fuzz.o fuzzutil.cc
 
+pyutil.o: pyutil.cc pyutil.h
+	$(CXX) $(CXXFLAGS) -g -O1 \
+	  `$$OUT/sanpy/bin/python-config --cflags` \
+	  -I../../mercurial -c -o pyutil.o pyutil.cc
+
 bdiff.o: ../../mercurial/bdiff.c
 	$(CC) $(CFLAGS) -fsanitize=fuzzer-no-link,address -c -o bdiff.o \
 	  ../../mercurial/bdiff.c
@@ -70,59 +75,86 b' xdiff_fuzzer: xdiff.cc fuzz-xdiffi.o fuz'
 	  fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o fuzzutil-oss-fuzz.o \
 	  -lFuzzingEngine -o $$OUT/xdiff_fuzzer
 
-# TODO use the $OUT env var instead of hardcoding /out
-/out/sanpy/bin/python:
-	cd /Python-2.7.15/ && ./configure --without-pymalloc --prefix=$$OUT/sanpy CFLAGS='-O1 -fno-omit-frame-pointer -g -fwrapv -fstack-protector-strong' LDFLAGS=-lasan  && ASAN_OPTIONS=detect_leaks=0 make && make install
-
-sanpy: /out/sanpy/bin/python
-
-manifest.o: sanpy ../../mercurial/cext/manifest.c
+manifest.o: ../../mercurial/cext/manifest.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o manifest.o ../../mercurial/cext/manifest.c
 
-charencode.o: sanpy ../../mercurial/cext/charencode.c
+charencode.o: ../../mercurial/cext/charencode.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o charencode.o ../../mercurial/cext/charencode.c
 
-parsers.o: sanpy ../../mercurial/cext/parsers.c
+parsers.o: ../../mercurial/cext/parsers.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o parsers.o ../../mercurial/cext/parsers.c
 
-dirs.o: sanpy ../../mercurial/cext/dirs.c
+dirs.o: ../../mercurial/cext/dirs.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o dirs.o ../../mercurial/cext/dirs.c
 
-pathencode.o: sanpy ../../mercurial/cext/pathencode.c
+pathencode.o: ../../mercurial/cext/pathencode.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o pathencode.o ../../mercurial/cext/pathencode.c
 
-revlog.o: sanpy ../../mercurial/cext/revlog.c
+revlog.o: ../../mercurial/cext/revlog.c
 	$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -I../../mercurial \
 	  -c -o revlog.o ../../mercurial/cext/revlog.c
 
-manifest_fuzzer: sanpy manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o
+manifest_fuzzer: manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
 	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
 	  -Wno-register -Wno-macro-redefined \
 	  -I../../mercurial manifest.cc \
-	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
 	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
 	  -o $$OUT/manifest_fuzzer
 
 manifest_corpus.zip:
 	python manifest_corpus.py $$OUT/manifest_fuzzer_seed_corpus.zip
 
+revlog_fuzzer: revlog.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
+	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
+	  -Wno-register -Wno-macro-redefined \
+	  -I../../mercurial revlog.cc \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
+	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
+	  -o $$OUT/revlog_fuzzer
+
+revlog_corpus.zip:
+	python revlog_corpus.py $$OUT/revlog_fuzzer_seed_corpus.zip
+
+dirstate_fuzzer: dirstate.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
+	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
+	  -Wno-register -Wno-macro-redefined \
+	  -I../../mercurial dirstate.cc \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
+	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
+	  -o $$OUT/dirstate_fuzzer
+
+dirstate_corpus.zip:
+	python dirstate_corpus.py $$OUT/dirstate_fuzzer_seed_corpus.zip
+
+fm1readmarkers_fuzzer: fm1readmarkers.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
+	$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
+	  -Wno-register -Wno-macro-redefined \
+	  -I../../mercurial fm1readmarkers.cc \
+	  manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
+	  -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
+	  -o $$OUT/fm1readmarkers_fuzzer
+
+fm1readmarkers_corpus.zip:
+	python fm1readmarkers_corpus.py $$OUT/fm1readmarkers_fuzzer_seed_corpus.zip
+
 clean:
 	$(RM) *.o *_fuzzer \
 	  bdiff \
 	  mpatch \
 	  xdiff
 
-oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip
+oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip revlog_fuzzer revlog_corpus.zip dirstate_fuzzer dirstate_corpus.zip fm1readmarkers_fuzzer fm1readmarkers_corpus.zip
 
-.PHONY: all clean oss-fuzz sanpy
+.PHONY: all clean oss-fuzz

contrib/fuzz/manifest.cc

@@ -3,43 +3,17 b''
 #include <stdlib.h>
 #include <unistd.h>
 
+#include "pyutil.h"
+
 #include <string>
 
 extern "C" {
 
-/* TODO: use Python 3 for this fuzzing? */
-PyMODINIT_FUNC initparsers(void);
-
-static char cpypath[8192] = "\0";
-
 static PyCodeObject *code;
-static PyObject *mainmod;
-static PyObject *globals;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
-	const std::string subdir = "/sanpy/lib/python2.7";
-	/* HACK ALERT: we need a full Python installation built without
-	   pymalloc and with ASAN, so we dump one in
-	   $OUT/sanpy/lib/python2.7. This helps us wire that up. */
-	std::string selfpath(*argv[0]);
-	std::string pypath;
-	auto pos = selfpath.rfind("/");
-	if (pos == std::string::npos) {
-		char wd[8192];
-		getcwd(wd, 8192);
-		pypath = std::string(wd) + subdir;
-	} else {
-		pypath = selfpath.substr(0, pos) + subdir;
-	}
-	strncpy(cpypath, pypath.c_str(), pypath.size());
-	setenv("PYTHONPATH", cpypath, 1);
-	setenv("PYTHONNOUSERSITE", "1", 1);
-	/* prevent Python from looking up users in the fuzz environment */
-	setenv("PYTHONUSERBASE", cpypath, 1);
-	Py_SetPythonHome(cpypath);
-	Py_InitializeEx(0);
-	initparsers();
+	contrib::initpy(*argv[0]);
 	code = (PyCodeObject *)Py_CompileString(R"py(
 from parsers import lazymanifest
 try:
@@ -60,8 +34,6 b' except Exception as e:'
   # print e
 )py",
 	                                        "fuzzer", Py_file_input);
-	mainmod = PyImport_AddModule("__main__");
-	globals = PyModule_GetDict(mainmod);
 	return 0;
 }
 
@@ -71,7 +43,7 b' int LLVMFuzzerTestOneInput(const uint8_t'
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "mdata", mtext);
-	PyObject *res = PyEval_EvalCode(code, globals, locals);
+	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}

contrib/fuzz/xdiff.cc

@@ -22,6 +22,11 b' int hunk_consumer(long a1, long a2, long'
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
+	// Don't allow fuzzer inputs larger than 100k, since we'll just bog
+	// down and not accomplish much.
+	if (Size > 100000) {
+		return 0;
+	}
 	auto maybe_inputs = SplitInputs(Data, Size);
 	if (!maybe_inputs) {
 		return 0;

contrib/hgclient.py

@@ -27,12 +27,18 b' else:'
     stringio = cStringIO.StringIO
     bprint = print
 
-def connectpipe(path=None):
+def connectpipe(path=None, extraargs=()):
     cmdline = [b'hg', b'serve', b'--cmdserver', b'pipe']
     if path:
         cmdline += [b'-R', path]
+    cmdline.extend(extraargs)
 
-    server = subprocess.Popen(cmdline, stdin=subprocess.PIPE,
+    def tonative(cmdline):
+        if os.name != r'nt':
+            return cmdline
+        return [arg.decode("utf-8") for arg in cmdline]
+
+    server = subprocess.Popen(tonative(cmdline), stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE)
 
     return server
@@ -114,6 +120,8 b' def runcommand(server, args, output=stdo'
             writeblock(server, input.read(data))
         elif ch == b'L':
             writeblock(server, input.readline(data))
+        elif ch == b'm':
+            bprint(b"message: %r" % data)
         elif ch == b'r':
             ret, = struct.unpack('>i', data)
             if ret != 0:
@@ -132,3 +140,8 b' def check(func, connect=connectpipe):'
     finally:
         server.stdin.close()
         server.wait()
+
+def checkwith(connect=connectpipe, **kwargs):
+    def wrap(func):
+        return check(func, lambda: connect(**kwargs))
+    return wrap

contrib/import-checker.py

@@ -40,8 +40,6 b' allowsymbolimports = ('
     # third-party imports should be directly imported
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
-    'mercurial.thirdparty.cbor',
-    'mercurial.thirdparty.cbor.cbor2',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
 )
@@ -260,10 +258,12 b' def list_stdlib_modules():'
                 break
         else:
             stdlib_prefixes.add(dirname)
+    sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
     for libpath in sys.path:
-        # We want to walk everything in sys.path that starts with
-        # something in stdlib_prefixes.
-        if not any(libpath.startswith(p) for p in stdlib_prefixes):
+        # We want to walk everything in sys.path that starts with something in
+        # stdlib_prefixes, but not directories from the hg sources.
+        if (os.path.abspath(libpath).startswith(sourceroot)
+            or not any(libpath.startswith(p) for p in stdlib_prefixes)):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
@@ -674,6 +674,8 b' def embedded(f, modname, src):'
         # "starts" is "line number" (1-origin), but embedded() is
         # expected to return "line offset" (0-origin). Therefore, this
         # yields "starts - 1".
+        if not isinstance(modname, str):
+            modname = modname.decode('utf8')
         yield code, "%s[%d]" % (modname, starts), name, starts - 1
 
 def sources(f, modname):
@@ -694,7 +696,7 b' def sources(f, modname):'
     if py or f.endswith('.t'):
         with open(f, 'rb') as src:
             for script, modname, t, line in embedded(f, modname, src):
-                yield script, modname, t, line
+                yield script, modname.encode('utf8'), t, line
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):

contrib/packaging/Makefile

@@ -14,7 +14,8 b' UBUNTU_CODENAMES := \\'
 FEDORA_RELEASES := \
   20 \
   21 \
-  28
+  28 \
+  29
 
 CENTOS_RELEASES := \
   5 \

contrib/packaging/docker/centos5

@@ -1,7 +1,7 b''
 FROM centos:centos5
 
-RUN groupadd -g 1000 build && \
-    useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
 
 RUN \
 	sed -i 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/*.repo && \

contrib/packaging/docker/centos6

@@ -1,7 +1,7 b''
 FROM centos:centos6
 
-RUN groupadd -g 1000 build && \
-    useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
 
 RUN yum install -y \
 	gcc \

contrib/packaging/docker/centos7

@@ -1,7 +1,7 b''
 FROM centos:centos7
 
-RUN groupadd -g 1000 build && \
-    useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
 
 RUN yum install -y \
 	gcc \

contrib/packaging/docker/fedora29

@@ -1,4 +1,4 b''
-FROM fedora:28
+FROM fedora:29
 
 RUN groupadd -g 1000 build && \
     useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build

contrib/packaging/dockerrpm

@@ -10,7 +10,15 b' DOCKER=$($BUILDDIR/hg-docker docker-path'
 
 CONTAINER=hg-docker-$PLATFORM
 
-$BUILDDIR/hg-docker build $BUILDDIR/docker/$PLATFORM $CONTAINER
+if [[ -z "${HG_DOCKER_OWN_USER}" ]]; then
+    DOCKERUID=1000
+    DOCKERGID=1000
+else
+    DOCKERUID=$(id -u)
+    DOCKERGID=$(id -g)
+fi
+
+$BUILDDIR/hg-docker build --build-arg UID=$DOCKERUID --build-arg GID=$DOCKERGID $BUILDDIR/docker/$PLATFORM $CONTAINER
 
 RPMBUILDDIR=$ROOTDIR/packages/$PLATFORM
 $ROOTDIR/contrib/packaging/buildrpm --rpmbuilddir $RPMBUILDDIR --prepare $*

contrib/packaging/hg-docker

@@ -47,7 +47,7 b' def get_dockerfile(path: pathlib.Path, a'
         df = fh.read()
 
     for k, v in args:
-        df = df.replace(b'%%%s%%' % k, v)
+        df = df.replace(bytes('%%%s%%' % k.decode(), 'utf-8'), v)
 
     return df
 
@@ -72,7 +72,12 b' def build_docker_image(dockerfile: pathl'
     ]
 
     print('executing: %r' % args)
-    subprocess.run(args, input=dockerfile, check=True)
+    p = subprocess.Popen(args, stdin=subprocess.PIPE)
+    p.communicate(input=dockerfile)
+    if p.returncode:
+        raise subprocess.CalledProcessException(
+                p.returncode, 'failed to build docker image: %s %s' \
+                % (p.stdout, p.stderr))
 
 def command_build(args):
     build_args = []

contrib/perf.py

@@ -24,8 +24,10 b' import functools'
 import gc
 import os
 import random
+import shutil
 import struct
 import sys
+import tempfile
 import threading
 import time
 from mercurial import (
@@ -35,6 +37,7 b' from mercurial import ('
     copies,
     error,
     extensions,
+    hg,
     mdiff,
     merge,
     revlog,
@@ -65,6 +68,11 b' try:'
     from mercurial import scmutil # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
+try:
+    from mercurial import setdiscovery # since 1.9 (or cb98fed52495)
+except ImportError:
+    pass
+
 
 def identity(a):
     return a
@@ -273,7 +281,9 b' def gettimer(ui, opts=None):'
     displayall = ui.configbool(b"perf", b"all-timing", False)
     return functools.partial(_timer, fm, displayall=displayall), fm
 
-def stub_timer(fm, func, title=None):
+def stub_timer(fm, func, setup=None, title=None):
+    if setup is not None:
+        setup()
     func()
 
 @contextlib.contextmanager
@@ -287,12 +297,14 b' def timeone():'
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
 
-def _timer(fm, func, title=None, displayall=False):
+def _timer(fm, func, setup=None, title=None, displayall=False):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
     while True:
+        if setup is not None:
+            setup()
         with timeone() as item:
             r = func()
         count += 1
@@ -453,11 +465,19 b' def repocleartagscachefunc(repo):'
 
 # utilities to clear cache
 
-def clearfilecache(repo, attrname):
-    unfi = repo.unfiltered()
-    if attrname in vars(unfi):
-        delattr(unfi, attrname)
-    unfi._filecache.pop(attrname, None)
+def clearfilecache(obj, attrname):
+    unfiltered = getattr(obj, 'unfiltered', None)
+    if unfiltered is not None:
+        obj = obj.unfiltered()
+    if attrname in vars(obj):
+        delattr(obj, attrname)
+    obj._filecache.pop(attrname, None)
+
+def clearchangelog(repo):
+    if repo is not repo.unfiltered():
+        object.__setattr__(repo, r'_clcachekey', None)
+        object.__setattr__(repo, r'_clcache', None)
+    clearfilecache(repo.unfiltered(), 'changelog')
 
 # perf commands
 
@@ -524,23 +544,23 b' def perfheads(ui, repo, **opts):'
     timer(d)
     fm.end()
 
-@command(b'perftags', formatteropts)
+@command(b'perftags', formatteropts+
+        [
+            (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
+        ])
 def perftags(ui, repo, **opts):
-    import mercurial.changelog
-    import mercurial.manifest
-
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
-    svfs = getsvfs(repo)
     repocleartagscache = repocleartagscachefunc(repo)
+    clearrevlogs = opts[b'clear_revlogs']
+    def s():
+        if clearrevlogs:
+            clearchangelog(repo)
+            clearfilecache(repo.unfiltered(), 'manifest')
+        repocleartagscache()
     def t():
-        repo.changelog = mercurial.changelog.changelog(svfs)
-        rootmanifest = mercurial.manifest.manifestrevlog(svfs)
-        repo.manifestlog = mercurial.manifest.manifestlog(svfs, repo,
-                                                          rootmanifest)
-        repocleartagscache()
         return len(repo.tags())
-    timer(t)
+    timer(t, setup=s)
     fm.end()
 
 @command(b'perfancestors', formatteropts)
@@ -567,15 +587,38 b' def perfancestorset(ui, repo, revset, **'
     timer(d)
     fm.end()
 
-@command(b'perfbookmarks', formatteropts)
+@command(b'perfdiscovery', formatteropts, b'PATH')
+def perfdiscovery(ui, repo, path, **opts):
+    """benchmark discovery between local repo and the peer at given path
+    """
+    repos = [repo, None]
+    timer, fm = gettimer(ui, opts)
+    path = ui.expandpath(path)
+
+    def s():
+        repos[1] = hg.peer(ui, opts, path)
+    def d():
+        setdiscovery.findcommonheads(ui, *repos)
+    timer(d, setup=s)
+    fm.end()
+
+@command(b'perfbookmarks', formatteropts +
+        [
+            (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
+        ])
 def perfbookmarks(ui, repo, **opts):
     """benchmark parsing bookmarks from disk to memory"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
-    def d():
+
+    clearrevlogs = opts[b'clear_revlogs']
+    def s():
+        if clearrevlogs:
+            clearchangelog(repo)
         clearfilecache(repo, b'_bookmarks')
+    def d():
         repo._bookmarks
-    timer(d)
+    timer(d, setup=s)
     fm.end()
 
 @command(b'perfbundleread', formatteropts, b'BUNDLE')
@@ -697,9 +740,9 b' def perfbundleread(ui, repo, bundlepath,'
         fm.end()
 
 @command(b'perfchangegroupchangelog', formatteropts +
-         [(b'', b'version', b'02', b'changegroup version'),
+         [(b'', b'cgversion', b'02', b'changegroup version'),
           (b'r', b'rev', b'', b'revisions to add to changegroup')])
-def perfchangegroupchangelog(ui, repo, version=b'02', rev=None, **opts):
+def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
     """Benchmark producing a changelog group for a changegroup.
 
     This measures the time spent processing the changelog during a
@@ -712,7 +755,7 b' def perfchangegroupchangelog(ui, repo, v'
     opts = _byteskwargs(opts)
     cl = repo.changelog
     nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
-    bundler = changegroup.getbundler(version, repo)
+    bundler = changegroup.getbundler(cgversion, repo)
 
     def d():
         state, chunks = bundler._generatechangelog(cl, nodes)
@@ -819,6 +862,7 b' def perfmergecalculate(ui, repo, rev, **'
 
 @command(b'perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
+    """benchmark the copy tracing logic"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
@@ -952,18 +996,48 b' def perfchangeset(ui, repo, rev, **opts)'
     timer(d)
     fm.end()
 
-@command(b'perfindex', formatteropts)
+@command(b'perfignore', formatteropts)
+def perfignore(ui, repo, **opts):
+    """benchmark operation related to computing ignore"""
+    opts = _byteskwargs(opts)
+    timer, fm = gettimer(ui, opts)
+    dirstate = repo.dirstate
+
+    def setupone():
+        dirstate.invalidate()
+        clearfilecache(dirstate, b'_ignore')
+
+    def runone():
+        dirstate._ignore
+
+    timer(runone, setup=setupone, title=b"load")
+    fm.end()
+
+@command(b'perfindex', [
+            (b'', b'rev', b'', b'revision to be looked up (default tip)'),
+         ] + formatteropts)
 def perfindex(ui, repo, **opts):
     import mercurial.revlog
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
-    n = repo[b"tip"].node()
-    svfs = getsvfs(repo)
+    if opts[b'rev'] is None:
+        n = repo[b"tip"].node()
+    else:
+        rev = scmutil.revsingle(repo, opts[b'rev'])
+        n = repo[rev].node()
+
+    unfi = repo.unfiltered()
+    # find the filecache func directly
+    # This avoid polluting the benchmark with the filecache logic
+    makecl = unfi.__class__.changelog.func
+    def setup():
+        # probably not necessary, but for good measure
+        clearchangelog(unfi)
     def d():
-        cl = mercurial.revlog.revlog(svfs, b"00changelog.i")
+        cl = makecl(unfi)
         cl.rev(n)
-    timer(d)
+    timer(d, setup=setup)
     fm.end()
 
 @command(b'perfstartup', formatteropts)
@@ -1144,6 +1218,82 b' def perftemplating(ui, repo, testedtempl'
     timer(format)
     fm.end()
 
+@command(b'perfhelper-pathcopies', formatteropts +
+         [
+          (b'r', b'revs', [], b'restrict search to these revisions'),
+          (b'', b'timing', False, b'provides extra data (costly)'),
+         ])
+def perfhelperpathcopies(ui, repo, revs=[], **opts):
+    """find statistic about potential parameters for the `perftracecopies`
+
+    This command find source-destination pair relevant for copytracing testing.
+    It report value for some of the parameters that impact copy tracing time.
+
+    If `--timing` is set, rename detection is run and the associated timing
+    will be reported. The extra details comes at the cost of a slower command
+    execution.
+
+    Since the rename detection is only run once, other factors might easily
+    affect the precision of the timing. However it should give a good
+    approximation of which revision pairs are very costly.
+    """
+    opts = _byteskwargs(opts)
+    fm = ui.formatter(b'perf', opts)
+    dotiming = opts[b'timing']
+
+    if dotiming:
+        header = '%12s %12s %12s %12s %12s %12s\n'
+        output = ("%(source)12s %(destination)12s "
+                  "%(nbrevs)12d %(nbmissingfiles)12d "
+                  "%(nbrenamedfiles)12d %(time)18.5f\n")
+        header_names = ("source", "destination", "nb-revs", "nb-files",
+                        "nb-renames", "time")
+        fm.plain(header % header_names)
+    else:
+        header = '%12s %12s %12s %12s\n'
+        output = ("%(source)12s %(destination)12s "
+                  "%(nbrevs)12d %(nbmissingfiles)12d\n")
+        fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
+
+    if not revs:
+        revs = ['all()']
+    revs = scmutil.revrange(repo, revs)
+
+    roi = repo.revs('merge() and %ld', revs)
+    for r in roi:
+        ctx = repo[r]
+        p1 = ctx.p1().rev()
+        p2 = ctx.p2().rev()
+        bases = repo.changelog._commonancestorsheads(p1, p2)
+        for p in (p1, p2):
+            for b in bases:
+                base = repo[b]
+                parent = repo[p]
+                missing = copies._computeforwardmissing(base, parent)
+                if not missing:
+                    continue
+                data = {
+                    b'source': base.hex(),
+                    b'destination': parent.hex(),
+                    b'nbrevs': len(repo.revs('%d::%d', b, p)),
+                    b'nbmissingfiles': len(missing),
+                }
+                if dotiming:
+                    begin = util.timer()
+                    renames = copies.pathcopies(base, parent)
+                    end = util.timer()
+                    # not very stable timing since we did only one run
+                    data['time'] = end - begin
+                    data['nbrenamedfiles'] = len(renames)
+                fm.startitem()
+                fm.data(**data)
+                out = data.copy()
+                out['source'] = fm.hexfunc(base.node())
+                out['destination'] = fm.hexfunc(parent.node())
+                fm.plain(output % out)
+
+    fm.end()
+
 @command(b'perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     opts = _byteskwargs(opts)
@@ -1402,7 +1552,7 b' def perfdiffwd(ui, repo, **opts):'
             ui.popbuffer()
         diffopt = diffopt.encode('ascii')
         title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
-        timer(d, title)
+        timer(d, title=title)
     fm.end()
 
 @command(b'perfrevlogindex', revlogopts + formatteropts,
@@ -1553,7 +1703,7 b' def perfrevlogrevisions(ui, repo, file_='
         dist = opts[b'dist']
 
         if reverse:
-            beginrev, endrev = endrev, beginrev
+            beginrev, endrev = endrev - 1, beginrev - 1
             dist = -1 * dist
 
         for x in _xrange(beginrev, endrev, dist):
@@ -1565,6 +1715,241 b' def perfrevlogrevisions(ui, repo, file_='
     timer(d)
     fm.end()
 
+@command(b'perfrevlogwrite', revlogopts + formatteropts +
+         [(b's', b'startrev', 1000, b'revision to start writing at'),
+          (b'', b'stoprev', -1, b'last revision to write'),
+          (b'', b'count', 3, b'last revision to write'),
+          (b'', b'details', False, b'print timing for every revisions tested'),
+          (b'', b'source', b'full', b'the kind of data feed in the revlog'),
+          (b'', b'lazydeltabase', True, b'try the provided delta first'),
+          (b'', b'clear-caches', True, b'clear revlog cache between calls'),
+         ],
+         b'-c|-m|FILE')
+def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
+    """Benchmark writing a series of revisions to a revlog.
+
+    Possible source values are:
+    * `full`: add from a full text (default).
+    * `parent-1`: add from a delta to the first parent
+    * `parent-2`: add from a delta to the second parent if it exists
+                  (use a delta from the first parent otherwise)
+    * `parent-smallest`: add from the smallest delta (either p1 or p2)
+    * `storage`: add from the existing precomputed deltas
+    """
+    opts = _byteskwargs(opts)
+
+    rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
+    rllen = getlen(ui)(rl)
+    if startrev < 0:
+        startrev = rllen + startrev
+    if stoprev < 0:
+        stoprev = rllen + stoprev
+
+    lazydeltabase = opts['lazydeltabase']
+    source = opts['source']
+    clearcaches = opts['clear_caches']
+    validsource = (b'full', b'parent-1', b'parent-2', b'parent-smallest',
+                   b'storage')
+    if source not in validsource:
+        raise error.Abort('invalid source type: %s' % source)
+
+    ### actually gather results
+    count = opts['count']
+    if count <= 0:
+        raise error.Abort('invalide run count: %d' % count)
+    allresults = []
+    for c in range(count):
+        timing = _timeonewrite(ui, rl, source, startrev, stoprev, c + 1,
+                               lazydeltabase=lazydeltabase,
+                               clearcaches=clearcaches)
+        allresults.append(timing)
+
+    ### consolidate the results in a single list
+    results = []
+    for idx, (rev, t) in enumerate(allresults[0]):
+        ts = [t]
+        for other in allresults[1:]:
+            orev, ot = other[idx]
+            assert orev == rev
+            ts.append(ot)
+        results.append((rev, ts))
+    resultcount = len(results)
+
+    ### Compute and display relevant statistics
+
+    # get a formatter
+    fm = ui.formatter(b'perf', opts)
+    displayall = ui.configbool(b"perf", b"all-timing", False)
+
+    # print individual details if requested
+    if opts['details']:
+        for idx, item in enumerate(results, 1):
+            rev, data = item
+            title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
+            formatone(fm, data, title=title, displayall=displayall)
+
+    # sorts results by median time
+    results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
+    # list of (name, index) to display)
+    relevants = [
+        ("min", 0),
+        ("10%", resultcount * 10 // 100),
+        ("25%", resultcount * 25 // 100),
+        ("50%", resultcount * 70 // 100),
+        ("75%", resultcount * 75 // 100),
+        ("90%", resultcount * 90 // 100),
+        ("95%", resultcount * 95 // 100),
+        ("99%", resultcount * 99 // 100),
+        ("99.9%", resultcount * 999 // 1000),
+        ("99.99%", resultcount * 9999 // 10000),
+        ("99.999%", resultcount * 99999 // 100000),
+        ("max", -1),
+    ]
+    if not ui.quiet:
+        for name, idx in relevants:
+            data = results[idx]
+            title = '%s of %d, rev %d' % (name, resultcount, data[0])
+            formatone(fm, data[1], title=title, displayall=displayall)
+
+    # XXX summing that many float will not be very precise, we ignore this fact
+    # for now
+    totaltime = []
+    for item in allresults:
+        totaltime.append((sum(x[1][0] for x in item),
+                          sum(x[1][1] for x in item),
+                          sum(x[1][2] for x in item),)
+        )
+    formatone(fm, totaltime, title="total time (%d revs)" % resultcount,
+              displayall=displayall)
+    fm.end()
+
+class _faketr(object):
+    def add(s, x, y, z=None):
+        return None
+
+def _timeonewrite(ui, orig, source, startrev, stoprev, runidx=None,
+                  lazydeltabase=True, clearcaches=True):
+    timings = []
+    tr = _faketr()
+    with _temprevlog(ui, orig, startrev) as dest:
+        dest._lazydeltabase = lazydeltabase
+        revs = list(orig.revs(startrev, stoprev))
+        total = len(revs)
+        topic = 'adding'
+        if runidx is not None:
+            topic += ' (run #%d)' % runidx
+         # Support both old and new progress API
+        if util.safehasattr(ui, 'makeprogress'):
+            progress = ui.makeprogress(topic, unit='revs', total=total)
+            def updateprogress(pos):
+                progress.update(pos)
+            def completeprogress():
+                progress.complete()
+        else:
+            def updateprogress(pos):
+                ui.progress(topic, pos, unit='revs', total=total)
+            def completeprogress():
+                ui.progress(topic, None, unit='revs', total=total)
+
+        for idx, rev in enumerate(revs):
+            updateprogress(idx)
+            addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
+            if clearcaches:
+                dest.index.clearcaches()
+                dest.clearcaches()
+            with timeone() as r:
+                dest.addrawrevision(*addargs, **addkwargs)
+            timings.append((rev, r[0]))
+        updateprogress(total)
+        completeprogress()
+    return timings
+
+def _getrevisionseed(orig, rev, tr, source):
+    from mercurial.node import nullid
+
+    linkrev = orig.linkrev(rev)
+    node = orig.node(rev)
+    p1, p2 = orig.parents(node)
+    flags = orig.flags(rev)
+    cachedelta = None
+    text = None
+
+    if source == b'full':
+        text = orig.revision(rev)
+    elif source == b'parent-1':
+        baserev = orig.rev(p1)
+        cachedelta = (baserev, orig.revdiff(p1, rev))
+    elif source == b'parent-2':
+        parent = p2
+        if p2 == nullid:
+            parent = p1
+        baserev = orig.rev(parent)
+        cachedelta = (baserev, orig.revdiff(parent, rev))
+    elif source == b'parent-smallest':
+        p1diff = orig.revdiff(p1, rev)
+        parent = p1
+        diff = p1diff
+        if p2 != nullid:
+            p2diff = orig.revdiff(p2, rev)
+            if len(p1diff) > len(p2diff):
+                parent = p2
+                diff = p2diff
+        baserev = orig.rev(parent)
+        cachedelta = (baserev, diff)
+    elif source == b'storage':
+        baserev = orig.deltaparent(rev)
+        cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
+
+    return ((text, tr, linkrev, p1, p2),
+            {'node': node, 'flags': flags, 'cachedelta': cachedelta})
+
[e-mail]
+def _temprevlog(ui, orig, truncaterev):
+    from mercurial import vfs as vfsmod
+
+    if orig._inline:
+        raise error.Abort('not supporting inline revlog (yet)')
+
+    origindexpath = orig.opener.join(orig.indexfile)
+    origdatapath = orig.opener.join(orig.datafile)
+    indexname = 'revlog.i'
+    dataname = 'revlog.d'
+
+    tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
+    try:
+        # copy the data file in a temporary directory
+        ui.debug('copying data in %s\n' % tmpdir)
+        destindexpath = os.path.join(tmpdir, 'revlog.i')
+        destdatapath = os.path.join(tmpdir, 'revlog.d')
+        shutil.copyfile(origindexpath, destindexpath)
+        shutil.copyfile(origdatapath, destdatapath)
+
+        # remove the data we want to add again
+        ui.debug('truncating data to be rewritten\n')
+        with open(destindexpath, 'ab') as index:
+            index.seek(0)
+            index.truncate(truncaterev * orig._io.size)
+        with open(destdatapath, 'ab') as data:
+            data.seek(0)
+            data.truncate(orig.start(truncaterev))
+
+        # instantiate a new revlog from the temporary copy
+        ui.debug('truncating adding to be rewritten\n')
+        vfs = vfsmod.vfs(tmpdir)
+        vfs.options = getattr(orig.opener, 'options', None)
+
+        dest = revlog.revlog(vfs,
+                             indexfile=indexname,
+                             datafile=dataname)
+        if dest._inline:
+            raise error.Abort('not supporting inline revlog (yet)')
+        # make sure internals are initialized
+        dest.revision(len(dest) - 1)
+        yield dest
+        del dest, vfs
+    finally:
+        shutil.rmtree(tmpdir, True)
+
 @command(b'perfrevlogchunks', revlogopts + formatteropts +
          [(b'e', b'engines', b'', b'compression engines to use'),
           (b's', b'startrev', 0, b'revision to start at')],
@@ -1692,10 +2077,11 b' def perfrevlogrevision(ui, repo, file_, '
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
-    2. Obtain the raw chunks for that delta chain
-    3. Decompress each raw chunk
-    4. Apply binary patches to obtain fulltext
-    5. Verify hash of fulltext
+    2. Slice the delta chain if applicable
+    3. Obtain the raw chunks for that delta chain
+    4. Decompress each raw chunk
+    5. Apply binary patches to obtain fulltext
+    6. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
@@ -1723,17 +2109,18 b' def perfrevlogrevision(ui, repo, file_, '
         inline = r._inline
         iosize = r._io.size
         buffer = util.buffer
-        offset = start(chain[0])
 
         chunks = []
         ladd = chunks.append
-
-        for rev in chain:
-            chunkstart = start(rev)
-            if inline:
-                chunkstart += (rev + 1) * iosize
-            chunklength = length(rev)
-            ladd(buffer(data, chunkstart - offset, chunklength))
+        for idx, item in enumerate(chain):
+            offset = start(item[0])
+            bits = data[idx]
+            for rev in item:
+                chunkstart = start(rev)
+                if inline:
+                    chunkstart += (rev + 1) * iosize
+                chunklength = length(rev)
+                ladd(buffer(bits, chunkstart - offset, chunklength))
 
         return chunks
 
@@ -1745,7 +2132,12 b' def perfrevlogrevision(ui, repo, file_, '
     def doread(chain):
         if not cache:
             r.clearcaches()
-        segmentforrevs(chain[0], chain[-1])
+        for item in slicedchain:
+            segmentforrevs(item[0], item[-1])
+
+    def doslice(r, chain, size):
+        for s in slicechunk(r, chain, targetsize=size):
+            pass
 
     def dorawchunks(data, chain):
         if not cache:
@@ -1772,9 +2164,19 b' def perfrevlogrevision(ui, repo, file_, '
             r.clearcaches()
         r.revision(node)
 
+    try:
+        from mercurial.revlogutils.deltas import slicechunk
+    except ImportError:
+        slicechunk = getattr(revlog, '_slicechunk', None)
+
+    size = r.length(rev)
     chain = r._deltachain(rev)[0]
-    data = segmentforrevs(chain[0], chain[-1])[1]
-    rawchunks = getrawchunks(data, chain)
+    if not getattr(r, '_withsparseread', False):
+        slicedchain = (chain,)
+    else:
+        slicedchain = tuple(slicechunk(r, chain, targetsize=size))
+    data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
+    rawchunks = getrawchunks(data, slicedchain)
     bins = r._chunks(chain)
     text = bytes(bins[0])
     bins = bins[1:]
@@ -1784,16 +2186,23 b' def perfrevlogrevision(ui, repo, file_, '
         (lambda: dorevision(), b'full'),
         (lambda: dodeltachain(rev), b'deltachain'),
         (lambda: doread(chain), b'read'),
-        (lambda: dorawchunks(data, chain), b'rawchunks'),
+    ]
+
+    if getattr(r, '_withsparseread', False):
+        slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
+        benches.append(slicing)
+
+    benches.extend([
+        (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
         (lambda: dodecompress(rawchunks), b'decompress'),
         (lambda: dopatch(text, bins), b'patch'),
         (lambda: dohash(text), b'hash'),
-    ]
+    ])
 
+    timer, fm = gettimer(ui, opts)
     for fn, title in benches:
-        timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
-        fm.end()
+    fm.end()
 
 @command(b'perfrevset',
          [(b'C', b'clear', False, b'clear volatile cache between each call.'),
@@ -1929,13 +2338,120 b' def perfbranchmap(ui, repo, *filternames'
         branchcachewrite.restore()
     fm.end()
 
+@command(b'perfbranchmapupdate', [
+     (b'', b'base', [], b'subset of revision to start from'),
+     (b'', b'target', [], b'subset of revision to end with'),
+     (b'', b'clear-caches', False, b'clear cache between each runs')
+    ] + formatteropts)
+def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
+    """benchmark branchmap update from for <base> revs to <target> revs
+
+    If `--clear-caches` is passed, the following items will be reset before
+    each update:
+        * the changelog instance and associated indexes
+        * the rev-branch-cache instance
+
+    Examples:
+
+       # update for the one last revision
+       $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
+
+       $ update for change coming with a new branch
+       $ hg perfbranchmapupdate --base 'stable' --target 'default'
+    """
+    from mercurial import branchmap
+    from mercurial import repoview
+    opts = _byteskwargs(opts)
+    timer, fm = gettimer(ui, opts)
+    clearcaches = opts[b'clear_caches']
+    unfi = repo.unfiltered()
+    x = [None] # used to pass data between closure
+
+    # we use a `list` here to avoid possible side effect from smartset
+    baserevs = list(scmutil.revrange(repo, base))
+    targetrevs = list(scmutil.revrange(repo, target))
+    if not baserevs:
+        raise error.Abort(b'no revisions selected for --base')
+    if not targetrevs:
+        raise error.Abort(b'no revisions selected for --target')
+
+    # make sure the target branchmap also contains the one in the base
+    targetrevs = list(set(baserevs) | set(targetrevs))
+    targetrevs.sort()
+
+    cl = repo.changelog
+    allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
+    allbaserevs.sort()
+    alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
+
+    newrevs = list(alltargetrevs.difference(allbaserevs))
+    newrevs.sort()
+
+    allrevs = frozenset(unfi.changelog.revs())
+    basefilterrevs = frozenset(allrevs.difference(allbaserevs))
+    targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
+
+    def basefilter(repo, visibilityexceptions=None):
+        return basefilterrevs
+
+    def targetfilter(repo, visibilityexceptions=None):
+        return targetfilterrevs
+
+    msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
+    ui.status(msg % (len(allbaserevs), len(newrevs)))
+    if targetfilterrevs:
+        msg = b'(%d revisions still filtered)\n'
+        ui.status(msg % len(targetfilterrevs))
+
+    try:
+        repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
+        repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
+
+        baserepo = repo.filtered(b'__perf_branchmap_update_base')
+        targetrepo = repo.filtered(b'__perf_branchmap_update_target')
+
+        # try to find an existing branchmap to reuse
+        subsettable = getbranchmapsubsettable()
+        candidatefilter = subsettable.get(None)
+        while candidatefilter is not None:
+            candidatebm = repo.filtered(candidatefilter).branchmap()
+            if candidatebm.validfor(baserepo):
+                filtered = repoview.filterrevs(repo, candidatefilter)
+                missing = [r for r in allbaserevs if r in filtered]
+                base = candidatebm.copy()
+                base.update(baserepo, missing)
+                break
+            candidatefilter = subsettable.get(candidatefilter)
+        else:
+            # no suitable subset where found
+            base = branchmap.branchcache()
+            base.update(baserepo, allbaserevs)
+
+        def setup():
+            x[0] = base.copy()
+            if clearcaches:
+                unfi._revbranchcache = None
+                clearchangelog(repo)
+
+        def bench():
+            x[0].update(targetrepo, newrevs)
+
+        timer(bench, setup=setup)
+        fm.end()
+    finally:
+        repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
+        repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
+
 @command(b'perfbranchmapload', [
      (b'f', b'filter', b'', b'Specify repoview filter'),
      (b'', b'list', False, b'List brachmap filter caches'),
+     (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
+
     ] + formatteropts)
-def perfbranchmapread(ui, repo, filter=b'', list=False, **opts):
+def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
     """benchmark reading the branchmap"""
     opts = _byteskwargs(opts)
+    clearrevlogs = opts[b'clear_revlogs']
 
     if list:
         for name, kind, st in repo.cachevfs.readdir(stat=True):
@@ -1944,16 +2460,31 b' def perfbranchmapread(ui, repo, filter=b'
                 ui.status(b'%s - %s\n'
                           % (filtername, util.bytecount(st.st_size)))
         return
-    if filter:
+    if not filter:
+        filter = None
+    subsettable = getbranchmapsubsettable()
+    if filter is None:
+        repo = repo.unfiltered()
+    else:
         repo = repoview.repoview(repo, filter)
-    else:
-        repo = repo.unfiltered()
+
+    repo.branchmap() # make sure we have a relevant, up to date branchmap
+
+    currentfilter = filter
     # try once without timer, the filter may not be cached
-    if branchmap.read(repo) is None:
-        raise error.Abort(b'No brachmap cached for %s repo'
-                          % (filter or b'unfiltered'))
+    while branchmap.read(repo) is None:
+        currentfilter = subsettable.get(currentfilter)
+        if currentfilter is None:
+            raise error.Abort(b'No branchmap cached for %s repo'
+                              % (filter or b'unfiltered'))
+        repo = repo.filtered(currentfilter)
     timer, fm = gettimer(ui, opts)
-    timer(lambda: branchmap.read(repo) and None)
+    def setup():
+        if clearrevlogs:
+            clearchangelog(repo)
+    def bench():
+        branchmap.read(repo)
+    timer(bench, setup=setup)
     fm.end()
 
 @command(b'perfloadmarkers')
@@ -2124,3 +2655,21 b' def uisetup(ui):'
                                   hint=b"use 3.5 or later")
             return orig(repo, cmd, file_, opts)
         extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
+
+@command(b'perfprogress', formatteropts + [
+    (b'', b'topic', b'topic', b'topic for progress messages'),
+    (b'c', b'total', 1000000, b'total value we are progressing to'),
+], norepo=True)
+def perfprogress(ui, topic=None, total=None, **opts):
+    """printing of progress bars"""
+    opts = _byteskwargs(opts)
+
+    timer, fm = gettimer(ui, opts)
+
+    def doprogress():
+        with ui.makeprogress(topic, total=total) as progress:
+            for i in pycompat.xrange(total):
+                progress.increment()
+
+    timer(doprogress)
+    fm.end()

contrib/python3-whitelist

@@ -1,4 +1,5 b''
 test-abort-checkin.t
+test-absorb-edit-lines.t
 test-absorb-filefixupstate.py
 test-absorb-phase.t
 test-absorb-rename.t
@@ -30,6 +31,7 b' test-bisect.t'
 test-bisect2.t
 test-bisect3.t
 test-blackbox.t
+test-bookflow.t
 test-bookmarks-current.t
 test-bookmarks-merge.t
 test-bookmarks-pushpull.t
@@ -62,6 +64,7 b' test-check-commit.t'
 test-check-config.py
 test-check-config.t
 test-check-execute.t
+test-check-help.t
 test-check-interfaces.py
 test-check-module-imports.t
 test-check-py3-compat.t
@@ -116,6 +119,7 b' test-convert-tagsbranch-topology.t'
 test-copy-move-merge.t
 test-copy.t
 test-copytrace-heuristics.t
+test-custom-filters.t
 test-debugbuilddag.t
 test-debugbundle.t
 test-debugcommands.t
@@ -193,9 +197,18 b' test-execute-bit.t'
 test-export.t
 test-extdata.t
 test-extdiff.t
+test-extension-timing.t
 test-extensions-afterloaded.t
 test-extensions-wrapfunction.py
 test-extra-filelog-entry.t
+test-fastannotate-corrupt.t
+test-fastannotate-diffopts.t
+test-fastannotate-hg.t
+test-fastannotate-perfhack.t
+test-fastannotate-protocol.t
+test-fastannotate-renames.t
+test-fastannotate-revmap.py
+test-fastannotate.t
 test-fetch.t
 test-filebranch.t
 test-filecache.py
@@ -206,6 +219,19 b' test-fix-topology.t'
 test-fix.t
 test-flags.t
 test-fncache.t
+test-gendoc-da.t
+test-gendoc-de.t
+test-gendoc-el.t
+test-gendoc-fr.t
+test-gendoc-it.t
+test-gendoc-ja.t
+test-gendoc-pt_BR.t
+test-gendoc-ro.t
+test-gendoc-ru.t
+test-gendoc-sv.t
+test-gendoc-zh_CN.t
+test-gendoc-zh_TW.t
+test-gendoc.t
 test-generaldelta.t
 test-getbundle.t
 test-git-export.t
@@ -217,6 +243,7 b' test-gpg.t'
 test-graft.t
 test-grep.t
 test-hardlinks.t
+test-help-hide.t
 test-help.t
 test-hg-parseurl.py
 test-hghave.t
@@ -261,6 +288,7 b' test-i18n.t'
 test-identify.t
 test-impexp-branch.t
 test-import-bypass.t
+test-import-context.t
 test-import-eol.t
 test-import-merge.t
 test-import-unknown.t
@@ -301,16 +329,22 b' test-largefiles-cache.t'
 test-largefiles-misc.t
 test-largefiles-small-disk.t
 test-largefiles-update.t
+test-largefiles-wireproto.t
 test-largefiles.t
+test-lfconvert.t
+test-lfs-bundle.t
 test-lfs-largefiles.t
 test-lfs-pointer.py
+test-lfs.t
 test-linelog.py
 test-linerange.py
 test-locate.t
 test-lock-badness.t
+test-log-exthook.t
 test-log-linerange.t
 test-log.t
 test-logexchange.t
+test-logtoprocess.t
 test-lrucachedict.py
 test-mactext.t
 test-mailmap.t
@@ -394,6 +428,8 b' test-narrow-pull.t'
 test-narrow-rebase.t
 test-narrow-shallow-merges.t
 test-narrow-shallow.t
+test-narrow-share.t
+test-narrow-sparse.t
 test-narrow-strip.t
 test-narrow-trackedcmd.t
 test-narrow-update.t
@@ -474,6 +510,7 b' test-push-checkheads-unpushed-D5.t'
 test-push-checkheads-unpushed-D6.t
 test-push-checkheads-unpushed-D7.t
 test-push-http.t
+test-push-race.t
 test-push-warn.t
 test-push.t
 test-pushvars.t
@@ -512,6 +549,28 b' test-releasenotes-formatting.t'
 test-releasenotes-merging.t
 test-releasenotes-parsing.t
 test-relink.t
+test-remotefilelog-bad-configs.t
+test-remotefilelog-bgprefetch.t
+test-remotefilelog-blame.t
+test-remotefilelog-bundle2.t
+test-remotefilelog-bundles.t
+test-remotefilelog-cacheprocess.t
+test-remotefilelog-clone-tree.t
+test-remotefilelog-clone.t
+test-remotefilelog-gcrepack.t
+test-remotefilelog-http.t
+test-remotefilelog-keepset.t
+test-remotefilelog-local.t
+test-remotefilelog-log.t
+test-remotefilelog-partial-shallow.t
+test-remotefilelog-permissions.t
+test-remotefilelog-permisssions.t
+test-remotefilelog-prefetch.t
+test-remotefilelog-pull-noshallow.t
+test-remotefilelog-share.t
+test-remotefilelog-sparse.t
+test-remotefilelog-tags.t
+test-remotefilelog-wireproto.t
 test-remove.t
 test-removeemptydirs.t
 test-rename-after-merge.t
@@ -541,11 +600,13 b' test-revset-outgoing.t'
 test-rollback.t
 test-run-tests.py
 test-run-tests.t
+test-rust-ancestor.py
 test-schemes.t
 test-serve.t
 test-setdiscovery.t
 test-share.t
 test-shelve.t
+test-shelve2.t
 test-show-stack.t
 test-show-work.t
 test-show.t

contrib/revsetbenchmarks.py

@@ -56,9 +56,11 b' def hg(cmd, repo=None):'
 def perf(revset, target=None, contexts=False):
     """run benchmark for this very revset"""
     try:
-        args = ['perfrevset', revset]
+        args = ['perfrevset']
         if contexts:
             args.append('--contexts')
+        args.append('--')
+        args.append(revset)
         output = hg(args, repo=target)
         return parseoutput(output)
     except subprocess.CalledProcessError as exc:

contrib/wix/help.wxs

@@ -47,6 +47,7 b''
             <File Id="internals.censor.txt"       Name="censor.txt" />
             <File Id="internals.changegroups.txt" Name="changegroups.txt" />
             <File Id="internals.config.txt"       Name="config.txt" />
+            <File Id="internals.extensions.txt"   Name="extensions.txt" />
             <File Id="internals.linelog.txt"      Name="linelog.txt" />
             <File Id="internals.requirements.txt" Name="requirements.txt" />
             <File Id="internals.revlogs.txt"      Name="revlogs.txt" />

doc/docchecker

@@ -9,18 +9,28 b''
 
 from __future__ import absolute_import, print_function
 
+import os
 import re
 import sys
 
-leadingline = re.compile(r'(^\s*)(\S.*)$')
+try:
+    import msvcrt
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
+except ImportError:
+    pass
+
+stdout = getattr(sys.stdout, 'buffer', sys.stdout)
+
+leadingline = re.compile(br'(^\s*)(\S.*)$')
 
 checks = [
-  (r""":hg:`[^`]*'[^`]*`""",
-    """warning: please avoid nesting ' in :hg:`...`"""),
-  (r'\w:hg:`',
-    'warning: please have a space before :hg:'),
-  (r"""(?:[^a-z][^'.])hg ([^,;"`]*'(?!hg)){2}""",
-    '''warning: please use " instead of ' for hg ... "..."'''),
+  (br""":hg:`[^`]*'[^`]*`""",
+   b"""warning: please avoid nesting ' in :hg:`...`"""),
+  (br'\w:hg:`',
+   b'warning: please have a space before :hg:'),
+  (br"""(?:[^a-z][^'.])hg ([^,;"`]*'(?!hg)){2}""",
+   b'''warning: please use " instead of ' for hg ... "..."'''),
 ]
 
 def check(line):
@@ -29,25 +39,25 b' def check(line):'
         if re.search(match, line):
             messages.append(msg)
     if messages:
-        print(line)
+        stdout.write(b'%s\n' % line)
         for msg in messages:
-            print(msg)
+            stdout.write(b'%s\n' % msg)
 
 def work(file):
-    (llead, lline) = ('', '')
+    (llead, lline) = (b'', b'')
 
     for line in file:
         # this section unwraps lines
         match = leadingline.match(line)
         if not match:
             check(lline)
-            (llead, lline) = ('', '')
+            (llead, lline) = (b'', b'')
             continue
 
         lead, line = match.group(1), match.group(2)
         if (lead == llead):
-            if (lline != ''):
-                lline += ' ' + line
+            if (lline != b''):
+                lline += b' ' + line
             else:
                 lline = line
         else:
@@ -58,9 +68,9 b' def work(file):'
 def main():
     for f in sys.argv[1:]:
         try:
-            with open(f) as file:
+            with open(f, 'rb') as file:
                 work(file)
         except BaseException as e:
-            print("failed to process %s: %s" % (f, e))
+            sys.stdout.write(r"failed to process %s: %s\n" % (f, e))
 
 main()

doc/gendoc.py

@@ -10,11 +10,18 b' import os'
 import sys
 import textwrap
 
+try:
+    import msvcrt
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
+except ImportError:
+    pass
+
 # This script is executed during installs and may not have C extensions
 # available. Relax C module requirements.
-os.environ['HGMODULEPOLICY'] = 'allow'
+os.environ[r'HGMODULEPOLICY'] = r'allow'
 # import from the live mercurial repo
-sys.path.insert(0, "..")
+sys.path.insert(0, r"..")
 from mercurial import demandimport; demandimport.enable()
 # Load util so that the locale path is set by i18n.setdatapath() before
 # calling _().
@@ -22,9 +29,11 b' from mercurial import util'
 util.datapath
 from mercurial import (
     commands,
+    encoding,
     extensions,
     help,
     minirst,
+    pycompat,
     ui as uimod,
 )
 from mercurial.i18n import (
@@ -39,19 +48,19 b' loaddoc = help.loaddoc'
 
 def get_desc(docstr):
     if not docstr:
-        return "", ""
+        return b"", b""
     # sanitize
-    docstr = docstr.strip("\n")
+    docstr = docstr.strip(b"\n")
     docstr = docstr.rstrip()
     shortdesc = docstr.splitlines()[0].strip()
 
-    i = docstr.find("\n")
+    i = docstr.find(b"\n")
     if i != -1:
         desc = docstr[i + 2:]
     else:
         desc = shortdesc
 
-    desc = textwrap.dedent(desc)
+    desc = textwrap.dedent(desc.decode('latin1')).encode('latin1')
 
     return (shortdesc, desc)
 
@@ -61,91 +70,93 b' def get_opts(opts):'
             shortopt, longopt, default, desc, optlabel = opt
         else:
             shortopt, longopt, default, desc = opt
-            optlabel = _("VALUE")
+            optlabel = _(b"VALUE")
         allopts = []
         if shortopt:
-            allopts.append("-%s" % shortopt)
+            allopts.append(b"-%s" % shortopt)
         if longopt:
-            allopts.append("--%s" % longopt)
+            allopts.append(b"--%s" % longopt)
         if isinstance(default, list):
-            allopts[-1] += " <%s[+]>" % optlabel
+            allopts[-1] += b" <%s[+]>" % optlabel
         elif (default is not None) and not isinstance(default, bool):
-            allopts[-1] += " <%s>" % optlabel
-        if '\n' in desc:
+            allopts[-1] += b" <%s>" % optlabel
+        if b'\n' in desc:
             # only remove line breaks and indentation
-            desc = ' '.join(l.lstrip() for l in desc.split('\n'))
-        desc += default and _(" (default: %s)") % default or ""
-        yield (", ".join(allopts), desc)
+            desc = b' '.join(l.lstrip() for l in desc.split(b'\n'))
+        desc += default and _(b" (default: %s)") % bytes(default) or b""
+        yield (b", ".join(allopts), desc)
 
 def get_cmd(cmd, cmdtable):
     d = {}
     attr = cmdtable[cmd]
-    cmds = cmd.lstrip("^").split("|")
+    cmds = cmd.lstrip(b"^").split(b"|")
 
-    d['cmd'] = cmds[0]
-    d['aliases'] = cmd.split("|")[1:]
-    d['desc'] = get_desc(gettext(attr[0].__doc__))
-    d['opts'] = list(get_opts(attr[1]))
+    d[b'cmd'] = cmds[0]
+    d[b'aliases'] = cmd.split(b"|")[1:]
+    d[b'desc'] = get_desc(gettext(pycompat.getdoc(attr[0])))
+    d[b'opts'] = list(get_opts(attr[1]))
 
-    s = 'hg ' + cmds[0]
+    s = b'hg ' + cmds[0]
     if len(attr) > 2:
-        if not attr[2].startswith('hg'):
-            s += ' ' + attr[2]
+        if not attr[2].startswith(b'hg'):
+            s += b' ' + attr[2]
         else:
             s = attr[2]
-    d['synopsis'] = s.strip()
+    d[b'synopsis'] = s.strip()
 
     return d
 
 def showdoc(ui):
     # print options
-    ui.write(minirst.section(_("Options")))
+    ui.write(minirst.section(_(b"Options")))
     multioccur = False
     for optstr, desc in get_opts(globalopts):
-        ui.write("%s\n    %s\n\n" % (optstr, desc))
-        if optstr.endswith("[+]>"):
+        ui.write(b"%s\n    %s\n\n" % (optstr, desc))
+        if optstr.endswith(b"[+]>"):
             multioccur = True
     if multioccur:
-        ui.write(_("\n[+] marked option can be specified multiple times\n"))
-        ui.write("\n")
+        ui.write(_(b"\n[+] marked option can be specified multiple times\n"))
+        ui.write(b"\n")
 
     # print cmds
-    ui.write(minirst.section(_("Commands")))
+    ui.write(minirst.section(_(b"Commands")))
     commandprinter(ui, table, minirst.subsection)
 
     # print help topics
     # The config help topic is included in the hgrc.5 man page.
-    helpprinter(ui, helptable, minirst.section, exclude=['config'])
+    helpprinter(ui, helptable, minirst.section, exclude=[b'config'])
 
-    ui.write(minirst.section(_("Extensions")))
-    ui.write(_("This section contains help for extensions that are "
-               "distributed together with Mercurial. Help for other "
-               "extensions is available in the help system."))
-    ui.write(("\n\n"
-             ".. contents::\n"
-             "   :class: htmlonly\n"
-             "   :local:\n"
-             "   :depth: 1\n\n"))
+    ui.write(minirst.section(_(b"Extensions")))
+    ui.write(_(b"This section contains help for extensions that are "
+               b"distributed together with Mercurial. Help for other "
+               b"extensions is available in the help system."))
+    ui.write((b"\n\n"
+              b".. contents::\n"
+              b"   :class: htmlonly\n"
+              b"   :local:\n"
+              b"   :depth: 1\n\n"))
 
     for extensionname in sorted(allextensionnames()):
         mod = extensions.load(ui, extensionname, None)
         ui.write(minirst.subsection(extensionname))
-        ui.write("%s\n\n" % gettext(mod.__doc__))
+        ui.write(b"%s\n\n" % gettext(pycompat.getdoc(mod)))
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
-            ui.write(minirst.subsubsection(_('Commands')))
+            ui.write(minirst.subsubsection(_(b'Commands')))
             commandprinter(ui, cmdtable, minirst.subsubsubsection)
 
 def showtopic(ui, topic):
     extrahelptable = [
-        (["common"], '', loaddoc('common'), help.TOPIC_CATEGORY_MISC),
-        (["hg.1"], '', loaddoc('hg.1'), help.TOPIC_CATEGORY_CONFIG),
-        (["hg-ssh.8"], '', loaddoc('hg-ssh.8'), help.TOPIC_CATEGORY_CONFIG),
-        (["hgignore.5"], '', loaddoc('hgignore.5'), help.TOPIC_CATEGORY_CONFIG),
-        (["hgrc.5"], '', loaddoc('hgrc.5'), help.TOPIC_CATEGORY_CONFIG),
-        (["hgignore.5.gendoc"], '', loaddoc('hgignore'),
+        ([b"common"], b'', loaddoc(b'common'), help.TOPIC_CATEGORY_MISC),
+        ([b"hg.1"], b'', loaddoc(b'hg.1'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hg-ssh.8"], b'', loaddoc(b'hg-ssh.8'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgignore.5"], b'', loaddoc(b'hgignore.5'),
          help.TOPIC_CATEGORY_CONFIG),
-        (["hgrc.5.gendoc"], '', loaddoc('config'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgrc.5"], b'', loaddoc(b'hgrc.5'), help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgignore.5.gendoc"], b'', loaddoc(b'hgignore'),
+         help.TOPIC_CATEGORY_CONFIG),
+        ([b"hgrc.5.gendoc"], b'', loaddoc(b'config'),
+         help.TOPIC_CATEGORY_CONFIG),
     ]
     helpprinter(ui, helptable + extrahelptable, None, include=[topic])
 
@@ -157,74 +168,73 b' def helpprinter(ui, helptable, sectionfu'
         if include and names[0] not in include:
             continue
         for name in names:
-            ui.write(".. _%s:\n" % name)
-        ui.write("\n")
+            ui.write(b".. _%s:\n" % name)
+        ui.write(b"\n")
         if sectionfunc:
             ui.write(sectionfunc(sec))
         if callable(doc):
             doc = doc(ui)
         ui.write(doc)
-        ui.write("\n")
+        ui.write(b"\n")
 
 def commandprinter(ui, cmdtable, sectionfunc):
     h = {}
     for c, attr in cmdtable.items():
-        f = c.split("|")[0]
-        f = f.lstrip("^")
+        f = c.split(b"|")[0]
+        f = f.lstrip(b"^")
         h[f] = c
     cmds = h.keys()
-    cmds.sort()
 
-    for f in cmds:
-        if f.startswith("debug"):
+    for f in sorted(cmds):
+        if f.startswith(b"debug"):
             continue
         d = get_cmd(h[f], cmdtable)
-        ui.write(sectionfunc(d['cmd']))
+        ui.write(sectionfunc(d[b'cmd']))
         # short description
-        ui.write(d['desc'][0])
+        ui.write(d[b'desc'][0])
         # synopsis
-        ui.write("::\n\n")
-        synopsislines = d['synopsis'].splitlines()
+        ui.write(b"::\n\n")
+        synopsislines = d[b'synopsis'].splitlines()
         for line in synopsislines:
             # some commands (such as rebase) have a multi-line
             # synopsis
-            ui.write("   %s\n" % line)
-        ui.write('\n')
+            ui.write(b"   %s\n" % line)
+        ui.write(b'\n')
         # description
-        ui.write("%s\n\n" % d['desc'][1])
+        ui.write(b"%s\n\n" % d[b'desc'][1])
         # options
-        opt_output = list(d['opts'])
+        opt_output = list(d[b'opts'])
         if opt_output:
             opts_len = max([len(line[0]) for line in opt_output])
-            ui.write(_("Options:\n\n"))
+            ui.write(_(b"Options:\n\n"))
             multioccur = False
             for optstr, desc in opt_output:
                 if desc:
-                    s = "%-*s  %s" % (opts_len, optstr, desc)
+                    s = b"%-*s  %s" % (opts_len, optstr, desc)
                 else:
                     s = optstr
-                ui.write("%s\n" % s)
-                if optstr.endswith("[+]>"):
+                ui.write(b"%s\n" % s)
+                if optstr.endswith(b"[+]>"):
                     multioccur = True
             if multioccur:
-                ui.write(_("\n[+] marked option can be specified"
-                           " multiple times\n"))
-            ui.write("\n")
+                ui.write(_(b"\n[+] marked option can be specified"
+                           b" multiple times\n"))
+            ui.write(b"\n")
         # aliases
-        if d['aliases']:
-            ui.write(_("    aliases: %s\n\n") % " ".join(d['aliases']))
+        if d[b'aliases']:
+            ui.write(_(b"    aliases: %s\n\n") % b" ".join(d[b'aliases']))
 
 
 def allextensionnames():
-    return extensions.enabled().keys() + extensions.disabled().keys()
+    return set(extensions.enabled().keys()) | set(extensions.disabled().keys())
 
 if __name__ == "__main__":
-    doc = 'hg.1.gendoc'
+    doc = b'hg.1.gendoc'
     if len(sys.argv) > 1:
-        doc = sys.argv[1]
+        doc = encoding.strtolocal(sys.argv[1])
 
     ui = uimod.ui.load()
-    if doc == 'hg.1.gendoc':
+    if doc == b'hg.1.gendoc':
         showdoc(ui)
     else:
-        showtopic(ui, sys.argv[1])
+        showtopic(ui, encoding.strtolocal(sys.argv[1]))

hgext/absorb.py

@@ -489,7 +489,8 b' class filefixupstate(object):'
             if l[colonpos - 1:colonpos + 2] != ' : ':
                 raise error.Abort(_('malformed line: %s') % l)
             linecontent = l[colonpos + 2:]
-            for i, ch in enumerate(l[leftpadpos:colonpos - 1]):
+            for i, ch in enumerate(
+                    pycompat.bytestr(l[leftpadpos:colonpos - 1])):
                 if ch == 'y':
                     contents[visiblefctxs[i][0]] += linecontent
         # chunkstats is hard to calculate if anything changes, therefore
@@ -971,9 +972,10 b' def absorb(ui, repo, stack=None, targetc'
                      label='absorb.description')
         fm.end()
     if not opts.get('dry_run'):
-        if not opts.get('apply_changes'):
-            if ui.promptchoice("apply changes (yn)? $$ &Yes $$ &No", default=1):
-                raise error.Abort(_('absorb cancelled\n'))
+        if (not opts.get('apply_changes') and
+            state.ctxaffected and
+            ui.promptchoice("apply changes (yn)? $$ &Yes $$ &No", default=1)):
+            raise error.Abort(_('absorb cancelled\n'))
 
         state.apply()
         if state.commit():

hgext/amend.py

@@ -36,6 +36,8 b' command = registrar.command(cmdtable)'
      ('e', 'edit', None, _('invoke editor on commit messages')),
      ('i', 'interactive', None, _('use interactive mode')),
      ('n', 'note', '', _('store a note on the amend')),
+     ('D', 'currentdate', None,
+      _('record the current date as commit date')),
     ] + cmdutil.walkopts + cmdutil.commitopts + cmdutil.commitopts2,
     _('[OPTION]... [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,

hgext/beautifygraph.py

@@ -31,8 +31,6 b" testedwith = 'ships-with-hg-core'"
 def prettyedge(before, edge, after):
     if edge == '~':
         return '\xE2\x95\xA7' # U+2567 ╧
-    if edge == 'X':
-        return '\xE2\x95\xB3' # U+2573 ╳
     if edge == '/':
         return '\xE2\x95\xB1' # U+2571 ╱
     if edge == '-':

hgext/blackbox.py

@@ -33,11 +33,15 b' Examples::'
   # rotate up to N log files when the current one gets too big
   maxfiles = 3
 
+  [blackbox]
+  # Include nanoseconds in log entries with %f (see Python function
+  # datetime.datetime.strftime)
+  date-format = '%Y-%m-%d @ %H:%M:%S.%f'
+
 """
 
 from __future__ import absolute_import
 
-import errno
 import re
 
 from mercurial.i18n import _
@@ -45,10 +49,8 b' from mercurial.node import hex'
 
 from mercurial import (
     encoding,
-    pycompat,
+    loggingutil,
     registrar,
-    ui as uimod,
-    util,
 )
 from mercurial.utils import (
     dateutil,
@@ -82,131 +84,69 b" configitem('blackbox', 'maxfiles',"
 configitem('blackbox', 'track',
     default=lambda: ['*'],
 )
+configitem('blackbox', 'date-format',
+    default='%Y/%m/%d %H:%M:%S',
+)
 
-lastui = None
+_lastlogger = loggingutil.proxylogger()
 
-def _openlogfile(ui, vfs):
-    def rotate(oldpath, newpath):
-        try:
-            vfs.unlink(newpath)
-        except OSError as err:
-            if err.errno != errno.ENOENT:
-                ui.debug("warning: cannot remove '%s': %s\n" %
-                         (newpath, err.strerror))
-        try:
-            if newpath:
-                vfs.rename(oldpath, newpath)
-        except OSError as err:
-            if err.errno != errno.ENOENT:
-                ui.debug("warning: cannot rename '%s' to '%s': %s\n" %
-                         (newpath, oldpath, err.strerror))
+class blackboxlogger(object):
+    def __init__(self, ui, repo):
+        self._repo = repo
+        self._trackedevents = set(ui.configlist('blackbox', 'track'))
+        self._maxfiles = ui.configint('blackbox', 'maxfiles')
+        self._maxsize = ui.configbytes('blackbox', 'maxsize')
+        self._inlog = False
 
-    maxsize = ui.configbytes('blackbox', 'maxsize')
-    name = 'blackbox.log'
-    if maxsize > 0:
+    def tracked(self, event):
+        return b'*' in self._trackedevents or event in self._trackedevents
+
+    def log(self, ui, event, msg, opts):
+        # self._log() -> ctx.dirty() may create new subrepo instance, which
+        # ui is derived from baseui. So the recursion guard in ui.log()
+        # doesn't work as it's local to the ui instance.
+        if self._inlog:
+            return
+        self._inlog = True
         try:
-            st = vfs.stat(name)
-        except OSError:
-            pass
-        else:
-            if st.st_size >= maxsize:
-                path = vfs.join(name)
-                maxfiles = ui.configint('blackbox', 'maxfiles')
-                for i in pycompat.xrange(maxfiles - 1, 1, -1):
-                    rotate(oldpath='%s.%d' % (path, i - 1),
-                           newpath='%s.%d' % (path, i))
-                rotate(oldpath=path,
-                       newpath=maxfiles > 0 and path + '.1')
-    return vfs(name, 'a')
-
-def wrapui(ui):
-    class blackboxui(ui.__class__):
-        @property
-        def _bbvfs(self):
-            vfs = None
-            repo = getattr(self, '_bbrepo', None)
-            if repo:
-                vfs = repo.vfs
-                if not vfs.isdir('.'):
-                    vfs = None
-            return vfs
-
-        @util.propertycache
-        def track(self):
-            return self.configlist('blackbox', 'track')
-
-        def debug(self, *msg, **opts):
-            super(blackboxui, self).debug(*msg, **opts)
-            if self.debugflag:
-                self.log('debug', '%s', ''.join(msg))
-
-        def log(self, event, *msg, **opts):
-            global lastui
-            super(blackboxui, self).log(event, *msg, **opts)
+            self._log(ui, event, msg, opts)
+        finally:
+            self._inlog = False
 
-            if not '*' in self.track and not event in self.track:
-                return
-
-            if self._bbvfs:
-                ui = self
-            else:
-                # certain ui instances exist outside the context of
-                # a repo, so just default to the last blackbox that
-                # was seen.
-                ui = lastui
-
-            if not ui:
-                return
-            vfs = ui._bbvfs
-            if not vfs:
-                return
+    def _log(self, ui, event, msg, opts):
+        default = ui.configdate('devel', 'default-date')
+        date = dateutil.datestr(default, ui.config('blackbox', 'date-format'))
+        user = procutil.getuser()
+        pid = '%d' % procutil.getpid()
+        rev = '(unknown)'
+        changed = ''
+        ctx = self._repo[None]
+        parents = ctx.parents()
+        rev = ('+'.join([hex(p.node()) for p in parents]))
+        if (ui.configbool('blackbox', 'dirty') and
+            ctx.dirty(missing=True, merge=False, branch=False)):
+            changed = '+'
+        if ui.configbool('blackbox', 'logsource'):
+            src = ' [%s]' % event
+        else:
+            src = ''
+        try:
+            fmt = '%s %s @%s%s (%s)%s> %s'
+            args = (date, user, rev, changed, pid, src, msg)
+            with loggingutil.openlogfile(
+                    ui, self._repo.vfs, name='blackbox.log',
+                    maxfiles=self._maxfiles, maxsize=self._maxsize) as fp:
+                fp.write(fmt % args)
+        except (IOError, OSError) as err:
+            # deactivate this to avoid failed logging again
+            self._trackedevents.clear()
+            ui.debug('warning: cannot write to blackbox.log: %s\n' %
+                     encoding.strtolocal(err.strerror))
+            return
+        _lastlogger.logger = self
 
-            repo = getattr(ui, '_bbrepo', None)
-            if not lastui or repo:
-                lastui = ui
-            if getattr(ui, '_bbinlog', False):
-                # recursion and failure guard
-                return
-            ui._bbinlog = True
-            default = self.configdate('devel', 'default-date')
-            date = dateutil.datestr(default, '%Y/%m/%d %H:%M:%S')
-            user = procutil.getuser()
-            pid = '%d' % procutil.getpid()
-            formattedmsg = msg[0] % msg[1:]
-            rev = '(unknown)'
-            changed = ''
-            if repo:
-                ctx = repo[None]
-                parents = ctx.parents()
-                rev = ('+'.join([hex(p.node()) for p in parents]))
-                if (ui.configbool('blackbox', 'dirty') and
-                    ctx.dirty(missing=True, merge=False, branch=False)):
-                    changed = '+'
-            if ui.configbool('blackbox', 'logsource'):
-                src = ' [%s]' % event
-            else:
-                src = ''
-            try:
-                fmt = '%s %s @%s%s (%s)%s> %s'
-                args = (date, user, rev, changed, pid, src, formattedmsg)
-                with _openlogfile(ui, vfs) as fp:
-                    fp.write(fmt % args)
-            except (IOError, OSError) as err:
-                self.debug('warning: cannot write to blackbox.log: %s\n' %
-                           encoding.strtolocal(err.strerror))
-                # do not restore _bbinlog intentionally to avoid failed
-                # logging again
-            else:
-                ui._bbinlog = False
-
-        def setrepo(self, repo):
-            self._bbrepo = repo
-
-    ui.__class__ = blackboxui
-    uimod.ui = blackboxui
-
-def uisetup(ui):
-    wrapui(ui)
+def uipopulate(ui):
+    ui.setlogger(b'blackbox', _lastlogger)
 
 def reposetup(ui, repo):
     # During 'hg pull' a httppeer repo is created to represent the remote repo.
@@ -215,14 +155,15 b' def reposetup(ui, repo):'
     if not repo.local():
         return
 
-    if util.safehasattr(ui, 'setrepo'):
-        ui.setrepo(repo)
+    # Since blackbox.log is stored in the repo directory, the logger should be
+    # instantiated per repository.
+    logger = blackboxlogger(ui, repo)
+    ui.setlogger(b'blackbox', logger)
 
-        # Set lastui even if ui.log is not called. This gives blackbox a
-        # fallback place to log.
-        global lastui
-        if lastui is None:
-            lastui = ui
+    # Set _lastlogger even if ui.log is not called. This gives blackbox a
+    # fallback place to log
+    if _lastlogger.logger is None:
+        _lastlogger.logger = logger
 
     repo._wlockfreeprefix.add('blackbox.log')
 

hgext/convert/filemap.py

@@ -270,6 +270,9 b' class filemap_source(common.converter_so'
             self.children[p] = self.children.get(p, 0) + 1
         return c
 
+    def numcommits(self):
+        return self.base.numcommits()
+
     def _cachedcommit(self, rev):
         if rev in self.commits:
             return self.commits[rev]
@@ -302,7 +305,18 b' class filemap_source(common.converter_so'
         for f in files:
             if self.filemapper(f):
                 return True
-        return False
+
+        # The include directive is documented to include nothing else (though
+        # valid branch closes are included).
+        if self.filemapper.include:
+            return False
+
+        # Allow empty commits in the source revision through.  The getchanges()
+        # method doesn't even bother calling this if it determines that the
+        # close marker is significant (i.e. all of the branch ancestors weren't
+        # eliminated).  Therefore if there *is* a close marker, getchanges()
+        # doesn't consider it significant, and this revision should be dropped.
+        return not files and 'close' not in self.commits[rev].extra
 
     def mark_not_wanted(self, rev, p):
         # Mark rev as not interesting and update data structures.

hgext/convert/hg.py

@@ -597,6 +597,9 b' class mercurial_source(common.converter_'
                              saverev=self.saverev,
                              phase=ctx.phase())
 
+    def numcommits(self):
+        return len(self.repo)
+
     def gettags(self):
         # This will get written to .hgtags, filter non global tags out.
         tags = [t for t in self.repo.tagslist()

hgext/extdiff.py

@@ -139,7 +139,7 b' def snapshot(ui, repo, files, node, tmpr'
         repo.ui.setconfig("ui", "archivemeta", False)
 
         archival.archive(repo, base, node, 'files',
-                         matchfn=scmutil.matchfiles(repo, files),
+                         match=scmutil.matchfiles(repo, files),
                          subrepos=listsubrepos)
 
         for fn in sorted(files):
@@ -152,6 +152,29 b' def snapshot(ui, repo, files, node, tmpr'
                 fnsandstat.append((dest, repo.wjoin(fn), os.lstat(dest)))
     return dirname, fnsandstat
 
+def formatcmdline(cmdline, repo_root, do3way,
+                  parent1, plabel1, parent2, plabel2, child, clabel):
+    # Function to quote file/dir names in the argument string.
+    # When not operating in 3-way mode, an empty string is
+    # returned for parent2
+    replace = {'parent': parent1, 'parent1': parent1, 'parent2': parent2,
+               'plabel1': plabel1, 'plabel2': plabel2,
+               'child': child, 'clabel': clabel,
+               'root': repo_root}
+    def quote(match):
+        pre = match.group(2)
+        key = match.group(3)
+        if not do3way and key == 'parent2':
+            return pre
+        return pre + procutil.shellquote(replace[key])
+
+    # Match parent2 first, so 'parent1?' will match both parent1 and parent
+    regex = (br'''(['"]?)([^\s'"$]*)'''
+             br'\$(parent2|parent1?|child|plabel1|plabel2|clabel|root)\1')
+    if not do3way and not re.search(regex, cmdline):
+        cmdline += ' $parent1 $child'
+    return re.sub(regex, quote, cmdline)
+
 def dodiff(ui, repo, cmdline, pats, opts):
     '''Do the actual diff:
 
@@ -281,28 +304,14 b' def dodiff(ui, repo, cmdline, pats, opts'
             label1b = None
             fnsandstat = []
 
-        # Function to quote file/dir names in the argument string.
-        # When not operating in 3-way mode, an empty string is
-        # returned for parent2
-        replace = {'parent': dir1a, 'parent1': dir1a, 'parent2': dir1b,
-                   'plabel1': label1a, 'plabel2': label1b,
-                   'clabel': label2, 'child': dir2,
-                   'root': repo.root}
-        def quote(match):
-            pre = match.group(2)
-            key = match.group(3)
-            if not do3way and key == 'parent2':
-                return pre
-            return pre + procutil.shellquote(replace[key])
-
-        # Match parent2 first, so 'parent1?' will match both parent1 and parent
-        regex = (br'''(['"]?)([^\s'"$]*)'''
-                 br'\$(parent2|parent1?|child|plabel1|plabel2|clabel|root)\1')
-        if not do3way and not re.search(regex, cmdline):
-            cmdline += ' $parent1 $child'
-        cmdline = re.sub(regex, quote, cmdline)
-
-        ui.debug('running %r in %s\n' % (pycompat.bytestr(cmdline), tmproot))
+        # Run the external tool on the 2 temp directories or the patches
+        cmdline = formatcmdline(
+            cmdline, repo.root, do3way=do3way,
+            parent1=dir1a, plabel1=label1a,
+            parent2=dir1b, plabel2=label1b,
+            child=dir2, clabel=label2)
+        ui.debug('running %r in %s\n' % (pycompat.bytestr(cmdline),
+                                         tmproot))
         ui.system(cmdline, cwd=tmproot, blockedtag='extdiff')
 
         for copy_fn, working_fn, st in fnsandstat:
@@ -383,8 +392,9 b' class savedcmd(object):'
 
     def __init__(self, path, cmdline):
         # We can't pass non-ASCII through docstrings (and path is
-        # in an unknown encoding anyway)
-        docpath = stringutil.escapestr(path)
+        # in an unknown encoding anyway), but avoid double separators on
+        # Windows
+        docpath = stringutil.escapestr(path).replace(b'\\\\', b'\\')
         self.__doc__ %= {r'path': pycompat.sysstr(stringutil.uirepr(docpath))}
         self._cmdline = cmdline
 

hgext/fastannotate/commands.py

@@ -261,8 +261,9 b' def debugbuildannotatecache(ui, repo, *p'
         repo.prefetchfastannotate(paths)
     else:
         # server, or full repo
+        progress = ui.makeprogress(_('building'), total=len(paths))
         for i, path in enumerate(paths):
-            ui.progress(_('building'), i, total=len(paths))
+            progress.update(i)
             with facontext.annotatecontext(repo, path) as actx:
                 try:
                     if actx.isuptodate(rev):
@@ -281,5 +282,4 b' def debugbuildannotatecache(ui, repo, *p'
                         # cache for other files.
                         ui.warn(_('fastannotate: %s: failed to '
                                   'build cache: %r\n') % (path, ex))
-        # clear the progress bar
-        ui.write()
+        progress.complete()

hgext/fastannotate/context.py

@@ -138,7 +138,7 b' def hashdiffopts(diffopts):'
         (k, getattr(diffopts, k))
         for k in mdiff.diffopts.defaults
     ))
-    return hashlib.sha1(diffoptstr).hexdigest()[:6]
+    return node.hex(hashlib.sha1(diffoptstr).digest())[:6]
 
 _defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
 
@@ -156,6 +156,7 b' class annotateopts(object):'
     }
 
     def __init__(self, **opts):
+        opts = pycompat.byteskwargs(opts)
         for k, v in self.defaults.iteritems():
             setattr(self, k, opts.get(k, v))
 
@@ -397,7 +398,8 b' class _annotatecontext(object):'
 
         # 3rd DFS does the actual annotate
         visit = initvisit[:]
-        progress = 0
+        progress = self.ui.makeprogress(('building cache'),
+                                        total=len(newmainbranch))
         while visit:
             f = visit[-1]
             if f in hist:
@@ -436,10 +438,7 b' class _annotatecontext(object):'
             del pcache[f]
 
             if ismainbranch: # need to write to linelog
-                if not self.ui.quiet:
-                    progress += 1
-                    self.ui.progress(_('building cache'), progress,
-                                     total=len(newmainbranch))
+                progress.increment()
                 bannotated = None
                 if len(pl) == 2 and self.opts.followmerge: # merge
                     bannotated = curr[0]
@@ -449,8 +448,7 b' class _annotatecontext(object):'
             elif showpath: # not append linelog, but we need to record path
                 self._node2path[f.node()] = f.path()
 
-        if progress: # clean progress bar
-            self.ui.write()
+        progress.complete()
 
         result = [
             ((self.revmap.rev2hsh(fr) if isinstance(fr, int) else fr.node()), l)
@@ -604,7 +602,7 b' class _annotatecontext(object):'
         the best case, the user provides a node and we don't need to read the
         filelog or construct any filecontext.
         """
-        if isinstance(f, str):
+        if isinstance(f, bytes):
             hsh = f
         else:
             hsh = f.node()
@@ -627,7 +625,7 b' class _annotatecontext(object):'
         if showpath:
             result = self._addpathtoresult(result)
         if showlines:
-            if isinstance(f, str): # f: node or fctx
+            if isinstance(f, bytes): # f: node or fctx
                 llrev = self.revmap.hsh2rev(f)
                 fctx = self._resolvefctx(f, self.revmap.rev2path(llrev))
             else:

hgext/fastannotate/formatter.py

@@ -39,23 +39,26 b' class defaultformatter(object):'
             orig = hexfunc
             hexfunc = lambda x: None if x is None else orig(x)
             wnode = hexfunc(repo[None].p1().node()) + '+'
-            wrev = str(repo[None].p1().rev())
+            wrev = '%d' % repo[None].p1().rev()
             wrevpad = ''
             if not opts.get('changeset'): # only show + if changeset is hidden
                 wrev += '+'
                 wrevpad = ' '
-            revenc = lambda x: wrev if x is None else str(x) + wrevpad
-            csetenc = lambda x: wnode if x is None else str(x) + ' '
+            revenc = lambda x: wrev if x is None else ('%d' % x) + wrevpad
+            def csetenc(x):
+                if x is None:
+                    return wnode
+                return pycompat.bytestr(x) + ' '
         else:
-            revenc = csetenc = str
+            revenc = csetenc = pycompat.bytestr
 
         # opt name, separator, raw value (for json/plain), encoder (for plain)
         opmap = [('user', ' ', lambda x: getctx(x).user(), ui.shortuser),
                  ('number', ' ', lambda x: getctx(x).rev(), revenc),
                  ('changeset', ' ', lambda x: hexfunc(x[0]), csetenc),
                  ('date', ' ', lambda x: getctx(x).date(), datefunc),
-                 ('file', ' ', lambda x: x[2], str),
-                 ('line_number', ':', lambda x: x[1] + 1, str)]
+                 ('file', ' ', lambda x: x[2], pycompat.bytestr),
+                 ('line_number', ':', lambda x: x[1] + 1, pycompat.bytestr)]
         fieldnamemap = {'number': 'rev', 'changeset': 'node'}
         funcmap = [(get, sep, fieldnamemap.get(op, op), enc)
                    for op, sep, get, enc in opmap
@@ -100,7 +103,7 b' class defaultformatter(object):'
                         result += ': ' + self.ui.label('-' + lines[i],
                                                        'diff.deleted')
 
-            if result[-1] != '\n':
+            if result[-1:] != '\n':
                 result += '\n'
 
         self.ui.write(result)
@@ -125,7 +128,7 b' class jsonformatter(defaultformatter):'
         if annotatedresult:
             self._writecomma()
 
-        pieces = [(name, map(f, annotatedresult))
+        pieces = [(name, pycompat.maplist(f, annotatedresult))
                   for f, sep, name, enc in self.funcmap]
         if lines is not None:
             pieces.append(('line', lines))

hgext/fastannotate/protocol.py

@@ -98,10 +98,10 b' def _parseresponse(payload):'
     state = 0 # 0: vfspath, 1: size
     vfspath = size = ''
     while i < l:
-        ch = payload[i]
+        ch = payload[i:i + 1]
         if ch == '\0':
             if state == 1:
-                result[vfspath] = buffer(payload, i + 1, int(size))
+                result[vfspath] = payload[i + 1:i + 1 + int(size)]
                 i += int(size)
                 state = 0
                 vfspath = size = ''

hgext/fastannotate/revmap.py

@@ -207,7 +207,7 b' class revmap(object):'
             path = self.rev2path(rev)
             if path is None:
                 raise error.CorruptedFileError('cannot find path for %s' % rev)
-            f.write(path + '\0')
+            f.write(path + b'\0')
         f.write(hsh)
 
     @staticmethod

hgext/fix.py

@@ -15,13 +15,15 b' formatting fixes to modified lines in C+'
   [fix]
   clang-format:command=clang-format --assume-filename={rootpath}
   clang-format:linerange=--lines={first}:{last}
-  clang-format:fileset=set:**.cpp or **.hpp
+  clang-format:pattern=set:**.cpp or **.hpp
 
 The :command suboption forms the first part of the shell command that will be
 used to fix a file. The content of the file is passed on standard input, and the
-fixed file content is expected on standard output. If there is any output on
-standard error, the file will not be affected. Some values may be substituted
-into the command::
+fixed file content is expected on standard output. Any output on standard error
+will be displayed as a warning. If the exit status is not zero, the file will
+not be affected. A placeholder warning is displayed if there is a non-zero exit
+status but no standard error output. Some values may be substituted into the
+command::
 
   {rootpath}  The path of the file being fixed, relative to the repo root
   {basename}  The name of the file being fixed, without the directory path
@@ -34,16 +36,42 b' substituted into the command::'
   {first}   The 1-based line number of the first line in the modified range
   {last}    The 1-based line number of the last line in the modified range
 
-The :fileset suboption determines which files will be passed through each
-configured tool. See :hg:`help fileset` for possible values. If there are file
-arguments to :hg:`fix`, the intersection of these filesets is used.
+The :pattern suboption determines which files will be passed through each
+configured tool. See :hg:`help patterns` for possible values. If there are file
+arguments to :hg:`fix`, the intersection of these patterns is used.
 
 There is also a configurable limit for the maximum size of file that will be
 processed by :hg:`fix`::
 
   [fix]
-  maxfilesize=2MB
+  maxfilesize = 2MB
+
+Normally, execution of configured tools will continue after a failure (indicated
+by a non-zero exit status). It can also be configured to abort after the first
+such failure, so that no files will be affected if any tool fails. This abort
+will also cause :hg:`fix` to exit with a non-zero status::
+
+  [fix]
+  failure = abort
 
+When multiple tools are configured to affect a file, they execute in an order
+defined by the :priority suboption. The priority suboption has a default value
+of zero for each tool. Tools are executed in order of descending priority. The
+execution order of tools with equal priority is unspecified. For example, you
+could use the 'sort' and 'head' utilities to keep only the 10 smallest numbers
+in a text file by ensuring that 'sort' runs before 'head'::
+
+  [fix]
+  sort:command = sort -n
+  head:command = head -n 10
+  sort:pattern = numbers.txt
+  head:pattern = numbers.txt
+  sort:priority = 2
+  head:priority = 1
+
+To account for changes made by each tool, the line numbers used for incremental
+formatting are recomputed before executing the next tool. So, each tool may see
+different values for the arguments added by the :linerange suboption.
 """
 
 from __future__ import absolute_import
@@ -90,16 +118,36 b' configtable = {}'
 configitem = registrar.configitem(configtable)
 
 # Register the suboptions allowed for each configured fixer.
-FIXER_ATTRS = ('command', 'linerange', 'fileset')
+FIXER_ATTRS = {
+    'command': None,
+    'linerange': None,
+    'fileset': None,
+    'pattern': None,
+    'priority': 0,
+}
 
-for key in FIXER_ATTRS:
-    configitem('fix', '.*(:%s)?' % key, default=None, generic=True)
+for key, default in FIXER_ATTRS.items():
+    configitem('fix', '.*(:%s)?' % key, default=default, generic=True)
 
 # A good default size allows most source code files to be fixed, but avoids
 # letting fixer tools choke on huge inputs, which could be surprising to the
 # user.
 configitem('fix', 'maxfilesize', default='2MB')
 
+# Allow fix commands to exit non-zero if an executed fixer tool exits non-zero.
+# This helps users do shell scripts that stop when a fixer tool signals a
+# problem.
+configitem('fix', 'failure', default='continue')
+
+def checktoolfailureaction(ui, message, hint=None):
+    """Abort with 'message' if fix.failure=abort"""
+    action = ui.config('fix', 'failure')
+    if action not in ('continue', 'abort'):
+        raise error.Abort(_('unknown fix.failure action: %s') % (action,),
+                          hint=_('use "continue" or "abort"'))
+    if action == 'abort':
+        raise error.Abort(message, hint=hint)
+
 allopt = ('', 'all', False, _('fix all non-public non-obsolete revisions'))
 baseopt = ('', 'base', [], _('revisions to diff against (overrides automatic '
                              'selection, and applies to every revision being '
@@ -465,9 +513,14 b' def fixfile(ui, opts, fixers, fixctx, pa'
                 showstderr(ui, fixctx.rev(), fixername, stderr)
             if proc.returncode == 0:
                 newdata = newerdata
-            elif not stderr:
-                showstderr(ui, fixctx.rev(), fixername,
-                           _('exited with status %d\n') % (proc.returncode,))
+            else:
+                if not stderr:
+                    message = _('exited with status %d\n') % (proc.returncode,)
+                    showstderr(ui, fixctx.rev(), fixername, message)
+                checktoolfailureaction(
+                    ui, _('no fixes will be applied'),
+                    hint=_('use --config fix.failure=continue to apply any '
+                           'successful fixes anyway'))
     return newdata
 
 def showstderr(ui, rev, fixername, stderr):
@@ -533,6 +586,17 b' def replacerev(ui, repo, ctx, filedata, '
     newp1node = replacements.get(p1ctx.node(), p1ctx.node())
     newp2node = replacements.get(p2ctx.node(), p2ctx.node())
 
+    # We don't want to create a revision that has no changes from the original,
+    # but we should if the original revision's parent has been replaced.
+    # Otherwise, we would produce an orphan that needs no actual human
+    # intervention to evolve. We can't rely on commit() to avoid creating the
+    # un-needed revision because the extra field added below produces a new hash
+    # regardless of file content changes.
+    if (not filedata and
+        p1ctx.node() not in replacements and
+        p2ctx.node() not in replacements):
+        return
+
     def filectxfn(repo, memctx, path):
         if path not in ctx:
             return None
@@ -549,6 +613,9 b' def replacerev(ui, repo, ctx, filedata, '
             isexec=fctx.isexec(),
             copied=copied)
 
+    extra = ctx.extra().copy()
+    extra['fix_source'] = ctx.hex()
+
     memctx = context.memctx(
         repo,
         parents=(newp1node, newp2node),
@@ -557,7 +624,7 b' def replacerev(ui, repo, ctx, filedata, '
         filectxfn=filectxfn,
         user=ctx.user(),
         date=ctx.date(),
-        extra=ctx.extra(),
+        extra=extra,
         branch=ctx.branch(),
         editor=None)
     sucnode = memctx.commit()
@@ -573,14 +640,21 b' def getfixers(ui):'
     Each value is a Fixer object with methods that implement the behavior of the
     fixer's config suboptions. Does not validate the config values.
     """
-    result = {}
+    fixers = {}
     for name in fixernames(ui):
-        result[name] = Fixer()
+        fixers[name] = Fixer()
         attrs = ui.configsuboptions('fix', name)[1]
-        for key in FIXER_ATTRS:
-            setattr(result[name], pycompat.sysstr('_' + key),
-                    attrs.get(key, ''))
-    return result
+        if 'fileset' in attrs and 'pattern' not in attrs:
+            ui.warn(_('the fix.tool:fileset config name is deprecated; '
+                      'please rename it to fix.tool:pattern\n'))
+            attrs['pattern'] = attrs['fileset']
+        for key, default in FIXER_ATTRS.items():
+            setattr(fixers[name], pycompat.sysstr('_' + key),
+                    attrs.get(key, default))
+        fixers[name]._priority = int(fixers[name]._priority)
+    return collections.OrderedDict(
+        sorted(fixers.items(), key=lambda item: item[1]._priority,
+               reverse=True))
 
 def fixernames(ui):
     """Returns the names of [fix] config options that have suboptions"""
@@ -595,7 +669,7 b' class Fixer(object):'
 
     def affects(self, opts, fixctx, path):
         """Should this fixer run on the file at the given path and context?"""
-        return scmutil.match(fixctx, [self._fileset], opts)(path)
+        return scmutil.match(fixctx, [self._pattern], opts)(path)
 
     def command(self, ui, path, rangesfn):
         """A shell command to use to invoke this fixer on the given file/lines

hgext/highlight/__init__.py

@@ -87,7 +87,7 b' def generate_css(web):'
     ]))
     return web.res.sendresponse()
 
-def extsetup():
+def extsetup(ui):
     # monkeypatch in the new version
     extensions.wrapfunction(webcommands, '_filerevision',
                             filerevision_highlight)

hgext/histedit.py

@@ -183,7 +183,17 b' unexpectedly::'
 
 from __future__ import absolute_import
 
+# chistedit dependencies that are not available everywhere
+try:
+    import fcntl
+    import termios
+except ImportError:
+    fcntl = None
+    termios = None
+
+import functools
 import os
+import struct
 
 from mercurial.i18n import _
 from mercurial import (
@@ -197,7 +207,7 b' from mercurial import ('
     exchange,
     extensions,
     hg,
-    lock,
+    logcmdutil,
     merge as mergemod,
     mergeutil,
     node,
@@ -210,11 +220,11 b' from mercurial import ('
     util,
 )
 from mercurial.utils import (
+    dateutil,
     stringutil,
 )
 
 pickle = util.pickle
-release = lock.release
 cmdtable = {}
 command = registrar.command(cmdtable)
 
@@ -235,6 +245,9 b" configitem('histedit', 'linelen',"
 configitem('histedit', 'singletransaction',
     default=False,
 )
+configitem('ui', 'interface.histedit',
+    default=None,
+)
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
@@ -294,21 +307,17 b' Commands:'
     return ''.join(['# %s\n' % l if l else '#\n' for l in lines])
 
 class histeditstate(object):
-    def __init__(self, repo, parentctxnode=None, actions=None, keep=None,
-            topmost=None, replacements=None, lock=None, wlock=None):
+    def __init__(self, repo):
         self.repo = repo
-        self.actions = actions
-        self.keep = keep
-        self.topmost = topmost
-        self.parentctxnode = parentctxnode
-        self.lock = lock
-        self.wlock = wlock
+        self.actions = None
+        self.keep = None
+        self.topmost = None
+        self.parentctxnode = None
+        self.lock = None
+        self.wlock = None
         self.backupfile = None
         self.stateobj = statemod.cmdstate(repo, 'histedit-state')
-        if replacements is None:
-            self.replacements = []
-        else:
-            self.replacements = replacements
+        self.replacements = []
 
     def read(self):
         """Load histedit state from disk and set fields appropriately."""
@@ -519,9 +528,12 b' class histeditaction(object):'
 
         editor = self.commiteditor()
         commit = commitfuncfor(repo, rulectx)
-
+        if repo.ui.configbool('rewrite', 'update-timestamp'):
+            date = dateutil.makedate()
+        else:
+            date = rulectx.date()
         commit(text=rulectx.description(), user=rulectx.user(),
-               date=rulectx.date(), extra=rulectx.extra(), editor=editor)
+               date=date, extra=rulectx.extra(), editor=editor)
 
     def commiteditor(self):
         """The editor to be used to edit the commit message."""
@@ -802,6 +814,10 b' class fold(histeditaction):'
             commitopts['date'] = ctx.date()
         else:
             commitopts['date'] = max(ctx.date(), oldctx.date())
+        # if date is to be updated to current
+        if ui.configbool('rewrite', 'update-timestamp'):
+            commitopts['date'] = dateutil.makedate()
+
         extra = ctx.extra().copy()
         # histedit_source
         # note: ctx is likely a temporary commit but that the best we can do
@@ -915,6 +931,562 b' def findoutgoing(ui, repo, remote=None, '
         raise error.Abort(msg, hint=hint)
     return repo[roots[0]].node()
 
+# Curses Support
+try:
+    import curses
+except ImportError:
+    curses = None
+
+KEY_LIST = ['pick', 'edit', 'fold', 'drop', 'mess', 'roll']
+ACTION_LABELS = {
+    'fold': '^fold',
+    'roll': '^roll',
+}
+
+COLOR_HELP, COLOR_SELECTED, COLOR_OK, COLOR_WARN  = 1, 2, 3, 4
+
+E_QUIT, E_HISTEDIT = 1, 2
+E_PAGEDOWN, E_PAGEUP, E_LINEUP, E_LINEDOWN, E_RESIZE = 3, 4, 5, 6, 7
+MODE_INIT, MODE_PATCH, MODE_RULES, MODE_HELP = 0, 1, 2, 3
+
+KEYTABLE = {
+    'global': {
+        'h':         'next-action',
+        'KEY_RIGHT': 'next-action',
+        'l':         'prev-action',
+        'KEY_LEFT':  'prev-action',
+        'q':         'quit',
+        'c':         'histedit',
+        'C':         'histedit',
+        'v':         'showpatch',
+        '?':         'help',
+    },
+    MODE_RULES: {
+        'd':         'action-drop',
+        'e':         'action-edit',
+        'f':         'action-fold',
+        'm':         'action-mess',
+        'p':         'action-pick',
+        'r':         'action-roll',
+        ' ':         'select',
+        'j':         'down',
+        'k':         'up',
+        'KEY_DOWN':  'down',
+        'KEY_UP':    'up',
+        'J':         'move-down',
+        'K':         'move-up',
+        'KEY_NPAGE': 'move-down',
+        'KEY_PPAGE': 'move-up',
+        '0':         'goto',  # Used for 0..9
+    },
+    MODE_PATCH: {
+        ' ':         'page-down',
+        'KEY_NPAGE': 'page-down',
+        'KEY_PPAGE': 'page-up',
+        'j':         'line-down',
+        'k':         'line-up',
+        'KEY_DOWN':  'line-down',
+        'KEY_UP':    'line-up',
+        'J':         'down',
+        'K':         'up',
+    },
+    MODE_HELP: {
+    },
+}
+
+def screen_size():
+    return struct.unpack('hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '    '))
+
+class histeditrule(object):
+    def __init__(self, ctx, pos, action='pick'):
+        self.ctx = ctx
+        self.action = action
+        self.origpos = pos
+        self.pos = pos
+        self.conflicts = []
+
+    def __str__(self):
+        # Some actions ('fold' and 'roll') combine a patch with a previous one.
+        # Add a marker showing which patch they apply to, and also omit the
+        # description for 'roll' (since it will get discarded). Example display:
+        #
+        #  #10 pick   316392:06a16c25c053   add option to skip tests
+        #  #11 ^roll  316393:71313c964cc5
+        #  #12 pick   316394:ab31f3973b0d   include mfbt for mozilla-config.h
+        #  #13 ^fold  316395:14ce5803f4c3   fix warnings
+        #
+        # The carets point to the changeset being folded into ("roll this
+        # changeset into the changeset above").
+        action = ACTION_LABELS.get(self.action, self.action)
+        h = self.ctx.hex()[0:12]
+        r = self.ctx.rev()
+        desc = self.ctx.description().splitlines()[0].strip()
+        if self.action == 'roll':
+            desc = ''
+        return "#{0:<2} {1:<6} {2}:{3}   {4}".format(
+                self.origpos, action, r, h, desc)
+
+    def checkconflicts(self, other):
+        if other.pos > self.pos and other.origpos <= self.origpos:
+            if set(other.ctx.files()) & set(self.ctx.files()) != set():
+                self.conflicts.append(other)
+                return self.conflicts
+
+        if other in self.conflicts:
+            self.conflicts.remove(other)
+        return self.conflicts
+
+# ============ EVENTS ===============
+def movecursor(state, oldpos, newpos):
+    '''Change the rule/changeset that the cursor is pointing to, regardless of
+    current mode (you can switch between patches from the view patch window).'''
+    state['pos'] = newpos
+
+    mode, _ = state['mode']
+    if mode == MODE_RULES:
+        # Scroll through the list by updating the view for MODE_RULES, so that
+        # even if we are not currently viewing the rules, switching back will
+        # result in the cursor's rule being visible.
+        modestate = state['modes'][MODE_RULES]
+        if newpos < modestate['line_offset']:
+            modestate['line_offset'] = newpos
+        elif newpos > modestate['line_offset'] + state['page_height'] - 1:
+            modestate['line_offset'] = newpos - state['page_height'] + 1
+
+    # Reset the patch view region to the top of the new patch.
+    state['modes'][MODE_PATCH]['line_offset'] = 0
+
+def changemode(state, mode):
+    curmode, _ = state['mode']
+    state['mode'] = (mode, curmode)
+
+def makeselection(state, pos):
+    state['selected'] = pos
+
+def swap(state, oldpos, newpos):
+    """Swap two positions and calculate necessary conflicts in
+    O(|newpos-oldpos|) time"""
+
+    rules = state['rules']
+    assert 0 <= oldpos < len(rules) and 0 <= newpos < len(rules)
+
+    rules[oldpos], rules[newpos] = rules[newpos], rules[oldpos]
+
+    # TODO: swap should not know about histeditrule's internals
+    rules[newpos].pos = newpos
+    rules[oldpos].pos = oldpos
+
+    start = min(oldpos, newpos)
+    end = max(oldpos, newpos)
+    for r in pycompat.xrange(start, end + 1):
+        rules[newpos].checkconflicts(rules[r])
+        rules[oldpos].checkconflicts(rules[r])
+
+    if state['selected']:
+        makeselection(state, newpos)
+
+def changeaction(state, pos, action):
+    """Change the action state on the given position to the new action"""
+    rules = state['rules']
+    assert 0 <= pos < len(rules)
+    rules[pos].action = action
+
+def cycleaction(state, pos, next=False):
+    """Changes the action state the next or the previous action from
+    the action list"""
+    rules = state['rules']
+    assert 0 <= pos < len(rules)
+    current = rules[pos].action
+
+    assert current in KEY_LIST
+
+    index = KEY_LIST.index(current)
+    if next:
+        index += 1
+    else:
+        index -= 1
+    changeaction(state, pos, KEY_LIST[index % len(KEY_LIST)])
+
+def changeview(state, delta, unit):
+    '''Change the region of whatever is being viewed (a patch or the list of
+    changesets). 'delta' is an amount (+/- 1) and 'unit' is 'page' or 'line'.'''
+    mode, _ = state['mode']
+    if mode != MODE_PATCH:
+        return
+    mode_state = state['modes'][mode]
+    num_lines = len(patchcontents(state))
+    page_height = state['page_height']
+    unit = page_height if unit == 'page' else 1
+    num_pages = 1 + (num_lines - 1) / page_height
+    max_offset = (num_pages - 1) * page_height
+    newline = mode_state['line_offset'] + delta * unit
+    mode_state['line_offset'] = max(0, min(max_offset, newline))
+
+def event(state, ch):
+    """Change state based on the current character input
+
+    This takes the current state and based on the current character input from
+    the user we change the state.
+    """
+    selected = state['selected']
+    oldpos = state['pos']
+    rules = state['rules']
+
+    if ch in (curses.KEY_RESIZE, "KEY_RESIZE"):
+        return E_RESIZE
+
+    lookup_ch = ch
+    if '0' <= ch <= '9':
+        lookup_ch = '0'
+
+    curmode, prevmode = state['mode']
+    action = KEYTABLE[curmode].get(lookup_ch, KEYTABLE['global'].get(lookup_ch))
+    if action is None:
+        return
+    if action in ('down', 'move-down'):
+        newpos = min(oldpos + 1, len(rules) - 1)
+        movecursor(state, oldpos, newpos)
+        if selected is not None or action == 'move-down':
+            swap(state, oldpos, newpos)
+    elif action in ('up', 'move-up'):
+        newpos = max(0, oldpos - 1)
+        movecursor(state, oldpos, newpos)
+        if selected is not None or action == 'move-up':
+            swap(state, oldpos, newpos)
+    elif action == 'next-action':
+        cycleaction(state, oldpos, next=True)
+    elif action == 'prev-action':
+        cycleaction(state, oldpos, next=False)
+    elif action == 'select':
+        selected = oldpos if selected is None else None
+        makeselection(state, selected)
+    elif action == 'goto' and int(ch) < len(rules) and len(rules) <= 10:
+        newrule = next((r for r in rules if r.origpos == int(ch)))
+        movecursor(state, oldpos, newrule.pos)
+        if selected is not None:
+            swap(state, oldpos, newrule.pos)
+    elif action.startswith('action-'):
+        changeaction(state, oldpos, action[7:])
+    elif action == 'showpatch':
+        changemode(state, MODE_PATCH if curmode != MODE_PATCH else prevmode)
+    elif action == 'help':
+        changemode(state, MODE_HELP if curmode != MODE_HELP else prevmode)
+    elif action == 'quit':
+        return E_QUIT
+    elif action == 'histedit':
+        return E_HISTEDIT
+    elif action == 'page-down':
+        return E_PAGEDOWN
+    elif action == 'page-up':
+        return E_PAGEUP
+    elif action == 'line-down':
+        return E_LINEDOWN
+    elif action == 'line-up':
+        return E_LINEUP
+
+def makecommands(rules):
+    """Returns a list of commands consumable by histedit --commands based on
+    our list of rules"""
+    commands = []
+    for rules in rules:
+        commands.append("{0} {1}\n".format(rules.action, rules.ctx))
+    return commands
+
+def addln(win, y, x, line, color=None):
+    """Add a line to the given window left padding but 100% filled with
+    whitespace characters, so that the color appears on the whole line"""
+    maxy, maxx = win.getmaxyx()
+    length = maxx - 1 - x
+    line = ("{0:<%d}" % length).format(str(line).strip())[:length]
+    if y < 0:
+        y = maxy + y
+    if x < 0:
+        x = maxx + x
+    if color:
+        win.addstr(y, x, line, color)
+    else:
+        win.addstr(y, x, line)
+
+def patchcontents(state):
+    repo = state['repo']
+    rule = state['rules'][state['pos']]
+    displayer = logcmdutil.changesetdisplayer(repo.ui, repo, {
+        'patch': True, 'verbose': True
+    }, buffered=True)
+    displayer.show(rule.ctx)
+    displayer.close()
+    return displayer.hunk[rule.ctx.rev()].splitlines()
+
+def _chisteditmain(repo, rules, stdscr):
+    # initialize color pattern
+    curses.init_pair(COLOR_HELP, curses.COLOR_WHITE, curses.COLOR_BLUE)
+    curses.init_pair(COLOR_SELECTED, curses.COLOR_BLACK, curses.COLOR_WHITE)
+    curses.init_pair(COLOR_WARN, curses.COLOR_BLACK, curses.COLOR_YELLOW)
+    curses.init_pair(COLOR_OK, curses.COLOR_BLACK, curses.COLOR_GREEN)
+
+    # don't display the cursor
+    try:
+        curses.curs_set(0)
+    except curses.error:
+        pass
+
+    def rendercommit(win, state):
+        """Renders the commit window that shows the log of the current selected
+        commit"""
+        pos = state['pos']
+        rules = state['rules']
+        rule = rules[pos]
+
+        ctx = rule.ctx
+        win.box()
+
+        maxy, maxx = win.getmaxyx()
+        length = maxx - 3
+
+        line = "changeset: {0}:{1:<12}".format(ctx.rev(), ctx)
+        win.addstr(1, 1, line[:length])
+
+        line = "user:      {0}".format(stringutil.shortuser(ctx.user()))
+        win.addstr(2, 1, line[:length])
+
+        bms = repo.nodebookmarks(ctx.node())
+        line = "bookmark:  {0}".format(' '.join(bms))
+        win.addstr(3, 1, line[:length])
+
+        line = "files:     {0}".format(','.join(ctx.files()))
+        win.addstr(4, 1, line[:length])
+
+        line = "summary:   {0}".format(ctx.description().splitlines()[0])
+        win.addstr(5, 1, line[:length])
+
+        conflicts = rule.conflicts
+        if len(conflicts) > 0:
+            conflictstr = ','.join(map(lambda r: str(r.ctx), conflicts))
+            conflictstr = "changed files overlap with {0}".format(conflictstr)
+        else:
+            conflictstr = 'no overlap'
+
+        win.addstr(6, 1, conflictstr[:length])
+        win.noutrefresh()
+
+    def helplines(mode):
+        if mode == MODE_PATCH:
+            help = """\
+?: help, k/up: line up, j/down: line down, v: stop viewing patch
+pgup: prev page, space/pgdn: next page, c: commit, q: abort
+"""
+        else:
+            help = """\
+?: help, k/up: move up, j/down: move down, space: select, v: view patch
+d: drop, e: edit, f: fold, m: mess, p: pick, r: roll
+pgup/K: move patch up, pgdn/J: move patch down, c: commit, q: abort
+"""
+        return help.splitlines()
+
+    def renderhelp(win, state):
+        maxy, maxx = win.getmaxyx()
+        mode, _ = state['mode']
+        for y, line in enumerate(helplines(mode)):
+            if y >= maxy:
+                break
+            addln(win, y, 0, line, curses.color_pair(COLOR_HELP))
+        win.noutrefresh()
+
+    def renderrules(rulesscr, state):
+        rules = state['rules']
+        pos = state['pos']
+        selected = state['selected']
+        start = state['modes'][MODE_RULES]['line_offset']
+
+        conflicts = [r.ctx for r in rules if r.conflicts]
+        if len(conflicts) > 0:
+            line = "potential conflict in %s" % ','.join(map(str, conflicts))
+            addln(rulesscr, -1, 0, line, curses.color_pair(COLOR_WARN))
+
+        for y, rule in enumerate(rules[start:]):
+            if y >= state['page_height']:
+                break
+            if len(rule.conflicts) > 0:
+                rulesscr.addstr(y, 0, " ", curses.color_pair(COLOR_WARN))
+            else:
+                rulesscr.addstr(y, 0, " ", curses.COLOR_BLACK)
+            if y + start == selected:
+                addln(rulesscr, y, 2, rule, curses.color_pair(COLOR_SELECTED))
+            elif y + start == pos:
+                addln(rulesscr, y, 2, rule, curses.A_BOLD)
+            else:
+                addln(rulesscr, y, 2, rule)
+        rulesscr.noutrefresh()
+
+    def renderstring(win, state, output):
+        maxy, maxx = win.getmaxyx()
+        length = min(maxy - 1, len(output))
+        for y in range(0, length):
+            win.addstr(y, 0, output[y])
+        win.noutrefresh()
+
+    def renderpatch(win, state):
+        start = state['modes'][MODE_PATCH]['line_offset']
+        renderstring(win, state, patchcontents(state)[start:])
+
+    def layout(mode):
+        maxy, maxx = stdscr.getmaxyx()
+        helplen = len(helplines(mode))
+        return {
+            'commit': (8, maxx),
+            'help': (helplen, maxx),
+            'main': (maxy - helplen - 8, maxx),
+        }
+
+    def drawvertwin(size, y, x):
+        win = curses.newwin(size[0], size[1], y, x)
+        y += size[0]
+        return win, y, x
+
+    state = {
+        'pos': 0,
+        'rules': rules,
+        'selected': None,
+        'mode': (MODE_INIT, MODE_INIT),
+        'page_height': None,
+        'modes': {
+            MODE_RULES: {
+                'line_offset': 0,
+            },
+            MODE_PATCH: {
+                'line_offset': 0,
+            }
+        },
+        'repo': repo,
+    }
+
+    # eventloop
+    ch = None
+    stdscr.clear()
+    stdscr.refresh()
+    while True:
+        try:
+            oldmode, _ = state['mode']
+            if oldmode == MODE_INIT:
+                changemode(state, MODE_RULES)
+            e = event(state, ch)
+
+            if e == E_QUIT:
+                return False
+            if e == E_HISTEDIT:
+                return state['rules']
+            else:
+                if e == E_RESIZE:
+                    size = screen_size()
+                    if size != stdscr.getmaxyx():
+                        curses.resizeterm(*size)
+
+                curmode, _ = state['mode']
+                sizes = layout(curmode)
+                if curmode != oldmode:
+                    state['page_height'] = sizes['main'][0]
+                    # Adjust the view to fit the current screen size.
+                    movecursor(state, state['pos'], state['pos'])
+
+                # Pack the windows against the top, each pane spread across the
+                # full width of the screen.
+                y, x = (0, 0)
+                helpwin, y, x = drawvertwin(sizes['help'], y, x)
+                mainwin, y, x = drawvertwin(sizes['main'], y, x)
+                commitwin, y, x = drawvertwin(sizes['commit'], y, x)
+
+                if e in (E_PAGEDOWN, E_PAGEUP, E_LINEDOWN, E_LINEUP):
+                    if e == E_PAGEDOWN:
+                        changeview(state, +1, 'page')
+                    elif e == E_PAGEUP:
+                        changeview(state, -1, 'page')
+                    elif e == E_LINEDOWN:
+                        changeview(state, +1, 'line')
+                    elif e == E_LINEUP:
+                        changeview(state, -1, 'line')
+
+                # start rendering
+                commitwin.erase()
+                helpwin.erase()
+                mainwin.erase()
+                if curmode == MODE_PATCH:
+                    renderpatch(mainwin, state)
+                elif curmode == MODE_HELP:
+                    renderstring(mainwin, state, __doc__.strip().splitlines())
+                else:
+                    renderrules(mainwin, state)
+                    rendercommit(commitwin, state)
+                renderhelp(helpwin, state)
+                curses.doupdate()
+                # done rendering
+                ch = stdscr.getkey()
+        except curses.error:
+            pass
+
+def _chistedit(ui, repo, *freeargs, **opts):
+    """interactively edit changeset history via a curses interface
+
+    Provides a ncurses interface to histedit. Press ? in chistedit mode
+    to see an extensive help. Requires python-curses to be installed."""
+
+    if curses is None:
+        raise error.Abort(_("Python curses library required"))
+
+    # disable color
+    ui._colormode = None
+
+    try:
+        keep = opts.get('keep')
+        revs = opts.get('rev', [])[:]
+        cmdutil.checkunfinished(repo)
+        cmdutil.bailifchanged(repo)
+
+        if os.path.exists(os.path.join(repo.path, 'histedit-state')):
+            raise error.Abort(_('history edit already in progress, try '
+                               '--continue or --abort'))
+        revs.extend(freeargs)
+        if not revs:
+            defaultrev = destutil.desthistedit(ui, repo)
+            if defaultrev is not None:
+                revs.append(defaultrev)
+        if len(revs) != 1:
+            raise error.Abort(
+                _('histedit requires exactly one ancestor revision'))
+
+        rr = list(repo.set('roots(%ld)', scmutil.revrange(repo, revs)))
+        if len(rr) != 1:
+            raise error.Abort(_('The specified revisions must have '
+                'exactly one common root'))
+        root = rr[0].node()
+
+        topmost, empty = repo.dirstate.parents()
+        revs = between(repo, root, topmost, keep)
+        if not revs:
+            raise error.Abort(_('%s is not an ancestor of working directory') %
+                             node.short(root))
+
+        ctxs = []
+        for i, r in enumerate(revs):
+            ctxs.append(histeditrule(repo[r], i))
+        rc = curses.wrapper(functools.partial(_chisteditmain, repo, ctxs))
+        curses.echo()
+        curses.endwin()
+        if rc is False:
+            ui.write(_("chistedit aborted\n"))
+            return 0
+        if type(rc) is list:
+            ui.status(_("running histedit\n"))
+            rules = makecommands(rc)
+            filename = repo.vfs.join('chistedit')
+            with open(filename, 'w+') as fp:
+                for r in rules:
+                    fp.write(r)
+            opts['commands'] = filename
+            return _texthistedit(ui, repo, *freeargs, **opts)
+    except KeyboardInterrupt:
+        pass
+    return -1
+
 @command('histedit',
     [('', 'commands', '',
       _('read history edits from the specified file'), _('FILE')),
@@ -1029,13 +1601,20 b' def histedit(ui, repo, *freeargs, **opts'
     for intentional "edit" command, but also for resolving unexpected
     conflicts).
     """
+    # kludge: _chistedit only works for starting an edit, not aborting
+    # or continuing, so fall back to regular _texthistedit for those
+    # operations.
+    if ui.interface('histedit') == 'curses' and  _getgoal(
+            pycompat.byteskwargs(opts)) == goalnew:
+        return _chistedit(ui, repo, *freeargs, **opts)
+    return _texthistedit(ui, repo, *freeargs, **opts)
+
+def _texthistedit(ui, repo, *freeargs, **opts):
     state = histeditstate(repo)
-    try:
-        state.wlock = repo.wlock()
-        state.lock = repo.lock()
+    with repo.wlock() as wlock, repo.lock() as lock:
+        state.wlock = wlock
+        state.lock = lock
         _histedit(ui, repo, state, *freeargs, **opts)
-    finally:
-        release(state.lock, state.wlock)
 
 goalcontinue = 'continue'
 goalabort = 'abort'
@@ -1043,11 +1622,11 b" goaleditplan = 'edit-plan'"
 goalnew = 'new'
 
 def _getgoal(opts):
-    if opts.get('continue'):
+    if opts.get(b'continue'):
         return goalcontinue
-    if opts.get('abort'):
+    if opts.get(b'abort'):
         return goalabort
-    if opts.get('edit_plan'):
+    if opts.get(b'edit_plan'):
         return goaleditplan
     return goalnew
 
@@ -1110,13 +1689,26 b' def _histedit(ui, repo, state, *freeargs'
     fm.startitem()
     goal = _getgoal(opts)
     revs = opts.get('rev', [])
-    # experimental config: ui.history-editing-backup
-    nobackup = not ui.configbool('ui', 'history-editing-backup')
+    nobackup = not ui.configbool('rewrite', 'backup-bundle')
     rules = opts.get('commands', '')
     state.keep = opts.get('keep', False)
 
     _validateargs(ui, repo, state, freeargs, opts, goal, rules, revs)
 
+    hastags = False
+    if revs:
+        revs = scmutil.revrange(repo, revs)
+        ctxs = [repo[rev] for rev in revs]
+        for ctx in ctxs:
+            tags = [tag for tag in ctx.tags() if tag != 'tip']
+            if not hastags:
+                hastags = len(tags)
+    if hastags:
+        if ui.promptchoice(_('warning: tags associated with the given'
+                             ' changeset will be lost after histedit.\n'
+                             'do you want to continue (yN)? $$ &Yes $$ &No'),
+                           default=1):
+            raise error.Abort(_('histedit cancelled\n'))
     # rebuild state
     if goal == goalcontinue:
         state.read()
@@ -1317,7 +1909,7 b' def _newhistedit(ui, repo, state, revs, '
     state.topmost = topmost
     state.replacements = []
 
-    ui.log("histedit", "%d actions to histedit", len(actions),
+    ui.log("histedit", "%d actions to histedit\n", len(actions),
            histedit_num_actions=len(actions))
 
     # Create a backup so we can always abort completely.

hgext/largefiles/__init__.py

@@ -107,9 +107,14 b' command.'
 from __future__ import absolute_import
 
 from mercurial import (
+    cmdutil,
+    extensions,
+    exthelper,
     hg,
+    httppeer,
     localrepo,
-    registrar,
+    sshpeer,
+    wireprotov1server,
 )
 
 from . import (
@@ -117,7 +122,6 b' from . import ('
     overrides,
     proto,
     reposetup,
-    uisetup as uisetupmod,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
@@ -126,29 +130,65 b' from . import ('
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
-configtable = {}
-configitem = registrar.configitem(configtable)
+eh = exthelper.exthelper()
+eh.merge(lfcommands.eh)
+eh.merge(overrides.eh)
+eh.merge(proto.eh)
 
-configitem('largefiles', 'minsize',
-    default=configitem.dynamicdefault,
+eh.configitem('largefiles', 'minsize',
+    default=eh.configitem.dynamicdefault,
 )
-configitem('largefiles', 'patterns',
+eh.configitem('largefiles', 'patterns',
     default=list,
 )
-configitem('largefiles', 'usercache',
+eh.configitem('largefiles', 'usercache',
     default=None,
 )
 
+cmdtable = eh.cmdtable
+configtable = eh.configtable
+extsetup = eh.finalextsetup
 reposetup = reposetup.reposetup
+uisetup = eh.finaluisetup
 
 def featuresetup(ui, supported):
     # don't die on seeing a repo with the largefiles requirement
     supported |= {'largefiles'}
 
-def uisetup(ui):
[e-mail]
+def _uisetup(ui):
     localrepo.featuresetupfuncs.add(featuresetup)
     hg.wirepeersetupfuncs.append(proto.wirereposetup)
-    uisetupmod.uisetup(ui)
+
+    cmdutil.outgoinghooks.add('largefiles', overrides.outgoinghook)
+    cmdutil.summaryremotehooks.add('largefiles', overrides.summaryremotehook)
+
+    # create the new wireproto commands ...
+    wireprotov1server.wireprotocommand('putlfile', 'sha', permission='push')(
+        proto.putlfile)
+    wireprotov1server.wireprotocommand('getlfile', 'sha', permission='pull')(
+        proto.getlfile)
+    wireprotov1server.wireprotocommand('statlfile', 'sha', permission='pull')(
+        proto.statlfile)
+    wireprotov1server.wireprotocommand('lheads', '', permission='pull')(
+        wireprotov1server.heads)
 
-cmdtable = lfcommands.cmdtable
-revsetpredicate = overrides.revsetpredicate
+    extensions.wrapfunction(wireprotov1server.commands['heads'], 'func',
+                            proto.heads)
+    # TODO also wrap wireproto.commandsv2 once heads is implemented there.
+
+    # can't do this in reposetup because it needs to have happened before
+    # wirerepo.__init__ is called
+    proto.ssholdcallstream = sshpeer.sshv1peer._callstream
+    proto.httpoldcallstream = httppeer.httppeer._callstream
+    sshpeer.sshv1peer._callstream = proto.sshrepocallstream
+    httppeer.httppeer._callstream = proto.httprepocallstream
+
+    # override some extensions' stuff as well
+    for name, module in extensions.extensions():
+        if name == 'rebase':
+            # TODO: teach exthelper to handle this
+            extensions.wrapfunction(module, 'rebase',
+                                    overrides.overriderebase)
+
+revsetpredicate = eh.revsetpredicate

hgext/largefiles/lfcommands.py

@@ -20,12 +20,12 b' from mercurial import ('
     cmdutil,
     context,
     error,
+    exthelper,
     hg,
     lock,
     match as matchmod,
     node,
     pycompat,
-    registrar,
     scmutil,
     util,
 )
@@ -44,10 +44,9 b' release = lock.release'
 
 # -- Commands ----------------------------------------------------------
 
-cmdtable = {}
-command = registrar.command(cmdtable)
+eh = exthelper.exthelper()
 
-@command('lfconvert',
[e-mail]('lfconvert',
     [('s', 'size', '',
       _('minimum size (MB) for files to be converted as largefiles'), 'SIZE'),
     ('', 'to-normal', False,
@@ -240,7 +239,7 b' def _lfconvert_addchangeset(rsrc, rdst, '
                 # largefile was modified, update standins
                 m = hashlib.sha1('')
                 m.update(ctx[f].data())
-                hash = m.hexdigest()
+                hash = node.hex(m.digest())
                 if f not in lfiletohash or lfiletohash[f] != hash:
                     rdst.wwrite(f, ctx[f].data(), ctx[f].flags())
                     executable = 'x' in ctx[f].flags()
@@ -560,7 +559,7 b' def updatelfiles(ui, repo, filelist=None'
             statuswriter(_('%d largefiles updated, %d removed\n') % (updated,
                 removed))
 
-@command('lfpull',
[e-mail]('lfpull',
     [('r', 'rev', [], _('pull largefiles for these revisions'))
     ] + cmdutil.remoteopts,
     _('-r REV... [-e CMD] [--remotecmd CMD] [SOURCE]'))
@@ -599,7 +598,7 b' def lfpull(ui, repo, source="default", *'
         numcached += len(cached)
     ui.status(_("%d largefiles cached\n") % numcached)
 
-@command('debuglfput',
[e-mail]('debuglfput',
     [] + cmdutil.remoteopts,
     _('FILE'))
 def debuglfput(ui, repo, filepath, **kwargs):

hgext/largefiles/overrides.py

@@ -14,18 +14,29 b' import os'
 
 from mercurial.i18n import _
 
+from mercurial.hgweb import (
+    webcommands,
+)
+
 from mercurial import (
     archival,
     cmdutil,
+    copies as copiesmod,
     error,
+    exchange,
+    exthelper,
+    filemerge,
     hg,
     logcmdutil,
     match as matchmod,
+    merge,
     pathutil,
     pycompat,
-    registrar,
     scmutil,
     smartset,
+    subrepo,
+    upgrade,
+    url as urlmod,
     util,
 )
 
@@ -35,6 +46,8 b' from . import ('
     storefactory,
 )
 
+eh = exthelper.exthelper()
+
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
 def composelargefilematcher(match, manifest):
@@ -248,16 +261,23 b' def removelargefiles(ui, repo, isaddremo'
 
 # For overriding mercurial.hgweb.webcommands so that largefiles will
 # appear at their right place in the manifests.
[e-mail](webcommands, 'decodepath')
 def decodepath(orig, path):
     return lfutil.splitstandin(path) or path
 
 # -- Wrappers: modify existing commands --------------------------------
 
[e-mail]('add',
+    opts=[('', 'large', None, _('add as largefile')),
+          ('', 'normal', None, _('add as normal file')),
+          ('', 'lfsize', '', _('add all files above this size (in megabytes) '
+                               'as largefiles (default: 10)'))])
 def overrideadd(orig, ui, repo, *pats, **opts):
     if opts.get(r'normal') and opts.get(r'large'):
         raise error.Abort(_('--normal cannot be used with --large'))
     return orig(ui, repo, *pats, **opts)
 
[e-mail](cmdutil, 'add')
 def cmdutiladd(orig, ui, repo, matcher, prefix, explicitonly, **opts):
     # The --normal flag short circuits this override
     if opts.get(r'normal'):
@@ -271,6 +291,7 b' def cmdutiladd(orig, ui, repo, matcher, '
     bad.extend(f for f in lbad)
     return bad
 
[e-mail](cmdutil, 'remove')
 def cmdutilremove(orig, ui, repo, matcher, prefix, after, force, subrepos,
                   dryrun):
     normalmatcher = composenormalfilematcher(matcher, repo[None].manifest())
@@ -279,6 +300,7 b' def cmdutilremove(orig, ui, repo, matche'
     return removelargefiles(ui, repo, False, matcher, dryrun, after=after,
                             force=force) or result
 
[e-mail](subrepo.hgsubrepo, 'status')
 def overridestatusfn(orig, repo, rev2, **opts):
     try:
         repo._repo.lfstatus = True
@@ -286,6 +308,7 b' def overridestatusfn(orig, repo, rev2, *'
     finally:
         repo._repo.lfstatus = False
 
[e-mail]('status')
 def overridestatus(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
@@ -293,6 +316,7 b' def overridestatus(orig, ui, repo, *pats'
     finally:
         repo.lfstatus = False
 
[e-mail](subrepo.hgsubrepo, 'dirty')
 def overridedirty(orig, repo, ignoreupdate=False, missing=False):
     try:
         repo._repo.lfstatus = True
@@ -300,6 +324,7 b' def overridedirty(orig, repo, ignoreupda'
     finally:
         repo._repo.lfstatus = False
 
[e-mail]('log')
 def overridelog(orig, ui, repo, *pats, **opts):
     def overridematchandpats(ctx, pats=(), opts=None, globbed=False,
             default='relpath', badfn=None):
@@ -406,6 +431,13 b' def overridelog(orig, ui, repo, *pats, *'
         restorematchandpatsfn()
         setattr(logcmdutil, '_makenofollowfilematcher', oldmakefilematcher)
 
[e-mail]('verify',
+    opts=[('', 'large', None,
+                _('verify that all largefiles in current revision exists')),
+          ('', 'lfa', None,
+                _('verify largefiles in all revisions, not just current')),
+          ('', 'lfc', None,
+                _('verify local largefile contents, not just existence'))])
 def overrideverify(orig, ui, repo, *pats, **opts):
     large = opts.pop(r'large', False)
     all = opts.pop(r'lfa', False)
@@ -416,6 +448,8 b' def overrideverify(orig, ui, repo, *pats'
         result = result or lfcommands.verifylfiles(ui, repo, all, contents)
     return result
 
[e-mail]('debugstate',
+    opts=[('', 'large', None, _('display largefiles dirstate'))])
 def overridedebugstate(orig, ui, repo, *pats, **opts):
     large = opts.pop(r'large', False)
     if large:
@@ -435,6 +469,7 b' def overridedebugstate(orig, ui, repo, *'
 # The overridden function filters the unknown files by removing any
 # largefiles. This makes the merge proceed and we can then handle this
 # case further in the overridden calculateupdates function below.
[e-mail](merge, '_checkunknownfile')
 def overridecheckunknownfile(origfn, repo, wctx, mctx, f, f2=None):
     if lfutil.standin(repo.dirstate.normalize(f)) in wctx:
         return False
@@ -466,6 +501,7 b' def overridecheckunknownfile(origfn, rep'
 # Finally, the merge.applyupdates function will then take care of
 # writing the files into the working copy and lfcommands.updatelfiles
 # will update the largefiles.
[e-mail](merge, 'calculateupdates')
 def overridecalculateupdates(origfn, repo, p1, p2, pas, branchmerge, force,
                              acceptremote, *args, **kwargs):
     overwrite = force and not branchmerge
@@ -534,6 +570,7 b' def overridecalculateupdates(origfn, rep'
 
     return actions, diverge, renamedelete
 
[e-mail](merge, 'recordupdates')
 def mergerecordupdates(orig, repo, actions, branchmerge):
     if 'lfmr' in actions:
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
@@ -549,6 +586,7 b' def mergerecordupdates(orig, repo, actio'
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits without prompting the user.
[e-mail](filemerge, '_filemerge')
 def overridefilemerge(origfn, premerge, repo, wctx, mynode, orig, fcd, fco, fca,
                       labels=None):
     if not lfutil.isstandin(orig) or fcd.isabsent() or fco.isabsent():
@@ -570,6 +608,7 b' def overridefilemerge(origfn, premerge, '
         repo.wwrite(fcd.path(), fco.data(), fco.flags())
     return True, 0, False
 
[e-mail](copiesmod, 'pathcopies')
 def copiespathcopies(orig, ctx1, ctx2, match=None):
     copies = orig(ctx1, ctx2, match=match)
     updated = {}
@@ -584,6 +623,7 b' def copiespathcopies(orig, ctx1, ctx2, m'
 # checks if the destination largefile already exists. It also keeps a
 # list of copied files so that the largefiles can be copied and the
 # dirstate updated.
[e-mail](cmdutil, 'copy')
 def overridecopy(orig, ui, repo, pats, opts, rename=False):
     # doesn't remove largefile on rename
     if len(pats) < 2:
@@ -729,6 +769,7 b' def overridecopy(orig, ui, repo, pats, o'
 # commits. Update the standins then run the original revert, changing
 # the matcher to hit standins instead of largefiles. Based on the
 # resulting standins update the largefiles.
[e-mail](cmdutil, 'revert')
 def overriderevert(orig, ui, repo, ctx, parents, *pats, **opts):
     # Because we put the standins in a bad state (by updating them)
     # and then return them to a correct state we need to lock to
@@ -799,6 +840,11 b' def overriderevert(orig, ui, repo, ctx, '
 
 # after pulling changesets, we need to take some extra care to get
 # largefiles updated remotely
[e-mail]('pull',
+    opts=[('', 'all-largefiles', None,
+                _('download all pulled versions of largefiles (DEPRECATED)')),
+          ('', 'lfrev', [],
+                _('download largefiles for these revisions'), _('REV'))])
 def overridepull(orig, ui, repo, source=None, **opts):
     revsprepull = len(repo)
     if not source:
@@ -822,6 +868,9 b' def overridepull(orig, ui, repo, source='
         ui.status(_("%d largefiles cached\n") % numcached)
     return result
 
[e-mail]('push',
+    opts=[('', 'lfrev', [],
+               _('upload largefiles for these revisions'), _('REV'))])
 def overridepush(orig, ui, repo, *args, **kwargs):
     """Override push command and store --lfrev parameters in opargs"""
     lfrevs = kwargs.pop(r'lfrev', None)
@@ -830,6 +879,7 b' def overridepush(orig, ui, repo, *args, '
         opargs['lfrevs'] = scmutil.revrange(repo, lfrevs)
     return orig(ui, repo, *args, **kwargs)
 
[e-mail](exchange, 'pushoperation')
 def exchangepushoperation(orig, *args, **kwargs):
     """Override pushoperation constructor and store lfrevs parameter"""
     lfrevs = kwargs.pop(r'lfrevs', None)
@@ -837,9 +887,7 b' def exchangepushoperation(orig, *args, *'
     pushop.lfrevs = lfrevs
     return pushop
 
-revsetpredicate = registrar.revsetpredicate()
-
-@revsetpredicate('pulled()')
[e-mail]('pulled()')
 def pulledrevsetsymbol(repo, subset, x):
     """Changesets that just has been pulled.
 
@@ -865,6 +913,9 b' def pulledrevsetsymbol(repo, subset, x):'
         raise error.Abort(_("pulled() only available in --lfrev"))
     return smartset.baseset([r for r in subset if r >= firstpulled])
 
[e-mail]('clone',
+    opts=[('', 'all-largefiles', None,
+               _('download all versions of all largefiles'))])
 def overrideclone(orig, ui, source, dest=None, **opts):
     d = dest
     if d is None:
@@ -876,6 +927,7 b' def overrideclone(orig, ui, source, dest'
 
     return orig(ui, source, dest, **opts)
 
[e-mail](hg, 'clone')
 def hgclone(orig, ui, opts, *args, **kwargs):
     result = orig(ui, opts, *args, **kwargs)
 
@@ -900,6 +952,7 b' def hgclone(orig, ui, opts, *args, **kwa'
 
     return result
 
[e-mail]('rebase', extension='rebase')
 def overriderebase(orig, ui, repo, **opts):
     if not util.safehasattr(repo, '_largefilesenabled'):
         return orig(ui, repo, **opts)
@@ -913,6 +966,7 b' def overriderebase(orig, ui, repo, **opt'
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
 
[e-mail]('archive')
 def overridearchivecmd(orig, ui, repo, dest, **opts):
     repo.unfiltered().lfstatus = True
 
@@ -921,6 +975,7 b' def overridearchivecmd(orig, ui, repo, d'
     finally:
         repo.unfiltered().lfstatus = False
 
[e-mail](webcommands, 'archive')
 def hgwebarchive(orig, web):
     web.repo.lfstatus = True
 
@@ -929,12 +984,13 b' def hgwebarchive(orig, web):'
     finally:
         web.repo.lfstatus = False
 
-def overridearchive(orig, repo, dest, node, kind, decode=True, matchfn=None,
[e-mail](archival, 'archive')
+def overridearchive(orig, repo, dest, node, kind, decode=True, match=None,
             prefix='', mtime=None, subrepos=None):
     # For some reason setting repo.lfstatus in hgwebarchive only changes the
     # unfiltered repo's attr, so check that as well.
     if not repo.lfstatus and not repo.unfiltered().lfstatus:
-        return orig(repo, dest, node, kind, decode, matchfn, prefix, mtime,
+        return orig(repo, dest, node, kind, decode, match, prefix, mtime,
                     subrepos)
 
     # No need to lock because we are only reading history and
@@ -955,7 +1011,7 b' def overridearchive(orig, repo, dest, no'
         prefix = archival.tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
-        if matchfn and not matchfn(name):
+        if match and not match(name):
             return
         data = getdata()
         if decode:
@@ -991,12 +1047,13 b' def overridearchive(orig, repo, dest, no'
     if subrepos:
         for subpath in sorted(ctx.substate):
             sub = ctx.workingsub(subpath)
-            submatch = matchmod.subdirmatcher(subpath, matchfn)
+            submatch = matchmod.subdirmatcher(subpath, match)
             sub._repo.lfstatus = True
             sub.archive(archiver, prefix, submatch)
 
     archiver.done()
 
[e-mail](subrepo.hgsubrepo, 'archive')
 def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
     lfenabled = util.safehasattr(repo._repo, '_largefilesenabled')
     if not lfenabled or not repo._repo.lfstatus:
@@ -1051,6 +1108,7 b' def hgsubrepoarchive(orig, repo, archive'
 # standin until a commit. cmdutil.bailifchanged() raises an exception
 # if the repo has uncommitted changes. Wrap it to also check if
 # largefiles were changed. This is used by bisect, backout and fetch.
[e-mail](cmdutil, 'bailifchanged')
 def overridebailifchanged(orig, repo, *args, **kwargs):
     orig(repo, *args, **kwargs)
     repo.lfstatus = True
@@ -1059,6 +1117,7 b' def overridebailifchanged(orig, repo, *a'
     if s.modified or s.added or s.removed or s.deleted:
         raise error.Abort(_('uncommitted changes'))
 
[e-mail](cmdutil, 'postcommitstatus')
 def postcommitstatus(orig, repo, *args, **kwargs):
     repo.lfstatus = True
     try:
@@ -1066,6 +1125,7 b' def postcommitstatus(orig, repo, *args, '
     finally:
         repo.lfstatus = False
 
[e-mail](cmdutil, 'forget')
 def cmdutilforget(orig, ui, repo, match, prefix, explicitonly, dryrun,
                   interactive):
     normalmatcher = composenormalfilematcher(match, repo[None].manifest())
@@ -1167,6 +1227,13 b' def outgoinghook(ui, repo, other, opts, '
                 showhashes(file)
             ui.status('\n')
 
[e-mail]('outgoing',
+    opts=[('', 'large', None, _('display outgoing largefiles'))])
+def _outgoingcmd(orig, *args, **kwargs):
+    # Nothing to do here other than add the extra help option- the hook above
+    # processes it.
+    return orig(*args, **kwargs)
+
 def summaryremotehook(ui, repo, opts, changes):
     largeopt = opts.get('large', False)
     if changes is None:
@@ -1196,6 +1263,8 b' def summaryremotehook(ui, repo, opts, ch'
             ui.status(_('largefiles: %d entities for %d files to upload\n')
                       % (len(lfhashes), len(toupload)))
 
[e-mail]('summary',
+    opts=[('', 'large', None, _('display outgoing largefiles'))])
 def overridesummary(orig, ui, repo, *pats, **opts):
     try:
         repo.lfstatus = True
@@ -1203,6 +1272,7 b' def overridesummary(orig, ui, repo, *pat'
     finally:
         repo.lfstatus = False
 
[e-mail](scmutil, 'addremove')
 def scmutiladdremove(orig, repo, matcher, prefix, opts=None):
     if opts is None:
         opts = {}
@@ -1242,6 +1312,7 b' def scmutiladdremove(orig, repo, matcher'
 
 # Calling purge with --all will cause the largefiles to be deleted.
 # Override repo.status to prevent this from happening.
[e-mail]('purge', extension='purge')
 def overridepurge(orig, ui, repo, *dirs, **opts):
     # XXX Monkey patching a repoview will not work. The assigned attribute will
     # be set on the unfiltered repo, but we will only lookup attributes in the
@@ -1267,6 +1338,7 b' def overridepurge(orig, ui, repo, *dirs,'
     orig(ui, repo, *dirs, **opts)
     repo.status = oldstatus
 
[e-mail]('rollback')
 def overriderollback(orig, ui, repo, **opts):
     with repo.wlock():
         before = repo.dirstate.parents()
@@ -1304,6 +1376,7 b' def overriderollback(orig, ui, repo, **o'
         lfdirstate.write()
     return result
 
[e-mail]('transplant', extension='transplant')
 def overridetransplant(orig, ui, repo, *revs, **opts):
     resuming = opts.get(r'continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
@@ -1315,6 +1388,7 b' def overridetransplant(orig, ui, repo, *'
         repo._lfcommithooks.pop()
     return result
 
[e-mail]('cat')
 def overridecat(orig, ui, repo, file1, *pats, **opts):
     opts = pycompat.byteskwargs(opts)
     ctx = scmutil.revsingle(repo, opts.get('rev'))
@@ -1375,6 +1449,7 b' def overridecat(orig, ui, repo, file1, *'
         err = 0
     return err
 
[e-mail](merge, 'update')
 def mergeupdate(orig, repo, node, branchmerge, force,
                 *args, **kwargs):
     matcher = kwargs.get(r'matcher', None)
@@ -1452,6 +1527,7 b' def mergeupdate(orig, repo, node, branch'
 
         return result
 
[e-mail](scmutil, 'marktouched')
 def scmutilmarktouched(orig, repo, files, *args, **kwargs):
     result = orig(repo, files, *args, **kwargs)
 
@@ -1466,6 +1542,8 b' def scmutilmarktouched(orig, repo, files'
 
     return result
 
[e-mail](upgrade, 'preservedrequirements')
[e-mail](upgrade, 'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if 'largefiles' in repo.requirements:
@@ -1473,6 +1551,8 b' def upgraderequirements(orig, repo):'
     return reqs
 
 _lfscheme = 'largefile://'
+
[e-mail](urlmod, 'open')
 def openlargefile(orig, ui, url_, data=None):
     if url_.startswith(_lfscheme):
         if data:

hgext/largefiles/proto.py

@@ -11,10 +11,12 b' from mercurial.i18n import _'
 
 from mercurial import (
     error,
+    exthelper,
     httppeer,
     util,
     wireprototypes,
     wireprotov1peer,
+    wireprotov1server,
 )
 
 from . import (
@@ -28,6 +30,8 b" LARGEFILES_REQUIRED_MSG = ('\\nThis repos"
                            '\n\nPlease enable it in your Mercurial config '
                            'file.\n')
 
+eh = exthelper.exthelper()
+
 # these will all be replaced by largefiles.uisetup
 ssholdcallstream = None
 httpoldcallstream = None
@@ -162,6 +166,7 b' def wirereposetup(ui, repo):'
     repo.__class__ = lfileswirerepository
 
 # advertise the largefiles=serve capability
[e-mail](wireprotov1server, '_capabilities')
 def _capabilities(orig, repo, proto):
     '''announce largefile server capability'''
     caps = orig(repo, proto)

hgext/lfs/__init__.py

@@ -129,30 +129,23 b' import sys'
 from mercurial.i18n import _
 
 from mercurial import (
-    bundle2,
-    changegroup,
-    cmdutil,
     config,
     context,
     error,
     exchange,
     extensions,
+    exthelper,
     filelog,
     filesetlang,
     localrepo,
     minifileset,
     node,
     pycompat,
-    registrar,
     repository,
     revlog,
     scmutil,
     templateutil,
-    upgrade,
     util,
-    vfs as vfsmod,
-    wireprotoserver,
-    wireprotov1server,
 )
 
 from . import (
@@ -167,45 +160,48 b' from . import ('
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
-configtable = {}
-configitem = registrar.configitem(configtable)
+eh = exthelper.exthelper()
+eh.merge(wrapper.eh)
+eh.merge(wireprotolfsserver.eh)
 
-configitem('experimental', 'lfs.serve',
+cmdtable = eh.cmdtable
+configtable = eh.configtable
+extsetup = eh.finalextsetup
+uisetup = eh.finaluisetup
+filesetpredicate = eh.filesetpredicate
+reposetup = eh.finalreposetup
+templatekeyword = eh.templatekeyword
+
+eh.configitem('experimental', 'lfs.serve',
     default=True,
 )
-configitem('experimental', 'lfs.user-agent',
+eh.configitem('experimental', 'lfs.user-agent',
     default=None,
 )
-configitem('experimental', 'lfs.disableusercache',
+eh.configitem('experimental', 'lfs.disableusercache',
     default=False,
 )
-configitem('experimental', 'lfs.worker-enable',
+eh.configitem('experimental', 'lfs.worker-enable',
     default=False,
 )
 
-configitem('lfs', 'url',
+eh.configitem('lfs', 'url',
     default=None,
 )
-configitem('lfs', 'usercache',
+eh.configitem('lfs', 'usercache',
     default=None,
 )
 # Deprecated
-configitem('lfs', 'threshold',
+eh.configitem('lfs', 'threshold',
     default=None,
 )
-configitem('lfs', 'track',
+eh.configitem('lfs', 'track',
     default='none()',
 )
-configitem('lfs', 'retry',
+eh.configitem('lfs', 'retry',
     default=5,
 )
 
-cmdtable = {}
-command = registrar.command(cmdtable)
-
-templatekeyword = registrar.templatekeyword()
-filesetpredicate = registrar.filesetpredicate()
-
 lfsprocessor = (
     wrapper.readfromstore,
     wrapper.writetostore,
@@ -216,10 +212,12 b' def featuresetup(ui, supported):'
     # don't die on seeing a repo with the lfs requirement
     supported |= {'lfs'}
 
-def uisetup(ui):
[e-mail]
+def _uisetup(ui):
     localrepo.featuresetupfuncs.add(featuresetup)
 
-def reposetup(ui, repo):
[e-mail]
+def _reposetup(ui, repo):
     # Nothing to do with a remote repo
     if not repo.local():
         return
@@ -246,7 +244,7 b' def reposetup(ui, repo):'
                 s = repo.set('%n:%n', _bin(kwargs[r'node']), _bin(last))
             else:
                 s = repo.set('%n', _bin(kwargs[r'node']))
-            match = repo.narrowmatch()
+            match = repo._storenarrowmatch
             for ctx in s:
                 # TODO: is there a way to just walk the files in the commit?
                 if any(ctx[f].islfs() for f in ctx.files()
@@ -305,6 +303,7 b' def _trackedmatcher(repo):'
 
     return _match
 
+# Called by remotefilelog
 def wrapfilelog(filelog):
     wrapfunction = extensions.wrapfunction
 
@@ -312,6 +311,7 b' def wrapfilelog(filelog):'
     wrapfunction(filelog, 'renamed', wrapper.filelogrenamed)
     wrapfunction(filelog, 'size', wrapper.filelogsize)
 
[e-mail](localrepo, 'resolverevlogstorevfsoptions')
 def _resolverevlogstorevfsoptions(orig, ui, requirements, features):
     opts = orig(ui, requirements, features)
     for name, module in extensions.extensions(ui):
@@ -326,38 +326,10 b' def _resolverevlogstorevfsoptions(orig, '
 
     return opts
 
-def extsetup(ui):
[e-mail]
+def _extsetup(ui):
     wrapfilelog(filelog.filelog)
 
-    wrapfunction = extensions.wrapfunction
-
-    wrapfunction(localrepo, 'makefilestorage', wrapper.localrepomakefilestorage)
-    wrapfunction(localrepo, 'resolverevlogstorevfsoptions',
-                 _resolverevlogstorevfsoptions)
-
-    wrapfunction(cmdutil, '_updatecatformatter', wrapper._updatecatformatter)
-    wrapfunction(scmutil, 'wrapconvertsink', wrapper.convertsink)
-
-    wrapfunction(upgrade, '_finishdatamigration',
-                 wrapper.upgradefinishdatamigration)
-
-    wrapfunction(upgrade, 'preservedrequirements',
-                 wrapper.upgraderequirements)
-
-    wrapfunction(upgrade, 'supporteddestrequirements',
-                 wrapper.upgraderequirements)
-
-    wrapfunction(changegroup,
-                 'allsupportedversions',
-                 wrapper.allsupportedversions)
-
-    wrapfunction(exchange, 'push', wrapper.push)
-    wrapfunction(wireprotov1server, '_capabilities', wrapper._capabilities)
-    wrapfunction(wireprotoserver, 'handlewsgirequest',
-                 wireprotolfsserver.handlewsgirequest)
-
-    wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp)
-    wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary)
     context.basefilectx.islfs = wrapper.filectxislfs
 
     scmutil.fileprefetchhooks.add('lfs', wrapper._prefetchfiles)
@@ -367,14 +339,7 b' def extsetup(ui):'
     # "packed1". Using "packed1" with lfs will likely cause trouble.
     exchange._bundlespeccontentopts["v2"]["cg.version"] = "03"
 
-    # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
-    # options and blob stores are passed from othervfs to the new readonlyvfs.
-    wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit)
-
-    # when writing a bundle via "hg bundle" command, upload related LFS blobs
-    wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle)
-
-@filesetpredicate('lfs()')
[e-mail]('lfs()')
 def lfsfileset(mctx, x):
     """File that uses LFS storage."""
     # i18n: "lfs" is a keyword
@@ -384,7 +349,7 b' def lfsfileset(mctx, x):'
         return wrapper.pointerfromctx(ctx, f, removed=True) is not None
     return mctx.predicate(lfsfilep, predrepr='<lfs>')
 
-@templatekeyword('lfs_files', requires={'ctx'})
[e-mail]('lfs_files', requires={'ctx'})
 def lfsfiles(context, mapping):
     """List of strings. All files modified, added, or removed by this
     changeset."""
@@ -409,8 +374,8 b' def lfsfiles(context, mapping):'
     f = templateutil._showcompatlist(context, mapping, 'lfs_file', files)
     return templateutil.hybrid(f, files, makemap, pycompat.identity)
 
-@command('debuglfsupload',
-         [('r', 'rev', [], _('upload large files introduced by REV'))])
[e-mail]('debuglfsupload',
+            [('r', 'rev', [], _('upload large files introduced by REV'))])
 def debuglfsupload(ui, repo, **opts):
     """upload lfs blobs added by the working copy parent or given revisions"""
     revs = opts.get(r'rev', [])

hgext/lfs/blobstore.py

@@ -7,6 +7,7 b''
 
 from __future__ import absolute_import
 
+import contextlib
 import errno
 import hashlib
 import json
@@ -17,7 +18,9 b' import socket'
 from mercurial.i18n import _
 
 from mercurial import (
+    encoding,
     error,
+    node,
     pathutil,
     pycompat,
     url as urlmod,
@@ -26,6 +29,10 b' from mercurial import ('
     worker,
 )
 
+from mercurial.utils import (
+    stringutil,
+)
+
 from ..largefiles import lfutil
 
 # 64 bytes for SHA256
@@ -150,7 +157,7 b' class local(object):'
                 fp.write(chunk)
                 sha256.update(chunk)
 
-            realoid = sha256.hexdigest()
+            realoid = node.hex(sha256.digest())
             if realoid != oid:
                 raise LfsCorruptionError(_('corrupt remote lfs object: %s')
                                          % oid)
@@ -200,7 +207,7 b' class local(object):'
             # Don't abort if corruption is detected, because `hg verify` will
             # give more useful info about the corruption- simply don't add the
             # hardlink.
-            if verify or hashlib.sha256(blob).hexdigest() == oid:
+            if verify or node.hex(hashlib.sha256(blob).digest()) == oid:
                 self.ui.note(_('lfs: found %s in the usercache\n') % oid)
                 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
         else:
@@ -224,13 +231,37 b' class local(object):'
             for chunk in util.filechunkiter(fp, size=1048576):
                 sha256.update(chunk)
 
-        return oid == sha256.hexdigest()
+        return oid == node.hex(sha256.digest())
 
     def has(self, oid):
         """Returns True if the local blobstore contains the requested blob,
         False otherwise."""
         return self.cachevfs.exists(oid) or self.vfs.exists(oid)
 
+def _urlerrorreason(urlerror):
+    '''Create a friendly message for the given URLError to be used in an
+    LfsRemoteError message.
+    '''
+    inst = urlerror
+
+    if isinstance(urlerror.reason, Exception):
+        inst = urlerror.reason
+
+    if util.safehasattr(inst, 'reason'):
+        try: # usually it is in the form (errno, strerror)
+            reason = inst.reason.args[1]
+        except (AttributeError, IndexError):
+            # it might be anything, for example a string
+            reason = inst.reason
+        if isinstance(reason, pycompat.unicode):
+            # SSLError of Python 2.7.9 contains a unicode
+            reason = encoding.unitolocal(reason)
+        return reason
+    elif getattr(inst, "strerror", None):
+        return encoding.strtolocal(inst.strerror)
+    else:
+        return stringutil.forcebytestr(urlerror)
+
 class _gitlfsremote(object):
 
     def __init__(self, repo, url):
@@ -263,16 +294,27 b' class _gitlfsremote(object):'
             'objects': objects,
             'operation': action,
         })
-        batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl,
-                                       data=requestdata)
+        url = '%s/objects/batch' % self.baseurl
+        batchreq = util.urlreq.request(url, data=requestdata)
         batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
         batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
         try:
-            rsp = self.urlopener.open(batchreq)
-            rawjson = rsp.read()
+            with contextlib.closing(self.urlopener.open(batchreq)) as rsp:
+                rawjson = rsp.read()
         except util.urlerr.httperror as ex:
-            raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)')
-                                 % (ex, action))
+            hints = {
+                400: _('check that lfs serving is enabled on %s and "%s" is '
+                       'supported') % (self.baseurl, action),
+                404: _('the "lfs.url" config may be used to override %s')
+                       % self.baseurl,
+            }
+            hint = hints.get(ex.code, _('api=%s, action=%s') % (url, action))
+            raise LfsRemoteError(_('LFS HTTP error: %s') % ex, hint=hint)
+        except util.urlerr.urlerror as ex:
+            hint = (_('the "lfs.url" config may be used to override %s')
+                    % self.baseurl)
+            raise LfsRemoteError(_('LFS error: %s') % _urlerrorreason(ex),
+                                 hint=hint)
         try:
             response = json.loads(rawjson)
         except ValueError:
@@ -379,30 +421,37 b' class _gitlfsremote(object):'
 
         response = b''
         try:
-            req = self.urlopener.open(request)
-
-            if self.ui.debugflag:
-                self.ui.debug('Status: %d\n' % req.status)
-                # lfs-test-server and hg serve return headers in different order
-                self.ui.debug('%s\n'
-                              % '\n'.join(sorted(str(req.info()).splitlines())))
+            with contextlib.closing(self.urlopener.open(request)) as req:
+                ui = self.ui  # Shorten debug lines
+                if self.ui.debugflag:
+                    ui.debug('Status: %d\n' % req.status)
+                    # lfs-test-server and hg serve return headers in different
+                    # order
+                    ui.debug('%s\n'
+                             % '\n'.join(sorted(str(req.info()).splitlines())))
 
-            if action == 'download':
-                # If downloading blobs, store downloaded data to local blobstore
-                localstore.download(oid, req)
-            else:
-                while True:
-                    data = req.read(1048576)
-                    if not data:
-                        break
-                    response += data
-                if response:
-                    self.ui.debug('lfs %s response: %s' % (action, response))
+                if action == 'download':
+                    # If downloading blobs, store downloaded data to local
+                    # blobstore
+                    localstore.download(oid, req)
+                else:
+                    while True:
+                        data = req.read(1048576)
+                        if not data:
+                            break
+                        response += data
+                    if response:
+                        ui.debug('lfs %s response: %s' % (action, response))
         except util.urlerr.httperror as ex:
             if self.ui.debugflag:
                 self.ui.debug('%s: %s\n' % (oid, ex.read()))
-            raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
+            raise LfsRemoteError(_('LFS HTTP error: %s (oid=%s, action=%s)')
                                  % (ex, oid, action))
+        except util.urlerr.urlerror as ex:
+            hint = (_('attempted connection to %s')
+                    % util.urllibcompat.getfullurl(request))
+            raise LfsRemoteError(_('LFS error: %s') % _urlerrorreason(ex),
+                                 hint=hint)
 
     def _batch(self, pointers, localstore, action):
         if action not in ['upload', 'download']:
@@ -539,7 +588,7 b' def _deduplicate(pointers):'
     return reduced.values()
 
 def _verify(oid, content):
-    realoid = hashlib.sha256(content).hexdigest()
+    realoid = node.hex(hashlib.sha256(content).digest())
     if realoid != oid:
         raise LfsCorruptionError(_('detected corrupt lfs object: %s') % oid,
                                  hint=_('run hg verify'))

hgext/lfs/wireprotolfsserver.py

@@ -17,8 +17,10 b' from mercurial.hgweb import ('
 )
 
 from mercurial import (
+    exthelper,
     pycompat,
     util,
+    wireprotoserver,
 )
 
 from . import blobstore
@@ -31,6 +33,9 b' HTTP_METHOD_NOT_ALLOWED = hgwebcommon.HT'
 HTTP_NOT_ACCEPTABLE = hgwebcommon.HTTP_NOT_ACCEPTABLE
 HTTP_UNSUPPORTED_MEDIA_TYPE = hgwebcommon.HTTP_UNSUPPORTED_MEDIA_TYPE
 
+eh = exthelper.exthelper()
+
[e-mail](wireprotoserver, 'handlewsgirequest')
 def handlewsgirequest(orig, rctx, req, res, checkperm):
     """Wrap wireprotoserver.handlewsgirequest() to possibly process an LFS
     request if it is left unprocessed by the wrapped method.

hgext/lfs/wrapper.py

@@ -13,10 +13,21 b' from mercurial.i18n import _'
 from mercurial.node import bin, hex, nullid, short
 
 from mercurial import (
+    bundle2,
+    changegroup,
+    cmdutil,
+    context,
     error,
+    exchange,
+    exthelper,
+    localrepo,
     repository,
     revlog,
+    scmutil,
+    upgrade,
     util,
+    vfs as vfsmod,
+    wireprotov1server,
 )
 
 from mercurial.utils import (
@@ -31,17 +42,22 b' from . import ('
     pointer,
 )
 
+eh = exthelper.exthelper()
+
[e-mail](localrepo, 'makefilestorage')
 def localrepomakefilestorage(orig, requirements, features, **kwargs):
     if b'lfs' in requirements:
         features.add(repository.REPO_FEATURE_LFS)
 
     return orig(requirements=requirements, features=features, **kwargs)
 
[e-mail](changegroup, 'allsupportedversions')
 def allsupportedversions(orig, ui):
     versions = orig(ui)
     versions.add('03')
     return versions
 
[e-mail](wireprotov1server, '_capabilities')
 def _capabilities(orig, repo, proto):
     '''Wrap server command to announce lfs server capability'''
     caps = orig(repo, proto)
@@ -130,6 +146,7 b' def _islfs(rlog, node=None, rev=None):'
     flags = rlog._revlog.flags(rev)
     return bool(flags & revlog.REVIDX_EXTSTORED)
 
+# Wrapping may also be applied by remotefilelog
 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
                        cachedelta=None, node=None,
                        flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
@@ -149,6 +166,7 b' def filelogaddrevision(orig, self, text,'
     return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
                 node=node, flags=flags, **kwds)
 
+# Wrapping may also be applied by remotefilelog
 def filelogrenamed(orig, self, node):
     if _islfs(self, node):
         rawtext = self._revlog.revision(node, raw=True)
@@ -161,6 +179,7 b' def filelogrenamed(orig, self, node):'
             return False
     return orig(self, node)
 
+# Wrapping may also be applied by remotefilelog
 def filelogsize(orig, self, rev):
     if _islfs(self, rev=rev):
         # fast path: use lfs metadata to answer size
@@ -169,6 +188,7 b' def filelogsize(orig, self, rev):'
         return int(metadata['size'])
     return orig(self, rev)
 
[e-mail](context.basefilectx, 'cmp')
 def filectxcmp(orig, self, fctx):
     """returns True if text is different than fctx"""
     # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
@@ -179,6 +199,7 b' def filectxcmp(orig, self, fctx):'
         return p1.oid() != p2.oid()
     return orig(self, fctx)
 
[e-mail](context.basefilectx, 'isbinary')
 def filectxisbinary(orig, self):
     if self.islfs():
         # fast path: use lfs metadata to answer isbinary
@@ -190,10 +211,12 b' def filectxisbinary(orig, self):'
 def filectxislfs(self):
     return _islfs(self.filelog(), self.filenode())
 
[e-mail](cmdutil, '_updatecatformatter')
 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
     orig(fm, ctx, matcher, path, decode)
     fm.data(rawdata=ctx[path].rawdata())
 
[e-mail](scmutil, 'wrapconvertsink')
 def convertsink(orig, sink):
     sink = orig(sink)
     if sink.repotype == 'hg':
@@ -219,6 +242,9 b' def convertsink(orig, sink):'
 
     return sink
 
+# bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
+# options and blob stores are passed from othervfs to the new readonlyvfs.
[e-mail](vfsmod.readonlyvfs, '__init__')
 def vfsinit(orig, self, othervfs):
     orig(self, othervfs)
     # copy lfs related options
@@ -290,6 +316,7 b' def prepush(pushop):'
     """
     return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
 
[e-mail](exchange, 'push')
 def push(orig, repo, remote, *args, **kwargs):
     """bail on push if the extension isn't enabled on remote when needed, and
     update the remote store based on the destination path."""
@@ -316,6 +343,8 b' def push(orig, repo, remote, *args, **kw'
     else:
         return orig(repo, remote, *args, **kwargs)
 
+# when writing a bundle via "hg bundle" command, upload related LFS blobs
[e-mail](bundle2, 'writenewbundle')
 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
                    *args, **kwargs):
     """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
@@ -393,6 +422,7 b' def uploadblobs(repo, pointers):'
     remoteblob = repo.svfs.lfsremoteblobstore
     remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
 
[e-mail](upgrade, '_finishdatamigration')
 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
     orig(ui, srcrepo, dstrepo, requirements)
 
@@ -407,6 +437,8 b' def upgradefinishdatamigration(orig, ui,'
                 ui.write(_('copying lfs blob %s\n') % oid)
                 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
 
[e-mail](upgrade, 'preservedrequirements')
[e-mail](upgrade, 'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if 'lfs' in repo.requirements:

hgext/logtoprocess.py

@@ -9,21 +9,21 b''
 This extension lets you specify a shell command per ui.log() event,
 sending all remaining arguments to as environment variables to that command.
 
-Each positional argument to the method results in a `MSG[N]` key in the
-environment, starting at 1 (so `MSG1`, `MSG2`, etc.). Each keyword argument
-is set as a `OPT_UPPERCASE_KEY` variable (so the key is uppercased, and
-prefixed with `OPT_`). The original event name is passed in the `EVENT`
-environment variable, and the process ID of mercurial is given in `HGPID`.
+Positional arguments construct a log message, which is passed in the `MSG1`
+environment variables. Each keyword argument is set as a `OPT_UPPERCASE_KEY`
+variable (so the key is uppercased, and prefixed with `OPT_`). The original
+event name is passed in the `EVENT` environment variable, and the process ID
+of mercurial is given in `HGPID`.
 
-So given a call `ui.log('foo', 'bar', 'baz', spam='eggs'), a script configured
-for the `foo` event can expect an environment with `MSG1=bar`, `MSG2=baz`, and
-`OPT_SPAM=eggs`.
+So given a call `ui.log('foo', 'bar %s\n', 'baz', spam='eggs'), a script
+configured for the `foo` event can expect an environment with `MSG1=bar baz`,
+and `OPT_SPAM=eggs`.
 
 Scripts are configured in the `[logtoprocess]` section, each key an event name.
 For example::
 
   [logtoprocess]
-  commandexception = echo "$MSG2$MSG3" > /var/log/mercurial_exceptions.log
+  commandexception = echo "$MSG1" > /var/log/mercurial_exceptions.log
 
 would log the warning message and traceback of any failed command dispatch.
 
@@ -34,14 +34,7 b' not ensure that they exit cleanly.'
 
 from __future__ import absolute_import
 
-import itertools
 import os
-import subprocess
-import sys
-
-from mercurial import (
-    pycompat,
-)
 
 from mercurial.utils import (
     procutil,
@@ -53,88 +46,30 b' from mercurial.utils import ('
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
-def uisetup(ui):
-    if pycompat.iswindows:
-        # no fork on Windows, but we can create a detached process
-        # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
-        # No stdlib constant exists for this value
-        DETACHED_PROCESS = 0x00000008
-        _creationflags = DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+class processlogger(object):
+    """Map log events to external commands
+
+    Arguments are passed on as environment variables.
+    """
+
+    def __init__(self, ui):
+        self._scripts = dict(ui.configitems(b'logtoprocess'))
+
+    def tracked(self, event):
+        return bool(self._scripts.get(event))
 
-        def runshellcommand(script, env):
-            # we can't use close_fds *and* redirect stdin. I'm not sure that we
-            # need to because the detached process has no console connection.
-            subprocess.Popen(
-                procutil.tonativestr(script),
-                shell=True, env=procutil.tonativeenv(env), close_fds=True,
-                creationflags=_creationflags)
-    else:
-        def runshellcommand(script, env):
-            # double-fork to completely detach from the parent process
-            # based on http://code.activestate.com/recipes/278731
-            pid = os.fork()
-            if pid:
-                # parent
-                return
-            # subprocess.Popen() forks again, all we need to add is
-            # flag the new process as a new session.
-            if sys.version_info < (3, 2):
-                newsession = {'preexec_fn': os.setsid}
-            else:
-                newsession = {'start_new_session': True}
-            try:
-                # connect std* to devnull to make sure the subprocess can't
-                # muck up these stream for mercurial.
-                # Connect all the streams to be more close to Windows behavior
-                # and pager will wait for scripts to end if we don't do that
-                nullrfd = open(os.devnull, 'r')
-                nullwfd = open(os.devnull, 'w')
-                subprocess.Popen(
-                    procutil.tonativestr(script),
-                    shell=True, stdin=nullrfd,
-                    stdout=nullwfd, stderr=nullwfd,
-                    env=procutil.tonativeenv(env),
-                    close_fds=True, **newsession)
-            finally:
-                # mission accomplished, this child needs to exit and not
-                # continue the hg process here.
-                os._exit(0)
+    def log(self, ui, event, msg, opts):
+        script = self._scripts[event]
+        env = {
+            b'EVENT': event,
+            b'HGPID': os.getpid(),
+            b'MSG1': msg,
+        }
+        # keyword arguments get prefixed with OPT_ and uppercased
+        env.update((b'OPT_%s' % key.upper(), value)
+                   for key, value in opts.items())
+        fullenv = procutil.shellenviron(env)
+        procutil.runbgcommand(script, fullenv, shell=True)
 
-    class logtoprocessui(ui.__class__):
-        def log(self, event, *msg, **opts):
-            """Map log events to external commands
-
-            Arguments are passed on as environment variables.
-
-            """
-            script = self.config('logtoprocess', event)
-            if script:
-                if msg:
-                    # try to format the log message given the remaining
-                    # arguments
-                    try:
-                        # Format the message as blackbox does
-                        formatted = msg[0] % msg[1:]
-                    except (TypeError, KeyError):
-                        # Failed to apply the arguments, ignore
-                        formatted = msg[0]
-                    messages = (formatted,) + msg[1:]
-                else:
-                    messages = msg
-                # positional arguments are listed as MSG[N] keys in the
-                # environment
-                msgpairs = (
-                    ('MSG{0:d}'.format(i), str(m))
-                    for i, m in enumerate(messages, 1))
-                # keyword arguments get prefixed with OPT_ and uppercased
-                optpairs = (
-                    ('OPT_{0}'.format(key.upper()), str(value))
-                    for key, value in opts.iteritems())
-                env = dict(itertools.chain(procutil.shellenviron().items(),
-                                           msgpairs, optpairs),
-                           EVENT=event, HGPID=str(os.getpid()))
-                runshellcommand(script, env)
-            return super(logtoprocessui, self).log(event, *msg, **opts)
-
-    # Replace the class for this instance and all clones created from it:
-    ui.__class__ = logtoprocessui
+def uipopulate(ui):
+    ui.setlogger(b'logtoprocess', processlogger(ui))

hgext/mq.py

@@ -139,6 +139,8 b' except KeyError:'
     class dummyui(object):
         def debug(self, msg):
             pass
+        def log(self, event, msgfmt, *msgargs, **opts):
+            pass
     stripext = extensions.load(dummyui(), 'strip', '')
 
 strip = stripext.strip

hgext/narrow/TODO.rst

@@ -1,6 +1,3 b''
-Integration with the share extension needs improvement. Right now
-we've seen some odd bugs.
-
 Address commentary in manifest.excludedmanifestrevlog.add -
 specifically we should improve the collaboration with core so that
 add() never gets called on an excluded directory and we can improve

hgext/narrow/narrowbundle2.py

@@ -20,7 +20,7 b' from mercurial import ('
     changegroup,
     error,
     exchange,
-    extensions,
+    localrepo,
     narrowspec,
     repair,
     repository,
@@ -31,10 +31,9 b' from mercurial.utils import ('
     stringutil,
 )
 
-NARROWCAP = 'narrow'
 _NARROWACL_SECTION = 'narrowhgacl'
-_CHANGESPECPART = NARROWCAP + ':changespec'
-_SPECPART = NARROWCAP + ':spec'
+_CHANGESPECPART = 'narrow:changespec'
+_SPECPART = 'narrow:spec'
 _SPECPART_INCLUDE = 'include'
 _SPECPART_EXCLUDE = 'exclude'
 _KILLNODESIGNAL = 'KILL'
@@ -44,12 +43,6 b" NARROWCAP = 'narrow'"
 _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
 _MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
 
-# When advertising capabilities, always include narrow clone support.
-def getrepocaps_narrow(orig, repo, **kwargs):
-    caps = orig(repo, **kwargs)
-    caps[NARROWCAP] = ['v0']
-    return caps
-
 # Serve a changegroup for a client with a narrow clone.
 def getbundlechangegrouppart_narrow(bundler, repo, source,
                                     bundlecaps=None, b2caps=None, heads=None,
@@ -158,6 +151,7 b' def _handlechangespec_2(op, inpart):'
         op.repo.requirements.add(repository.NARROW_REQUIREMENT)
         op.repo._writerequirements()
     op.repo.setnarrowpats(includepats, excludepats)
+    narrowspec.copytoworkingcopy(op.repo)
 
 @bundle2.parthandler(_CHANGESPECPART)
 def _handlechangespec(op, inpart):
@@ -187,18 +181,15 b' def _handlechangespec(op, inpart):'
 
     if clkills:
         # preserve bookmarks that repair.strip() would otherwise strip
-        bmstore = repo._bookmarks
+        op._bookmarksbackup = repo._bookmarks
         class dummybmstore(dict):
             def applychanges(self, repo, tr, changes):
                 pass
-            def recordchange(self, tr): # legacy version
-                pass
-        repo._bookmarks = dummybmstore()
+        localrepo.localrepository._bookmarks.set(repo, dummybmstore())
         chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
                                  topic='widen')
-        repo._bookmarks = bmstore
         if chgrpfile:
-            op._widen_uninterr = repo.ui.uninterruptable()
+            op._widen_uninterr = repo.ui.uninterruptible()
             op._widen_uninterr.__enter__()
             # presence of _widen_bundle attribute activates widen handler later
             op._widen_bundle = chgrpfile
@@ -252,16 +243,12 b' def handlechangegroup_widen(op, inpart):'
 
 def setup():
     """Enable narrow repo support in bundle2-related extension points."""
-    extensions.wrapfunction(bundle2, 'getrepocaps', getrepocaps_narrow)
-
     getbundleargs = wireprototypes.GETBUNDLE_ARGUMENTS
 
     getbundleargs['narrow'] = 'boolean'
     getbundleargs['depth'] = 'plain'
     getbundleargs['oldincludepats'] = 'csv'
     getbundleargs['oldexcludepats'] = 'csv'
-    getbundleargs['includepats'] = 'csv'
-    getbundleargs['excludepats'] = 'csv'
     getbundleargs['known'] = 'csv'
 
     # Extend changegroup serving to handle requests from narrow clients.
@@ -284,5 +271,10 b' def setup():'
         origcghandler(op, inpart)
         if util.safehasattr(op, '_widen_bundle'):
             handlechangegroup_widen(op, inpart)
+        if util.safehasattr(op, '_bookmarksbackup'):
+            localrepo.localrepository._bookmarks.set(op.repo,
+                                                     op._bookmarksbackup)
+            del op._bookmarksbackup
+
     wrappedcghandler.params = origcghandler.params
     bundle2.parthandlermapping['changegroup'] = wrappedcghandler

hgext/narrow/narrowcommands.py

@@ -20,7 +20,6 b' from mercurial import ('
     exchange,
     extensions,
     hg,
-    merge,
     narrowspec,
     node,
     pycompat,
@@ -141,8 +140,10 b' def pullbundle2extraprepare(orig, pullop'
     include, exclude = repo.narrowpats
     kwargs['oldincludepats'] = include
     kwargs['oldexcludepats'] = exclude
-    kwargs['includepats'] = include
-    kwargs['excludepats'] = exclude
+    if include:
+        kwargs['includepats'] = include
+    if exclude:
+        kwargs['excludepats'] = exclude
     # calculate known nodes only in ellipses cases because in non-ellipses cases
     # we have all the nodes
     if wireprototypes.ELLIPSESCAP in pullop.remote.capabilities():
@@ -158,16 +159,6 b' def pullbundle2extraprepare(orig, pullop'
 extensions.wrapfunction(exchange,'_pullbundle2extraprepare',
                         pullbundle2extraprepare)
 
-# This is an extension point for filesystems that need to do something other
-# than just blindly unlink the files. It's not clear what arguments would be
-# useful, so we're passing in a fair number of them, some of them redundant.
-def _narrowcleanupwdir(repo, oldincludes, oldexcludes, newincludes, newexcludes,
-                       oldmatch, newmatch):
-    for f in repo.dirstate:
-        if not newmatch(f):
-            repo.dirstate.drop(f)
-            repo.wvfs.unlinkpath(f)
-
 def _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
             newincludes, newexcludes, force):
     oldmatch = narrowspec.match(repo.root, oldincludes, oldexcludes)
@@ -205,7 +196,7 b' def _narrow(ui, repo, remote, commoninc,'
                               hint=_('use --force-delete-local-changes to '
                                      'ignore'))
 
-    with ui.uninterruptable():
+    with ui.uninterruptible():
         if revstostrip:
             tostrip = [unfi.changelog.node(r) for r in revstostrip]
             if repo['.'].node() in tostrip:
@@ -213,7 +204,9 b' def _narrow(ui, repo, remote, commoninc,'
                 urev = max(repo.revs('(::%n) - %ln + null',
                                      repo['.'].node(), visibletostrip))
                 hg.clean(repo, urev)
-            repair.strip(ui, unfi, tostrip, topic='narrow')
+            overrides = {('devel', 'strip-obsmarkers'): False}
+            with ui.configoverride(overrides, 'narrow'):
+                repair.strip(ui, unfi, tostrip, topic='narrow')
 
         todelete = []
         for f, f2, size in repo.store.datafiles():
@@ -237,22 +230,23 b' def _narrow(ui, repo, remote, commoninc,'
 
         repo.destroying()
 
-        with repo.transaction("narrowing"):
+        with repo.transaction('narrowing'):
+            # Update narrowspec before removing revlogs, so repo won't be
+            # corrupt in case of crash
+            repo.setnarrowpats(newincludes, newexcludes)
+
             for f in todelete:
                 ui.status(_('deleting %s\n') % f)
                 util.unlinkpath(repo.svfs.join(f))
                 repo.store.markremoved(f)
 
-            _narrowcleanupwdir(repo, oldincludes, oldexcludes, newincludes,
-                               newexcludes, oldmatch, newmatch)
-            repo.setnarrowpats(newincludes, newexcludes)
+            narrowspec.updateworkingcopy(repo, assumeclean=True)
+            narrowspec.copytoworkingcopy(repo)
 
         repo.destroyed()
 
 def _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
            newincludes, newexcludes):
-    newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
-
     # for now we assume that if a server has ellipses enabled, we will be
     # exchanging ellipses nodes. In future we should add ellipses as a client
     # side requirement (maybe) to distinguish a client is shallow or not and
@@ -277,7 +271,7 b' def _widen(ui, repo, remote, commoninc, '
     # silence the devel-warning of applying an empty changegroup
     overrides = {('devel', 'all-warnings'): False}
 
-    with ui.uninterruptable():
+    with ui.uninterruptible():
         common = commoninc[0]
         if ellipsesremote:
             ds = repo.dirstate
@@ -308,19 +302,10 b' def _widen(ui, repo, remote, commoninc, '
                 bundle2.processbundle(repo, bundle,
                         transactiongetter=tgetter)
 
-        repo.setnewnarrowpats()
-        actions = {k: [] for k in 'a am f g cd dc r dm dg m e k p pr'.split()}
-        addgaction = actions['g'].append
-
-        mf = repo['.'].manifest().matches(newmatch)
-        for f, fn in mf.iteritems():
-            if f not in repo.dirstate:
-                addgaction((f, (mf.flags(f), False),
-                            "add from widened narrow clone"))
-
-        merge.applyupdates(repo, actions, wctx=repo[None],
-                           mctx=repo['.'], overwrite=False)
-        merge.recordupdates(repo, actions, branchmerge=False)
+        with repo.transaction('widening'):
+            repo.setnewnarrowpats()
+            narrowspec.updateworkingcopy(repo)
+            narrowspec.copytoworkingcopy(repo)
 
 # TODO(rdamazio): Make new matcher format and update description
 @command('tracked',
@@ -332,6 +317,8 b' def _widen(ui, repo, remote, commoninc, '
      ('', 'clear', False, _('whether to replace the existing narrowspec')),
      ('', 'force-delete-local-changes', False,
        _('forces deletion of local changes when narrowing')),
+     ('', 'update-working-copy', False,
+      _('update working copy when the store has changed')),
     ] + commands.remoteopts,
     _('[OPTIONS]... [REMOTE]'),
     inferrepo=True)
@@ -361,15 +348,13 b' def trackedcmd(ui, repo, remotepath=None'
     """
     opts = pycompat.byteskwargs(opts)
     if repository.NARROW_REQUIREMENT not in repo.requirements:
-        ui.warn(_('The narrow command is only supported on respositories cloned'
-                  ' with --narrow.\n'))
-        return 1
+        raise error.Abort(_('the narrow command is only supported on '
+                            'respositories cloned with --narrow'))
 
     # Before supporting, decide whether it "hg tracked --clear" should mean
     # tracking no paths or all paths.
     if opts['clear']:
-        ui.warn(_('The --clear option is not yet supported.\n'))
-        return 1
+        raise error.Abort(_('the --clear option is not yet supported'))
 
     # import rules from a file
     newrules = opts.get('import_rules')
@@ -392,27 +377,48 b' def trackedcmd(ui, repo, remotepath=None'
     removedincludes = narrowspec.parsepatterns(opts['removeinclude'])
     addedexcludes = narrowspec.parsepatterns(opts['addexclude'])
     removedexcludes = narrowspec.parsepatterns(opts['removeexclude'])
+
+    update_working_copy = opts['update_working_copy']
+    only_show = not (addedincludes or removedincludes or addedexcludes or
+                     removedexcludes or newrules or update_working_copy)
+
+    oldincludes, oldexcludes = repo.narrowpats
+
+    # filter the user passed additions and deletions into actual additions and
+    # deletions of excludes and includes
+    addedincludes -= oldincludes
+    removedincludes &= oldincludes
+    addedexcludes -= oldexcludes
+    removedexcludes &= oldexcludes
+
     widening = addedincludes or removedexcludes
     narrowing = removedincludes or addedexcludes
-    only_show = not widening and not narrowing
 
     # Only print the current narrowspec.
     if only_show:
-        include, exclude = repo.narrowpats
-
         ui.pager('tracked')
         fm = ui.formatter('narrow', opts)
-        for i in sorted(include):
+        for i in sorted(oldincludes):
             fm.startitem()
             fm.write('status', '%s ', 'I', label='narrow.included')
             fm.write('pat', '%s\n', i, label='narrow.included')
-        for i in sorted(exclude):
+        for i in sorted(oldexcludes):
             fm.startitem()
             fm.write('status', '%s ', 'X', label='narrow.excluded')
             fm.write('pat', '%s\n', i, label='narrow.excluded')
         fm.end()
         return 0
 
+    if update_working_copy:
+        with repo.wlock(), repo.lock(), repo.transaction('narrow-wc'):
+            narrowspec.updateworkingcopy(repo)
+            narrowspec.copytoworkingcopy(repo)
+        return 0
+
+    if not widening and not narrowing:
+        ui.status(_("nothing to widen or narrow\n"))
+        return 0
+
     with repo.wlock(), repo.lock():
         cmdutil.bailifchanged(repo)
 
@@ -432,7 +438,6 b' def trackedcmd(ui, repo, remotepath=None'
 
         commoninc = discovery.findcommonincoming(repo, remote)
 
-        oldincludes, oldexcludes = repo.narrowpats
         if narrowing:
             newincludes = oldincludes - removedincludes
             newexcludes = oldexcludes | addedexcludes

hgext/phabricator.py

@@ -41,6 +41,7 b' Config::'
 
 from __future__ import absolute_import
 
+import contextlib
 import itertools
 import json
 import operator
@@ -58,6 +59,7 b' from mercurial import ('
     obsutil,
     parser,
     patch,
+    phases,
     registrar,
     scmutil,
     smartset,
@@ -121,7 +123,7 b' colortable = {'
      )),
 ]
 
-def vcrcommand(name, flags, spec):
+def vcrcommand(name, flags, spec, helpcategory=None):
     fullflags = flags + _VCR_FLAGS
     def decorate(fn):
         def inner(*args, **kwargs):
@@ -143,7 +145,7 b' def vcrcommand(name, flags, spec):'
             return fn(*args, **kwargs)
         inner.__name__ = fn.__name__
         inner.__doc__ = fn.__doc__
-        return command(name, fullflags, spec)(inner)
+        return command(name, fullflags, spec, helpcategory=helpcategory)(inner)
     return decorate
 
 def urlencodenested(params):
@@ -214,7 +216,8 b' def callconduit(repo, name, params):'
     else:
         urlopener = urlmod.opener(repo.ui, authinfo)
         request = util.urlreq.request(url, data=data)
-        body = urlopener.open(request).read()
+        with contextlib.closing(urlopener.open(request)) as rsp:
+            body = rsp.read()
     repo.ui.debug(b'Conduit Response: %s\n' % body)
     parsed = json.loads(body)
     if parsed.get(r'error_code'):
@@ -465,7 +468,8 b' def userphids(repo, names):'
           (b'', b'amend', True, _(b'update commit messages')),
           (b'', b'reviewer', [], _(b'specify reviewers')),
           (b'', b'confirm', None, _(b'ask for confirmation before sending'))],
-         _(b'REV [OPTIONS]'))
+         _(b'REV [OPTIONS]'),
+         helpcategory=command.CATEGORY_IMPORT_EXPORT)
 def phabsend(ui, repo, *revs, **opts):
     """upload changesets to Phabricator
 
@@ -581,6 +585,10 b' def phabsend(ui, repo, *revs, **opts):'
                 newdesc = encoding.unitolocal(newdesc)
                 # Make sure commit message contain "Differential Revision"
                 if old.description() != newdesc:
+                    if old.phase() == phases.public:
+                        ui.warn(_("warning: not updating public commit %s\n")
+                                % scmutil.formatchangeid(old))
+                        continue
                     parents = [
                         mapping.get(old.p1().node(), (old.p1(),))[0],
                         mapping.get(old.p2().node(), (old.p2(),))[0],
@@ -919,7 +927,8 b' def readpatch(repo, drevs, write):'
 
 @vcrcommand(b'phabread',
          [(b'', b'stack', False, _(b'read dependencies'))],
-         _(b'DREVSPEC [OPTIONS]'))
+         _(b'DREVSPEC [OPTIONS]'),
+         helpcategory=command.CATEGORY_IMPORT_EXPORT)
 def phabread(ui, repo, spec, **opts):
     """print patches from Phabricator suitable for importing
 
@@ -950,7 +959,8 b' def phabread(ui, repo, spec, **opts):'
           (b'', b'abandon', False, _(b'abandon revisions')),
           (b'', b'reclaim', False, _(b'reclaim revisions')),
           (b'm', b'comment', b'', _(b'comment on the last revision')),
-          ], _(b'DREVSPEC [OPTIONS]'))
+          ], _(b'DREVSPEC [OPTIONS]'),
+          helpcategory=command.CATEGORY_IMPORT_EXPORT)
 def phabupdate(ui, repo, spec, **opts):
     """update Differential Revision in batch
 
@@ -987,3 +997,17 b' def template_review(context, mapping):'
             b'url': m.group(b'url'),
             b'id': b"D{}".format(m.group(b'id')),
         })
+    else:
+        tags = ctx.repo().nodetags(ctx.node())
+        for t in tags:
+            if _differentialrevisiontagre.match(t):
+                url = ctx.repo().ui.config(b'phabricator', b'url')
+                if not url.endswith(b'/'):
+                    url += b'/'
+                url += t
+
+                return templateutil.hybriddict({
+                    b'url': url,
+                    b'id': t,
+                })
+    return None

hgext/rebase.py

@@ -177,7 +177,7 b' class rebaseruntime(object):'
         if e:
             self.extrafns = [e]
 
-        self.backupf = ui.configbool('ui', 'history-editing-backup')
+        self.backupf = ui.configbool('rewrite', 'backup-bundle')
         self.keepf = opts.get('keep', False)
         self.keepbranchesf = opts.get('keepbranches', False)
         self.obsoletenotrebased = {}
@@ -347,9 +347,7 b' class rebaseruntime(object):'
 
         if isabort:
             backup = backup and self.backupf
-            return abort(self.repo, self.originalwd, self.destmap, self.state,
-                         activebookmark=self.activebookmark, backup=backup,
-                         suppwarns=suppwarns)
+            return self._abort(backup=backup, suppwarns=suppwarns)
 
     def _preparenewrebase(self, destmap):
         if not destmap:
@@ -404,7 +402,9 b' class rebaseruntime(object):'
         else:
             self.wctx = self.repo[None]
             self.repo.ui.debug("rebasing on disk\n")
-        self.repo.ui.log("rebase", "", rebase_imm_used=self.inmemory)
+        self.repo.ui.log("rebase",
+                         "using in-memory rebase: %r\n", self.inmemory,
+                         rebase_imm_used=self.inmemory)
 
     def _performrebase(self, tr):
         self._assignworkingcopy()
@@ -573,8 +573,8 b' class rebaseruntime(object):'
                 ui.debug('rebased as %s\n' % short(newnode))
             else:
                 if not self.collapsef:
-                    ui.warn(_('note: rebase of %d:%s created no changes '
-                              'to commit\n') % (rev, ctx))
+                    ui.warn(_('note: not rebasing %s, its destination already '
+                              'has all its changes\n') % desc)
                     self.skipped.add(rev)
                 self.state[rev] = p1
                 ui.debug('next revision set to %d\n' % p1)
@@ -651,6 +651,63 b' class rebaseruntime(object):'
             repo['.'].node() == repo._bookmarks[self.activebookmark]):
                 bookmarks.activate(repo, self.activebookmark)
 
+    def _abort(self, backup=True, suppwarns=False):
+        '''Restore the repository to its original state.'''
+
+        repo = self.repo
+        try:
+            # If the first commits in the rebased set get skipped during the
+            # rebase, their values within the state mapping will be the dest
+            # rev id. The rebased list must must not contain the dest rev
+            # (issue4896)
+            rebased = [s for r, s in self.state.items()
+                       if s >= 0 and s != r and s != self.destmap[r]]
+            immutable = [d for d in rebased if not repo[d].mutable()]
+            cleanup = True
+            if immutable:
+                repo.ui.warn(_("warning: can't clean up public changesets %s\n")
+                             % ', '.join(bytes(repo[r]) for r in immutable),
+                             hint=_("see 'hg help phases' for details"))
+                cleanup = False
+
+            descendants = set()
+            if rebased:
+                descendants = set(repo.changelog.descendants(rebased))
+            if descendants - set(rebased):
+                repo.ui.warn(_("warning: new changesets detected on "
+                               "destination branch, can't strip\n"))
+                cleanup = False
+
+            if cleanup:
+                shouldupdate = False
+                if rebased:
+                    strippoints = [
+                        c.node() for c in repo.set('roots(%ld)', rebased)]
+
+                updateifonnodes = set(rebased)
+                updateifonnodes.update(self.destmap.values())
+                updateifonnodes.add(self.originalwd)
+                shouldupdate = repo['.'].rev() in updateifonnodes
+
+                # Update away from the rebase if necessary
+                if shouldupdate or needupdate(repo, self.state):
+                    mergemod.update(repo, self.originalwd, branchmerge=False,
+                                    force=True)
+
+                # Strip from the first rebased revision
+                if rebased:
+                    repair.strip(repo.ui, repo, strippoints, backup=backup)
+
+            if self.activebookmark and self.activebookmark in repo._bookmarks:
+                bookmarks.activate(repo, self.activebookmark)
+
+        finally:
+            clearstatus(repo)
+            clearcollapsemsg(repo)
+            if not suppwarns:
+                repo.ui.warn(_('rebase aborted\n'))
+        return 0
+
 @command('rebase',
     [('s', 'source', '',
      _('rebase the specified changeset and descendants'), _('REV')),
@@ -1080,7 +1137,8 b' def _definedestmap(ui, repo, inmemory, d'
             return None
 
     rebasingwcp = repo['.'].rev() in rebaseset
-    ui.log("rebase", "", rebase_rebasing_wcp=rebasingwcp)
+    ui.log("rebase", "rebasing working copy parent: %r\n", rebasingwcp,
+           rebase_rebasing_wcp=rebasingwcp)
     if inmemory and rebasingwcp:
         # Check these since we did not before.
         cmdutil.checkunfinished(repo)
@@ -1606,64 +1664,6 b' def needupdate(repo, state):'
 
     return False
 
-def abort(repo, originalwd, destmap, state, activebookmark=None, backup=True,
-          suppwarns=False):
-    '''Restore the repository to its original state.  Additional args:
-
-    activebookmark: the name of the bookmark that should be active after the
-        restore'''
-
-    try:
-        # If the first commits in the rebased set get skipped during the rebase,
-        # their values within the state mapping will be the dest rev id. The
-        # rebased list must must not contain the dest rev (issue4896)
-        rebased = [s for r, s in state.items()
-                   if s >= 0 and s != r and s != destmap[r]]
-        immutable = [d for d in rebased if not repo[d].mutable()]
-        cleanup = True
-        if immutable:
-            repo.ui.warn(_("warning: can't clean up public changesets %s\n")
-                        % ', '.join(bytes(repo[r]) for r in immutable),
-                        hint=_("see 'hg help phases' for details"))
-            cleanup = False
-
-        descendants = set()
-        if rebased:
-            descendants = set(repo.changelog.descendants(rebased))
-        if descendants - set(rebased):
-            repo.ui.warn(_("warning: new changesets detected on destination "
-                           "branch, can't strip\n"))
-            cleanup = False
-
-        if cleanup:
-            shouldupdate = False
-            if rebased:
-                strippoints = [
-                        c.node() for c in repo.set('roots(%ld)', rebased)]
-
-            updateifonnodes = set(rebased)
-            updateifonnodes.update(destmap.values())
-            updateifonnodes.add(originalwd)
-            shouldupdate = repo['.'].rev() in updateifonnodes
-
-            # Update away from the rebase if necessary
-            if shouldupdate or needupdate(repo, state):
-                mergemod.update(repo, originalwd, branchmerge=False, force=True)
-
-            # Strip from the first rebased revision
-            if rebased:
-                repair.strip(repo.ui, repo, strippoints, backup=backup)
-
-        if activebookmark and activebookmark in repo._bookmarks:
-            bookmarks.activate(repo, activebookmark)
-
-    finally:
-        clearstatus(repo)
-        clearcollapsemsg(repo)
-        if not suppwarns:
-            repo.ui.warn(_('rebase aborted\n'))
-    return 0
-
 def sortsource(destmap):
     """yield source revisions in an order that we only rebase things once