merge default into stable for 3.9 code freeze

hgext/journal.py

@@ -0,0 +1,506 b''
+# journal.py
+#
+# Copyright 2014-2016 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+"""Track previous positions of bookmarks (EXPERIMENTAL)
+
+This extension adds a new command: `hg journal`, which shows you where
+bookmarks were previously located.
+
+"""
+
+from __future__ import absolute_import
+
+import collections
+import errno
+import os
+import weakref
+
+from mercurial.i18n import _
+
+from mercurial import (
+    bookmarks,
+    cmdutil,
+    commands,
+    dirstate,
+    dispatch,
+    error,
+    extensions,
+    hg,
+    localrepo,
+    lock,
+    node,
+    util,
+)
+
+from . import share
+
+cmdtable = {}
+command = cmdutil.command(cmdtable)
+
+# Note for extension authors: ONLY specify testedwith = 'internal' for
+# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
+# be specifying the version(s) of Mercurial they are tested with, or
+# leave the attribute unspecified.
+testedwith = 'internal'
+
+# storage format version; increment when the format changes
+storageversion = 0
+
+# namespaces
+bookmarktype = 'bookmark'
+wdirparenttype = 'wdirparent'
+# In a shared repository, what shared feature name is used
+# to indicate this namespace is shared with the source?
+sharednamespaces = {
+    bookmarktype: hg.sharedbookmarks,
+}
+
+# Journal recording, register hooks and storage object
+def extsetup(ui):
+    extensions.wrapfunction(dispatch, 'runcommand', runcommand)
+    extensions.wrapfunction(bookmarks.bmstore, '_write', recordbookmarks)
+    extensions.wrapfunction(
+        dirstate.dirstate, '_writedirstate', recorddirstateparents)
+    extensions.wrapfunction(
+        localrepo.localrepository.dirstate, 'func', wrapdirstate)
+    extensions.wrapfunction(hg, 'postshare', wrappostshare)
+    extensions.wrapfunction(hg, 'copystore', unsharejournal)
+
+def reposetup(ui, repo):
+    if repo.local():
+        repo.journal = journalstorage(repo)
+
+def runcommand(orig, lui, repo, cmd, fullargs, *args):
+    """Track the command line options for recording in the journal"""
+    journalstorage.recordcommand(*fullargs)
+    return orig(lui, repo, cmd, fullargs, *args)
+
+# hooks to record dirstate changes
+def wrapdirstate(orig, repo):
+    """Make journal storage available to the dirstate object"""
+    dirstate = orig(repo)
+    if util.safehasattr(repo, 'journal'):
+        dirstate.journalstorage = repo.journal
+    return dirstate
+
+def recorddirstateparents(orig, dirstate, dirstatefp):
+    """Records all dirstate parent changes in the journal."""
+    if util.safehasattr(dirstate, 'journalstorage'):
+        old = [node.nullid, node.nullid]
+        nodesize = len(node.nullid)
+        try:
+            # The only source for the old state is in the dirstate file still
+            # on disk; the in-memory dirstate object only contains the new
+            # state. dirstate._opendirstatefile() switches beteen .hg/dirstate
+            # and .hg/dirstate.pending depending on the transaction state.
+            with dirstate._opendirstatefile() as fp:
+                state = fp.read(2 * nodesize)
+            if len(state) == 2 * nodesize:
+                old = [state[:nodesize], state[nodesize:]]
+        except IOError:
+            pass
+
+        new = dirstate.parents()
+        if old != new:
+            # only record two hashes if there was a merge
+            oldhashes = old[:1] if old[1] == node.nullid else old
+            newhashes = new[:1] if new[1] == node.nullid else new
+            dirstate.journalstorage.record(
+                wdirparenttype, '.', oldhashes, newhashes)
+
+    return orig(dirstate, dirstatefp)
+
+# hooks to record bookmark changes (both local and remote)
+def recordbookmarks(orig, store, fp):
+    """Records all bookmark changes in the journal."""
+    repo = store._repo
+    if util.safehasattr(repo, 'journal'):
+        oldmarks = bookmarks.bmstore(repo)
+        for mark, value in store.iteritems():
+            oldvalue = oldmarks.get(mark, node.nullid)
+            if value != oldvalue:
+                repo.journal.record(bookmarktype, mark, oldvalue, value)
+    return orig(store, fp)
+
+# shared repository support
+def _readsharedfeatures(repo):
+    """A set of shared features for this repository"""
+    try:
+        return set(repo.vfs.read('shared').splitlines())
+    except IOError as inst:
+        if inst.errno != errno.ENOENT:
+            raise
+        return set()
+
+def _mergeentriesiter(*iterables, **kwargs):
+    """Given a set of sorted iterables, yield the next entry in merged order
+
+    Note that by default entries go from most recent to oldest.
+    """
+    order = kwargs.pop('order', max)
+    iterables = [iter(it) for it in iterables]
+    # this tracks still active iterables; iterables are deleted as they are
+    # exhausted, which is why this is a dictionary and why each entry also
+    # stores the key. Entries are mutable so we can store the next value each
+    # time.
+    iterable_map = {}
+    for key, it in enumerate(iterables):
+        try:
+            iterable_map[key] = [next(it), key, it]
+        except StopIteration:
+            # empty entry, can be ignored
+            pass
+
+    while iterable_map:
+        value, key, it = order(iterable_map.itervalues())
+        yield value
+        try:
+            iterable_map[key][0] = next(it)
+        except StopIteration:
+            # this iterable is empty, remove it from consideration
+            del iterable_map[key]
+
+def wrappostshare(orig, sourcerepo, destrepo, **kwargs):
+    """Mark this shared working copy as sharing journal information"""
+    orig(sourcerepo, destrepo, **kwargs)
+    with destrepo.vfs('shared', 'a') as fp:
+        fp.write('journal\n')
+
+def unsharejournal(orig, ui, repo, repopath):
+    """Copy shared journal entries into this repo when unsharing"""
+    if (repo.path == repopath and repo.shared() and
+            util.safehasattr(repo, 'journal')):
+        sharedrepo = share._getsrcrepo(repo)
+        sharedfeatures = _readsharedfeatures(repo)
+        if sharedrepo and sharedfeatures > set(['journal']):
+            # there is a shared repository and there are shared journal entries
+            # to copy. move shared date over from source to destination but
+            # move the local file first
+            if repo.vfs.exists('journal'):
+                journalpath = repo.join('journal')
+                util.rename(journalpath, journalpath + '.bak')
+            storage = repo.journal
+            local = storage._open(
+                repo.vfs, filename='journal.bak', _newestfirst=False)
+            shared = (
+                e for e in storage._open(sharedrepo.vfs, _newestfirst=False)
+                if sharednamespaces.get(e.namespace) in sharedfeatures)
+            for entry in _mergeentriesiter(local, shared, order=min):
+                storage._write(repo.vfs, entry)
+
+    return orig(ui, repo, repopath)
+
+class journalentry(collections.namedtuple(
+        'journalentry',
+        'timestamp user command namespace name oldhashes newhashes')):
+    """Individual journal entry
+
+    * timestamp: a mercurial (time, timezone) tuple
+    * user: the username that ran the command
+    * namespace: the entry namespace, an opaque string
+    * name: the name of the changed item, opaque string with meaning in the
+      namespace
+    * command: the hg command that triggered this record
+    * oldhashes: a tuple of one or more binary hashes for the old location
+    * newhashes: a tuple of one or more binary hashes for the new location
+
+    Handles serialisation from and to the storage format. Fields are
+    separated by newlines, hashes are written out in hex separated by commas,
+    timestamp and timezone are separated by a space.
+
+    """
+    @classmethod
+    def fromstorage(cls, line):
+        (time, user, command, namespace, name,
+         oldhashes, newhashes) = line.split('\n')
+        timestamp, tz = time.split()
+        timestamp, tz = float(timestamp), int(tz)
+        oldhashes = tuple(node.bin(hash) for hash in oldhashes.split(','))
+        newhashes = tuple(node.bin(hash) for hash in newhashes.split(','))
+        return cls(
+            (timestamp, tz), user, command, namespace, name,
+            oldhashes, newhashes)
+
+    def __str__(self):
+        """String representation for storage"""
+        time = ' '.join(map(str, self.timestamp))
+        oldhashes = ','.join([node.hex(hash) for hash in self.oldhashes])
+        newhashes = ','.join([node.hex(hash) for hash in self.newhashes])
+        return '\n'.join((
+            time, self.user, self.command, self.namespace, self.name,
+            oldhashes, newhashes))
+
+class journalstorage(object):
+    """Storage for journal entries
+
+    Entries are divided over two files; one with entries that pertain to the
+    local working copy *only*, and one with entries that are shared across
+    multiple working copies when shared using the share extension.
+
+    Entries are stored with NUL bytes as separators. See the journalentry
+    class for the per-entry structure.
+
+    The file format starts with an integer version, delimited by a NUL.
+
+    This storage uses a dedicated lock; this makes it easier to avoid issues
+    with adding entries that added when the regular wlock is unlocked (e.g.
+    the dirstate).
+
+    """
+    _currentcommand = ()
+    _lockref = None
+
+    def __init__(self, repo):
+        self.user = util.getuser()
+        self.ui = repo.ui
+        self.vfs = repo.vfs
+
+        # is this working copy using a shared storage?
+        self.sharedfeatures = self.sharedvfs = None
+        if repo.shared():
+            features = _readsharedfeatures(repo)
+            sharedrepo = share._getsrcrepo(repo)
+            if sharedrepo is not None and 'journal' in features:
+                self.sharedvfs = sharedrepo.vfs
+                self.sharedfeatures = features
+
+    # track the current command for recording in journal entries
+    @property
+    def command(self):
+        commandstr = ' '.join(
+            map(util.shellquote, journalstorage._currentcommand))
+        if '\n' in commandstr:
+            # truncate multi-line commands
+            commandstr = commandstr.partition('\n')[0] + ' ...'
+        return commandstr
+
+    @classmethod
+    def recordcommand(cls, *fullargs):
+        """Set the current hg arguments, stored with recorded entries"""
+        # Set the current command on the class because we may have started
+        # with a non-local repo (cloning for example).
+        cls._currentcommand = fullargs
+
+    def jlock(self, vfs):
+        """Create a lock for the journal file"""
+        if self._lockref and self._lockref():
+            raise error.Abort(_('journal lock does not support nesting'))
+        desc = _('journal of %s') % vfs.base
+        try:
+            l = lock.lock(vfs, 'journal.lock', 0, desc=desc)
+        except error.LockHeld as inst:
+            self.ui.warn(
+                _("waiting for lock on %s held by %r\n") % (desc, inst.locker))
+            # default to 600 seconds timeout
+            l = lock.lock(
+                vfs, 'journal.lock',
+                int(self.ui.config("ui", "timeout", "600")), desc=desc)
+            self.ui.warn(_("got lock after %s seconds\n") % l.delay)
+        self._lockref = weakref.ref(l)
+        return l
+
+    def record(self, namespace, name, oldhashes, newhashes):
+        """Record a new journal entry
+
+        * namespace: an opaque string; this can be used to filter on the type
+          of recorded entries.
+        * name: the name defining this entry; for bookmarks, this is the
+          bookmark name. Can be filtered on when retrieving entries.
+        * oldhashes and newhashes: each a single binary hash, or a list of
+          binary hashes. These represent the old and new position of the named
+          item.
+
+        """
+        if not isinstance(oldhashes, list):
+            oldhashes = [oldhashes]
+        if not isinstance(newhashes, list):
+            newhashes = [newhashes]
+
+        entry = journalentry(
+            util.makedate(), self.user, self.command, namespace, name,
+            oldhashes, newhashes)
+
+        vfs = self.vfs
+        if self.sharedvfs is not None:
+            # write to the shared repository if this feature is being
+            # shared between working copies.
+            if sharednamespaces.get(namespace) in self.sharedfeatures:
+                vfs = self.sharedvfs
+
+        self._write(vfs, entry)
+
+    def _write(self, vfs, entry):
+        with self.jlock(vfs):
+            version = None
+            # open file in amend mode to ensure it is created if missing
+            with vfs('journal', mode='a+b', atomictemp=True) as f:
+                f.seek(0, os.SEEK_SET)
+                # Read just enough bytes to get a version number (up to 2
+                # digits plus separator)
+                version = f.read(3).partition('\0')[0]
+                if version and version != str(storageversion):
+                    # different version of the storage. Exit early (and not
+                    # write anything) if this is not a version we can handle or
+                    # the file is corrupt. In future, perhaps rotate the file
+                    # instead?
+                    self.ui.warn(
+                        _("unsupported journal file version '%s'\n") % version)
+                    return
+                if not version:
+                    # empty file, write version first
+                    f.write(str(storageversion) + '\0')
+                f.seek(0, os.SEEK_END)
+                f.write(str(entry) + '\0')
+
+    def filtered(self, namespace=None, name=None):
+        """Yield all journal entries with the given namespace or name
+
+        Both the namespace and the name are optional; if neither is given all
+        entries in the journal are produced.
+
+        Matching supports regular expressions by using the `re:` prefix
+        (use `literal:` to match names or namespaces that start with `re:`)
+
+        """
+        if namespace is not None:
+            namespace = util.stringmatcher(namespace)[-1]
+        if name is not None:
+            name = util.stringmatcher(name)[-1]
+        for entry in self:
+            if namespace is not None and not namespace(entry.namespace):
+                continue
+            if name is not None and not name(entry.name):
+                continue
+            yield entry
+
+    def __iter__(self):
+        """Iterate over the storage
+
+        Yields journalentry instances for each contained journal record.
+
+        """
+        local = self._open(self.vfs)
+
+        if self.sharedvfs is None:
+            return local
+
+        # iterate over both local and shared entries, but only those
+        # shared entries that are among the currently shared features
+        shared = (
+            e for e in self._open(self.sharedvfs)
+            if sharednamespaces.get(e.namespace) in self.sharedfeatures)
+        return _mergeentriesiter(local, shared)
+
+    def _open(self, vfs, filename='journal', _newestfirst=True):
+        if not vfs.exists(filename):
+            return
+
+        with vfs(filename) as f:
+            raw = f.read()
+
+        lines = raw.split('\0')
+        version = lines and lines[0]
+        if version != str(storageversion):
+            version = version or _('not available')
+            raise error.Abort(_("unknown journal file version '%s'") % version)
+
+        # Skip the first line, it's a version number. Normally we iterate over
+        # these in reverse order to list newest first; only when copying across
+        # a shared storage do we forgo reversing.
+        lines = lines[1:]
+        if _newestfirst:
+            lines = reversed(lines)
+        for line in lines:
+            if not line:
+                continue
+            yield journalentry.fromstorage(line)
+
+# journal reading
+# log options that don't make sense for journal
+_ignoreopts = ('no-merges', 'graph')
+@command(
+    'journal', [
+        ('', 'all', None, 'show history for all names'),
+        ('c', 'commits', None, 'show commit metadata'),
+    ] + [opt for opt in commands.logopts if opt[1] not in _ignoreopts],
+    '[OPTION]... [BOOKMARKNAME]')
+def journal(ui, repo, *args, **opts):
+    """show the previous position of bookmarks and the working copy
+
+    The journal is used to see the previous commits that bookmarks and the
+    working copy pointed to. By default the previous locations for the working
+    copy.  Passing a bookmark name will show all the previous positions of
+    that bookmark. Use the --all switch to show previous locations for all
+    bookmarks and the working copy; each line will then include the bookmark
+    name, or '.' for the working copy, as well.
+
+    If `name` starts with `re:`, the remainder of the name is treated as
+    a regular expression. To match a name that actually starts with `re:`,
+    use the prefix `literal:`.
+
+    By default hg journal only shows the commit hash and the command that was
+    running at that time. -v/--verbose will show the prior hash, the user, and
+    the time at which it happened.
+
+    Use -c/--commits to output log information on each commit hash; at this
+    point you can use the usual `--patch`, `--git`, `--stat` and `--template`
+    switches to alter the log output for these.
+
+    `hg journal -T json` can be used to produce machine readable output.
+
+    """
+    name = '.'
+    if opts.get('all'):
+        if args:
+            raise error.Abort(
+                _("You can't combine --all and filtering on a name"))
+        name = None
+    if args:
+        name = args[0]
+
+    fm = ui.formatter('journal', opts)
+
+    if opts.get("template") != "json":
+        if name is None:
+            displayname = _('the working copy and bookmarks')
+        else:
+            displayname = "'%s'" % name
+        ui.status(_("previous locations of %s:\n") % displayname)
+
+    limit = cmdutil.loglimit(opts)
+    entry = None
+    for count, entry in enumerate(repo.journal.filtered(name=name)):
+        if count == limit:
+            break
+        newhashesstr = ','.join([node.short(hash) for hash in entry.newhashes])
+        oldhashesstr = ','.join([node.short(hash) for hash in entry.oldhashes])
+
+        fm.startitem()
+        fm.condwrite(ui.verbose, 'oldhashes', '%s -> ', oldhashesstr)
+        fm.write('newhashes', '%s', newhashesstr)
+        fm.condwrite(ui.verbose, 'user', ' %-8s', entry.user)
+        fm.condwrite(
+            opts.get('all') or name.startswith('re:'),
+            'name', '  %-8s', entry.name)
+
+        timestring = util.datestr(entry.timestamp, '%Y-%m-%d %H:%M %1%2')
+        fm.condwrite(ui.verbose, 'date', ' %s', timestring)
+        fm.write('command', '  %s\n', entry.command)
+
+        if opts.get("commits"):
+            displayer = cmdutil.show_changeset(ui, repo, opts, buffered=False)
+            for hash in entry.newhashes:
+                try:
+                    ctx = repo[hash]
+                    displayer.show(ctx)
+                except error.RepoLookupError as e:
+                    fm.write('repolookuperror', "%s\n\n", str(e))
+            displayer.close()
+
+    fm.end()
+
+    if entry is None:
+        ui.status(_("no recorded locations\n"))

mercurial/bdiff.h

@@ -0,0 +1,21 b''
+#ifndef _HG_BDIFF_H_
+#define _HG_BDIFF_H_
+
+struct bdiff_line {
+	int hash, n, e;
+	ssize_t len;
+	const char *l;
+};
+
+struct bdiff_hunk;
+struct bdiff_hunk {
+	int a1, a2, b1, b2;
+	struct bdiff_hunk *next;
+};
+
+int bdiff_splitlines(const char *a, ssize_t len, struct bdiff_line **lr);
+int bdiff_diff(struct bdiff_line *a, int an, struct bdiff_line *b, int bn,
+	struct bdiff_hunk *base);
+void bdiff_freehunks(struct bdiff_hunk *l);
+
+#endif

mercurial/bdiff_module.c

@@ -0,0 +1,203 b''
+/*
+ bdiff.c - efficient binary diff extension for Mercurial
+
+ Copyright 2005, 2006 Matt Mackall <[e-mail]>
+
+ This software may be used and distributed according to the terms of
+ the GNU General Public License, incorporated herein by reference.
+
+ Based roughly on Python difflib
+*/
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#include "bdiff.h"
+#include "bitmanipulation.h"
+
+
+static PyObject *blocks(PyObject *self, PyObject *args)
+{
+	PyObject *sa, *sb, *rl = NULL, *m;
+	struct bdiff_line *a, *b;
+	struct bdiff_hunk l, *h;
+	int an, bn, count, pos = 0;
+
+	l.next = NULL;
+
+	if (!PyArg_ParseTuple(args, "SS:bdiff", &sa, &sb))
+		return NULL;
+
+	an = bdiff_splitlines(PyBytes_AsString(sa), PyBytes_Size(sa), &a);
+	bn = bdiff_splitlines(PyBytes_AsString(sb), PyBytes_Size(sb), &b);
+
+	if (!a || !b)
+		goto nomem;
+
+	count = bdiff_diff(a, an, b, bn, &l);
+	if (count < 0)
+		goto nomem;
+
+	rl = PyList_New(count);
+	if (!rl)
+		goto nomem;
+
+	for (h = l.next; h; h = h->next) {
+		m = Py_BuildValue("iiii", h->a1, h->a2, h->b1, h->b2);
+		PyList_SetItem(rl, pos, m);
+		pos++;
+	}
+
+nomem:
+	free(a);
+	free(b);
+	bdiff_freehunks(l.next);
+	return rl ? rl : PyErr_NoMemory();
+}
+
+static PyObject *bdiff(PyObject *self, PyObject *args)
+{
+	char *sa, *sb, *rb;
+	PyObject *result = NULL;
+	struct bdiff_line *al, *bl;
+	struct bdiff_hunk l, *h;
+	int an, bn, count;
+	Py_ssize_t len = 0, la, lb;
+	PyThreadState *_save;
+
+	l.next = NULL;
+
+	if (!PyArg_ParseTuple(args, "s#s#:bdiff", &sa, &la, &sb, &lb))
+		return NULL;
+
+	if (la > UINT_MAX || lb > UINT_MAX) {
+		PyErr_SetString(PyExc_ValueError, "bdiff inputs too large");
+		return NULL;
+	}
+
+	_save = PyEval_SaveThread();
+	an = bdiff_splitlines(sa, la, &al);
+	bn = bdiff_splitlines(sb, lb, &bl);
+	if (!al || !bl)
+		goto nomem;
+
+	count = bdiff_diff(al, an, bl, bn, &l);
+	if (count < 0)
+		goto nomem;
+
+	/* calculate length of output */
+	la = lb = 0;
+	for (h = l.next; h; h = h->next) {
+		if (h->a1 != la || h->b1 != lb)
+			len += 12 + bl[h->b1].l - bl[lb].l;
+		la = h->a2;
+		lb = h->b2;
+	}
+	PyEval_RestoreThread(_save);
+	_save = NULL;
+
+	result = PyBytes_FromStringAndSize(NULL, len);
+
+	if (!result)
+		goto nomem;
+
+	/* build binary patch */
+	rb = PyBytes_AsString(result);
+	la = lb = 0;
+
+	for (h = l.next; h; h = h->next) {
+		if (h->a1 != la || h->b1 != lb) {
+			len = bl[h->b1].l - bl[lb].l;
+			putbe32((uint32_t)(al[la].l - al->l), rb);
+			putbe32((uint32_t)(al[h->a1].l - al->l), rb + 4);
+			putbe32((uint32_t)len, rb + 8);
+			memcpy(rb + 12, bl[lb].l, len);
+			rb += 12 + len;
+		}
+		la = h->a2;
+		lb = h->b2;
+	}
+
+nomem:
+	if (_save)
+		PyEval_RestoreThread(_save);
+	free(al);
+	free(bl);
+	bdiff_freehunks(l.next);
+	return result ? result : PyErr_NoMemory();
+}
+
+/*
+ * If allws != 0, remove all whitespace (' ', \t and \r). Otherwise,
+ * reduce whitespace sequences to a single space and trim remaining whitespace
+ * from end of lines.
+ */
+static PyObject *fixws(PyObject *self, PyObject *args)
+{
+	PyObject *s, *result = NULL;
+	char allws, c;
+	const char *r;
+	Py_ssize_t i, rlen, wlen = 0;
+	char *w;
+
+	if (!PyArg_ParseTuple(args, "Sb:fixws", &s, &allws))
+		return NULL;
+	r = PyBytes_AsString(s);
+	rlen = PyBytes_Size(s);
+
+	w = (char *)malloc(rlen ? rlen : 1);
+	if (!w)
+		goto nomem;
+
+	for (i = 0; i != rlen; i++) {
+		c = r[i];
+		if (c == ' ' || c == '\t' || c == '\r') {
+			if (!allws && (wlen == 0 || w[wlen - 1] != ' '))
+				w[wlen++] = ' ';
+		} else if (c == '\n' && !allws
+			  && wlen > 0 && w[wlen - 1] == ' ') {
+			w[wlen - 1] = '\n';
+		} else {
+			w[wlen++] = c;
+		}
+	}
+
+	result = PyBytes_FromStringAndSize(w, wlen);
+
+nomem:
+	free(w);
+	return result ? result : PyErr_NoMemory();
+}
+
+
+static char mdiff_doc[] = "Efficient binary diff.";
+
+static PyMethodDef methods[] = {
+	{"bdiff", bdiff, METH_VARARGS, "calculate a binary diff\n"},
+	{"blocks", blocks, METH_VARARGS, "find a list of matching lines\n"},
+	{"fixws", fixws, METH_VARARGS, "normalize diff whitespaces\n"},
+	{NULL, NULL}
+};
+
+#ifdef IS_PY3K
+static struct PyModuleDef bdiff_module = {
+	PyModuleDef_HEAD_INIT,
+	"bdiff",
+	mdiff_doc,
+	-1,
+	methods
+};
+
+PyMODINIT_FUNC PyInit_bdiff(void)
+{
+	return PyModule_Create(&bdiff_module);
+}
+#else
+PyMODINIT_FUNC initbdiff(void)
+{
+	Py_InitModule3("bdiff", methods, mdiff_doc);
+}
+#endif

mercurial/bitmanipulation.h

@@ -0,0 +1,53 b''
+#ifndef _HG_BITMANIPULATION_H_
+#define _HG_BITMANIPULATION_H_
+
+#include "compat.h"
+
+static inline uint32_t getbe32(const char *c)
+{
+	const unsigned char *d = (const unsigned char *)c;
+
+	return ((d[0] << 24) |
+		(d[1] << 16) |
+		(d[2] << 8) |
+		(d[3]));
+}
+
+static inline int16_t getbeint16(const char *c)
+{
+	const unsigned char *d = (const unsigned char *)c;
+
+	return ((d[0] << 8) |
+		(d[1]));
+}
+
+static inline uint16_t getbeuint16(const char *c)
+{
+	const unsigned char *d = (const unsigned char *)c;
+
+	return ((d[0] << 8) |
+		(d[1]));
+}
+
+static inline void putbe32(uint32_t x, char *c)
+{
+	c[0] = (x >> 24) & 0xff;
+	c[1] = (x >> 16) & 0xff;
+	c[2] = (x >> 8) & 0xff;
+	c[3] = (x) & 0xff;
+}
+
+static inline double getbefloat64(const char *c)
+{
+	const unsigned char *d = (const unsigned char *)c;
+	double ret;
+	int i;
+	uint64_t t = 0;
+	for (i = 0; i < 8; i++) {
+		t = (t<<8) + d[i];
+	}
+	memcpy(&ret, &t, sizeof(t));
+	return ret;
+}
+
+#endif

mercurial/compat.h

@@ -0,0 +1,43 b''
+#ifndef _HG_COMPAT_H_
+#define _HG_COMPAT_H_
+
+#ifdef _WIN32
+#ifdef _MSC_VER
+/* msvc 6.0 has problems */
+#define inline __inline
+#if defined(_WIN64)
+typedef __int64 ssize_t;
+#else
+typedef int ssize_t;
+#endif
+typedef signed char int8_t;
+typedef short int16_t;
+typedef long int32_t;
+typedef __int64 int64_t;
+typedef unsigned char uint8_t;
+typedef unsigned short uint16_t;
+typedef unsigned long uint32_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+#else
+/* not windows */
+#include <sys/types.h>
+#if defined __BEOS__ && !defined __HAIKU__
+#include <ByteOrder.h>
+#else
+#include <arpa/inet.h>
+#endif
+#include <inttypes.h>
+#endif
+
+#if defined __hpux || defined __SUNPRO_C || defined _AIX
+#define inline
+#endif
+
+#ifdef __linux
+#define inline __inline
+#endif
+
+#endif

mercurial/policy.py

@@ -0,0 +1,45 b''
+# policy.py - module policy logic for Mercurial.
+#
+# Copyright 2015 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import os
+import sys
+
+# Rules for how modules can be loaded. Values are:
+#
+#    c - require C extensions
+#    allow - allow pure Python implementation when C loading fails
+#    cffi - required cffi versions (implemented within pure module)
+#    cffi-allow - allow pure Python implementation if cffi version is missing
+#    py - only load pure Python modules
+#
+# By default, require the C extensions for performance reasons.
+policy = 'c'
+policynoc = ('cffi', 'cffi-allow', 'py')
+policynocffi = ('c', 'py')
+
+try:
+    from . import __modulepolicy__
+    policy = __modulepolicy__.modulepolicy
+except ImportError:
+    pass
+
+# PyPy doesn't load C extensions.
+#
+# The canonical way to do this is to test platform.python_implementation().
+# But we don't import platform and don't bloat for it here.
+if '__pypy__' in sys.builtin_module_names:
+    policy = 'cffi'
+
+# Our C extensions aren't yet compatible with Python 3. So use pure Python
+# on Python 3 for now.
+if sys.version_info[0] >= 3:
+    policy = 'py'
+
+# Environment variable can always force settings.
+policy = os.environ.get('HGMODULEPOLICY', policy)

Makefile

@@ -156,7 +156,7 b' i18n/hg.pot: $(PYFILES) $(DOCFILES) i18n'
 # Packaging targets
 
 osx:
-	python setup.py install --optimize=1 \
+	/usr/bin/python2.7 setup.py install --optimize=1 \
 	  --root=build/mercurial/ --prefix=/usr/local/ \
 	  --install-lib=/Library/Python/2.7/site-packages/
 	make -C doc all install DESTDIR="$(PWD)/build/mercurial/"

contrib/bash_completion

@@ -184,7 +184,7 b' shopt -s extglob'
             return
         fi
 
-        opts=$(_hg_cmd debugcomplete --options "$cmd")
+        opts=$(HGPLAINEXCEPT=alias _hg_cmd debugcomplete --options "$cmd")
 
         COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$opts' -- "$cur"))
         _hg_fix_wordlist

contrib/bdiff-torture.py

@@ -1,7 +1,9 b''
 # Randomized torture test generation for bdiff
 
 from __future__ import absolute_import, print_function
-import random, sys
+import random
+import sys
+
 from mercurial import (
     bdiff,
     mpatch,

contrib/check-code.py

@@ -26,6 +26,15 b' import optparse'
 import os
 import re
 import sys
+if sys.version_info[0] < 3:
+    opentext = open
+else:
+    def opentext(f):
+        return open(f, encoding='ascii')
+try:
+    xrange
+except NameError:
+    xrange = range
 try:
     import re2
 except ImportError:
@@ -41,26 +50,26 b' def compilere(pat, multiline=False):'
             pass
     return re.compile(pat)
 
+# check "rules depending on implementation of repquote()" in each
+# patterns (especially pypats), before changing around repquote()
+_repquotefixedmap = {' ': ' ', '\n': '\n', '.': 'p', ':': 'q',
+                     '%': '%', '\\': 'b', '*': 'A', '+': 'P', '-': 'M'}
+def _repquoteencodechr(i):
+    if i > 255:
+        return 'u'
+    c = chr(i)
+    if c in _repquotefixedmap:
+        return _repquotefixedmap[c]
+    if c.isalpha():
+        return 'x'
+    if c.isdigit():
+        return 'n'
+    return 'o'
+_repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256))
+
 def repquote(m):
-    fromc = '.:'
-    tochr = 'pq'
-    def encodechr(i):
-        if i > 255:
-            return 'u'
-        c = chr(i)
-        if c in ' \n':
-            return c
-        if c.isalpha():
-            return 'x'
-        if c.isdigit():
-            return 'n'
-        try:
-            return tochr[fromc.find(c)]
-        except (ValueError, IndexError):
-            return 'o'
     t = m.group('text')
-    tt = ''.join(encodechr(i) for i in xrange(256))
-    t = t.translate(tt)
+    t = t.translate(_repquotett)
     return m.group('quote') + t + m.group('quote')
 
 def reppython(m):
@@ -103,7 +112,7 b' testpats = ['
     (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
     (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
     (r'ls.*-\w*R', "don't use 'ls -R', use 'find'"),
-    (r'printf.*[^\\]\\([1-9]|0\d)', "don't use 'printf \NNN', use Python"),
+    (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"),
     (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
     (r'\$\(.*\)', "don't use $(expr), use `expr`"),
     (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
@@ -114,7 +123,7 b' testpats = ['
     (r'export .*=', "don't export and assign at once"),
     (r'^source\b', "don't use 'source', use '.'"),
     (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
-    (r'ls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
+    (r'\bls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
     (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
     (r'^stop\(\)', "don't use 'stop' as a shell function name"),
     (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
@@ -133,6 +142,7 b' testpats = ['
     (r'\|&', "don't use |&, use 2>&1"),
     (r'\w =  +\w', "only one space after = allowed"),
     (r'\bsed\b.*[^\\]\\n', "don't use 'sed ... \\n', use a \\ and a newline"),
+    (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'")
   ],
   # warnings
   [
@@ -179,6 +189,8 b' utestpats = ['
     (r'^  .*: largefile \S+ not available from file:.*/.*[^)]$', winglobmsg),
     (r'^  .*file://\$TESTTMP',
      'write "file:/*/$TESTTMP" + (glob) to match on windows too'),
+    (r'^  [^$>].*27\.0\.0\.1.*[^)]$',
+     'use (glob) to match localhost IP on hosts without 127.0.0.1 too'),
     (r'^  (cat|find): .*: No such file or directory',
      'use test -f to test for file existence'),
     (r'^  diff -[^ -]*p',
@@ -197,8 +209,8 b' utestpats = ['
   ],
   # warnings
   [
-    (r'^  [^*?/\n]* \(glob\)$',
-     "glob match with no glob character (?*/)"),
+    (r'^  (?!.*127\.0\.0\.1)[^*?/\n]* \(glob\)$',
+     "glob match with no glob string (?, *, /, and 127.0.0.1)"),
   ]
 ]
 
@@ -214,7 +226,7 b' for i in [0, 1]:'
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
-    (r"( *)(#([^\n]*\S)?)", repcomment),
+    (r"( +)(#([^\n]*\S)?)", repcomment),
 ]
 
 pypats = [
@@ -238,7 +250,6 b' pypats = ['
     (r'^\s+(\w|\.)+=\w[^,()\n]*$', "missing whitespace in assignment"),
     (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
     (r'.{81}', "line too long"),
-    (r' x+[xo][\'"]\n\s+[\'"]x', 'string join across lines with no space'),
     (r'[^\n]\Z', "no trailing newline"),
     (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
 #    (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=',
@@ -305,8 +316,6 b' pypats = ['
     (r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
      'legacy exception syntax; use "as" instead of ","'),
     (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
-    (r'ui\.(status|progress|write|note|warn)\([\'\"]x',
-     "missing _() in ui message (use () to hide false-positives)"),
     (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
     (r'\b__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
     (r'os\.path\.join\(.*, *(""|\'\')\)',
@@ -318,9 +327,37 b' pypats = ['
     (r'^import Queue', "don't use Queue, use util.queue + util.empty"),
     (r'^import cStringIO', "don't use cStringIO.StringIO, use util.stringio"),
     (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"),
+    (r'^import SocketServer', "don't use SockerServer, use util.socketserver"),
+    (r'^import urlparse', "don't use urlparse, use util.urlparse"),
+    (r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"),
+    (r'^import cPickle', "don't use cPickle, use util.pickle"),
+    (r'^import pickle', "don't use pickle, use util.pickle"),
+    (r'^import httplib', "don't use httplib, use util.httplib"),
+    (r'^import BaseHTTPServer', "use util.httpserver instead"),
+    (r'\.next\(\)', "don't use .next(), use next(...)"),
+
+    # rules depending on implementation of repquote()
+    (r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
+     'string join across lines with no space'),
+    (r'''(?x)ui\.(status|progress|write|note|warn)\(
+         [ \t\n#]*
+         (?# any strings/comments might precede a string, which
+           # contains translatable message)
+         ((['"]|\'\'\'|""")[ \npq%bAPMxno]*(['"]|\'\'\'|""")[ \t\n#]+)*
+         (?# sequence consisting of below might precede translatable message
+           # - formatting string: "% 10s", "%05d", "% -3.2f", "%*s", "%%" ...
+           # - escaped character: "\\", "\n", "\0" ...
+           # - character other than '%', 'b' as '\', and 'x' as alphabet)
+         (['"]|\'\'\'|""")
+         ((%([ n]?[PM]?([np]+|A))?x)|%%|b[bnx]|[ \nnpqAPMo])*x
+         (?# this regexp can't use [^...] style,
+           # because _preparepats forcibly adds "\n" into [^...],
+           # even though this regexp wants match it against "\n")''',
+     "missing _() in ui message (use () to hide false-positives)"),
   ],
   # warnings
   [
+    # rules depending on implementation of repquote()
     (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
   ]
 ]
@@ -365,9 +402,13 b' cpats = ['
     (r'^\s*#import\b', "use only #include in standard C code"),
     (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"),
     (r'strcat\(', "don't use strcat"),
+
+    # rules depending on implementation of repquote()
   ],
   # warnings
-  []
+  [
+    # rules depending on implementation of repquote()
+  ]
 ]
 
 cfilters = [
@@ -433,7 +474,6 b' def _preparepats():'
         filters = c[3]
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
-_preparepats()
 
 class norepeatlogger(object):
     def __init__(self):
@@ -486,12 +526,15 b' def checkfile(f, logfunc=_defaultlogger.'
     result = True
 
     try:
-        fp = open(f)
+        with opentext(f) as fp:
+            try:
+                pre = post = fp.read()
+            except UnicodeDecodeError as e:
+                print("%s while reading %s" % (e, f))
+                return result
     except IOError as e:
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
-    pre = post = fp.read()
-    fp.close()
 
     for name, match, magic, filters, pats in checks:
         if debug:
@@ -578,7 +621,7 b' def checkfile(f, logfunc=_defaultlogger.'
 
     return result
 
-if __name__ == "__main__":
+def main():
     parser = optparse.OptionParser("%prog [options] [files]")
     parser.add_option("-w", "--warnings", action="store_true",
                       help="include warning-level checks")
@@ -600,10 +643,15 b' if __name__ == "__main__":'
     else:
         check = args
 
+    _preparepats()
+
     ret = 0
     for f in check:
         if not checkfile(f, maxerr=options.per_file, warnings=options.warnings,
                          blame=options.blame, debug=options.debug,
                          lineno=options.lineno):
             ret = 1
-    sys.exit(ret)
+    return ret
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/check-commit

@@ -15,7 +15,11 b''
 #
 # See also: https://mercurial-scm.org/wiki/ContributingChanges
 
-import re, sys, os
+from __future__ import absolute_import, print_function
+
+import os
+import re
+import sys
 
 commitheader = r"^(?:# [^\n]*\n)*"
 afterheader = commitheader + r"(?!#)"
@@ -69,9 +73,9 b' def checkcommit(commit, node=None):'
                     break
                 if not printed:
                     printed = True
-                    print "node: %s" % node
-                print "%d: %s" % (n, msg)
-                print " %s" % nonempty(l, last)[:-1]
+                    print("node: %s" % node)
+                print("%d: %s" % (n, msg))
+                print(" %s" % nonempty(l, last)[:-1])
                 if "BYPASS" not in os.environ:
                     exitcode = 1
                 del hits[0]

contrib/check-py3-compat.py

@@ -61,7 +61,20 b' def check_compat_py3(f):'
                 imp.load_module(name, fh, '', ('py', 'r', imp.PY_SOURCE))
             except Exception as e:
                 exc_type, exc_value, tb = sys.exc_info()
-                frame = traceback.extract_tb(tb)[-1]
+                # We walk the stack and ignore frames from our custom importer,
+                # import mechanisms, and stdlib modules. This kinda/sorta
+                # emulates CPython behavior in import.c while also attempting
+                # to pin blame on a Mercurial file.
+                for frame in reversed(traceback.extract_tb(tb)):
+                    if frame.name == '_call_with_frames_removed':
+                        continue
+                    if 'importlib' in frame.filename:
+                        continue
+                    if 'mercurial/__init__.py' in frame.filename:
+                        continue
+                    if frame.filename.startswith(sys.prefix):
+                        continue
+                    break
 
                 if frame.filename:
                     filename = os.path.basename(frame.filename)

contrib/chg/README

@@ -28,3 +28,5 b' The following variables are available fo'
 
  * CHGDEBUG enables debug messages.
  * CHGSOCKNAME specifies the socket path of the background cmdserver.
+ * CHGTIMEOUT specifies how many seconds chg will wait before giving up
+   connecting to a cmdserver. If it is 0, chg will wait forever. Default: 60

contrib/chg/chg.c

@@ -249,7 +249,13 b' static hgclient_t *retryconnectcmdserver'
 	int pst = 0;
 
 	debugmsg("try connect to %s repeatedly", opts->sockname);
-	for (unsigned int i = 0; i < 10 * 100; i++) {
+
+	unsigned int timeoutsec = 60;  /* default: 60 seconds */
+	const char *timeoutenv = getenv("CHGTIMEOUT");
+	if (timeoutenv)
+		sscanf(timeoutenv, "%u", &timeoutsec);
+
+	for (unsigned int i = 0; !timeoutsec || i < timeoutsec * 100; i++) {
 		hgclient_t *hgc = hgc_open(opts->sockname);
 		if (hgc)
 			return hgc;
@@ -332,6 +338,7 b' static void killcmdserver(const struct c'
 	}
 }
 
+static pid_t pagerpid = 0;
 static pid_t peerpid = 0;
 
 static void forwardsignal(int sig)
@@ -374,6 +381,17 b' error:'
 	abortmsgerrno("failed to handle stop signal");
 }
 
+static void handlechildsignal(int sig UNUSED_)
+{
+	if (peerpid == 0 || pagerpid == 0)
+		return;
+	/* if pager exits, notify the server with SIGPIPE immediately.
+	 * otherwise the server won't get SIGPIPE if it does not write
+	 * anything. (issue5278) */
+	if (waitpid(pagerpid, NULL, WNOHANG) == pagerpid)
+		kill(peerpid, SIGPIPE);
+}
+
 static void setupsignalhandler(pid_t pid)
 {
 	if (pid <= 0)
@@ -410,6 +428,11 b' static void setupsignalhandler(pid_t pid'
 	sa.sa_flags = SA_RESTART;
 	if (sigaction(SIGTSTP, &sa, NULL) < 0)
 		goto error;
+	/* get notified when pager exits */
+	sa.sa_handler = handlechildsignal;
+	sa.sa_flags = SA_RESTART;
+	if (sigaction(SIGCHLD, &sa, NULL) < 0)
+		goto error;
 
 	return;
 
@@ -417,21 +440,56 b' error:'
 	abortmsgerrno("failed to set up signal handlers");
 }
 
-/* This implementation is based on hgext/pager.py (pre 369741ef7253) */
-static void setuppager(hgclient_t *hgc, const char *const args[],
+static void restoresignalhandler()
+{
+	struct sigaction sa;
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_handler = SIG_DFL;
+	sa.sa_flags = SA_RESTART;
+	if (sigemptyset(&sa.sa_mask) < 0)
+		goto error;
+
+	if (sigaction(SIGHUP, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGTERM, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGWINCH, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGCONT, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGTSTP, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGCHLD, &sa, NULL) < 0)
+		goto error;
+
+	/* ignore Ctrl+C while shutting down to make pager exits cleanly */
+	sa.sa_handler = SIG_IGN;
+	if (sigaction(SIGINT, &sa, NULL) < 0)
+		goto error;
+
+	peerpid = 0;
+	return;
+
+error:
+	abortmsgerrno("failed to restore signal handlers");
+}
+
+/* This implementation is based on hgext/pager.py (post 369741ef7253)
+ * Return 0 if pager is not started, or pid of the pager */
+static pid_t setuppager(hgclient_t *hgc, const char *const args[],
 		       size_t argsize)
 {
 	const char *pagercmd = hgc_getpager(hgc, args, argsize);
 	if (!pagercmd)
-		return;
+		return 0;
 
 	int pipefds[2];
 	if (pipe(pipefds) < 0)
-		return;
+		return 0;
 	pid_t pid = fork();
 	if (pid < 0)
 		goto error;
-	if (pid == 0) {
+	if (pid > 0) {
 		close(pipefds[0]);
 		if (dup2(pipefds[1], fileno(stdout)) < 0)
 			goto error;
@@ -441,7 +499,7 b' static void setuppager(hgclient_t *hgc, '
 		}
 		close(pipefds[1]);
 		hgc_attachio(hgc);  /* reattach to pager */
-		return;
+		return pid;
 	} else {
 		dup2(pipefds[0], fileno(stdin));
 		close(pipefds[0]);
@@ -451,13 +509,27 b' static void setuppager(hgclient_t *hgc, '
 		if (r < 0) {
 			abortmsgerrno("cannot start pager '%s'", pagercmd);
 		}
-		return;
+		return 0;
 	}
 
 error:
 	close(pipefds[0]);
 	close(pipefds[1]);
 	abortmsgerrno("failed to prepare pager");
+	return 0;
+}
+
+static void waitpager(pid_t pid)
+{
+	/* close output streams to notify the pager its input ends */
+	fclose(stdout);
+	fclose(stderr);
+	while (1) {
+		pid_t ret = waitpid(pid, NULL, 0);
+		if (ret == -1 && errno == EINTR)
+			continue;
+		break;
+	}
 }
 
 /* Run instructions sent from the server like unlink and set redirect path
@@ -585,9 +657,13 b' int main(int argc, const char *argv[], c'
 	}
 
 	setupsignalhandler(hgc_peerpid(hgc));
-	setuppager(hgc, argv + 1, argc - 1);
+	pagerpid = setuppager(hgc, argv + 1, argc - 1);
 	int exitcode = hgc_runcommand(hgc, argv + 1, argc - 1);
+	restoresignalhandler();
 	hgc_close(hgc);
 	freecmdserveropts(&opts);
+	if (pagerpid)
+		waitpager(pagerpid);
+
 	return exitcode;
 }

contrib/chg/hgclient.c

@@ -63,6 +63,7 b' typedef struct {'
 
 struct hgclient_tag_ {
 	int sockfd;
+	pid_t pgid;
 	pid_t pid;
 	context_t ctx;
 	unsigned int capflags;
@@ -125,10 +126,15 b' static void readchannel(hgclient_t *hgc)'
 		return;  /* assumes input request */
 
 	size_t cursize = 0;
+	int emptycount = 0;
 	while (cursize < hgc->ctx.datasize) {
 		rsize = recv(hgc->sockfd, hgc->ctx.data + cursize,
 			     hgc->ctx.datasize - cursize, 0);
-		if (rsize < 0)
+		/* rsize == 0 normally indicates EOF, while it's also a valid
+		 * packet size for unix socket. treat it as EOF and abort if
+		 * we get many empty responses in a row. */
+		emptycount = (rsize == 0 ? emptycount + 1 : 0);
+		if (rsize < 0 || emptycount > 20)
 			abortmsg("failed to read data block");
 		cursize += rsize;
 	}
@@ -339,6 +345,8 b' static void readhello(hgclient_t *hgc)'
 			u = dataend;
 		if (strncmp(s, "capabilities:", t - s + 1) == 0) {
 			hgc->capflags = parsecapabilities(t + 2, u);
+		} else if (strncmp(s, "pgid:", t - s + 1) == 0) {
+			hgc->pgid = strtol(t + 2, NULL, 10);
 		} else if (strncmp(s, "pid:", t - s + 1) == 0) {
 			hgc->pid = strtol(t + 2, NULL, 10);
 		}
@@ -463,6 +471,12 b' void hgc_close(hgclient_t *hgc)'
 	free(hgc);
 }
 
+pid_t hgc_peerpgid(const hgclient_t *hgc)
+{
+	assert(hgc);
+	return hgc->pgid;
+}
+
 pid_t hgc_peerpid(const hgclient_t *hgc)
 {
 	assert(hgc);

contrib/chg/hgclient.h

@@ -18,6 +18,7 b' typedef struct hgclient_tag_ hgclient_t;'
 hgclient_t *hgc_open(const char *sockname);
 void hgc_close(hgclient_t *hgc);
 
+pid_t hgc_peerpgid(const hgclient_t *hgc);
 pid_t hgc_peerpid(const hgclient_t *hgc);
 
 const char **hgc_validate(hgclient_t *hgc, const char *const args[],

contrib/chg/util.h

@@ -12,8 +12,10 b''
 
 #ifdef __GNUC__
 #define PRINTF_FORMAT_ __attribute__((format(printf, 1, 2)))
+#define UNUSED_ __attribute__((unused))
 #else
 #define PRINTF_FORMAT_
+#define UNUSED_
 #endif
 
 void abortmsg(const char *fmt, ...) PRINTF_FORMAT_;

contrib/debugshell.py

@@ -52,7 +52,7 b' def debugshell(ui, repo, **opts):'
         with demandimport.deactivated():
             __import__(pdbmap[debugger])
     except ImportError:
-        ui.warn("%s debugger specified but %s module was not found\n"
+        ui.warn(("%s debugger specified but %s module was not found\n")
                 % (debugger, pdbmap[debugger]))
         debugger = 'pdb'
 

contrib/dirstatenonnormalcheck.py

@@ -25,10 +25,10 b' def checkconsistency(ui, orig, dmap, _no'
     """Compute nonnormalset from dmap, check that it matches _nonnormalset"""
     nonnormalcomputedmap = nonnormalentries(dmap)
     if _nonnormalset != nonnormalcomputedmap:
-        ui.develwarn("%s call to %s\n" % (label, orig))
-        ui.develwarn("inconsistency in nonnormalset\n")
-        ui.develwarn("[nonnormalset] %s\n" % _nonnormalset)
-        ui.develwarn("[map] %s\n" % nonnormalcomputedmap)
+        ui.develwarn("%s call to %s\n" % (label, orig), config='dirstate')
+        ui.develwarn("inconsistency in nonnormalset\n", config='dirstate')
+        ui.develwarn("[nonnormalset] %s\n" % _nonnormalset, config='dirstate')
+        ui.develwarn("[map] %s\n" % nonnormalcomputedmap, config='dirstate')
 
 def _checkdirstate(orig, self, arg):
     """Check nonnormal set consistency before and after the call to orig"""

contrib/dumprevlog

@@ -2,8 +2,14 b''
 # Dump revlogs as raw data stream
 # $ find .hg/store/ -name "*.i" | xargs dumprevlog > repo.dump
 
+from __future__ import absolute_import, print_function
+
 import sys
-from mercurial import revlog, node, util
+from mercurial import (
+    node,
+    revlog,
+    util,
+)
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     util.setbinary(fp)
@@ -11,15 +17,15 b' for fp in (sys.stdin, sys.stdout, sys.st'
 for f in sys.argv[1:]:
     binopen = lambda fn: open(fn, 'rb')
     r = revlog.revlog(binopen, f)
-    print "file:", f
+    print("file:", f)
     for i in r:
         n = r.node(i)
         p = r.parents(n)
         d = r.revision(n)
-        print "node:", node.hex(n)
-        print "linkrev:", r.linkrev(i)
-        print "parents:", node.hex(p[0]), node.hex(p[1])
-        print "length:", len(d)
-        print "-start-"
-        print d
-        print "-end-"
+        print("node:", node.hex(n))
+        print("linkrev:", r.linkrev(i))
+        print("parents:", node.hex(p[0]), node.hex(p[1]))
+        print("length:", len(d))
+        print("-start-")
+        print(d)
+        print("-end-")

contrib/import-checker.py

@@ -11,8 +11,9 b' import sys'
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
-import BaseHTTPServer
-import zlib
+if True: # disable lexical sorting checks
+    import BaseHTTPServer
+    import zlib
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
@@ -126,22 +127,32 b' def fromlocalfunc(modulename, localmods)'
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
+    >>> fromlocal('foo1', 1)
+    ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
+    >>> fromlocal2('bar2', 1)
+    False
+    >>> fromlocal2('bar', 2)
+    ('foo.bar', 'foo.bar.__init__', True)
     """
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
     def fromlocal(name, level=0):
-        # name is None when relative imports are used.
-        if name is None:
+        # name is false value when relative imports are used.
+        if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
-            # Check relative name first.
-            candidates = [prefix + name, name]
+            if not level:
+                # Check relative name first.
+                candidates = [prefix + name, name]
+            else:
+                candidates = ['.'.join(modulename.split('.')[:-level]) +
+                              '.' + name]
 
         for n in candidates:
             if n in localmods:
@@ -175,6 +186,9 b' def list_stdlib_modules():'
 
     >>> 'cStringIO' in mods
     True
+
+    >>> 'cffi' in mods
+    True
     """
     for m in sys.builtin_module_names:
         yield m
@@ -187,6 +201,8 b' def list_stdlib_modules():'
         yield m
     for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
         yield m
+    for m in ['cffi']:
+        yield m
     stdlib_prefixes = set([sys.prefix, sys.exec_prefix])
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
@@ -360,7 +376,7 b' def verify_modern_convention(module, roo'
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
-      must be performed before other relative imports. This rule only
+      must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed.
     * Certain modules must be aliased to alternate names to avoid aliasing
@@ -371,8 +387,8 b' def verify_modern_convention(module, roo'
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
-    # Whether a relative, non-symbol import has been seen.
-    seennonsymbolrelative = False
+    # Whether a local/non-stdlib, non-symbol import has been seen.
+    seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     # Relative import levels encountered so far.
@@ -446,26 +462,26 b' def verify_modern_convention(module, roo'
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
+            found = fromlocal(node.module, node.level)
+            if found and found[2]:  # node.module is a package
+                prefix = found[0] + '.'
+                symbols = [n.name for n in node.names
+                           if not fromlocal(prefix + n.name)]
+            else:
+                symbols = [n.name for n in node.names]
             if node.module and node.col_offset == root_col_offset:
-                found = fromlocal(node.module, node.level)
-                if found and found[2]:  # node.module is a package
-                    prefix = found[0] + '.'
-                    symbols = [n.name for n in node.names
-                               if not fromlocal(prefix + n.name)]
-                else:
-                    symbols = [n.name for n in node.names]
-
                 if symbols and fullname not in allowsymbolimports:
                     yield msg('direct symbol import %s from %s',
                               ', '.join(symbols), fullname)
 
-                if symbols and seennonsymbolrelative:
+                if symbols and seennonsymbollocal:
                     yield msg('symbol import follows non-symbol import: %s',
                               fullname)
+            if not symbols and fullname not in stdlib_modules:
+                seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
-                seennonsymbolrelative = True
 
                 # Only allow 1 group per level.
                 if (node.level in seenlevels
@@ -652,7 +668,7 b' def sources(f, modname):'
     the input file.
     """
     py = False
-    if f.endswith('.py'):
+    if not f.endswith('.t'):
         with open(f) as src:
             yield src.read(), modname, f, 0
             py = True

contrib/macosx/Welcome.html

@@ -1,5 +1,5 b''
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
-<!-- This is the second screen displayed during the install. -->
+<!-- This is the first screen displayed during the install. -->
 <html>
 <head>
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">

contrib/perf.py

@@ -1,6 +1,23 b''
 # perf.py - performance test routines
 '''helper extension to measure performance'''
 
+# "historical portability" policy of perf.py:
+#
+# We have to do:
+# - make perf.py "loadable" with as wide Mercurial version as possible
+#   This doesn't mean that perf commands work correctly with that Mercurial.
+#   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
+# - make historical perf command work correctly with as wide Mercurial
+#   version as possible
+#
+# We have to do, if possible with reasonable cost:
+# - make recent perf command for historical feature work correctly
+#   with early Mercurial
+#
+# We don't have to do:
+# - make perf command for recent feature work correctly with early
+#   Mercurial
+
 from __future__ import absolute_import
 import functools
 import os
@@ -8,25 +25,97 b' import random'
 import sys
 import time
 from mercurial import (
-    branchmap,
     cmdutil,
     commands,
     copies,
     error,
+    extensions,
     mdiff,
     merge,
-    obsolete,
-    repoview,
     revlog,
-    scmutil,
     util,
 )
 
-formatteropts = commands.formatteropts
-revlogopts = commands.debugrevlogopts
+# for "historical portability":
+# try to import modules separately (in dict order), and ignore
+# failure, because these aren't available with early Mercurial
+try:
+    from mercurial import branchmap # since 2.5 (or bcee63733aad)
+except ImportError:
+    pass
+try:
+    from mercurial import obsolete # since 2.3 (or ad0d6c2b3279)
+except ImportError:
+    pass
+try:
+    from mercurial import repoview # since 2.5 (or 3a6ddacb7198)
+except ImportError:
+    pass
+try:
+    from mercurial import scmutil # since 1.9 (or 8b252e826c68)
+except ImportError:
+    pass
+
+# for "historical portability":
+# define util.safehasattr forcibly, because util.safehasattr has been
+# available since 1.9.3 (or 94b200a11cf7)
+_undefined = object()
+def safehasattr(thing, attr):
+    return getattr(thing, attr, _undefined) is not _undefined
+setattr(util, 'safehasattr', safehasattr)
+
+# for "historical portability":
+# use locally defined empty option list, if formatteropts isn't
+# available, because commands.formatteropts has been available since
+# 3.2 (or 7a7eed5176a4), even though formatting itself has been
+# available since 2.2 (or ae5f92e154d3)
+formatteropts = getattr(commands, "formatteropts", [])
+
+# for "historical portability":
+# use locally defined option list, if debugrevlogopts isn't available,
+# because commands.debugrevlogopts has been available since 3.7 (or
+# 5606f7d0d063), even though cmdutil.openrevlog() has been available
+# since 1.9 (or a79fea6b3e77).
+revlogopts = getattr(commands, "debugrevlogopts", [
+        ('c', 'changelog', False, ('open changelog')),
+        ('m', 'manifest', False, ('open manifest')),
+        ('', 'dir', False, ('open directory manifest')),
+        ])
 
 cmdtable = {}
-command = cmdutil.command(cmdtable)
+
+# for "historical portability":
+# define parsealiases locally, because cmdutil.parsealiases has been
+# available since 1.5 (or 6252852b4332)
+def parsealiases(cmd):
+    return cmd.lstrip("^").split("|")
+
+if safehasattr(cmdutil, 'command'):
+    import inspect
+    command = cmdutil.command(cmdtable)
+    if 'norepo' not in inspect.getargspec(command)[0]:
+        # for "historical portability":
+        # wrap original cmdutil.command, because "norepo" option has
+        # been available since 3.1 (or 75a96326cecb)
+        _command = command
+        def command(name, options=(), synopsis=None, norepo=False):
+            if norepo:
+                commands.norepo += ' %s' % ' '.join(parsealiases(name))
+            return _command(name, list(options), synopsis)
+else:
+    # for "historical portability":
+    # define "@command" annotation locally, because cmdutil.command
+    # has been available since 1.9 (or 2daa5179e73f)
+    def command(name, options=(), synopsis=None, norepo=False):
+        def decorator(func):
+            if synopsis:
+                cmdtable[name] = func, list(options), synopsis
+            else:
+                cmdtable[name] = func, list(options)
+            if norepo:
+                commands.norepo += ' %s' % ' '.join(parsealiases(name))
+            return func
+        return decorator
 
 def getlen(ui):
     if ui.configbool("perf", "stub"):
@@ -796,3 +885,18 b' def perflrucache(ui, size=4, gets=10000,'
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
+
+def uisetup(ui):
+    if (util.safehasattr(cmdutil, 'openrevlog') and
+        not util.safehasattr(commands, 'debugrevlogopts')):
+        # for "historical portability":
+        # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
+        # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
+        # openrevlog() should cause failure, because it has been
+        # available since 3.5 (or 49c583ca48c4).
+        def openrevlog(orig, repo, cmd, file_, opts):
+            if opts.get('dir') and not util.safehasattr(repo, 'dirlog'):
+                raise error.Abort("This version doesn't support --dir option",
+                                  hint="use 3.5 or later")
+            return orig(repo, cmd, file_, opts)
+        extensions.wrapfunction(cmdutil, 'openrevlog', openrevlog)

contrib/revsetbenchmarks.py

@@ -10,41 +10,32 b''
 
 from __future__ import absolute_import, print_function
 import math
+import optparse  # cannot use argparse, python 2.7 only
 import os
 import re
+import subprocess
 import sys
-from subprocess import (
-    CalledProcessError,
-    check_call,
-    PIPE,
-    Popen,
-    STDOUT,
-)
-# cannot use argparse, python 2.7 only
-from optparse import (
-    OptionParser,
-)
 
 DEFAULTVARIANTS = ['plain', 'min', 'max', 'first', 'last',
                    'reverse', 'reverse+first', 'reverse+last',
                    'sort', 'sort+first', 'sort+last']
 
 def check_output(*args, **kwargs):
-    kwargs.setdefault('stderr', PIPE)
-    kwargs.setdefault('stdout', PIPE)
-    proc = Popen(*args, **kwargs)
+    kwargs.setdefault('stderr', subprocess.PIPE)
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    proc = subprocess.Popen(*args, **kwargs)
     output, error = proc.communicate()
     if proc.returncode != 0:
-        raise CalledProcessError(proc.returncode, ' '.join(args[0]))
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(args[0]))
     return output
 
 def update(rev):
     """update the repo to a revision"""
     try:
-        check_call(['hg', 'update', '--quiet', '--check', str(rev)])
+        subprocess.check_call(['hg', 'update', '--quiet', '--check', str(rev)])
         check_output(['make', 'local'],
                      stderr=None)  # suppress output except for error/warning
-    except CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         print('update to revision %s failed, aborting'%rev, file=sys.stderr)
         sys.exit(exc.returncode)
 
@@ -60,7 +51,7 b' def hg(cmd, repo=None):'
     fullcmd += ['--config',
                 'extensions.perf=' + os.path.join(contribdir, 'perf.py')]
     fullcmd += cmd
-    return check_output(fullcmd, stderr=STDOUT)
+    return check_output(fullcmd, stderr=subprocess.STDOUT)
 
 def perf(revset, target=None, contexts=False):
     """run benchmark for this very revset"""
@@ -70,7 +61,7 b' def perf(revset, target=None, contexts=F'
             args.append('--contexts')
         output = hg(args, repo=target)
         return parseoutput(output)
-    except CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         print('abort: cannot run revset benchmark: %s'%exc.cmd, file=sys.stderr)
         if getattr(exc, 'output', None) is None: # no output before 2.7
             print('(no output)', file=sys.stderr)
@@ -103,9 +94,9 b' def printrevision(rev):'
     """print data about a revision"""
     sys.stdout.write("Revision ")
     sys.stdout.flush()
-    check_call(['hg', 'log', '--rev', str(rev), '--template',
-                '{if(tags, " ({tags})")} '
-                '{rev}:{node|short}: {desc|firstline}\n'])
+    subprocess.check_call(['hg', 'log', '--rev', str(rev), '--template',
+                           '{if(tags, " ({tags})")} '
+                           '{rev}:{node|short}: {desc|firstline}\n'])
 
 def idxwidth(nbidx):
     """return the max width of number used for index
@@ -215,7 +206,7 b' def getrevs(spec):'
     """get the list of rev matched by a revset"""
     try:
         out = check_output(['hg', 'log', '--template={rev}\n', '--rev', spec])
-    except CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         print("abort, can't get revision from %s"%spec, file=sys.stderr)
         sys.exit(exc.returncode)
     return [r for r in out.split() if r]
@@ -234,8 +225,8 b' summary output is provided. Use it to de'
 point regressions. Revsets to run are specified in a file (or from stdin), one
 revsets per line. Line starting with '#' will be ignored, allowing insertion of
 comments."""
-parser = OptionParser(usage="usage: %prog [options] <revs>",
-                      description=helptext)
+parser = optparse.OptionParser(usage="usage: %prog [options] <revs>",
+                               description=helptext)
 parser.add_option("-f", "--file",
                   help="read revset from FILE (stdin if omitted)",
                   metavar="FILE")

contrib/synthrepo.py

@@ -45,6 +45,13 b' import os'
 import random
 import sys
 import time
+
+from mercurial.i18n import _
+from mercurial.node import (
+    nullid,
+    nullrev,
+    short,
+)
 from mercurial import (
     cmdutil,
     context,
@@ -54,12 +61,6 b' from mercurial import ('
     scmutil,
     util,
 )
-from mercurial.i18n import _
-from mercurial.node import (
-    nullid,
-    nullrev,
-    short,
-)
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
@@ -506,7 +507,7 b' def renamedirs(dirs, words):'
             head = rename(head)
         else:
             head = ''
-        renamed = os.path.join(head, wordgen.next())
+        renamed = os.path.join(head, next(wordgen))
         replacements[dirpath] = renamed
         return renamed
     result = []

contrib/undumprevlog

@@ -3,8 +3,16 b''
 # $ hg init
 # $ undumprevlog < repo.dump
 
+from __future__ import absolute_import
+
 import sys
-from mercurial import revlog, node, scmutil, util, transaction
+from mercurial import (
+    node,
+    revlog,
+    scmutil,
+    transaction,
+    util,
+)
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     util.setbinary(fp)

contrib/win32/hgwebdir_wsgi.py

@@ -79,6 +79,8 b''
 # - Restart the web server and see if things are running.
 #
 
+from __future__ import absolute_import
+
 # Configuration file location
 hgweb_config = r'c:\your\directory\wsgi.config'
 
@@ -87,7 +89,6 b' path_strip = 0 # Strip this many path '
 path_prefix = 1  # This many path elements are prefixes (depends on the
                  # virtual path of the IIS application).
 
-from __future__ import absolute_import
 import sys
 
 # Adjust python path if this is not a system-wide install

contrib/win32/mercurial.ini

@@ -46,7 +46,6 b' editor = notepad'
 ;extdiff =
 ;fetch =
 ;gpg =
-;hgcia =
 ;hgk =
 ;highlight = 
 ;histedit =

doc/docchecker

@@ -6,8 +6,11 b''
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import, print_function
+
+import re
 import sys
-import re
 
 leadingline = re.compile(r'(^\s*)(\S.*)$')
 

doc/gendoc.py

@@ -117,11 +117,11 b' def showdoc(ui):'
     ui.write(_("This section contains help for extensions that are "
                "distributed together with Mercurial. Help for other "
                "extensions is available in the help system."))
-    ui.write("\n\n"
+    ui.write(("\n\n"
              ".. contents::\n"
              "   :class: htmlonly\n"
              "   :local:\n"
-             "   :depth: 1\n\n")
+             "   :depth: 1\n\n"))
 
     for extensionname in sorted(allextensionnames()):
         mod = extensions.load(ui, extensionname, None)

doc/hgmanpage.py

@@ -415,7 +415,7 b' class Translator(nodes.NodeVisitor):'
         else:
             self._docinfo[name] = node.astext()
         self._docinfo_keys.append(name)
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def depart_docinfo_item(self, node):
         pass
@@ -469,7 +469,7 b' class Translator(nodes.NodeVisitor):'
 
     def visit_citation_reference(self, node):
         self.body.append('['+node.astext()+']')
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_classifier(self, node):
         pass
@@ -489,7 +489,7 b' class Translator(nodes.NodeVisitor):'
     def visit_comment(self, node,
                       sub=re.compile('-(?=-)').sub):
         self.body.append(self.comment(node.astext()))
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_contact(self, node):
         self.visit_docinfo_item(node, 'contact')
@@ -643,7 +643,7 b' class Translator(nodes.NodeVisitor):'
             name_normalized = self._field_name.lower().replace(" ","_")
             self._docinfo_names[name_normalized] = self._field_name
             self.visit_docinfo_item(node, name_normalized)
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
 
     def depart_field_body(self, node):
         pass
@@ -657,7 +657,7 b' class Translator(nodes.NodeVisitor):'
     def visit_field_name(self, node):
         if self._in_docinfo:
             self._field_name = node.astext()
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
         else:
             self.body.append(self.defs['field_name'][0])
 
@@ -693,7 +693,7 b' class Translator(nodes.NodeVisitor):'
 
     def visit_footnote_reference(self, node):
         self.body.append('['+self.deunicode(node.astext())+']')
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def depart_footnote_reference(self, node):
         pass
@@ -705,7 +705,7 b' class Translator(nodes.NodeVisitor):'
         pass
 
     def visit_header(self, node):
-        raise NotImplementedError, node.astext()
+        raise NotImplementedError(node.astext())
 
     def depart_header(self, node):
         pass
@@ -742,7 +742,7 b' class Translator(nodes.NodeVisitor):'
         if 'uri' in node.attributes:
             text.append(node.attributes['uri'])
         self.body.append('[image: %s]\n' % ('/'.join(text)))
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_important(self, node):
         self.visit_admonition(node, 'important')
@@ -753,7 +753,7 b' class Translator(nodes.NodeVisitor):'
         # footnote and citation
         if (isinstance(node.parent, nodes.footnote)
             or isinstance(node.parent, nodes.citation)):
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
         self.document.reporter.warning('"unsupported "label"',
                 base_node=node)
         self.body.append('[')
@@ -793,7 +793,7 b' class Translator(nodes.NodeVisitor):'
     def visit_list_item(self, node):
         # man 7 man argues to use ".IP" instead of ".TP"
         self.body.append('.IP %s %d\n' % (
-                self._list_char[-1].next(),
+                next(self._list_char[-1]),
                 self._list_char[-1].get_width(),))
 
     def depart_list_item(self, node):
@@ -814,7 +814,7 b' class Translator(nodes.NodeVisitor):'
         self.body.append(self.defs['literal_block'][1])
 
     def visit_meta(self, node):
-        raise NotImplementedError, node.astext()
+        raise NotImplementedError(node.astext())
 
     def depart_meta(self, node):
         pass
@@ -924,7 +924,7 b' class Translator(nodes.NodeVisitor):'
         if node.get('format') == 'manpage':
             self.body.append(node.astext() + "\n")
         # Keep non-manpage raw text out of output:
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_reference(self, node):
         """E.g. link or email address."""
@@ -963,7 +963,7 b' class Translator(nodes.NodeVisitor):'
 
     def visit_substitution_definition(self, node):
         """Internal only."""
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_substitution_reference(self, node):
         self.document.reporter.warning('"substitution_reference" not supported',
@@ -1009,7 +1009,7 b' class Translator(nodes.NodeVisitor):'
 
     def visit_target(self, node):
         # targets are in-document hyper targets, without any use for man-pages.
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_tbody(self, node):
         pass
@@ -1053,7 +1053,7 b' class Translator(nodes.NodeVisitor):'
             self._docinfo['title'] = node.astext()
             # document title for .TH
             self._docinfo['title_upper'] = node.astext().upper()
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
         elif self.section_level == 1:
             self.body.append('.SH ')
             for n in node.traverse(nodes.Text):

hg

@@ -11,9 +11,11 b' import os'
 import sys
 
 if os.environ.get('HGUNICODEPEDANTRY', False):
-    reload(sys)
-    sys.setdefaultencoding("undefined")
-
+    try:
+        reload(sys)
+        sys.setdefaultencoding("undefined")
+    except NameError:
+        pass
 
 libdir = '@LIBDIR@'
 
@@ -26,9 +28,9 b" if libdir != '@' 'LIBDIR' '@':"
 
 # enable importing on demand to reduce startup time
 try:
-    from mercurial import demandimport; demandimport.enable()
+    if sys.version_info[0] < 3:
+        from mercurial import demandimport; demandimport.enable()
 except ImportError:
-    import sys
     sys.stderr.write("abort: couldn't find mercurial libraries in [%s]\n" %
                      ' '.join(sys.path))
     sys.stderr.write("(check your install and PYTHONPATH)\n")

hgext/automv.py

@@ -26,6 +26,7 b' The threshold at which a file is conside'
 
 from __future__ import absolute_import
 
+from mercurial.i18n import _
 from mercurial import (
     commands,
     copies,
@@ -34,7 +35,6 b' from mercurial import ('
     scmutil,
     similar
 )
-from mercurial.i18n import _
 
 def extsetup(ui):
     entry = extensions.wrapcommand(

hgext/bugzilla.py

@@ -281,8 +281,6 b' from __future__ import absolute_import'
 
 import re
 import time
-import urlparse
-import xmlrpclib
 
 from mercurial.i18n import _
 from mercurial.node import short
@@ -293,6 +291,9 b' from mercurial import ('
     util,
 )
 
+urlparse = util.urlparse
+xmlrpclib = util.xmlrpclib
+
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or

hgext/chgserver.py

@@ -40,18 +40,15 b' Config'
 
 from __future__ import absolute_import
 
-import SocketServer
 import errno
-import gc
+import hashlib
 import inspect
 import os
-import random
 import re
+import signal
 import struct
 import sys
-import threading
 import time
-import traceback
 
 from mercurial.i18n import _
 
@@ -76,10 +73,11 b" testedwith = 'internal'"
 
 def _hashlist(items):
     """return sha1 hexdigest for a list"""
-    return util.sha1(str(items)).hexdigest()
+    return hashlib.sha1(str(items)).hexdigest()
 
 # sensitive config sections affecting confighash
 _configsections = [
+    'alias',  # affects global state commands.table
     'extdiff',  # uisetup will register new commands
     'extensions',
 ]
@@ -150,6 +148,10 b' def _mtimehash(paths):'
 
     for chgserver, it is designed that once mtimehash changes, the server is
     considered outdated immediately and should no longer provide service.
+
+    mtimehash is not included in confighash because we only know the paths of
+    extensions after importing them (there is imp.find_module but that faces
+    race conditions). We need to calculate confighash without importing.
     """
     def trystat(path):
         try:
@@ -213,18 +215,6 b' def _setuppagercmd(ui, options, cmd):'
         ui.setconfig('ui', 'interactive', False, 'pager')
         return p
 
-_envvarre = re.compile(r'\$[a-zA-Z_]+')
-
-def _clearenvaliases(cmdtable):
-    """Remove stale command aliases referencing env vars; variable expansion
-    is done at dispatch.addaliases()"""
-    for name, tab in cmdtable.items():
-        cmddef = tab[0]
-        if (isinstance(cmddef, dispatch.cmdalias) and
-            not cmddef.definition.startswith('!') and  # shell alias
-            _envvarre.search(cmddef.definition)):
-            del cmdtable[name]
-
 def _newchgui(srcui, csystem):
     class chgui(srcui.__class__):
         def __init__(self, src=None):
@@ -357,6 +347,7 b' class chgcmdserver(commandserver.server)'
             self.capabilities['validate'] = chgcmdserver.validate
 
     def cleanup(self):
+        super(chgcmdserver, self).cleanup()
         # dispatch._runcatch() does not flush outputs if exception is not
         # handled by dispatch._dispatch()
         self.ui.flush()
@@ -508,6 +499,11 b' class chgcmdserver(commandserver.server)'
 
         pagercmd = _setuppagercmd(self.ui, options, cmd)
         if pagercmd:
+            # Python's SIGPIPE is SIG_IGN by default. change to SIG_DFL so
+            # we can exit if the pipe to the pager is closed
+            if util.safehasattr(signal, 'SIGPIPE') and \
+                    signal.getsignal(signal.SIGPIPE) == signal.SIG_IGN:
+                signal.signal(signal.SIGPIPE, signal.SIG_DFL)
             self.cresult.write(pagercmd)
         else:
             self.cresult.write('\0')
@@ -525,7 +521,6 b' class chgcmdserver(commandserver.server)'
         _log('setenv: %r\n' % sorted(newenv.keys()))
         os.environ.clear()
         os.environ.update(newenv)
-        _clearenvaliases(commands.table)
 
     capabilities = commandserver.server.capabilities.copy()
     capabilities.update({'attachio': attachio,
@@ -534,174 +529,110 b' class chgcmdserver(commandserver.server)'
                          'setenv': setenv,
                          'setumask': setumask})
 
-# copied from mercurial/commandserver.py
-class _requesthandler(SocketServer.StreamRequestHandler):
-    def handle(self):
-        # use a different process group from the master process, making this
-        # process pass kernel "is_current_pgrp_orphaned" check so signals like
-        # SIGTSTP, SIGTTIN, SIGTTOU are not ignored.
-        os.setpgid(0, 0)
-        # change random state otherwise forked request handlers would have a
-        # same state inherited from parent.
-        random.seed()
-        ui = self.server.ui
-        repo = self.server.repo
-        sv = None
-        try:
-            sv = chgcmdserver(ui, repo, self.rfile, self.wfile, self.connection,
-                              self.server.hashstate, self.server.baseaddress)
-            try:
-                sv.serve()
-            # handle exceptions that may be raised by command server. most of
-            # known exceptions are caught by dispatch.
-            except error.Abort as inst:
-                ui.warn(_('abort: %s\n') % inst)
-            except IOError as inst:
-                if inst.errno != errno.EPIPE:
-                    raise
-            except KeyboardInterrupt:
-                pass
-            finally:
-                sv.cleanup()
-        except: # re-raises
-            # also write traceback to error channel. otherwise client cannot
-            # see it because it is written to server's stderr by default.
-            if sv:
-                cerr = sv.cerr
-            else:
-                cerr = commandserver.channeledoutput(self.wfile, 'e')
-            traceback.print_exc(file=cerr)
-            raise
-        finally:
-            # trigger __del__ since ForkingMixIn uses os._exit
-            gc.collect()
-
 def _tempaddress(address):
     return '%s.%d.tmp' % (address, os.getpid())
 
 def _hashaddress(address, hashstr):
     return '%s-%s' % (address, hashstr)
 
-class AutoExitMixIn:  # use old-style to comply with SocketServer design
-    lastactive = time.time()
-    idletimeout = 3600  # default 1 hour
+class chgunixservicehandler(object):
+    """Set of operations for chg services"""
+
+    pollinterval = 1  # [sec]
 
-    def startautoexitthread(self):
-        # note: the auto-exit check here is cheap enough to not use a thread,
-        # be done in serve_forever. however SocketServer is hook-unfriendly,
-        # you simply cannot hook serve_forever without copying a lot of code.
-        # besides, serve_forever's docstring suggests using thread.
-        thread = threading.Thread(target=self._autoexitloop)
-        thread.daemon = True
-        thread.start()
+    def __init__(self, ui):
+        self.ui = ui
+        self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600)
+        self._lastactive = time.time()
+
+    def bindsocket(self, sock, address):
+        self._inithashstate(address)
+        self._checkextensions()
+        self._bind(sock)
+        self._createsymlink()
 
-    def _autoexitloop(self, interval=1):
-        while True:
-            time.sleep(interval)
-            if not self.issocketowner():
-                _log('%s is not owned, exiting.\n' % self.server_address)
-                break
-            if time.time() - self.lastactive > self.idletimeout:
-                _log('being idle too long. exiting.\n')
-                break
-        self.shutdown()
+    def _inithashstate(self, address):
+        self._baseaddress = address
+        if self.ui.configbool('chgserver', 'skiphash', False):
+            self._hashstate = None
+            self._realaddress = address
+            return
+        self._hashstate = hashstate.fromui(self.ui)
+        self._realaddress = _hashaddress(address, self._hashstate.confighash)
 
-    def process_request(self, request, address):
-        self.lastactive = time.time()
-        return SocketServer.ForkingMixIn.process_request(
-            self, request, address)
+    def _checkextensions(self):
+        if not self._hashstate:
+            return
+        if extensions.notloaded():
+            # one or more extensions failed to load. mtimehash becomes
+            # meaningless because we do not know the paths of those extensions.
+            # set mtimehash to an illegal hash value to invalidate the server.
+            self._hashstate.mtimehash = ''
 
-    def server_bind(self):
+    def _bind(self, sock):
         # use a unique temp address so we can stat the file and do ownership
         # check later
-        tempaddress = _tempaddress(self.server_address)
-        # use relative path instead of full path at bind() if possible, since
-        # AF_UNIX path has very small length limit (107 chars) on common
-        # platforms (see sys/un.h)
-        dirname, basename = os.path.split(tempaddress)
-        bakwdfd = None
-        if dirname:
-            bakwdfd = os.open('.', os.O_DIRECTORY)
-            os.chdir(dirname)
-        self.socket.bind(basename)
-        self._socketstat = os.stat(basename)
+        tempaddress = _tempaddress(self._realaddress)
+        util.bindunixsocket(sock, tempaddress)
+        self._socketstat = os.stat(tempaddress)
         # rename will replace the old socket file if exists atomically. the
         # old server will detect ownership change and exit.
-        util.rename(basename, self.server_address)
-        if bakwdfd:
-            os.fchdir(bakwdfd)
-            os.close(bakwdfd)
+        util.rename(tempaddress, self._realaddress)
 
-    def issocketowner(self):
+    def _createsymlink(self):
+        if self._baseaddress == self._realaddress:
+            return
+        tempaddress = _tempaddress(self._baseaddress)
+        os.symlink(os.path.basename(self._realaddress), tempaddress)
+        util.rename(tempaddress, self._baseaddress)
+
+    def _issocketowner(self):
         try:
-            stat = os.stat(self.server_address)
+            stat = os.stat(self._realaddress)
             return (stat.st_ino == self._socketstat.st_ino and
                     stat.st_mtime == self._socketstat.st_mtime)
         except OSError:
             return False
 
-    def unlinksocketfile(self):
-        if not self.issocketowner():
+    def unlinksocket(self, address):
+        if not self._issocketowner():
             return
         # it is possible to have a race condition here that we may
         # remove another server's socket file. but that's okay
         # since that server will detect and exit automatically and
         # the client will start a new server on demand.
         try:
-            os.unlink(self.server_address)
+            os.unlink(self._realaddress)
         except OSError as exc:
             if exc.errno != errno.ENOENT:
                 raise
 
-class chgunixservice(commandserver.unixservice):
-    def init(self):
-        if self.repo:
-            # one chgserver can serve multiple repos. drop repo infomation
-            self.ui.setconfig('bundle', 'mainreporoot', '', 'repo')
-            self.repo = None
-        self._inithashstate()
-        self._checkextensions()
-        class cls(AutoExitMixIn, SocketServer.ForkingMixIn,
-                  SocketServer.UnixStreamServer):
-            ui = self.ui
-            repo = self.repo
-            hashstate = self.hashstate
-            baseaddress = self.baseaddress
-        self.server = cls(self.address, _requesthandler)
-        self.server.idletimeout = self.ui.configint(
-            'chgserver', 'idletimeout', self.server.idletimeout)
-        self.server.startautoexitthread()
-        self._createsymlink()
+    def printbanner(self, address):
+        # no "listening at" message should be printed to simulate hg behavior
+        pass
+
+    def shouldexit(self):
+        if not self._issocketowner():
+            self.ui.debug('%s is not owned, exiting.\n' % self._realaddress)
+            return True
+        if time.time() - self._lastactive > self._idletimeout:
+            self.ui.debug('being idle too long. exiting.\n')
+            return True
+        return False
 
-    def _inithashstate(self):
-        self.baseaddress = self.address
-        if self.ui.configbool('chgserver', 'skiphash', False):
-            self.hashstate = None
-            return
-        self.hashstate = hashstate.fromui(self.ui)
-        self.address = _hashaddress(self.address, self.hashstate.confighash)
+    def newconnection(self):
+        self._lastactive = time.time()
+
+    def createcmdserver(self, repo, conn, fin, fout):
+        return chgcmdserver(self.ui, repo, fin, fout, conn,
+                            self._hashstate, self._baseaddress)
 
-    def _checkextensions(self):
-        if not self.hashstate:
-            return
-        if extensions.notloaded():
-            # one or more extensions failed to load. mtimehash becomes
-            # meaningless because we do not know the paths of those extensions.
-            # set mtimehash to an illegal hash value to invalidate the server.
-            self.hashstate.mtimehash = ''
-
-    def _createsymlink(self):
-        if self.baseaddress == self.address:
-            return
-        tempaddress = _tempaddress(self.baseaddress)
-        os.symlink(os.path.basename(self.address), tempaddress)
-        util.rename(tempaddress, self.baseaddress)
-
-    def run(self):
-        try:
-            self.server.serve_forever()
-        finally:
-            self.server.unlinksocketfile()
+def chgunixservice(ui, repo, opts):
+    if repo:
+        # one chgserver can serve multiple repos. drop repo infomation
+        ui.setconfig('bundle', 'mainreporoot', '', 'repo')
+    h = chgunixservicehandler(ui)
+    return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h)
 
 def uisetup(ui):
     commandserver._servicemap['chgunix'] = chgunixservice

hgext/color.py

@@ -156,6 +156,8 b' If ``pagermode`` is not defined, the ``m'
 from __future__ import absolute_import
 
 import os
+
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
@@ -165,7 +167,6 b' from mercurial import ('
     ui as uimod,
     util,
 )
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)

hgext/convert/__init__.py

@@ -9,11 +9,11 b''
 
 from __future__ import absolute_import
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     registrar,
 )
-from mercurial.i18n import _
 
 from . import (
     convcmd,

hgext/convert/bzr.py

@@ -10,11 +10,12 b''
 from __future__ import absolute_import
 
 import os
+
+from mercurial.i18n import _
 from mercurial import (
     demandimport,
     error
 )
-from mercurial.i18n import _
 from . import common
 
 # these do not work with demandimport, blacklist

hgext/convert/common.py

@@ -7,20 +7,20 b''
 from __future__ import absolute_import
 
 import base64
-import cPickle as pickle
 import datetime
 import errno
 import os
 import re
 import subprocess
 
+from mercurial.i18n import _
 from mercurial import (
     error,
     phases,
     util,
 )
-from mercurial.i18n import _
 
+pickle = util.pickle
 propertycache = util.propertycache
 
 def encodeargs(args):

hgext/convert/convcmd.py

@@ -10,13 +10,13 b' import os'
 import shlex
 import shutil
 
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     hg,
     util,
 )
-from mercurial.i18n import _
 
 from . import (
     bzr,

hgext/convert/cvs.py

@@ -11,12 +11,12 b' import os'
 import re
 import socket
 
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     util,
 )
-from mercurial.i18n import _
 
 from . import (
     common,

hgext/convert/cvsps.py

@@ -6,15 +6,16 b''
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
-import cPickle as pickle
 import os
 import re
 
+from mercurial.i18n import _
 from mercurial import (
     hook,
     util,
 )
-from mercurial.i18n import _
+
+pickle = util.pickle
 
 class logentry(object):
     '''Class logentry has the following attributes:

hgext/convert/filemap.py

@@ -7,10 +7,11 b' from __future__ import absolute_import'
 
 import posixpath
 import shlex
+
+from mercurial.i18n import _
 from mercurial import (
     error,
 )
-from mercurial.i18n import _
 from . import common
 SKIPREV = common.SKIPREV
 

hgext/convert/git.py

@@ -7,12 +7,13 b''
 from __future__ import absolute_import
 
 import os
+
+from mercurial.i18n import _
 from mercurial import (
     config,
     error,
     node as nodemod,
 )
-from mercurial.i18n import _
 
 from . import (
     common,

hgext/convert/gnuarch.py

@@ -12,12 +12,13 b' import os'
 import shutil
 import stat
 import tempfile
+
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     util,
 )
-from mercurial.i18n import _
 from . import common
 
 class gnuarch_source(common.converter_source, common.commandline):

hgext/convert/hg.py

@@ -22,6 +22,7 b' import os'
 import re
 import time
 
+from mercurial.i18n import _
 from mercurial import (
     bookmarks,
     context,
@@ -37,7 +38,6 b' from mercurial import ('
 )
 stringio = util.stringio
 
-from mercurial.i18n import _
 from . import common
 mapfile = common.mapfile
 NoRepo = common.NoRepo

hgext/convert/monotone.py

@@ -10,11 +10,11 b' from __future__ import absolute_import'
 import os
 import re
 
+from mercurial.i18n import _
 from mercurial import (
     error,
     util,
 )
-from mercurial.i18n import _
 
 from . import common
 

hgext/convert/p4.py

@@ -9,11 +9,11 b' from __future__ import absolute_import'
 import marshal
 import re
 
+from mercurial.i18n import _
 from mercurial import (
     error,
     util,
 )
-from mercurial.i18n import _
 
 from . import common
 

hgext/convert/subversion.py

@@ -3,13 +3,13 b''
 # Copyright(C) 2007 Daniel Holth et al
 from __future__ import absolute_import
 
-import cPickle as pickle
 import os
 import re
 import sys
 import tempfile
 import xml.dom.minidom
 
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
@@ -17,10 +17,10 b' from mercurial import ('
     strutil,
     util,
 )
-from mercurial.i18n import _
 
 from . import common
 
+pickle = util.pickle
 stringio = util.stringio
 propertycache = util.propertycache
 urlerr = util.urlerr

hgext/factotum.py

@@ -102,8 +102,7 b' def monkeypatch_method(cls):'
 
 @monkeypatch_method(passwordmgr)
 def find_user_password(self, realm, authuri):
-    user, passwd = urlreq.httppasswordmgrwithdefaultrealm.find_user_password(
-        self, realm, authuri)
+    user, passwd = self.passwddb.find_user_password(realm, authuri)
     if user and passwd:
         self._writedebug(user, passwd)
         return (user, passwd)

hgext/fetch.py

@@ -7,12 +7,23 b''
 
 '''pull, update and merge in one command (DEPRECATED)'''
 
+from __future__ import absolute_import
+
 from mercurial.i18n import _
-from mercurial.node import short
-from mercurial import commands, cmdutil, hg, util, error
-from mercurial.lock import release
-from mercurial import exchange
+from mercurial.node import (
+    short,
+)
+from mercurial import (
+    cmdutil,
+    commands,
+    error,
+    exchange,
+    hg,
+    lock,
+    util,
+)
 
+release = lock.release
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'internal' for

hgext/fsmonitor/__init__.py

@@ -91,10 +91,12 b' will disable itself if any of those are '
 
 from __future__ import absolute_import
 
+import hashlib
 import os
 import stat
 import sys
 
+from mercurial.i18n import _
 from mercurial import (
     context,
     extensions,
@@ -105,7 +107,6 b' from mercurial import ('
     util,
 )
 from mercurial import match as matchmod
-from mercurial.i18n import _
 
 from . import (
     state,
@@ -141,7 +142,7 b' def _hashignore(ignore):'
     copy.
 
     """
-    sha1 = util.sha1()
+    sha1 = hashlib.sha1()
     if util.safehasattr(ignore, 'includepat'):
         sha1.update(ignore.includepat)
     sha1.update('\0\0')

hgext/fsmonitor/state.py

@@ -12,8 +12,8 b' import os'
 import socket
 import struct
 
+from mercurial.i18n import _
 from mercurial import pathutil
-from mercurial.i18n import _
 
 _version = 4
 _versionformat = ">I"

hgext/gpg.py

@@ -5,10 +5,21 b''
 
 '''commands to sign and verify changesets'''
 
-import os, tempfile, binascii
-from mercurial import util, commands, match, cmdutil, error
-from mercurial import node as hgnode
+from __future__ import absolute_import
+
+import binascii
+import os
+import tempfile
+
 from mercurial.i18n import _
+from mercurial import (
+    cmdutil,
+    commands,
+    error,
+    match,
+    node as hgnode,
+    util,
+)
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
@@ -187,7 +198,7 b' def sigcheck(ui, repo, rev):'
         return
 
     # print summary
-    ui.write("%s is signed by:\n" % hgnode.short(rev))
+    ui.write(_("%s is signed by:\n") % hgnode.short(rev))
     for key in keys:
         ui.write(" %s\n" % keystr(ui, key))
 

hgext/graphlog.py

@@ -15,8 +15,13 b' commands. When this options is given, an'
 revision graph is also shown.
 '''
 
+from __future__ import absolute_import
+
 from mercurial.i18n import _
-from mercurial import cmdutil, commands
+from mercurial import (
+    cmdutil,
+    commands,
+)
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)

hgext/hgk.py

@@ -34,10 +34,23 b' Revisions context menu will now display '
 vdiff on hovered and selected revisions.
 '''
 
+from __future__ import absolute_import
+
 import os
-from mercurial import cmdutil, commands, patch, scmutil, obsolete
-from mercurial.node import nullid, nullrev, short
+
 from mercurial.i18n import _
+from mercurial.node import (
+    nullid,
+    nullrev,
+    short,
+)
+from mercurial import (
+    cmdutil,
+    commands,
+    obsolete,
+    patch,
+    scmutil,
+)
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
@@ -68,13 +81,13 b' def difftree(ui, repo, node1=None, node2'
 
         for f in modified:
             # TODO get file permissions
-            ui.write(":100664 100664 %s %s M\t%s\t%s\n" %
+            ui.write((":100664 100664 %s %s M\t%s\t%s\n") %
                      (short(mmap[f]), short(mmap2[f]), f, f))
         for f in added:
-            ui.write(":000000 100664 %s %s N\t%s\t%s\n" %
+            ui.write((":000000 100664 %s %s N\t%s\t%s\n") %
                      (empty, short(mmap2[f]), f, f))
         for f in removed:
-            ui.write(":100664 000000 %s %s D\t%s\t%s\n" %
+            ui.write((":100664 000000 %s %s D\t%s\t%s\n") %
                      (short(mmap[f]), empty, f, f))
     ##
 

hgext/highlight/__init__.py

@@ -26,9 +26,21 b' Pygments will try very hard to identify '
 match (even matches with a low confidence score) will be used.
 """
 
-import highlight
-from mercurial.hgweb import webcommands, webutil, common
-from mercurial import extensions, encoding, fileset
+from __future__ import absolute_import
+
+from . import highlight
+from mercurial.hgweb import (
+    common,
+    webcommands,
+    webutil,
+)
+
+from mercurial import (
+    encoding,
+    extensions,
+    fileset,
+)
+
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or

hgext/highlight/highlight.py

@@ -8,14 +8,27 b''
 # The original module was split in an interface and an implementation
 # file to defer pygments loading and speedup extension setup.
 
+from __future__ import absolute_import
+
+import pygments
+import pygments.formatters
+import pygments.lexers
+import pygments.util
+
 from mercurial import demandimport
 demandimport.ignore.extend(['pkgutil', 'pkg_resources', '__main__'])
-from mercurial import util, encoding
+
+from mercurial import (
+    encoding,
+    util,
+)
 
-from pygments import highlight
-from pygments.util import ClassNotFound
-from pygments.lexers import guess_lexer, guess_lexer_for_filename, TextLexer
-from pygments.formatters import HtmlFormatter
+highlight = pygments.highlight
+ClassNotFound = pygments.util.ClassNotFound
+guess_lexer = pygments.lexers.guess_lexer
+guess_lexer_for_filename = pygments.lexers.guess_lexer_for_filename
+TextLexer = pygments.lexers.TextLexer
+HtmlFormatter = pygments.formatters.HtmlFormatter
 
 SYNTAX_CSS = ('\n<link rel="stylesheet" href="{url}highlightcss" '
               'type="text/css" />')
@@ -68,7 +81,7 b' def pygmentize(field, fctx, style, tmpl,'
     coloriter = (s.encode(encoding.encoding, 'replace')
                  for s in colorized.splitlines())
 
-    tmpl.filters['colorize'] = lambda x: coloriter.next()
+    tmpl.filters['colorize'] = lambda x: next(coloriter)
 
     oldl = tmpl.cache[field]
     newl = oldl.replace('line|escape', 'line|colorize')

hgext/histedit.py

@@ -169,30 +169,35 b' the drop to be implicit for missing comm'
 
 """
 
-import pickle
+from __future__ import absolute_import
+
 import errno
 import os
 import sys
 
-from mercurial import bundle2
-from mercurial import cmdutil
-from mercurial import discovery
-from mercurial import error
-from mercurial import copies
-from mercurial import context
-from mercurial import destutil
-from mercurial import exchange
-from mercurial import extensions
-from mercurial import hg
-from mercurial import node
-from mercurial import repair
-from mercurial import scmutil
-from mercurial import util
-from mercurial import obsolete
-from mercurial import merge as mergemod
-from mercurial.lock import release
 from mercurial.i18n import _
+from mercurial import (
+    bundle2,
+    cmdutil,
+    context,
+    copies,
+    destutil,
+    discovery,
+    error,
+    exchange,
+    extensions,
+    hg,
+    lock,
+    merge as mergemod,
+    node,
+    obsolete,
+    repair,
+    scmutil,
+    util,
+)
 
+pickle = util.pickle
+release = lock.release
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 
@@ -415,9 +420,7 b' class histeditaction(object):'
         <hash> <rev> <summary>
         """
         ctx = self.repo[self.node]
-        summary = ''
-        if ctx.description():
-            summary = ctx.description().splitlines()[0]
+        summary = _getsummary(ctx)
         line = '%s %s %d %s' % (self.verb, ctx, ctx.rev(), summary)
         # trim to 75 columns by default so it's not stupidly wide in my editor
         # (the 5 more are left for verb)
@@ -1264,6 +1267,14 b' def _newhistedit(ui, repo, state, revs, '
                                     'histedit')
     state.backupfile = backupfile
 
+def _getsummary(ctx):
+    # a common pattern is to extract the summary but default to the empty
+    # string
+    summary = ctx.description() or ''
+    if summary:
+        summary = summary.splitlines()[0]
+    return summary
+
 def bootstrapcontinue(ui, state, opts):
     repo = state.repo
     if state.actions:
@@ -1304,6 +1315,40 b' def ruleeditor(repo, ui, actions, editco'
 
     rules are in the format [ [act, ctx], ...] like in state.rules
     """
+    if repo.ui.configbool("experimental", "histedit.autoverb"):
+        newact = util.sortdict()
+        for act in actions:
+            ctx = repo[act.node]
+            summary = _getsummary(ctx)
+            fword = summary.split(' ', 1)[0].lower()
+            added = False
+
+            # if it doesn't end with the special character '!' just skip this
+            if fword.endswith('!'):
+                fword = fword[:-1]
+                if fword in primaryactions | secondaryactions | tertiaryactions:
+                    act.verb = fword
+                    # get the target summary
+                    tsum = summary[len(fword) + 1:].lstrip()
+                    # safe but slow: reverse iterate over the actions so we
+                    # don't clash on two commits having the same summary
+                    for na, l in reversed(list(newact.iteritems())):
+                        actx = repo[na.node]
+                        asum = _getsummary(actx)
+                        if asum == tsum:
+                            added = True
+                            l.append(act)
+                            break
+
+            if not added:
+                newact[act] = []
+
+        # copy over and flatten the new list
+        actions = []
+        for na, l in newact.iteritems():
+            actions.append(na)
+            actions += l
+
     rules = '\n'.join([act.torule() for act in actions])
     rules += '\n\n'
     rules += editcomment

hgext/keyword.py

@@ -89,8 +89,8 b' import os'
 import re
 import tempfile
 
+from mercurial.i18n import _
 from mercurial.hgweb import webcommands
-from mercurial.i18n import _
 
 from mercurial import (
     cmdutil,
@@ -455,7 +455,7 b' def demo(ui, repo, *args, **opts):'
 
     uisetup(ui)
     reposetup(ui, repo)
-    ui.write('[extensions]\nkeyword =\n')
+    ui.write(('[extensions]\nkeyword =\n'))
     demoitems('keyword', ui.configitems('keyword'))
     demoitems('keywordset', ui.configitems('keywordset'))
     demoitems('keywordmaps', kwmaps.iteritems())
@@ -735,7 +735,7 b' def reposetup(ui, repo):'
     def kwfilectx_cmp(orig, self, fctx):
         # keyword affects data size, comparing wdir and filelog size does
         # not make sense
-        if (fctx._filerev is None and
+        if (fctx._filenode is None and
             (self._repo._encodefilterpats or
              kwt.match(fctx.path()) and 'l' not in fctx.flags() or
              self.size() - 4 == fctx.size()) or

hgext/largefiles/__init__.py

@@ -104,14 +104,20 b' largefile. To add the first largefile to'
 explicitly do so with the --large flag passed to the :hg:`add`
 command.
 '''
+from __future__ import absolute_import
 
-from mercurial import hg, localrepo
+from mercurial import (
+    hg,
+    localrepo,
+)
 
-import lfcommands
-import proto
-import reposetup
-import uisetup as uisetupmod
-import overrides
+from . import (
+    lfcommands,
+    overrides,
+    proto,
+    reposetup,
+    uisetup as uisetupmod,
+)
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

hgext/largefiles/basestore.py

@@ -7,13 +7,13 b''
 # GNU General Public License version 2 or any later version.
 
 '''base class for store implementations and store-related utility code'''
+from __future__ import absolute_import
 
-import re
-
-from mercurial import util, node, hg, error
 from mercurial.i18n import _
 
-import lfutil
+from mercurial import node, util
+
+from . import lfutil
 
 class StoreError(Exception):
     '''Raised when there is a problem getting files from or putting
@@ -116,19 +116,26 b' class basestore(object):'
         '''Verify the existence (and, optionally, contents) of every big
         file revision referenced by every changeset in revs.
         Return 0 if all is well, non-zero on any errors.'''
-        failed = False
 
         self.ui.status(_('searching %d changesets for largefiles\n') %
                        len(revs))
         verified = set()                # set of (filename, filenode) tuples
-
+        filestocheck = []               # list of (cset, filename, expectedhash)
         for rev in revs:
             cctx = self.repo[rev]
             cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
 
             for standin in cctx:
-                if self._verifyfile(cctx, cset, contents, standin, verified):
-                    failed = True
+                filename = lfutil.splitstandin(standin)
+                if filename:
+                    fctx = cctx[standin]
+                    key = (filename, fctx.filenode())
+                    if key not in verified:
+                        verified.add(key)
+                        expectedhash = fctx.data()[0:40]
+                        filestocheck.append((cset, filename, expectedhash))
+
+        failed = self._verifyfiles(contents, filestocheck)
 
         numrevs = len(verified)
         numlfiles = len(set([fname for (fname, fnode) in verified]))
@@ -150,72 +157,10 b' class basestore(object):'
         exist in the store).'''
         raise NotImplementedError('abstract method')
 
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        '''Perform the actual verification of a file in the store.
-        'cset' is only used in warnings.
+    def _verifyfiles(self, contents, filestocheck):
+        '''Perform the actual verification of files in the store.
         'contents' controls verification of content hash.
-        'standin' is the standin path of the largefile to verify.
-        'verified' is maintained as a set of already verified files.
-        Returns _true_ if it is a standin and any problems are found!
+        'filestocheck' is list of files to check.
+        Returns _true_ if any problems are found!
         '''
         raise NotImplementedError('abstract method')
-
-import localstore, wirestore
-
-_storeprovider = {
-    'file':  [localstore.localstore],
-    'http':  [wirestore.wirestore],
-    'https': [wirestore.wirestore],
-    'ssh': [wirestore.wirestore],
-    }
-
-_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
-
-# During clone this function is passed the src's ui object
-# but it needs the dest's ui object so it can read out of
-# the config file. Use repo.ui instead.
-def _openstore(repo, remote=None, put=False):
-    ui = repo.ui
-
-    if not remote:
-        lfpullsource = getattr(repo, 'lfpullsource', None)
-        if lfpullsource:
-            path = ui.expandpath(lfpullsource)
-        elif put:
-            path = ui.expandpath('default-push', 'default')
-        else:
-            path = ui.expandpath('default')
-
-        # ui.expandpath() leaves 'default-push' and 'default' alone if
-        # they cannot be expanded: fallback to the empty string,
-        # meaning the current directory.
-        if path == 'default-push' or path == 'default':
-            path = ''
-            remote = repo
-        else:
-            path, _branches = hg.parseurl(path)
-            remote = hg.peer(repo, {}, path)
-
-    # The path could be a scheme so use Mercurial's normal functionality
-    # to resolve the scheme to a repository and use its path
-    path = util.safehasattr(remote, 'url') and remote.url() or remote.path
-
-    match = _scheme_re.match(path)
-    if not match:                       # regular filesystem path
-        scheme = 'file'
-    else:
-        scheme = match.group(1)
-
-    try:
-        storeproviders = _storeprovider[scheme]
-    except KeyError:
-        raise error.Abort(_('unsupported URL scheme %r') % scheme)
-
-    for classobj in storeproviders:
-        try:
-            return classobj(ui, repo, remote)
-        except lfutil.storeprotonotcapable:
-            pass
-
-    raise error.Abort(_('%s does not appear to be a largefile store') %
-                     util.hidepassword(path))

hgext/largefiles/lfcommands.py

@@ -7,20 +7,39 b''
 # GNU General Public License version 2 or any later version.
 
 '''High-level command function for lfconvert, plus the cmdtable.'''
+from __future__ import absolute_import
 
-import os, errno
+import errno
+import hashlib
+import os
 import shutil
 
-from mercurial import util, match as match_, hg, node, context, error, \
-    cmdutil, scmutil, commands
 from mercurial.i18n import _
-from mercurial.lock import release
 
-from hgext.convert import convcmd
-from hgext.convert import filemap
+from mercurial import (
+    cmdutil,
+    commands,
+    context,
+    error,
+    hg,
+    lock,
+    match as matchmod,
+    node,
+    scmutil,
+    util,
+)
 
-import lfutil
-import basestore
+from ..convert import (
+    convcmd,
+    filemap,
+)
+
+from . import (
+    lfutil,
+    storefactory
+)
+
+release = lock.release
 
 # -- Commands ----------------------------------------------------------
 
@@ -92,7 +111,7 b' def lfconvert(ui, src, dest, *pats, **op'
             if not pats:
                 pats = ui.configlist(lfutil.longname, 'patterns', default=[])
             if pats:
-                matcher = match_.match(rsrc.root, '', list(pats))
+                matcher = matchmod.match(rsrc.root, '', list(pats))
             else:
                 matcher = None
 
@@ -211,7 +230,7 b' def _lfconvert_addchangeset(rsrc, rdst, '
                         raise error.Abort(_('largefile %s becomes symlink') % f)
 
                 # largefile was modified, update standins
-                m = util.sha1('')
+                m = hashlib.sha1('')
                 m.update(ctx[f].data())
                 hash = m.hexdigest()
                 if f not in lfiletohash or lfiletohash[f] != hash:
@@ -337,7 +356,7 b' def uploadlfiles(ui, rsrc, rdst, files):'
     if not files:
         return
 
-    store = basestore._openstore(rsrc, rdst, put=True)
+    store = storefactory.openstore(rsrc, rdst, put=True)
 
     at = 0
     ui.debug("sending statlfile command for %d largefiles\n" % len(files))
@@ -368,7 +387,7 b' def verifylfiles(ui, repo, all=False, co'
     else:
         revs = ['.']
 
-    store = basestore._openstore(repo)
+    store = storefactory.openstore(repo)
     return store.verify(revs, contents=contents)
 
 def cachelfiles(ui, repo, node, filelist=None):
@@ -394,7 +413,7 b' def cachelfiles(ui, repo, node, filelist'
             toget.append((lfile, expectedhash))
 
     if toget:
-        store = basestore._openstore(repo)
+        store = storefactory.openstore(repo)
         ret = store.get(toget)
         return ret
 

hgext/largefiles/lfutil.py

@@ -7,21 +7,30 b''
 # GNU General Public License version 2 or any later version.
 
 '''largefiles utility code: must not import other modules in this package.'''
+from __future__ import absolute_import
 
+import copy
+import hashlib
 import os
 import platform
 import stat
-import copy
+
+from mercurial.i18n import _
 
-from mercurial import dirstate, httpconnection, match as match_, util, scmutil
-from mercurial.i18n import _
-from mercurial import node, error
+from mercurial import (
+    dirstate,
+    error,
+    httpconnection,
+    match as matchmod,
+    node,
+    scmutil,
+    util,
+)
 
 shortname = '.hglf'
 shortnameslash = shortname + '/'
 longname = 'largefiles'
 
-
 # -- Private worker functions ------------------------------------------
 
 def getminsize(ui, assumelfiles, opt, default=10):
@@ -152,7 +161,7 b' def openlfdirstate(ui, repo, create=True'
 
 def lfdirstatestatus(lfdirstate, repo):
     wctx = repo['.']
-    match = match_.always(repo.root, repo.getcwd())
+    match = matchmod.always(repo.root, repo.getcwd())
     unsure, s = lfdirstate.status(match, [], False, False, False)
     modified, clean = s.modified, s.clean
     for lfile in unsure:
@@ -180,12 +189,11 b' def listlfiles(repo, rev=None, matcher=N'
             if rev is not None or repo.dirstate[f] != '?']
 
 def instore(repo, hash, forcelocal=False):
-    '''Return true if a largefile with the given hash exists in the user
-    cache.'''
+    '''Return true if a largefile with the given hash exists in the store'''
     return os.path.exists(storepath(repo, hash, forcelocal))
 
 def storepath(repo, hash, forcelocal=False):
-    '''Return the correct location in the repository largefiles cache for a
+    '''Return the correct location in the repository largefiles store for a
     file with the given hash.'''
     if not forcelocal and repo.shared():
         return repo.vfs.reljoin(repo.sharedpath, longname, hash)
@@ -251,7 +259,6 b' def copyalltostore(repo, node):'
             realfile = splitstandin(filename)
             copytostore(repo, ctx.node(), realfile)
 
-
 def copytostoreabsolute(repo, file, hash):
     if inusercache(repo.ui, hash):
         link(usercachepath(repo.ui, hash), storepath(repo, hash))
@@ -350,7 +357,7 b' def writestandin(repo, standin, hash, ex'
 def copyandhash(instream, outfile):
     '''Read bytes from instream (iterable) and write them to outfile,
     computing the SHA-1 hash of the data along the way. Return the hash.'''
-    hasher = util.sha1('')
+    hasher = hashlib.sha1('')
     for data in instream:
         hasher.update(data)
         outfile.write(data)
@@ -362,7 +369,7 b' def hashrepofile(repo, file):'
 def hashfile(file):
     if not os.path.exists(file):
         return ''
-    hasher = util.sha1('')
+    hasher = hashlib.sha1('')
     fd = open(file, 'rb')
     for data in util.filechunkiter(fd, 128 * 1024):
         hasher.update(data)
@@ -391,7 +398,7 b' def urljoin(first, second, *arg):'
 def hexsha1(data):
     """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
     object data"""
-    h = util.sha1()
+    h = hashlib.sha1()
     for chunk in util.filechunkiter(data):
         h.update(chunk)
     return h.hexdigest()
@@ -533,7 +540,7 b' def updatestandinsbymatch(repo, match):'
         # otherwise to update all standins if the largefiles are
         # large.
         lfdirstate = openlfdirstate(ui, repo)
-        dirtymatch = match_.always(repo.root, repo.getcwd())
+        dirtymatch = matchmod.always(repo.root, repo.getcwd())
         unsure, s = lfdirstate.status(dirtymatch, [], False, False,
                                       False)
         modifiedfiles = unsure + s.modified + s.added + s.removed

hgext/largefiles/localstore.py

@@ -7,11 +7,14 b''
 # GNU General Public License version 2 or any later version.
 
 '''store class for local filesystem'''
+from __future__ import absolute_import
 
 from mercurial.i18n import _
 
-import lfutil
-import basestore
+from . import (
+    basestore,
+    lfutil,
+)
 
 class localstore(basestore.basestore):
     '''localstore first attempts to grab files out of the store in the remote
@@ -33,7 +36,6 b' class localstore(basestore.basestore):'
             retval[hash] = lfutil.instore(self.remote, hash)
         return retval
 
-
     def _getfile(self, tmpfile, filename, hash):
         path = lfutil.findfile(self.remote, hash)
         if not path:
@@ -42,29 +44,23 b' class localstore(basestore.basestore):'
         with open(path, 'rb') as fd:
             return lfutil.copyandhash(fd, tmpfile)
 
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        filename = lfutil.splitstandin(standin)
-        if not filename:
-            return False
-        fctx = cctx[standin]
-        key = (filename, fctx.filenode())
-        if key in verified:
-            return False
-
-        expecthash = fctx.data()[0:40]
-        storepath, exists = lfutil.findstorepath(self.remote, expecthash)
-        verified.add(key)
-        if not exists:
-            self.ui.warn(
-                _('changeset %s: %s references missing %s\n')
-                % (cset, filename, storepath))
-            return True                 # failed
-
-        if contents:
-            actualhash = lfutil.hashfile(storepath)
-            if actualhash != expecthash:
+    def _verifyfiles(self, contents, filestocheck):
+        failed = False
+        for cset, filename, expectedhash in filestocheck:
+            storepath, exists = lfutil.findstorepath(self.repo, expectedhash)
+            if not exists:
+                storepath, exists = lfutil.findstorepath(
+                    self.remote, expectedhash)
+            if not exists:
                 self.ui.warn(
-                    _('changeset %s: %s references corrupted %s\n')
+                    _('changeset %s: %s references missing %s\n')
                     % (cset, filename, storepath))
-                return True             # failed
-        return False
+                failed = True
+            elif contents:
+                actualhash = lfutil.hashfile(storepath)
+                if actualhash != expectedhash:
+                    self.ui.warn(
+                        _('changeset %s: %s references corrupted %s\n')
+                        % (cset, filename, storepath))
+                    failed = True
+        return failed

hgext/largefiles/overrides.py

@@ -7,17 +7,31 b''
 # GNU General Public License version 2 or any later version.
 
 '''Overridden Mercurial commands and functions for the largefiles extension'''
+from __future__ import absolute_import
 
-import os
 import copy
+import os
 
-from mercurial import hg, util, cmdutil, scmutil, match as match_, \
-        archival, pathutil, registrar, revset, error
 from mercurial.i18n import _
 
-import lfutil
-import lfcommands
-import basestore
+from mercurial import (
+    archival,
+    cmdutil,
+    error,
+    hg,
+    match as matchmod,
+    pathutil,
+    registrar,
+    revset,
+    scmutil,
+    util,
+)
+
+from . import (
+    lfcommands,
+    lfutil,
+    storefactory,
+)
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
@@ -99,13 +113,13 b' def addlargefiles(ui, repo, isaddremove,'
     if lfutil.islfilesrepo(repo):
         lfpats = ui.configlist(lfutil.longname, 'patterns', default=[])
         if lfpats:
-            lfmatcher = match_.match(repo.root, '', list(lfpats))
+            lfmatcher = matchmod.match(repo.root, '', list(lfpats))
 
     lfnames = []
     m = matcher
 
     wctx = repo[None]
-    for f in repo.walk(match_.badmatch(m, lambda x, y: None)):
+    for f in repo.walk(matchmod.badmatch(m, lambda x, y: None)):
         exact = m.exact(f)
         lfile = lfutil.standin(f) in wctx
         nfile = f in wctx
@@ -307,7 +321,7 b' def overridelog(orig, ui, repo, *pats, *'
             if pat.startswith('set:'):
                 return pat
 
-            kindpat = match_._patsplit(pat, None)
+            kindpat = matchmod._patsplit(pat, None)
 
             if kindpat[0] is not None:
                 return kindpat[0] + ':' + tostandin(kindpat[1])
@@ -532,7 +546,6 b' def mergerecordupdates(orig, repo, actio'
 
     return orig(repo, actions, branchmerge)
 
-
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits without prompting the user.
 def overridefilemerge(origfn, premerge, repo, mynode, orig, fcd, fco, fca,
@@ -626,7 +639,7 b' def overridecopy(orig, ui, repo, pats, o'
             # The patterns were previously mangled to add the standin
             # directory; we need to remove that now
             for pat in pats:
-                if match_.patkind(pat) is None and lfutil.shortname in pat:
+                if matchmod.patkind(pat) is None and lfutil.shortname in pat:
                     newpats.append(pat.replace(lfutil.shortname, ''))
                 else:
                     newpats.append(pat)
@@ -644,7 +657,7 b' def overridecopy(orig, ui, repo, pats, o'
         oldmatch = installmatchfn(overridematch)
         listpats = []
         for pat in pats:
-            if match_.patkind(pat) is not None:
+            if matchmod.patkind(pat) is not None:
                 listpats.append(pat)
             else:
                 listpats.append(makestandin(pat))
@@ -977,7 +990,7 b' def overridearchive(orig, repo, dest, no'
     if subrepos:
         for subpath in sorted(ctx.substate):
             sub = ctx.workingsub(subpath)
-            submatch = match_.subdirmatcher(subpath, matchfn)
+            submatch = matchmod.subdirmatcher(subpath, matchfn)
             sub._repo.lfstatus = True
             sub.archive(archiver, prefix, submatch)
 
@@ -1025,7 +1038,7 b' def hgsubrepoarchive(orig, repo, archive'
 
     for subpath in sorted(ctx.substate):
         sub = ctx.workingsub(subpath)
-        submatch = match_.subdirmatcher(subpath, match)
+        submatch = matchmod.subdirmatcher(subpath, match)
         sub._repo.lfstatus = True
         sub.archive(archiver, prefix + repo._path + '/', submatch)
 
@@ -1109,7 +1122,7 b' def _getoutgoings(repo, other, missing, '
             lfhashes.add(lfhash)
     lfutil.getlfilestoupload(repo, missing, dedup)
     if lfhashes:
-        lfexists = basestore._openstore(repo, other).exists(lfhashes)
+        lfexists = storefactory.openstore(repo, other).exists(lfhashes)
         for fn, lfhash in knowns:
             if not lfexists[lfhash]: # lfhash doesn't exist on "other"
                 addfunc(fn, lfhash)
@@ -1190,7 +1203,7 b' def scmutiladdremove(orig, repo, matcher'
         return orig(repo, matcher, prefix, opts, dry_run, similarity)
     # Get the list of missing largefiles so we can remove them
     lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
-    unsure, s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [],
+    unsure, s = lfdirstate.status(matchmod.always(repo.root, repo.getcwd()), [],
                                   False, False, False)
 
     # Call into the normal remove code, but the removing of the standin, we want
@@ -1338,7 +1351,7 b' def overridecat(orig, ui, repo, file1, *'
         else:
             hash = lfutil.readstandin(repo, lf, ctx.rev())
             if not lfutil.inusercache(repo.ui, hash):
-                store = basestore._openstore(repo)
+                store = storefactory.openstore(repo)
                 success, missing = store.get([(lf, hash)])
                 if len(success) != 1:
                     raise error.Abort(
@@ -1375,7 +1388,7 b' def mergeupdate(orig, repo, node, branch'
         # (*1) deprecated, but used internally (e.g: "rebase --collapse")
 
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
-        unsure, s = lfdirstate.status(match_.always(repo.root,
+        unsure, s = lfdirstate.status(matchmod.always(repo.root,
                                                     repo.getcwd()),
                                       [], False, False, False)
         pctx = repo['.']

hgext/largefiles/proto.py

@@ -2,18 +2,27 b''
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
 
 import os
 import re
 
-from mercurial import error, httppeer, util, wireproto
 from mercurial.i18n import _
 
+from mercurial import (
+    error,
+    httppeer,
+    util,
+    wireproto,
+)
+
+from . import (
+    lfutil,
+)
+
 urlerr = util.urlerr
 urlreq = util.urlreq
 
-import lfutil
-
 LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
                            '\n\nPlease enable it in your Mercurial config '
                            'file.\n')

hgext/largefiles/remotestore.py

@@ -5,20 +5,30 b''
 # GNU General Public License version 2 or any later version.
 
 '''remote largefile store; the base class for wirestore'''
+from __future__ import absolute_import
 
-from mercurial import util, wireproto, error
 from mercurial.i18n import _
 
+from mercurial import (
+    error,
+    util,
+    wireproto,
+)
+
+from . import (
+    basestore,
+    lfutil,
+    localstore,
+)
+
 urlerr = util.urlerr
 urlreq = util.urlreq
 
-import lfutil
-import basestore
-
 class remotestore(basestore.basestore):
     '''a largefile store accessed over a network'''
     def __init__(self, ui, repo, url):
         super(remotestore, self).__init__(ui, repo, url)
+        self._lstore = localstore.localstore(self.ui, self.repo, self.repo)
 
     def put(self, source, hash):
         if self.sendfile(source, hash):
@@ -65,34 +75,43 b' class remotestore(basestore.basestore):'
 
         return lfutil.copyandhash(chunks, tmpfile)
 
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        filename = lfutil.splitstandin(standin)
-        if not filename:
-            return False
-        fctx = cctx[standin]
-        key = (filename, fctx.filenode())
-        if key in verified:
-            return False
+    def _hashesavailablelocally(self, hashes):
+        existslocallymap = self._lstore.exists(hashes)
+        localhashes = [hash for hash in hashes if existslocallymap[hash]]
+        return localhashes
 
-        verified.add(key)
+    def _verifyfiles(self, contents, filestocheck):
+        failed = False
+        expectedhashes = [expectedhash
+                          for cset, filename, expectedhash in filestocheck]
+        localhashes = self._hashesavailablelocally(expectedhashes)
+        stats = self._stat([expectedhash for expectedhash in expectedhashes
+                            if expectedhash not in localhashes])
 
-        expecthash = fctx.data()[0:40]
-        stat = self._stat([expecthash])[expecthash]
-        if not stat:
-            return False
-        elif stat == 1:
-            self.ui.warn(
-                _('changeset %s: %s: contents differ\n')
-                % (cset, filename))
-            return True # failed
-        elif stat == 2:
-            self.ui.warn(
-                _('changeset %s: %s missing\n')
-                % (cset, filename))
-            return True # failed
-        else:
-            raise RuntimeError('verify failed: unexpected response from '
-                               'statlfile (%r)' % stat)
+        for cset, filename, expectedhash in filestocheck:
+            if expectedhash in localhashes:
+                filetocheck = (cset, filename, expectedhash)
+                verifyresult = self._lstore._verifyfiles(contents,
+                                                         [filetocheck])
+                if verifyresult:
+                    failed = True
+            else:
+                stat = stats[expectedhash]
+                if stat:
+                    if stat == 1:
+                        self.ui.warn(
+                            _('changeset %s: %s: contents differ\n')
+                            % (cset, filename))
+                        failed = True
+                    elif stat == 2:
+                        self.ui.warn(
+                            _('changeset %s: %s missing\n')
+                            % (cset, filename))
+                        failed = True
+                    else:
+                        raise RuntimeError('verify failed: unexpected response '
+                                           'from statlfile (%r)' % stat)
+        return failed
 
     def batch(self):
         '''Support for remote batching.'''

hgext/largefiles/reposetup.py

@@ -7,14 +7,23 b''
 # GNU General Public License version 2 or any later version.
 
 '''setup for largefiles repositories: reposetup'''
+from __future__ import absolute_import
+
 import copy
 
-from mercurial import error, match as match_, error
 from mercurial.i18n import _
-from mercurial import scmutil, localrepo
 
-import lfcommands
-import lfutil
+from mercurial import (
+    error,
+    localrepo,
+    match as matchmod,
+    scmutil,
+)
+
+from . import (
+    lfcommands,
+    lfutil,
+)
 
 def reposetup(ui, repo):
     # wire repositories should be given new wireproto functions
@@ -94,7 +103,7 b' def reposetup(ui, repo):'
             parentworking = working and ctx1 == self['.']
 
             if match is None:
-                match = match_.always(self.root, self.getcwd())
+                match = matchmod.always(self.root, self.getcwd())
 
             wlock = None
             try:

hgext/largefiles/storefactory.py

@@ -1,180 +1,28 b''
-# Copyright 2009-2010 Gregory P. Ward
-# Copyright 2009-2010 Intelerad Medical Systems Incorporated
-# Copyright 2010-2011 Fog Creek Software
-# Copyright 2010-2011 Unity Technologies
-#
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-'''base class for store implementations and store-related utility code'''
+from __future__ import absolute_import
 
 import re
 
-from mercurial import util, node, hg, error
 from mercurial.i18n import _
 
-import lfutil
-
-class StoreError(Exception):
-    '''Raised when there is a problem getting files from or putting
-    files to a central store.'''
-    def __init__(self, filename, hash, url, detail):
-        self.filename = filename
-        self.hash = hash
-        self.url = url
-        self.detail = detail
-
-    def longmessage(self):
-        return (_("error getting id %s from url %s for file %s: %s\n") %
-                 (self.hash, util.hidepassword(self.url), self.filename,
-                  self.detail))
-
-    def __str__(self):
-        return "%s: %s" % (util.hidepassword(self.url), self.detail)
-
-class basestore(object):
-    def __init__(self, ui, repo, url):
-        self.ui = ui
-        self.repo = repo
-        self.url = url
-
-    def put(self, source, hash):
-        '''Put source file into the store so it can be retrieved by hash.'''
-        raise NotImplementedError('abstract method')
-
-    def exists(self, hashes):
-        '''Check to see if the store contains the given hashes. Given an
-        iterable of hashes it returns a mapping from hash to bool.'''
-        raise NotImplementedError('abstract method')
-
-    def get(self, files):
-        '''Get the specified largefiles from the store and write to local
-        files under repo.root.  files is a list of (filename, hash)
-        tuples.  Return (success, missing), lists of files successfully
-        downloaded and those not found in the store.  success is a list
-        of (filename, hash) tuples; missing is a list of filenames that
-        we could not get.  (The detailed error message will already have
-        been presented to the user, so missing is just supplied as a
-        summary.)'''
-        success = []
-        missing = []
-        ui = self.ui
-
-        at = 0
-        available = self.exists(set(hash for (_filename, hash) in files))
-        for filename, hash in files:
-            ui.progress(_('getting largefiles'), at, unit=_('files'),
-                total=len(files))
-            at += 1
-            ui.note(_('getting %s:%s\n') % (filename, hash))
-
-            if not available.get(hash):
-                ui.warn(_('%s: largefile %s not available from %s\n')
-                        % (filename, hash, util.hidepassword(self.url)))
-                missing.append(filename)
-                continue
-
-            if self._gethash(filename, hash):
-                success.append((filename, hash))
-            else:
-                missing.append(filename)
-
-        ui.progress(_('getting largefiles'), None)
-        return (success, missing)
-
-    def _gethash(self, filename, hash):
-        """Get file with the provided hash and store it in the local repo's
-        store and in the usercache.
-        filename is for informational messages only.
-        """
-        util.makedirs(lfutil.storepath(self.repo, ''))
-        storefilename = lfutil.storepath(self.repo, hash)
+from mercurial import (
+    error,
+    hg,
+    util,
+)
 
-        tmpname = storefilename + '.tmp'
-        tmpfile = util.atomictempfile(tmpname,
-                                      createmode=self.repo.store.createmode)
-
-        try:
-            gothash = self._getfile(tmpfile, filename, hash)
-        except StoreError as err:
-            self.ui.warn(err.longmessage())
-            gothash = ""
-        tmpfile.close()
-
-        if gothash != hash:
-            if gothash != "":
-                self.ui.warn(_('%s: data corruption (expected %s, got %s)\n')
-                             % (filename, hash, gothash))
-            util.unlink(tmpname)
-            return False
-
-        util.rename(tmpname, storefilename)
-        lfutil.linktousercache(self.repo, hash)
-        return True
-
-    def verify(self, revs, contents=False):
-        '''Verify the existence (and, optionally, contents) of every big
-        file revision referenced by every changeset in revs.
-        Return 0 if all is well, non-zero on any errors.'''
-        failed = False
-
-        self.ui.status(_('searching %d changesets for largefiles\n') %
-                       len(revs))
-        verified = set()                # set of (filename, filenode) tuples
-
-        for rev in revs:
-            cctx = self.repo[rev]
-            cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
-
-            for standin in cctx:
-                if self._verifyfile(cctx, cset, contents, standin, verified):
-                    failed = True
-
-        numrevs = len(verified)
-        numlfiles = len(set([fname for (fname, fnode) in verified]))
-        if contents:
-            self.ui.status(
-                _('verified contents of %d revisions of %d largefiles\n')
-                % (numrevs, numlfiles))
-        else:
-            self.ui.status(
-                _('verified existence of %d revisions of %d largefiles\n')
-                % (numrevs, numlfiles))
-        return int(failed)
-
-    def _getfile(self, tmpfile, filename, hash):
-        '''Fetch one revision of one file from the store and write it
-        to tmpfile.  Compute the hash of the file on-the-fly as it
-        downloads and return the hash.  Close tmpfile.  Raise
-        StoreError if unable to download the file (e.g. it does not
-        exist in the store).'''
-        raise NotImplementedError('abstract method')
-
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        '''Perform the actual verification of a file in the store.
-        'cset' is only used in warnings.
-        'contents' controls verification of content hash.
-        'standin' is the standin path of the largefile to verify.
-        'verified' is maintained as a set of already verified files.
-        Returns _true_ if it is a standin and any problems are found!
-        '''
-        raise NotImplementedError('abstract method')
-
-import localstore, wirestore
-
-_storeprovider = {
-    'file':  [localstore.localstore],
-    'http':  [wirestore.wirestore],
-    'https': [wirestore.wirestore],
-    'ssh': [wirestore.wirestore],
-    }
-
-_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
+from . import (
+    lfutil,
+    localstore,
+    wirestore,
+)
 
 # During clone this function is passed the src's ui object
 # but it needs the dest's ui object so it can read out of
 # the config file. Use repo.ui instead.
-def _openstore(repo, remote=None, put=False):
+def openstore(repo, remote=None, put=False):
     ui = repo.ui
 
     if not remote:
@@ -219,3 +67,12 b' def _openstore(repo, remote=None, put=Fa'
 
     raise error.Abort(_('%s does not appear to be a largefile store') %
                      util.hidepassword(path))
+
+_storeprovider = {
+    'file':  [localstore.localstore],
+    'http':  [wirestore.wirestore],
+    'https': [wirestore.wirestore],
+    'ssh': [wirestore.wirestore],
+    }
+
+_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')

hgext/largefiles/uisetup.py

@@ -7,14 +7,36 b''
 # GNU General Public License version 2 or any later version.
 
 '''setup for largefiles extension: uisetup'''
+from __future__ import absolute_import
 
-from mercurial import archival, cmdutil, commands, extensions, filemerge, hg, \
-    httppeer, merge, scmutil, sshpeer, wireproto, subrepo, copies, exchange
 from mercurial.i18n import _
-from mercurial.hgweb import hgweb_mod, webcommands
+
+from mercurial.hgweb import (
+    hgweb_mod,
+    webcommands,
+)
 
-import overrides
-import proto
+from mercurial import (
+    archival,
+    cmdutil,
+    commands,
+    copies,
+    exchange,
+    extensions,
+    filemerge,
+    hg,
+    httppeer,
+    merge,
+    scmutil,
+    sshpeer,
+    subrepo,
+    wireproto,
+)
+
+from . import (
+    overrides,
+    proto,
+)
 
 def uisetup(ui):
     # Disable auto-status for some commands which assume that all

hgext/largefiles/wirestore.py

@@ -4,9 +4,12 b''
 # GNU General Public License version 2 or any later version.
 
 '''largefile store working over Mercurial's wire protocol'''
+from __future__ import absolute_import
 
-import lfutil
-import remotestore
+from . import (
+    lfutil,
+    remotestore,
+)
 
 class wirestore(remotestore.remotestore):
     def __init__(self, ui, repo, remote):

hgext/logtoprocess.py

@@ -92,7 +92,7 b' def uisetup(ui):'
             Arguments are passed on as environment variables.
 
             """
-            script = ui.config('logtoprocess', event)
+            script = self.config('logtoprocess', event)
             if script:
                 if msg:
                     # try to format the log message given the remaining

hgext/mq.py

@@ -62,19 +62,39 b' This extension used to provide a strip c'
 in the strip extension.
 '''
 
+from __future__ import absolute_import
+
+import errno
+import os
+import re
+import shutil
 from mercurial.i18n import _
-from mercurial.node import bin, hex, short, nullid, nullrev
-from mercurial.lock import release
-from mercurial import commands, cmdutil, hg, scmutil, util, revset
-from mercurial import dispatch
-from mercurial import extensions, error, phases
-from mercurial import patch as patchmod
-from mercurial import lock as lockmod
-from mercurial import localrepo
-from mercurial import registrar
-from mercurial import subrepo
-import os, re, errno, shutil
-
+from mercurial.node import (
+    bin,
+    hex,
+    nullid,
+    nullrev,
+    short,
+)
+from mercurial import (
+    cmdutil,
+    commands,
+    dispatch,
+    error,
+    extensions,
+    hg,
+    localrepo,
+    lock as lockmod,
+    patch as patchmod,
+    phases,
+    registrar,
+    revset,
+    scmutil,
+    subrepo,
+    util,
+)
+
+release = lockmod.release
 seriesopts = [('s', 'summary', None, _('print first line of patch header'))]
 
 cmdtable = {}

hgext/notify.py

@@ -139,6 +139,7 b' import fnmatch'
 import socket
 import time
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     error,
@@ -146,7 +147,6 b' from mercurial import ('
     patch,
     util,
 )
-from mercurial.i18n import _
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
@@ -363,7 +363,7 b' class notifier(object):'
             s = patch.diffstat(difflines)
             # s may be nil, don't include the header if it is
             if s:
-                self.ui.write('\ndiffstat:\n\n%s' % s)
+                self.ui.write(_('\ndiffstat:\n\n%s') % s)
 
         if maxdiff == 0:
             return

hgext/pager.py

@@ -66,6 +66,7 b' import signal'
 import subprocess
 import sys
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
@@ -73,7 +74,6 b' from mercurial import ('
     extensions,
     util,
     )
-from mercurial.i18n import _
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

hgext/patchbomb.py

@@ -71,6 +71,7 b' import os'
 import socket
 import tempfile
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
@@ -83,7 +84,6 b' from mercurial import ('
     util,
 )
 stringio = util.stringio
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
@@ -708,13 +708,7 b' def email(ui, repo, *revs, **opts):'
                 fp.close()
         else:
             if not sendmail:
-                verifycert = ui.config('smtp', 'verifycert', 'strict')
-                if opts.get('insecure'):
-                    ui.setconfig('smtp', 'verifycert', 'loose', 'patchbomb')
-                try:
-                    sendmail = mail.connect(ui, mbox=mbox)
-                finally:
-                    ui.setconfig('smtp', 'verifycert', verifycert, 'patchbomb')
+                sendmail = mail.connect(ui, mbox=mbox)
             ui.status(_('sending '), subj, ' ...\n')
             ui.progress(_('sending'), i, item=subj, total=len(msgs),
                         unit=_('emails'))

hgext/purge.py

@@ -27,6 +27,7 b' from __future__ import absolute_import'
 
 import os
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
@@ -34,7 +35,6 b' from mercurial import ('
     scmutil,
     util,
 )
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
@@ -84,13 +84,13 b' def purge(ui, repo, *dirs, **opts):'
     list of files that this program would delete, use the --print
     option.
     '''
-    act = not opts['print']
+    act = not opts.get('print')
     eol = '\n'
-    if opts['print0']:
+    if opts.get('print0'):
         eol = '\0'
         act = False # --print0 implies --print
-    removefiles = opts['files']
-    removedirs = opts['dirs']
+    removefiles = opts.get('files')
+    removedirs = opts.get('dirs')
     if not removefiles and not removedirs:
         removefiles = True
         removedirs = True
@@ -101,7 +101,7 b' def purge(ui, repo, *dirs, **opts):'
                 remove_func(repo.wjoin(name))
             except OSError:
                 m = _('%s cannot be removed') % name
-                if opts['abort_on_err']:
+                if opts.get('abort_on_err'):
                     raise error.Abort(m)
                 ui.warn(_('warning: %s\n') % m)
         else:
@@ -111,7 +111,7 b' def purge(ui, repo, *dirs, **opts):'
     if removedirs:
         directories = []
         match.explicitdir = match.traversedir = directories.append
-    status = repo.status(match=match, ignored=opts['all'], unknown=True)
+    status = repo.status(match=match, ignored=opts.get('all'), unknown=True)
 
     if removefiles:
         for f in sorted(status.unknown + status.ignored):

hgext/rebase.py

@@ -14,14 +14,42 b' For more information:'
 https://mercurial-scm.org/wiki/RebaseExtension
 '''
 
-from mercurial import hg, util, repair, merge, cmdutil, commands, bookmarks
-from mercurial import extensions, patch, scmutil, phases, obsolete, error
-from mercurial import copies, destutil, repoview, registrar, revset
-from mercurial.commands import templateopts
-from mercurial.node import nullrev, nullid, hex, short
-from mercurial.lock import release
+from __future__ import absolute_import
+
+import errno
+import os
+
 from mercurial.i18n import _
-import os, errno
+from mercurial.node import (
+    hex,
+    nullid,
+    nullrev,
+    short,
+)
+from mercurial import (
+    bookmarks,
+    cmdutil,
+    commands,
+    copies,
+    destutil,
+    error,
+    extensions,
+    hg,
+    lock,
+    merge,
+    obsolete,
+    patch,
+    phases,
+    registrar,
+    repair,
+    repoview,
+    revset,
+    scmutil,
+    util,
+)
+
+release = lock.release
+templateopts = commands.templateopts
 
 # The following constants are used throughout the rebase module. The ordering of
 # their values must be maintained.
@@ -91,6 +119,394 b' def _revsetdestrebase(repo, subset, x):'
         sourceset = revset.getset(repo, revset.fullreposet(repo), x)
     return subset & revset.baseset([_destrebase(repo, sourceset)])
 
+class rebaseruntime(object):
+    """This class is a container for rebase runtime state"""
+    def __init__(self, repo, ui, opts=None):
+        if opts is None:
+            opts = {}
+
+        self.repo = repo
+        self.ui = ui
+        self.opts = opts
+        self.originalwd = None
+        self.external = nullrev
+        # Mapping between the old revision id and either what is the new rebased
+        # revision or what needs to be done with the old revision. The state
+        # dict will be what contains most of the rebase progress state.
+        self.state = {}
+        self.activebookmark = None
+        self.currentbookmarks = None
+        self.target = None
+        self.skipped = set()
+        self.targetancestors = set()
+
+        self.collapsef = opts.get('collapse', False)
+        self.collapsemsg = cmdutil.logmessage(ui, opts)
+        self.date = opts.get('date', None)
+
+        e = opts.get('extrafn') # internal, used by e.g. hgsubversion
+        self.extrafns = [_savegraft]
+        if e:
+            self.extrafns = [e]
+
+        self.keepf = opts.get('keep', False)
+        self.keepbranchesf = opts.get('keepbranches', False)
+        # keepopen is not meant for use on the command line, but by
+        # other extensions
+        self.keepopen = opts.get('keepopen', False)
+        self.obsoletenotrebased = {}
+
+    def restorestatus(self):
+        """Restore a previously stored status"""
+        repo = self.repo
+        keepbranches = None
+        target = None
+        collapse = False
+        external = nullrev
+        activebookmark = None
+        state = {}
+
+        try:
+            f = repo.vfs("rebasestate")
+            for i, l in enumerate(f.read().splitlines()):
+                if i == 0:
+                    originalwd = repo[l].rev()
+                elif i == 1:
+                    target = repo[l].rev()
+                elif i == 2:
+                    external = repo[l].rev()
+                elif i == 3:
+                    collapse = bool(int(l))
+                elif i == 4:
+                    keep = bool(int(l))
+                elif i == 5:
+                    keepbranches = bool(int(l))
+                elif i == 6 and not (len(l) == 81 and ':' in l):
+                    # line 6 is a recent addition, so for backwards
+                    # compatibility check that the line doesn't look like the
+                    # oldrev:newrev lines
+                    activebookmark = l
+                else:
+                    oldrev, newrev = l.split(':')
+                    if newrev in (str(nullmerge), str(revignored),
+                                  str(revprecursor), str(revpruned)):
+                        state[repo[oldrev].rev()] = int(newrev)
+                    elif newrev == nullid:
+                        state[repo[oldrev].rev()] = revtodo
+                        # Legacy compat special case
+                    else:
+                        state[repo[oldrev].rev()] = repo[newrev].rev()
+
+        except IOError as err:
+            if err.errno != errno.ENOENT:
+                raise
+            cmdutil.wrongtooltocontinue(repo, _('rebase'))
+
+        if keepbranches is None:
+            raise error.Abort(_('.hg/rebasestate is incomplete'))
+
+        skipped = set()
+        # recompute the set of skipped revs
+        if not collapse:
+            seen = set([target])
+            for old, new in sorted(state.items()):
+                if new != revtodo and new in seen:
+                    skipped.add(old)
+                seen.add(new)
+        repo.ui.debug('computed skipped revs: %s\n' %
+                        (' '.join(str(r) for r in sorted(skipped)) or None))
+        repo.ui.debug('rebase status resumed\n')
+        _setrebasesetvisibility(repo, state.keys())
+
+        self.originalwd = originalwd
+        self.target = target
+        self.state = state
+        self.skipped = skipped
+        self.collapsef = collapse
+        self.keepf = keep
+        self.keepbranchesf = keepbranches
+        self.external = external
+        self.activebookmark = activebookmark
+
+    def _handleskippingobsolete(self, rebaserevs, obsoleterevs, target):
+        """Compute structures necessary for skipping obsolete revisions
+
+        rebaserevs:     iterable of all revisions that are to be rebased
+        obsoleterevs:   iterable of all obsolete revisions in rebaseset
+        target:         a destination revision for the rebase operation
+        """
+        self.obsoletenotrebased = {}
+        if not self.ui.configbool('experimental', 'rebaseskipobsolete',
+                                  default=True):
+            return
+        rebaseset = set(rebaserevs)
+        obsoleteset = set(obsoleterevs)
+        self.obsoletenotrebased = _computeobsoletenotrebased(self.repo,
+                                    obsoleteset, target)
+        skippedset = set(self.obsoletenotrebased)
+        _checkobsrebase(self.repo, self.ui, obsoleteset, rebaseset, skippedset)
+
+    def _prepareabortorcontinue(self, isabort):
+        try:
+            self.restorestatus()
+            self.collapsemsg = restorecollapsemsg(self.repo)
+        except error.RepoLookupError:
+            if isabort:
+                clearstatus(self.repo)
+                clearcollapsemsg(self.repo)
+                self.repo.ui.warn(_('rebase aborted (no revision is removed,'
+                                    ' only broken state is cleared)\n'))
+                return 0
+            else:
+                msg = _('cannot continue inconsistent rebase')
+                hint = _('use "hg rebase --abort" to clear broken state')
+                raise error.Abort(msg, hint=hint)
+        if isabort:
+            return abort(self.repo, self.originalwd, self.target,
+                         self.state, activebookmark=self.activebookmark)
+
+        obsrevs = (r for r, st in self.state.items() if st == revprecursor)
+        self._handleskippingobsolete(self.state.keys(), obsrevs, self.target)
+
+    def _preparenewrebase(self, dest, rebaseset):
+        if dest is None:
+            return _nothingtorebase()
+
+        allowunstable = obsolete.isenabled(self.repo, obsolete.allowunstableopt)
+        if (not (self.keepf or allowunstable)
+              and self.repo.revs('first(children(%ld) - %ld)',
+                                 rebaseset, rebaseset)):
+            raise error.Abort(
+                _("can't remove original changesets with"
+                  " unrebased descendants"),
+                hint=_('use --keep to keep original changesets'))
+
+        obsrevs = _filterobsoleterevs(self.repo, rebaseset)
+        self._handleskippingobsolete(rebaseset, obsrevs, dest)
+
+        result = buildstate(self.repo, dest, rebaseset, self.collapsef,
+                            self.obsoletenotrebased)
+
+        if not result:
+            # Empty state built, nothing to rebase
+            self.ui.status(_('nothing to rebase\n'))
+            return _nothingtorebase()
+
+        root = min(rebaseset)
+        if not self.keepf and not self.repo[root].mutable():
+            raise error.Abort(_("can't rebase public changeset %s")
+                             % self.repo[root],
+                             hint=_('see "hg help phases" for details'))
+
+        (self.originalwd, self.target, self.state) = result
+        if self.collapsef:
+            self.targetancestors = self.repo.changelog.ancestors(
+                                        [self.target],
+                                        inclusive=True)
+            self.external = externalparent(self.repo, self.state,
+                                              self.targetancestors)
+
+        if dest.closesbranch() and not self.keepbranchesf:
+            self.ui.status(_('reopening closed branch head %s\n') % dest)
+
+    def _performrebase(self):
+        repo, ui, opts = self.repo, self.ui, self.opts
+        if self.keepbranchesf:
+            # insert _savebranch at the start of extrafns so if
+            # there's a user-provided extrafn it can clobber branch if
+            # desired
+            self.extrafns.insert(0, _savebranch)
+            if self.collapsef:
+                branches = set()
+                for rev in self.state:
+                    branches.add(repo[rev].branch())
+                    if len(branches) > 1:
+                        raise error.Abort(_('cannot collapse multiple named '
+                            'branches'))
+
+        # Rebase
+        if not self.targetancestors:
+            self.targetancestors = repo.changelog.ancestors([self.target],
+                                                               inclusive=True)
+
+        # Keep track of the current bookmarks in order to reset them later
+        self.currentbookmarks = repo._bookmarks.copy()
+        self.activebookmark = self.activebookmark or repo._activebookmark
+        if self.activebookmark:
+            bookmarks.deactivate(repo)
+
+        sortedrevs = sorted(self.state)
+        total = len(self.state)
+        pos = 0
+        for rev in sortedrevs:
+            ctx = repo[rev]
+            desc = '%d:%s "%s"' % (ctx.rev(), ctx,
+                                   ctx.description().split('\n', 1)[0])
+            names = repo.nodetags(ctx.node()) + repo.nodebookmarks(ctx.node())
+            if names:
+                desc += ' (%s)' % ' '.join(names)
+            pos += 1
+            if self.state[rev] == revtodo:
+                ui.status(_('rebasing %s\n') % desc)
+                ui.progress(_("rebasing"), pos, ("%d:%s" % (rev, ctx)),
+                            _('changesets'), total)
+                p1, p2, base = defineparents(repo, rev, self.target,
+                                             self.state,
+                                             self.targetancestors,
+                                             self.obsoletenotrebased)
+                storestatus(repo, self.originalwd, self.target,
+                            self.state, self.collapsef, self.keepf,
+                            self.keepbranchesf, self.external,
+                            self.activebookmark)
+                storecollapsemsg(repo, self.collapsemsg)
+                if len(repo[None].parents()) == 2:
+                    repo.ui.debug('resuming interrupted rebase\n')
+                else:
+                    try:
+                        ui.setconfig('ui', 'forcemerge', opts.get('tool', ''),
+                                     'rebase')
+                        stats = rebasenode(repo, rev, p1, base, self.state,
+                                           self.collapsef, self.target)
+                        if stats and stats[3] > 0:
+                            raise error.InterventionRequired(
+                                _('unresolved conflicts (see hg '
+                                  'resolve, then hg rebase --continue)'))
+                    finally:
+                        ui.setconfig('ui', 'forcemerge', '', 'rebase')
+                if not self.collapsef:
+                    merging = p2 != nullrev
+                    editform = cmdutil.mergeeditform(merging, 'rebase')
+                    editor = cmdutil.getcommiteditor(editform=editform, **opts)
+                    newnode = concludenode(repo, rev, p1, p2,
+                                           extrafn=_makeextrafn(self.extrafns),
+                                           editor=editor,
+                                           keepbranches=self.keepbranchesf,
+                                           date=self.date)
+                else:
+                    # Skip commit if we are collapsing
+                    repo.dirstate.beginparentchange()
+                    repo.setparents(repo[p1].node())
+                    repo.dirstate.endparentchange()
+                    newnode = None
+                # Update the state
+                if newnode is not None:
+                    self.state[rev] = repo[newnode].rev()
+                    ui.debug('rebased as %s\n' % short(newnode))
+                else:
+                    if not self.collapsef:
+                        ui.warn(_('note: rebase of %d:%s created no changes '
+                                  'to commit\n') % (rev, ctx))
+                        self.skipped.add(rev)
+                    self.state[rev] = p1
+                    ui.debug('next revision set to %s\n' % p1)
+            elif self.state[rev] == nullmerge:
+                ui.debug('ignoring null merge rebase of %s\n' % rev)
+            elif self.state[rev] == revignored:
+                ui.status(_('not rebasing ignored %s\n') % desc)
+            elif self.state[rev] == revprecursor:
+                targetctx = repo[self.obsoletenotrebased[rev]]
+                desctarget = '%d:%s "%s"' % (targetctx.rev(), targetctx,
+                             targetctx.description().split('\n', 1)[0])
+                msg = _('note: not rebasing %s, already in destination as %s\n')
+                ui.status(msg % (desc, desctarget))
+            elif self.state[rev] == revpruned:
+                msg = _('note: not rebasing %s, it has no successor\n')
+                ui.status(msg % desc)
+            else:
+                ui.status(_('already rebased %s as %s\n') %
+                          (desc, repo[self.state[rev]]))
+
+        ui.progress(_('rebasing'), None)
+        ui.note(_('rebase merging completed\n'))
+
+    def _finishrebase(self):
+        repo, ui, opts = self.repo, self.ui, self.opts
+        if self.collapsef and not self.keepopen:
+            p1, p2, _base = defineparents(repo, min(self.state),
+                                          self.target, self.state,
+                                          self.targetancestors,
+                                          self.obsoletenotrebased)
+            editopt = opts.get('edit')
+            editform = 'rebase.collapse'
+            if self.collapsemsg:
+                commitmsg = self.collapsemsg
+            else:
+                commitmsg = 'Collapsed revision'
+                for rebased in self.state:
+                    if rebased not in self.skipped and\
+                       self.state[rebased] > nullmerge:
+                        commitmsg += '\n* %s' % repo[rebased].description()
+                editopt = True
+            editor = cmdutil.getcommiteditor(edit=editopt, editform=editform)
+            revtoreuse = max(self.state)
+            newnode = concludenode(repo, revtoreuse, p1, self.external,
+                                   commitmsg=commitmsg,
+                                   extrafn=_makeextrafn(self.extrafns),
+                                   editor=editor,
+                                   keepbranches=self.keepbranchesf,
+                                   date=self.date)
+            if newnode is None:
+                newrev = self.target
+            else:
+                newrev = repo[newnode].rev()
+            for oldrev in self.state.iterkeys():
+                if self.state[oldrev] > nullmerge:
+                    self.state[oldrev] = newrev
+
+        if 'qtip' in repo.tags():
+            updatemq(repo, self.state, self.skipped, **opts)
+
+        if self.currentbookmarks:
+            # Nodeids are needed to reset bookmarks
+            nstate = {}
+            for k, v in self.state.iteritems():
+                if v > nullmerge:
+                    nstate[repo[k].node()] = repo[v].node()
+                elif v == revprecursor:
+                    succ = self.obsoletenotrebased[k]
+                    nstate[repo[k].node()] = repo[succ].node()
+            # XXX this is the same as dest.node() for the non-continue path --
+            # this should probably be cleaned up
+            targetnode = repo[self.target].node()
+
+        # restore original working directory
+        # (we do this before stripping)
+        newwd = self.state.get(self.originalwd, self.originalwd)
+        if newwd == revprecursor:
+            newwd = self.obsoletenotrebased[self.originalwd]
+        elif newwd < 0:
+            # original directory is a parent of rebase set root or ignored
+            newwd = self.originalwd
+        if newwd not in [c.rev() for c in repo[None].parents()]:
+            ui.note(_("update back to initial working directory parent\n"))
+            hg.updaterepo(repo, newwd, False)
+
+        if not self.keepf:
+            collapsedas = None
+            if self.collapsef:
+                collapsedas = newnode
+            clearrebased(ui, repo, self.state, self.skipped, collapsedas)
+
+        with repo.transaction('bookmark') as tr:
+            if self.currentbookmarks:
+                updatebookmarks(repo, targetnode, nstate,
+                                self.currentbookmarks, tr)
+                if self.activebookmark not in repo._bookmarks:
+                    # active bookmark was divergent one and has been deleted
+                    self.activebookmark = None
+        clearstatus(repo)
+        clearcollapsemsg(repo)
+
+        ui.note(_("rebase completed\n"))
+        util.unlinkpath(repo.sjoin('undo'), ignoremissing=True)
+        if self.skipped:
+            skippedlen = len(self.skipped)
+            ui.note(_("%d revisions have been skipped\n") % skippedlen)
+
+        if (self.activebookmark and
+            repo['.'].node() == repo._bookmarks[self.activebookmark]):
+                bookmarks.activate(repo, self.activebookmark)
+
 @command('rebase',
     [('s', 'source', '',
      _('rebase the specified changeset and descendants'), _('REV')),
@@ -201,16 +617,7 b' def rebase(ui, repo, **opts):'
     unresolved conflicts.
 
     """
-    originalwd = target = None
-    activebookmark = None
-    external = nullrev
-    # Mapping between the old revision id and either what is the new rebased
-    # revision or what needs to be done with the old revision. The state dict
-    # will be what contains most of the rebase progress state.
-    state = {}
-    skipped = set()
-    targetancestors = set()
-
+    rbsrt = rebaseruntime(repo, ui, opts)
 
     lock = wlock = None
     try:
@@ -227,19 +634,6 b' def rebase(ui, repo, **opts):'
         destspace = opts.get('_destspace')
         contf = opts.get('continue')
         abortf = opts.get('abort')
-        collapsef = opts.get('collapse', False)
-        collapsemsg = cmdutil.logmessage(ui, opts)
-        date = opts.get('date', None)
-        e = opts.get('extrafn') # internal, used by e.g. hgsubversion
-        extrafns = [_savegraft]
-        if e:
-            extrafns = [e]
-        keepf = opts.get('keep', False)
-        keepbranchesf = opts.get('keepbranches', False)
-        # keepopen is not meant for use on the command line, but by
-        # other extensions
-        keepopen = opts.get('keepopen', False)
-
         if opts.get('interactive'):
             try:
                 if extensions.find('histedit'):
@@ -251,14 +645,14 b' def rebase(ui, repo, **opts):'
                     "'histedit' extension (see \"%s\")") % help
             raise error.Abort(msg)
 
-        if collapsemsg and not collapsef:
+        if rbsrt.collapsemsg and not rbsrt.collapsef:
             raise error.Abort(
                 _('message can only be specified with collapse'))
 
         if contf or abortf:
             if contf and abortf:
                 raise error.Abort(_('cannot use both abort and continue'))
-            if collapsef:
+            if rbsrt.collapsef:
                 raise error.Abort(
                     _('cannot use collapse with continue or abort'))
             if srcf or basef or destf:
@@ -267,265 +661,18 b' def rebase(ui, repo, **opts):'
             if abortf and opts.get('tool', False):
                 ui.warn(_('tool option will be ignored\n'))
 
-            try:
-                (originalwd, target, state, skipped, collapsef, keepf,
-                 keepbranchesf, external, activebookmark) = restorestatus(repo)
-                collapsemsg = restorecollapsemsg(repo)
-            except error.RepoLookupError:
-                if abortf:
-                    clearstatus(repo)
-                    clearcollapsemsg(repo)
-                    repo.ui.warn(_('rebase aborted (no revision is removed,'
-                                   ' only broken state is cleared)\n'))
-                    return 0
-                else:
-                    msg = _('cannot continue inconsistent rebase')
-                    hint = _('use "hg rebase --abort" to clear broken state')
-                    raise error.Abort(msg, hint=hint)
-            if abortf:
-                return abort(repo, originalwd, target, state,
-                             activebookmark=activebookmark)
-
-            obsoletenotrebased = {}
-            if ui.configbool('experimental', 'rebaseskipobsolete',
-                             default=True):
-                rebaseobsrevs = set([r for r, status in state.items()
-                                     if status == revprecursor])
-                rebasesetrevs = set(state.keys())
-                obsoletenotrebased = _computeobsoletenotrebased(repo,
-                                                                rebaseobsrevs,
-                                                                target)
-                rebaseobsskipped = set(obsoletenotrebased)
-                _checkobsrebase(repo, ui, rebaseobsrevs, rebasesetrevs,
-                                rebaseobsskipped)
+            retcode = rbsrt._prepareabortorcontinue(abortf)
+            if retcode is not None:
+                return retcode
         else:
             dest, rebaseset = _definesets(ui, repo, destf, srcf, basef, revf,
                                           destspace=destspace)
-            if dest is None:
-                return _nothingtorebase()
-
-            allowunstable = obsolete.isenabled(repo, obsolete.allowunstableopt)
-            if (not (keepf or allowunstable)
-                  and repo.revs('first(children(%ld) - %ld)',
-                                rebaseset, rebaseset)):
-                raise error.Abort(
-                    _("can't remove original changesets with"
-                      " unrebased descendants"),
-                    hint=_('use --keep to keep original changesets'))
-
-            obsoletenotrebased = {}
-            if ui.configbool('experimental', 'rebaseskipobsolete',
-                             default=True):
-                rebasesetrevs = set(rebaseset)
-                rebaseobsrevs = _filterobsoleterevs(repo, rebasesetrevs)
-                obsoletenotrebased = _computeobsoletenotrebased(repo,
-                                                                rebaseobsrevs,
-                                                                dest)
-                rebaseobsskipped = set(obsoletenotrebased)
-                _checkobsrebase(repo, ui, rebaseobsrevs,
-                                              rebasesetrevs,
-                                              rebaseobsskipped)
-
-            result = buildstate(repo, dest, rebaseset, collapsef,
-                                obsoletenotrebased)
-
-            if not result:
-                # Empty state built, nothing to rebase
-                ui.status(_('nothing to rebase\n'))
-                return _nothingtorebase()
-
-            root = min(rebaseset)
-            if not keepf and not repo[root].mutable():
-                raise error.Abort(_("can't rebase public changeset %s")
-                                 % repo[root],
-                                 hint=_('see "hg help phases" for details'))
-
-            originalwd, target, state = result
-            if collapsef:
-                targetancestors = repo.changelog.ancestors([target],
-                                                           inclusive=True)
-                external = externalparent(repo, state, targetancestors)
-
-            if dest.closesbranch() and not keepbranchesf:
-                ui.status(_('reopening closed branch head %s\n') % dest)
-
-        if keepbranchesf:
-            # insert _savebranch at the start of extrafns so if
-            # there's a user-provided extrafn it can clobber branch if
-            # desired
-            extrafns.insert(0, _savebranch)
-            if collapsef:
-                branches = set()
-                for rev in state:
-                    branches.add(repo[rev].branch())
-                    if len(branches) > 1:
-                        raise error.Abort(_('cannot collapse multiple named '
-                            'branches'))
-
-        # Rebase
-        if not targetancestors:
-            targetancestors = repo.changelog.ancestors([target], inclusive=True)
-
-        # Keep track of the current bookmarks in order to reset them later
-        currentbookmarks = repo._bookmarks.copy()
-        activebookmark = activebookmark or repo._activebookmark
-        if activebookmark:
-            bookmarks.deactivate(repo)
-
-        extrafn = _makeextrafn(extrafns)
+            retcode = rbsrt._preparenewrebase(dest, rebaseset)
+            if retcode is not None:
+                return retcode
 
-        sortedstate = sorted(state)
-        total = len(sortedstate)
-        pos = 0
-        for rev in sortedstate:
-            ctx = repo[rev]
-            desc = '%d:%s "%s"' % (ctx.rev(), ctx,
-                                   ctx.description().split('\n', 1)[0])
-            names = repo.nodetags(ctx.node()) + repo.nodebookmarks(ctx.node())
-            if names:
-                desc += ' (%s)' % ' '.join(names)
-            pos += 1
-            if state[rev] == revtodo:
-                ui.status(_('rebasing %s\n') % desc)
-                ui.progress(_("rebasing"), pos, ("%d:%s" % (rev, ctx)),
-                            _('changesets'), total)
-                p1, p2, base = defineparents(repo, rev, target, state,
-                                             targetancestors)
-                storestatus(repo, originalwd, target, state, collapsef, keepf,
-                            keepbranchesf, external, activebookmark)
-                storecollapsemsg(repo, collapsemsg)
-                if len(repo[None].parents()) == 2:
-                    repo.ui.debug('resuming interrupted rebase\n')
-                else:
-                    try:
-                        ui.setconfig('ui', 'forcemerge', opts.get('tool', ''),
-                                     'rebase')
-                        stats = rebasenode(repo, rev, p1, base, state,
-                                           collapsef, target)
-                        if stats and stats[3] > 0:
-                            raise error.InterventionRequired(
-                                _('unresolved conflicts (see hg '
-                                  'resolve, then hg rebase --continue)'))
-                    finally:
-                        ui.setconfig('ui', 'forcemerge', '', 'rebase')
-                if not collapsef:
-                    merging = p2 != nullrev
-                    editform = cmdutil.mergeeditform(merging, 'rebase')
-                    editor = cmdutil.getcommiteditor(editform=editform, **opts)
-                    newnode = concludenode(repo, rev, p1, p2, extrafn=extrafn,
-                                           editor=editor,
-                                           keepbranches=keepbranchesf,
-                                           date=date)
-                else:
-                    # Skip commit if we are collapsing
-                    repo.dirstate.beginparentchange()
-                    repo.setparents(repo[p1].node())
-                    repo.dirstate.endparentchange()
-                    newnode = None
-                # Update the state
-                if newnode is not None:
-                    state[rev] = repo[newnode].rev()
-                    ui.debug('rebased as %s\n' % short(newnode))
-                else:
-                    if not collapsef:
-                        ui.warn(_('note: rebase of %d:%s created no changes '
-                                  'to commit\n') % (rev, ctx))
-                        skipped.add(rev)
-                    state[rev] = p1
-                    ui.debug('next revision set to %s\n' % p1)
-            elif state[rev] == nullmerge:
-                ui.debug('ignoring null merge rebase of %s\n' % rev)
-            elif state[rev] == revignored:
-                ui.status(_('not rebasing ignored %s\n') % desc)
-            elif state[rev] == revprecursor:
-                targetctx = repo[obsoletenotrebased[rev]]
-                desctarget = '%d:%s "%s"' % (targetctx.rev(), targetctx,
-                             targetctx.description().split('\n', 1)[0])
-                msg = _('note: not rebasing %s, already in destination as %s\n')
-                ui.status(msg % (desc, desctarget))
-            elif state[rev] == revpruned:
-                msg = _('note: not rebasing %s, it has no successor\n')
-                ui.status(msg % desc)
-            else:
-                ui.status(_('already rebased %s as %s\n') %
-                          (desc, repo[state[rev]]))
-
-        ui.progress(_('rebasing'), None)
-        ui.note(_('rebase merging completed\n'))
-
-        if collapsef and not keepopen:
-            p1, p2, _base = defineparents(repo, min(state), target,
-                                          state, targetancestors)
-            editopt = opts.get('edit')
-            editform = 'rebase.collapse'
-            if collapsemsg:
-                commitmsg = collapsemsg
-            else:
-                commitmsg = 'Collapsed revision'
-                for rebased in state:
-                    if rebased not in skipped and state[rebased] > nullmerge:
-                        commitmsg += '\n* %s' % repo[rebased].description()
-                editopt = True
-            editor = cmdutil.getcommiteditor(edit=editopt, editform=editform)
-            newnode = concludenode(repo, rev, p1, external, commitmsg=commitmsg,
-                                   extrafn=extrafn, editor=editor,
-                                   keepbranches=keepbranchesf,
-                                   date=date)
-            if newnode is None:
-                newrev = target
-            else:
-                newrev = repo[newnode].rev()
-            for oldrev in state.iterkeys():
-                if state[oldrev] > nullmerge:
-                    state[oldrev] = newrev
-
-        if 'qtip' in repo.tags():
-            updatemq(repo, state, skipped, **opts)
-
-        if currentbookmarks:
-            # Nodeids are needed to reset bookmarks
-            nstate = {}
-            for k, v in state.iteritems():
-                if v > nullmerge:
-                    nstate[repo[k].node()] = repo[v].node()
-            # XXX this is the same as dest.node() for the non-continue path --
-            # this should probably be cleaned up
-            targetnode = repo[target].node()
-
-        # restore original working directory
-        # (we do this before stripping)
-        newwd = state.get(originalwd, originalwd)
-        if newwd < 0:
-            # original directory is a parent of rebase set root or ignored
-            newwd = originalwd
-        if newwd not in [c.rev() for c in repo[None].parents()]:
-            ui.note(_("update back to initial working directory parent\n"))
-            hg.updaterepo(repo, newwd, False)
-
-        if not keepf:
-            collapsedas = None
-            if collapsef:
-                collapsedas = newnode
-            clearrebased(ui, repo, state, skipped, collapsedas)
-
-        with repo.transaction('bookmark') as tr:
-            if currentbookmarks:
-                updatebookmarks(repo, targetnode, nstate, currentbookmarks, tr)
-                if activebookmark not in repo._bookmarks:
-                    # active bookmark was divergent one and has been deleted
-                    activebookmark = None
-        clearstatus(repo)
-        clearcollapsemsg(repo)
-
-        ui.note(_("rebase completed\n"))
-        util.unlinkpath(repo.sjoin('undo'), ignoremissing=True)
-        if skipped:
-            ui.note(_("%d revisions have been skipped\n") % len(skipped))
-
-        if (activebookmark and
-            repo['.'].node() == repo._bookmarks[activebookmark]):
-                bookmarks.activate(repo, activebookmark)
-
+        rbsrt._performrebase()
+        rbsrt._finishrebase()
     finally:
         release(lock, wlock)
 
@@ -733,21 +880,12 b' def _checkobsrebase(repo, ui,'
               "experimental.allowdivergence=True")
         raise error.Abort(msg % (",".join(divhashes),), hint=h)
 
-    # - plain prune (no successor) changesets are rebased
-    # - split changesets are not rebased if at least one of the
-    # changeset resulting from the split is an ancestor of dest
-    rebaseset = rebasesetrevs - rebaseobsskipped
-    if rebasesetrevs and not rebaseset:
-        msg = _('all requested changesets have equivalents '
-                'or were marked as obsolete')
-        hint = _('to force the rebase, set the config '
-                 'experimental.rebaseskipobsolete to False')
-        raise error.Abort(msg, hint=hint)
-
-def defineparents(repo, rev, target, state, targetancestors):
+def defineparents(repo, rev, target, state, targetancestors,
+                  obsoletenotrebased):
     'Return the new parent relationship of the revision that will be rebased'
     parents = repo[rev].parents()
     p1 = p2 = nullrev
+    rp1 = None
 
     p1n = parents[0].rev()
     if p1n in targetancestors:
@@ -771,6 +909,8 b' def defineparents(repo, rev, target, sta'
         if p2n in state:
             if p1 == target: # p1n in targetancestors or external
                 p1 = state[p2n]
+                if p1 == revprecursor:
+                    rp1 = obsoletenotrebased[p2n]
             elif state[p2n] in revskipped:
                 p2 = nearestrebased(repo, p2n, state)
                 if p2 is None:
@@ -784,7 +924,7 b' def defineparents(repo, rev, target, sta'
                         'would have 3 parents') % rev)
             p2 = p2n
     repo.ui.debug(" future parents are %d and %d\n" %
-                            (repo[p1].rev(), repo[p2].rev()))
+                            (repo[rp1 or p1].rev(), repo[p2].rev()))
 
     if not any(p.rev() in state for p in parents):
         # Case (1) root changeset of a non-detaching rebase set.
@@ -828,6 +968,8 b' def defineparents(repo, rev, target, sta'
         # make it feasible to consider different cases separately. In these
         # other cases we currently just leave it to the user to correctly
         # resolve an impossible merge using a wrong ancestor.
+        #
+        # xx, p1 could be -4, and both parents could probably be -4...
         for p in repo[rev].parents():
             if state.get(p.rev()) == p1:
                 base = p.rev()
@@ -838,7 +980,7 b' def defineparents(repo, rev, target, sta'
             # Raise because this function is called wrong (see issue 4106)
             raise AssertionError('no base found to rebase on '
                                  '(defineparents called wrong)')
-    return p1, p2, base
+    return rp1 or p1, p2, base
 
 def isagitpatch(repo, patchname):
     'Return true if the given patch is in git format'
@@ -952,68 +1094,6 b' def clearstatus(repo):'
     _clearrebasesetvisibiliy(repo)
     util.unlinkpath(repo.join("rebasestate"), ignoremissing=True)
 
-def restorestatus(repo):
-    'Restore a previously stored status'
-    keepbranches = None
-    target = None
-    collapse = False
-    external = nullrev
-    activebookmark = None
-    state = {}
-
-    try:
-        f = repo.vfs("rebasestate")
-        for i, l in enumerate(f.read().splitlines()):
-            if i == 0:
-                originalwd = repo[l].rev()
-            elif i == 1:
-                target = repo[l].rev()
-            elif i == 2:
-                external = repo[l].rev()
-            elif i == 3:
-                collapse = bool(int(l))
-            elif i == 4:
-                keep = bool(int(l))
-            elif i == 5:
-                keepbranches = bool(int(l))
-            elif i == 6 and not (len(l) == 81 and ':' in l):
-                # line 6 is a recent addition, so for backwards compatibility
-                # check that the line doesn't look like the oldrev:newrev lines
-                activebookmark = l
-            else:
-                oldrev, newrev = l.split(':')
-                if newrev in (str(nullmerge), str(revignored),
-                              str(revprecursor), str(revpruned)):
-                    state[repo[oldrev].rev()] = int(newrev)
-                elif newrev == nullid:
-                    state[repo[oldrev].rev()] = revtodo
-                    # Legacy compat special case
-                else:
-                    state[repo[oldrev].rev()] = repo[newrev].rev()
-
-    except IOError as err:
-        if err.errno != errno.ENOENT:
-            raise
-        cmdutil.wrongtooltocontinue(repo, _('rebase'))
-
-    if keepbranches is None:
-        raise error.Abort(_('.hg/rebasestate is incomplete'))
-
-    skipped = set()
-    # recompute the set of skipped revs
-    if not collapse:
-        seen = set([target])
-        for old, new in sorted(state.items()):
-            if new != revtodo and new in seen:
-                skipped.add(old)
-            seen.add(new)
-    repo.ui.debug('computed skipped revs: %s\n' %
-                    (' '.join(str(r) for r in sorted(skipped)) or None))
-    repo.ui.debug('rebase status resumed\n')
-    _setrebasesetvisibility(repo, state.keys())
-    return (originalwd, target, state, skipped,
-            collapse, keep, keepbranches, external, activebookmark)
-
 def needupdate(repo, state):
     '''check whether we should `update --clean` away from a merge, or if
     somehow the working dir got forcibly updated, e.g. by older hg'''
@@ -1336,7 +1416,9 b' def summaryhook(ui, repo):'
     if not os.path.exists(repo.join('rebasestate')):
         return
     try:
-        state = restorestatus(repo)[2]
+        rbsrt = rebaseruntime(repo, ui, {})
+        rbsrt.restorestatus()
+        state = rbsrt.state
     except error.RepoLookupError:
         # i18n: column positioning for "hg summary"
         msg = _('rebase: (use "hg rebase --abort" to clear broken state)\n')

hgext/record.py

@@ -12,13 +12,13 b' The feature provided by this extension h'
 
 from __future__ import absolute_import
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     error,
     extensions,
 )
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)

hgext/relink.py

@@ -11,13 +11,13 b' from __future__ import absolute_import'
 import os
 import stat
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     error,
     hg,
     util,
 )
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)

hgext/schemes.py

@@ -43,6 +43,8 b' from __future__ import absolute_import'
 
 import os
 import re
+
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     error,
@@ -51,7 +53,6 b' from mercurial import ('
     templater,
     util,
 )
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)

hgext/share.py

@@ -37,10 +37,22 b' The following ``share.`` config options '
     The default naming mode is "identity."
 '''
 
+from __future__ import absolute_import
+
+import errno
 from mercurial.i18n import _
-from mercurial import cmdutil, commands, hg, util, extensions, bookmarks, error
-from mercurial.hg import repository, parseurl
-import errno
+from mercurial import (
+    bookmarks,
+    cmdutil,
+    commands,
+    error,
+    extensions,
+    hg,
+    util,
+)
+
+repository = hg.repository
+parseurl = hg.parseurl
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
@@ -135,7 +147,7 b' def _hassharedbookmarks(repo):'
         if inst.errno != errno.ENOENT:
             raise
         return False
-    return 'bookmarks' in shared
+    return hg.sharedbookmarks in shared
 
 def _getsrcrepo(repo):
     """
@@ -145,10 +157,15 b' def _getsrcrepo(repo):'
     if repo.sharedpath == repo.path:
         return None
 
+    if util.safehasattr(repo, 'srcrepo') and repo.srcrepo:
+        return repo.srcrepo
+
     # the sharedpath always ends in the .hg; we want the path to the repo
     source = repo.vfs.split(repo.sharedpath)[0]
     srcurl, branches = parseurl(source)
-    return repository(repo.ui, srcurl)
+    srcrepo = repository(repo.ui, srcurl)
+    repo.srcrepo = srcrepo
+    return srcrepo
 
 def getbkfile(orig, repo):
     if _hassharedbookmarks(repo):

hgext/shelve.py

@@ -25,6 +25,8 b' from __future__ import absolute_import'
 import collections
 import errno
 import itertools
+
+from mercurial.i18n import _
 from mercurial import (
     bundle2,
     bundlerepo,
@@ -45,7 +47,6 b' from mercurial import ('
     templatefilters,
     util,
 )
-from mercurial.i18n import _
 
 from . import (
     rebase,
@@ -164,21 +165,26 b' class shelvedstate(object):'
                 raise error.Abort(_('this version of shelve is incompatible '
                                    'with the version used in this repo'))
             name = fp.readline().strip()
-            wctx = fp.readline().strip()
-            pendingctx = fp.readline().strip()
+            wctx = nodemod.bin(fp.readline().strip())
+            pendingctx = nodemod.bin(fp.readline().strip())
             parents = [nodemod.bin(h) for h in fp.readline().split()]
             stripnodes = [nodemod.bin(h) for h in fp.readline().split()]
             branchtorestore = fp.readline().strip()
+        except (ValueError, TypeError) as err:
+            raise error.CorruptedState(str(err))
         finally:
             fp.close()
 
-        obj = cls()
-        obj.name = name
-        obj.wctx = repo[nodemod.bin(wctx)]
-        obj.pendingctx = repo[nodemod.bin(pendingctx)]
-        obj.parents = parents
-        obj.stripnodes = stripnodes
-        obj.branchtorestore = branchtorestore
+        try:
+            obj = cls()
+            obj.name = name
+            obj.wctx = repo[wctx]
+            obj.pendingctx = repo[pendingctx]
+            obj.parents = parents
+            obj.stripnodes = stripnodes
+            obj.branchtorestore = branchtorestore
+        except error.RepoLookupError as err:
+            raise error.CorruptedState(str(err))
 
         return obj
 
@@ -225,28 +231,10 b' def cleanupoldbackups(repo):'
 def _aborttransaction(repo):
     '''Abort current transaction for shelve/unshelve, but keep dirstate
     '''
-    backupname = 'dirstate.shelve'
-    dirstatebackup = None
-    try:
-        # create backup of (un)shelved dirstate, because aborting transaction
-        # should restore dirstate to one at the beginning of the
-        # transaction, which doesn't include the result of (un)shelving
-        fp = repo.vfs.open(backupname, "w")
-        dirstatebackup = backupname
-        # clearing _dirty/_dirtypl of dirstate by _writedirstate below
-        # is unintentional. but it doesn't cause problem in this case,
-        # because no code path refers them until transaction is aborted.
-        repo.dirstate._writedirstate(fp) # write in-memory changes forcibly
-
-        tr = repo.currenttransaction()
-        tr.abort()
-
-        # restore to backuped dirstate
-        repo.vfs.rename(dirstatebackup, 'dirstate')
-        dirstatebackup = None
-    finally:
-        if dirstatebackup:
-            repo.vfs.unlink(dirstatebackup)
+    tr = repo.currenttransaction()
+    repo.dirstate.savebackup(tr, suffix='.shelve')
+    tr.abort()
+    repo.dirstate.restorebackup(None, suffix='.shelve')
 
 def createcmd(ui, repo, pats, opts):
     """subcommand that creates a new shelve"""
@@ -683,6 +671,20 b' def _dounshelve(ui, repo, *shelved, **op'
             if err.errno != errno.ENOENT:
                 raise
             cmdutil.wrongtooltocontinue(repo, _('unshelve'))
+        except error.CorruptedState as err:
+            ui.debug(str(err) + '\n')
+            if continuef:
+                msg = _('corrupted shelved state file')
+                hint = _('please run hg unshelve --abort to abort unshelve '
+                         'operation')
+                raise error.Abort(msg, hint=hint)
+            elif abortf:
+                msg = _('could not read shelved state file, your working copy '
+                        'may be in an unexpected state\nplease update to some '
+                        'commit\n')
+                ui.warn(msg)
+                shelvedstate.clear(repo)
+            return
 
         if abortf:
             return unshelveabort(ui, repo, state, opts)

hgext/strip.py

@@ -5,6 +5,7 b' repository. See the command help for det'
 """
 from __future__ import absolute_import
 
+from mercurial.i18n import _
 from mercurial import (
     bookmarks as bookmarksmod,
     cmdutil,
@@ -17,7 +18,6 b' from mercurial import ('
     scmutil,
     util,
 )
-from mercurial.i18n import _
 nullid = nodemod.nullid
 release = lockmod.release
 

hgext/win32mbcs.py

@@ -49,11 +49,11 b' from __future__ import absolute_import'
 import os
 import sys
 
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
 )
-from mercurial.i18n import _
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
@@ -192,5 +192,5 b' def extsetup(ui):'
         # command line options is not yet applied when
         # extensions.loadall() is called.
         if '--debug' in sys.argv:
-            ui.write("[win32mbcs] activated with encoding: %s\n"
+            ui.write(("[win32mbcs] activated with encoding: %s\n")
                      % _encoding)

hgext/win32text.py

@@ -41,10 +41,16 b' pushed or pulled::'
   # or pretxnchangegroup.cr = python:hgext.win32text.forbidcr
 '''
 
+from __future__ import absolute_import
+
+import re
 from mercurial.i18n import _
-from mercurial.node import short
-from mercurial import util
-import re
+from mercurial.node import (
+    short,
+)
+from mercurial import (
+    util,
+)
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

i18n/hggettext

@@ -20,7 +20,11 b' Use xgettext like normal to extract stri'
 join the message cataloges to get the final catalog.
 """
 
-import os, sys, inspect
+from __future__ import absolute_import, print_function
+
+import inspect
+import os
+import sys
 
 
 def escape(s):
@@ -95,7 +99,7 b' def docstrings(path):'
     if mod.__doc__:
         src = open(path).read()
         lineno = 1 + offset(src, mod.__doc__, path, 7)
-        print poentry(path, lineno, mod.__doc__)
+        print(poentry(path, lineno, mod.__doc__))
 
     functions = list(getattr(mod, 'i18nfunctions', []))
     functions = [(f, True) for f in functions]
@@ -115,12 +119,12 b' def docstrings(path):'
             if rstrip:
                 doc = doc.rstrip()
             lineno += offset(src, doc, name, 1)
-            print poentry(path, lineno, doc)
+            print(poentry(path, lineno, doc))
 
 
 def rawtext(path):
     src = open(path).read()
-    print poentry(path, 1, src)
+    print(poentry(path, 1, src))
 
 
 if __name__ == "__main__":

i18n/polib.py

@@ -13,6 +13,8 b' modify entries, comments or metadata, et'
 :func:`~polib.mofile` convenience functions.
 """
 
+from __future__ import absolute_import
+
 __author__    = 'David Jean Louis <[e-mail]>'
 __version__   = '0.6.4'
 __all__       = ['pofile', 'POFile', 'POEntry', 'mofile', 'MOFile', 'MOEntry',

i18n/posplit

@@ -5,9 +5,11 b''
 # license: MIT/X11/Expat
 #
 
+from __future__ import absolute_import, print_function
+
+import polib
 import re
 import sys
-import polib
 
 def addentry(po, entry, cache):
     e = cache.get(entry.msgid)
@@ -67,8 +69,8 b' if __name__ == "__main__":'
                             continue
                         else:
                             # lines following directly, unexpected
-                            print 'Warning: text follows line with directive' \
-                                  ' %s' % directive
+                            print('Warning: text follows line with directive' \
+                                  ' %s' % directive)
                     comment = 'do not translate: .. %s::' % directive
                     if not newentry.comment:
                         newentry.comment = comment

mercurial/__init__.py

@@ -12,36 +12,13 b' import os'
 import sys
 import zipimport
 
+from . import (
+    policy
+)
+
 __all__ = []
 
-# Rules for how modules can be loaded. Values are:
-#
-#    c - require C extensions
-#    allow - allow pure Python implementation when C loading fails
-#    py - only load pure Python modules
-#
-# By default, require the C extensions for performance reasons.
-modulepolicy = 'c'
-try:
-    from . import __modulepolicy__
-    modulepolicy = __modulepolicy__.modulepolicy
-except ImportError:
-    pass
-
-# PyPy doesn't load C extensions.
-#
-# The canonical way to do this is to test platform.python_implementation().
-# But we don't import platform and don't bloat for it here.
-if '__pypy__' in sys.builtin_module_names:
-    modulepolicy = 'py'
-
-# Our C extensions aren't yet compatible with Python 3. So use pure Python
-# on Python 3 for now.
-if sys.version_info[0] >= 3:
-    modulepolicy = 'py'
-
-# Environment variable can always force settings.
-modulepolicy = os.environ.get('HGMODULEPOLICY', modulepolicy)
+modulepolicy = policy.policy
 
 # Modules that have both Python and C implementations. See also the
 # set of .py files under mercurial/pure/.
@@ -82,7 +59,7 b' class hgimporter(object):'
                 return zl
 
             try:
-                if modulepolicy == 'py':
+                if modulepolicy in policy.policynoc:
                     raise ImportError()
 
                 zl = ziploader('mercurial')
@@ -109,7 +86,7 b' class hgimporter(object):'
         stem = name.split('.')[-1]
 
         try:
-            if modulepolicy == 'py':
+            if modulepolicy in policy.policynoc:
                 raise ImportError()
 
             modinfo = imp.find_module(stem, mercurial.__path__)
@@ -144,9 +121,238 b' class hgimporter(object):'
         sys.modules[name] = mod
         return mod
 
+# Python 3 uses a custom module loader that transforms source code between
+# source file reading and compilation. This is done by registering a custom
+# finder that changes the spec for Mercurial modules to use a custom loader.
+if sys.version_info[0] >= 3:
+    from . import pure
+    import importlib
+    import io
+    import token
+    import tokenize
+
+    class hgpathentryfinder(importlib.abc.MetaPathFinder):
+        """A sys.meta_path finder that uses a custom module loader."""
+        def find_spec(self, fullname, path, target=None):
+            # Only handle Mercurial-related modules.
+            if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')):
+                return None
+
+            # This assumes Python 3 doesn't support loading C modules.
+            if fullname in _dualmodules:
+                stem = fullname.split('.')[-1]
+                fullname = 'mercurial.pure.%s' % stem
+                target = pure
+                assert len(path) == 1
+                path = [os.path.join(path[0], 'pure')]
+
+            # Try to find the module using other registered finders.
+            spec = None
+            for finder in sys.meta_path:
+                if finder == self:
+                    continue
+
+                spec = finder.find_spec(fullname, path, target=target)
+                if spec:
+                    break
+
+            # This is a Mercurial-related module but we couldn't find it
+            # using the previously-registered finders. This likely means
+            # the module doesn't exist.
+            if not spec:
+                return None
+
+            if fullname.startswith('mercurial.pure.'):
+                spec.name = spec.name.replace('.pure.', '.')
+
+            # TODO need to support loaders from alternate specs, like zip
+            # loaders.
+            spec.loader = hgloader(spec.name, spec.origin)
+            return spec
+
+    def replacetokens(tokens):
+        """Transform a stream of tokens from raw to Python 3.
+
+        It is called by the custom module loading machinery to rewrite
+        source/tokens between source decoding and compilation.
+
+        Returns a generator of possibly rewritten tokens.
+
+        The input token list may be mutated as part of processing. However,
+        its changes do not necessarily match the output token stream.
+
+        REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
+        OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
+        """
+        for i, t in enumerate(tokens):
+            # Convert most string literals to byte literals. String literals
+            # in Python 2 are bytes. String literals in Python 3 are unicode.
+            # Most strings in Mercurial are bytes and unicode strings are rare.
+            # Rather than rewrite all string literals to use ``b''`` to indicate
+            # byte strings, we apply this token transformer to insert the ``b``
+            # prefix nearly everywhere.
+            if t.type == token.STRING:
+                s = t.string
+
+                # Preserve docstrings as string literals. This is inconsistent
+                # with regular unprefixed strings. However, the
+                # "from __future__" parsing (which allows a module docstring to
+                # exist before it) doesn't properly handle the docstring if it
+                # is b''' prefixed, leading to a SyntaxError. We leave all
+                # docstrings as unprefixed to avoid this. This means Mercurial
+                # components touching docstrings need to handle unicode,
+                # unfortunately.
+                if s[0:3] in ("'''", '"""'):
+                    yield t
+                    continue
+
+                # If the first character isn't a quote, it is likely a string
+                # prefixing character (such as 'b', 'u', or 'r'. Ignore.
+                if s[0] not in ("'", '"'):
+                    yield t
+                    continue
+
+                # String literal. Prefix to make a b'' string.
+                yield tokenize.TokenInfo(t.type, 'b%s' % s, t.start, t.end,
+                                          t.line)
+                continue
+
+            try:
+                nexttoken = tokens[i + 1]
+            except IndexError:
+                nexttoken = None
+
+            try:
+                prevtoken = tokens[i - 1]
+            except IndexError:
+                prevtoken = None
+
+            # This looks like a function call.
+            if (t.type == token.NAME and nexttoken and
+                nexttoken.type == token.OP and nexttoken.string == '('):
+                fn = t.string
+
+                # *attr() builtins don't accept byte strings to 2nd argument.
+                # Rewrite the token to include the unicode literal prefix so
+                # the string transformer above doesn't add the byte prefix.
+                if fn in ('getattr', 'setattr', 'hasattr', 'safehasattr'):
+                    try:
+                        # (NAME, 'getattr')
+                        # (OP, '(')
+                        # (NAME, 'foo')
+                        # (OP, ',')
+                        # (NAME|STRING, foo)
+                        st = tokens[i + 4]
+                        if (st.type == token.STRING and
+                            st.string[0] in ("'", '"')):
+                            rt = tokenize.TokenInfo(st.type, 'u%s' % st.string,
+                                                    st.start, st.end, st.line)
+                            tokens[i + 4] = rt
+                    except IndexError:
+                        pass
+
+                # .encode() and .decode() on str/bytes/unicode don't accept
+                # byte strings on Python 3. Rewrite the token to include the
+                # unicode literal prefix so the string transformer above doesn't
+                # add the byte prefix.
+                if (fn in ('encode', 'decode') and
+                    prevtoken.type == token.OP and prevtoken.string == '.'):
+                    # (OP, '.')
+                    # (NAME, 'encode')
+                    # (OP, '(')
+                    # (STRING, 'utf-8')
+                    # (OP, ')')
+                    try:
+                        st = tokens[i + 2]
+                        if (st.type == token.STRING and
+                            st.string[0] in ("'", '"')):
+                            rt = tokenize.TokenInfo(st.type, 'u%s' % st.string,
+                                                    st.start, st.end, st.line)
+                            tokens[i + 2] = rt
+                    except IndexError:
+                        pass
+
+            # Emit unmodified token.
+            yield t
+
+    # Header to add to bytecode files. This MUST be changed when
+    # ``replacetoken`` or any mechanism that changes semantics of module
+    # loading is changed. Otherwise cached bytecode may get loaded without
+    # the new transformation mechanisms applied.
+    BYTECODEHEADER = b'HG\x00\x01'
+
+    class hgloader(importlib.machinery.SourceFileLoader):
+        """Custom module loader that transforms source code.
+
+        When the source code is converted to a code object, we transform
+        certain patterns to be Python 3 compatible. This allows us to write code
+        that is natively Python 2 and compatible with Python 3 without
+        making the code excessively ugly.
+
+        We do this by transforming the token stream between parse and compile.
+
+        Implementing transformations invalidates caching assumptions made
+        by the built-in importer. The built-in importer stores a header on
+        saved bytecode files indicating the Python/bytecode version. If the
+        version changes, the cached bytecode is ignored. The Mercurial
+        transformations could change at any time. This means we need to check
+        that cached bytecode was generated with the current transformation
+        code or there could be a mismatch between cached bytecode and what
+        would be generated from this class.
+
+        We supplement the bytecode caching layer by wrapping ``get_data``
+        and ``set_data``. These functions are called when the
+        ``SourceFileLoader`` retrieves and saves bytecode cache files,
+        respectively. We simply add an additional header on the file. As
+        long as the version in this file is changed when semantics change,
+        cached bytecode should be invalidated when transformations change.
+
+        The added header has the form ``HG<VERSION>``. That is a literal
+        ``HG`` with 2 binary bytes indicating the transformation version.
+        """
+        def get_data(self, path):
+            data = super(hgloader, self).get_data(path)
+
+            if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
+                return data
+
+            # There should be a header indicating the Mercurial transformation
+            # version. If it doesn't exist or doesn't match the current version,
+            # we raise an OSError because that is what
+            # ``SourceFileLoader.get_code()`` expects when loading bytecode
+            # paths to indicate the cached file is "bad."
+            if data[0:2] != b'HG':
+                raise OSError('no hg header')
+            if data[0:4] != BYTECODEHEADER:
+                raise OSError('hg header version mismatch')
+
+            return data[4:]
+
+        def set_data(self, path, data, *args, **kwargs):
+            if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
+                data = BYTECODEHEADER + data
+
+            return super(hgloader, self).set_data(path, data, *args, **kwargs)
+
+        def source_to_code(self, data, path):
+            """Perform token transformation before compilation."""
+            buf = io.BytesIO(data)
+            tokens = tokenize.tokenize(buf.readline)
+            data = tokenize.untokenize(replacetokens(list(tokens)))
+            # Python's built-in importer strips frames from exceptions raised
+            # for this code. Unfortunately, that mechanism isn't extensible
+            # and our frame will be blamed for the import failure. There
+            # are extremely hacky ways to do frame stripping. We haven't
+            # implemented them because they are very ugly.
+            return super(hgloader, self).source_to_code(data, path)
+
 # We automagically register our custom importer as a side-effect of loading.
 # This is necessary to ensure that any entry points are able to import
 # mercurial.* modules without having to perform this registration themselves.
-if not any(isinstance(x, hgimporter) for x in sys.meta_path):
+if sys.version_info[0] >= 3:
+    _importercls = hgpathentryfinder
+else:
+    _importercls = hgimporter
+if not any(isinstance(x, _importercls) for x in sys.meta_path):
     # meta_path is used before any implicit finders and before sys.path.
-    sys.meta_path.insert(0, hgimporter())
+    sys.meta_path.insert(0, _importercls())

mercurial/ancestor.py

@@ -291,7 +291,7 b' class lazyancestors(object):'
     def __nonzero__(self):
         """False if the set is empty, True otherwise."""
         try:
-            iter(self).next()
+            next(iter(self))
             return True
         except StopIteration:
             return False

mercurial/bdiff.c

@@ -9,37 +9,25 b''
  Based roughly on Python difflib
 */
 
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
 
-#include "util.h"
-
-struct line {
-	int hash, n, e;
-	Py_ssize_t len;
-	const char *l;
-};
+#include "compat.h"
+#include "bitmanipulation.h"
+#include "bdiff.h"
 
 struct pos {
 	int pos, len;
 };
 
-struct hunk;
-struct hunk {
-	int a1, a2, b1, b2;
-	struct hunk *next;
-};
-
-static int splitlines(const char *a, Py_ssize_t len, struct line **lr)
+int bdiff_splitlines(const char *a, ssize_t len, struct bdiff_line **lr)
 {
 	unsigned hash;
 	int i;
 	const char *p, *b = a;
 	const char * const plast = a + len - 1;
-	struct line *l;
+	struct bdiff_line *l;
 
 	/* count the lines */
 	i = 1; /* extra line for sentinel */
@@ -47,7 +35,7 b' static int splitlines(const char *a, Py_'
 		if (*p == '\n' || p == plast)
 			i++;
 
-	*lr = l = (struct line *)malloc(sizeof(struct line) * i);
+	*lr = l = (struct bdiff_line *)malloc(sizeof(struct bdiff_line) * i);
 	if (!l)
 		return -1;
 
@@ -75,12 +63,13 b' static int splitlines(const char *a, Py_'
 	return i - 1;
 }
 
-static inline int cmp(struct line *a, struct line *b)
+static inline int cmp(struct bdiff_line *a, struct bdiff_line *b)
 {
 	return a->hash != b->hash || a->len != b->len || memcmp(a->l, b->l, a->len);
 }
 
-static int equatelines(struct line *a, int an, struct line *b, int bn)
+static int equatelines(struct bdiff_line *a, int an, struct bdiff_line *b,
+	int bn)
 {
 	int i, j, buckets = 1, t, scale;
 	struct pos *h = NULL;
@@ -145,7 +134,8 b' static int equatelines(struct line *a, i'
 	return 1;
 }
 
-static int longest_match(struct line *a, struct line *b, struct pos *pos,
+static int longest_match(struct bdiff_line *a, struct bdiff_line *b,
+			struct pos *pos,
 			 int a1, int a2, int b1, int b2, int *omi, int *omj)
 {
 	int mi = a1, mj = b1, mk = 0, i, j, k, half;
@@ -206,8 +196,9 b' static int longest_match(struct line *a,'
 	return mk;
 }
 
-static struct hunk *recurse(struct line *a, struct line *b, struct pos *pos,
-			    int a1, int a2, int b1, int b2, struct hunk *l)
+static struct bdiff_hunk *recurse(struct bdiff_line *a, struct bdiff_line *b,
+				struct pos *pos,
+			    int a1, int a2, int b1, int b2, struct bdiff_hunk *l)
 {
 	int i, j, k;
 
@@ -222,7 +213,7 b' static struct hunk *recurse(struct line '
 		if (!l)
 			return NULL;
 
-		l->next = (struct hunk *)malloc(sizeof(struct hunk));
+		l->next = (struct bdiff_hunk *)malloc(sizeof(struct bdiff_hunk));
 		if (!l->next)
 			return NULL;
 
@@ -239,10 +230,10 b' static struct hunk *recurse(struct line '
 	}
 }
 
-static int diff(struct line *a, int an, struct line *b, int bn,
-		 struct hunk *base)
+int bdiff_diff(struct bdiff_line *a, int an, struct bdiff_line *b,
+		int bn, struct bdiff_hunk *base)
 {
-	struct hunk *curr;
+	struct bdiff_hunk *curr;
 	struct pos *pos;
 	int t, count = 0;
 
@@ -258,7 +249,7 b' static int diff(struct line *a, int an, '
 			return -1;
 
 		/* sentinel end hunk */
-		curr->next = (struct hunk *)malloc(sizeof(struct hunk));
+		curr->next = (struct bdiff_hunk *)malloc(sizeof(struct bdiff_hunk));
 		if (!curr->next)
 			return -1;
 		curr = curr->next;
@@ -271,7 +262,7 b' static int diff(struct line *a, int an, '
 
 	/* normalize the hunk list, try to push each hunk towards the end */
 	for (curr = base->next; curr; curr = curr->next) {
-		struct hunk *next = curr->next;
+		struct bdiff_hunk *next = curr->next;
 
 		if (!next)
 			break;
@@ -293,195 +284,13 b' static int diff(struct line *a, int an, '
 	return count;
 }
 
-static void freehunks(struct hunk *l)
+void bdiff_freehunks(struct bdiff_hunk *l)
 {
-	struct hunk *n;
+	struct bdiff_hunk *n;
 	for (; l; l = n) {
 		n = l->next;
 		free(l);
 	}
 }
 
-static PyObject *blocks(PyObject *self, PyObject *args)
-{
-	PyObject *sa, *sb, *rl = NULL, *m;
-	struct line *a, *b;
-	struct hunk l, *h;
-	int an, bn, count, pos = 0;
 
-	l.next = NULL;
-
-	if (!PyArg_ParseTuple(args, "SS:bdiff", &sa, &sb))
-		return NULL;
-
-	an = splitlines(PyBytes_AsString(sa), PyBytes_Size(sa), &a);
-	bn = splitlines(PyBytes_AsString(sb), PyBytes_Size(sb), &b);
-
-	if (!a || !b)
-		goto nomem;
-
-	count = diff(a, an, b, bn, &l);
-	if (count < 0)
-		goto nomem;
-
-	rl = PyList_New(count);
-	if (!rl)
-		goto nomem;
-
-	for (h = l.next; h; h = h->next) {
-		m = Py_BuildValue("iiii", h->a1, h->a2, h->b1, h->b2);
-		PyList_SetItem(rl, pos, m);
-		pos++;
-	}
-
-nomem:
-	free(a);
-	free(b);
-	freehunks(l.next);
-	return rl ? rl : PyErr_NoMemory();
-}
-
-static PyObject *bdiff(PyObject *self, PyObject *args)
-{
-	char *sa, *sb, *rb;
-	PyObject *result = NULL;
-	struct line *al, *bl;
-	struct hunk l, *h;
-	int an, bn, count;
-	Py_ssize_t len = 0, la, lb;
-	PyThreadState *_save;
-
-	l.next = NULL;
-
-	if (!PyArg_ParseTuple(args, "s#s#:bdiff", &sa, &la, &sb, &lb))
-		return NULL;
-
-	if (la > UINT_MAX || lb > UINT_MAX) {
-		PyErr_SetString(PyExc_ValueError, "bdiff inputs too large");
-		return NULL;
-	}
-
-	_save = PyEval_SaveThread();
-	an = splitlines(sa, la, &al);
-	bn = splitlines(sb, lb, &bl);
-	if (!al || !bl)
-		goto nomem;
-
-	count = diff(al, an, bl, bn, &l);
-	if (count < 0)
-		goto nomem;
-
-	/* calculate length of output */
-	la = lb = 0;
-	for (h = l.next; h; h = h->next) {
-		if (h->a1 != la || h->b1 != lb)
-			len += 12 + bl[h->b1].l - bl[lb].l;
-		la = h->a2;
-		lb = h->b2;
-	}
-	PyEval_RestoreThread(_save);
-	_save = NULL;
-
-	result = PyBytes_FromStringAndSize(NULL, len);
-
-	if (!result)
-		goto nomem;
-
-	/* build binary patch */
-	rb = PyBytes_AsString(result);
-	la = lb = 0;
-
-	for (h = l.next; h; h = h->next) {
-		if (h->a1 != la || h->b1 != lb) {
-			len = bl[h->b1].l - bl[lb].l;
-			putbe32((uint32_t)(al[la].l - al->l), rb);
-			putbe32((uint32_t)(al[h->a1].l - al->l), rb + 4);
-			putbe32((uint32_t)len, rb + 8);
-			memcpy(rb + 12, bl[lb].l, len);
-			rb += 12 + len;
-		}
-		la = h->a2;
-		lb = h->b2;
-	}
-
-nomem:
-	if (_save)
-		PyEval_RestoreThread(_save);
-	free(al);
-	free(bl);
-	freehunks(l.next);
-	return result ? result : PyErr_NoMemory();
-}
-
-/*
- * If allws != 0, remove all whitespace (' ', \t and \r). Otherwise,
- * reduce whitespace sequences to a single space and trim remaining whitespace
- * from end of lines.
- */
-static PyObject *fixws(PyObject *self, PyObject *args)
-{
-	PyObject *s, *result = NULL;
-	char allws, c;
-	const char *r;
-	Py_ssize_t i, rlen, wlen = 0;
-	char *w;
-
-	if (!PyArg_ParseTuple(args, "Sb:fixws", &s, &allws))
-		return NULL;
-	r = PyBytes_AsString(s);
-	rlen = PyBytes_Size(s);
-
-	w = (char *)malloc(rlen ? rlen : 1);
-	if (!w)
-		goto nomem;
-
-	for (i = 0; i != rlen; i++) {
-		c = r[i];
-		if (c == ' ' || c == '\t' || c == '\r') {
-			if (!allws && (wlen == 0 || w[wlen - 1] != ' '))
-				w[wlen++] = ' ';
-		} else if (c == '\n' && !allws
-			  && wlen > 0 && w[wlen - 1] == ' ') {
-			w[wlen - 1] = '\n';
-		} else {
-			w[wlen++] = c;
-		}
-	}
-
-	result = PyBytes_FromStringAndSize(w, wlen);
-
-nomem:
-	free(w);
-	return result ? result : PyErr_NoMemory();
-}
-
-
-static char mdiff_doc[] = "Efficient binary diff.";
-
-static PyMethodDef methods[] = {
-	{"bdiff", bdiff, METH_VARARGS, "calculate a binary diff\n"},
-	{"blocks", blocks, METH_VARARGS, "find a list of matching lines\n"},
-	{"fixws", fixws, METH_VARARGS, "normalize diff whitespaces\n"},
-	{NULL, NULL}
-};
-
-#ifdef IS_PY3K
-static struct PyModuleDef bdiff_module = {
-	PyModuleDef_HEAD_INIT,
-	"bdiff",
-	mdiff_doc,
-	-1,
-	methods
-};
-
-PyMODINIT_FUNC PyInit_bdiff(void)
-{
-	return PyModule_Create(&bdiff_module);
-}
-#else
-PyMODINIT_FUNC initbdiff(void)
-{
-	Py_InitModule3("bdiff", methods, mdiff_doc);
-}
-#endif
-

mercurial/bookmarks.py

@@ -17,6 +17,7 b' from .node import ('
 )
 from . import (
     encoding,
+    error,
     lock as lockmod,
     obsolete,
     util,
@@ -109,39 +110,6 b' class bmstore(dict):'
                             location='plain')
         tr.hookargs['bookmark_moved'] = '1'
 
-    def write(self):
-        '''Write bookmarks
-
-        Write the given bookmark => hash dictionary to the .hg/bookmarks file
-        in a format equal to those of localtags.
-
-        We also store a backup of the previous state in undo.bookmarks that
-        can be copied back on rollback.
-        '''
-        msg = 'bm.write() is deprecated, use bm.recordchange(transaction)'
-        self._repo.ui.deprecwarn(msg, '3.7')
-        # TODO: writing the active bookmark should probably also use a
-        # transaction.
-        self._writeactive()
-        if self._clean:
-            return
-        repo = self._repo
-        if (repo.ui.configbool('devel', 'all-warnings')
-                or repo.ui.configbool('devel', 'check-locks')):
-            l = repo._wlockref and repo._wlockref()
-            if l is None or not l.held:
-                repo.ui.develwarn('bookmarks write with no wlock')
-
-        tr = repo.currenttransaction()
-        if tr:
-            self.recordchange(tr)
-            # invalidatevolatilesets() is omitted because this doesn't
-            # write changes out actually
-            return
-
-        self._writerepo(repo)
-        repo.invalidatevolatilesets()
-
     def _writerepo(self, repo):
         """Factored out for extensibility"""
         rbm = repo._bookmarks
@@ -150,7 +118,8 b' class bmstore(dict):'
             rbm._writeactive()
 
         with repo.wlock():
-            file_ = repo.vfs('bookmarks', 'w', atomictemp=True)
+            file_ = repo.vfs('bookmarks', 'w', atomictemp=True,
+                             checkambig=True)
             try:
                 self._write(file_)
             except: # re-raises
@@ -164,7 +133,8 b' class bmstore(dict):'
             return
         with self._repo.wlock():
             if self._active is not None:
-                f = self._repo.vfs('bookmarks.current', 'w', atomictemp=True)
+                f = self._repo.vfs('bookmarks.current', 'w', atomictemp=True,
+                                   checkambig=True)
                 try:
                     f.write(encoding.fromlocal(self._active))
                 finally:
@@ -185,7 +155,10 b' class bmstore(dict):'
 
     def expandname(self, bname):
         if bname == '.':
-            return self.active
+            if self.active:
+                return self.active
+            else:
+                raise error.Abort(_("no active bookmark"))
         return bname
 
 def _readactive(repo, marks):

mercurial/branchmap.py

@@ -363,7 +363,7 b' class revbranchcache(object):'
             bndata = repo.vfs.read(_rbcnames)
             self._rbcsnameslen = len(bndata) # for verification before writing
             self._names = [encoding.tolocal(bn) for bn in bndata.split('\0')]
-        except (IOError, OSError) as inst:
+        except (IOError, OSError):
             if readonly:
                 # don't try to use cache - fall back to the slow path
                 self.branchinfo = self._branchinfo
@@ -402,10 +402,9 b' class revbranchcache(object):'
         if rev == nullrev:
             return changelog.branchinfo(rev)
 
-        # if requested rev is missing, add and populate all missing revs
+        # if requested rev isn't allocated, grow and cache the rev info
         if len(self._rbcrevs) < rbcrevidx + _rbcrecsize:
-            self._rbcrevs.extend('\0' * (len(changelog) * _rbcrecsize -
-                                         len(self._rbcrevs)))
+            return self._branchinfo(rev)
 
         # fast path: extract data from cache, use it if node is matching
         reponode = changelog.node(rev)[:_rbcnodelen]
@@ -452,6 +451,10 b' class revbranchcache(object):'
         rbcrevidx = rev * _rbcrecsize
         rec = array('c')
         rec.fromstring(pack(_rbcrecfmt, node, branchidx))
+        if len(self._rbcrevs) < rbcrevidx + _rbcrecsize:
+            self._rbcrevs.extend('\0' *
+                                 (len(self._repo.changelog) * _rbcrecsize -
+                                  len(self._rbcrevs)))
         self._rbcrevs[rbcrevidx:rbcrevidx + _rbcrecsize] = rec
         self._rbcrevslen = min(self._rbcrevslen, rev)
 

mercurial/bundle2.py

@@ -690,7 +690,7 b' class unbundle20(unpackermixin):'
 
     def _processallparams(self, paramsblock):
         """"""
-        params = {}
+        params = util.sortdict()
         for p in paramsblock.split(' '):
             p = p.split('=', 1)
             p = [urlreq.unquote(i) for i in p]
@@ -1115,8 +1115,8 b' class unbundlepart(unpackermixin):'
         self.mandatoryparams = tuple(mandatoryparams)
         self.advisoryparams  = tuple(advisoryparams)
         # user friendly UI
-        self.params = dict(self.mandatoryparams)
-        self.params.update(dict(self.advisoryparams))
+        self.params = util.sortdict(self.mandatoryparams)
+        self.params.update(self.advisoryparams)
         self.mandatorykeys = frozenset(p[0] for p in mandatoryparams)
 
     def _payloadchunks(self, chunknum=0):
@@ -1294,6 +1294,9 b' def writebundle(ui, cg, filename, bundle'
         bundle.setcompression(compression)
         part = bundle.newpart('changegroup', data=cg.getchunks())
         part.addparam('version', cg.version)
+        if 'clcount' in cg.extras:
+            part.addparam('nbchanges', str(cg.extras['clcount']),
+                          mandatory=False)
         chunkiter = bundle.getchunks()
     else:
         # compression argument is only for the bundle2 case

mercurial/bundlerepo.py

@@ -291,7 +291,7 b' class bundlerepository(localrepo.localre'
                                                     ".cg%sun" % version)
 
             if cgstream is None:
-                raise error.Abort('No changegroups found')
+                raise error.Abort(_('No changegroups found'))
             cgstream.seek(0)
 
             self.bundle = changegroup.getunbundler(version, cgstream, 'UN')

mercurial/changegroup.py

@@ -135,7 +135,7 b' class cg1unpacker(object):'
     version = '01'
     _grouplistcount = 1 # One list of files after the manifests
 
-    def __init__(self, fh, alg):
+    def __init__(self, fh, alg, extras=None):
         if alg == 'UN':
             alg = None # get more modern without breaking too much
         if not alg in util.decompressors:
@@ -145,6 +145,7 b' class cg1unpacker(object):'
             alg = '_truncatedBZ'
         self._stream = util.decompressors[alg](fh)
         self._type = alg
+        self.extras = extras or {}
         self.callback = None
 
     # These methods (compressed, read, seek, tell) all appear to only
@@ -530,6 +531,17 b' class cg1packer(object):'
     def fileheader(self, fname):
         return chunkheader(len(fname)) + fname
 
+    # Extracted both for clarity and for overriding in extensions.
+    def _sortgroup(self, revlog, nodelist, lookup):
+        """Sort nodes for change group and turn them into revnums."""
+        # for generaldelta revlogs, we linearize the revs; this will both be
+        # much quicker and generate a much smaller bundle
+        if (revlog._generaldelta and self._reorder is None) or self._reorder:
+            dag = dagutil.revlogdag(revlog)
+            return dag.linearize(set(revlog.rev(n) for n in nodelist))
+        else:
+            return sorted([revlog.rev(n) for n in nodelist])
+
     def group(self, nodelist, revlog, lookup, units=None):
         """Calculate a delta group, yielding a sequence of changegroup chunks
         (strings).
@@ -549,14 +561,7 b' class cg1packer(object):'
             yield self.close()
             return
 
-        # for generaldelta revlogs, we linearize the revs; this will both be
-        # much quicker and generate a much smaller bundle
-        if (revlog._generaldelta and self._reorder is None) or self._reorder:
-            dag = dagutil.revlogdag(revlog)
-            revs = set(revlog.rev(n) for n in nodelist)
-            revs = dag.linearize(revs)
-        else:
-            revs = sorted([revlog.rev(n) for n in nodelist])
+        revs = self._sortgroup(revlog, nodelist, lookup)
 
         # add the parent of the first rev
         p = revlog.parentrevs(revs[0])[0]
@@ -724,10 +729,11 b' class cg1packer(object):'
             dir = min(tmfnodes)
             nodes = tmfnodes[dir]
             prunednodes = self.prune(dirlog(dir), nodes, commonrevs)
-            for x in self._packmanifests(dir, prunednodes,
-                                         makelookupmflinknode(dir)):
-                size += len(x)
-                yield x
+            if not dir or prunednodes:
+                for x in self._packmanifests(dir, prunednodes,
+                                             makelookupmflinknode(dir)):
+                    size += len(x)
+                    yield x
             del tmfnodes[dir]
         self._verbosenote(_('%8.i (manifests)\n') % size)
         yield self._manifestsdone()
@@ -895,8 +901,8 b' def getbundler(version, repo, bundlecaps'
     assert version in supportedoutgoingversions(repo)
     return _packermap[version][0](repo, bundlecaps)
 
-def getunbundler(version, fh, alg):
-    return _packermap[version][1](fh, alg)
+def getunbundler(version, fh, alg, extras=None):
+    return _packermap[version][1](fh, alg, extras=extras)
 
 def _changegroupinfo(repo, nodes, source):
     if repo.ui.verbose or source == 'bundle':
@@ -924,7 +930,8 b' def getsubsetraw(repo, outgoing, bundler'
 
 def getsubset(repo, outgoing, bundler, source, fastpath=False):
     gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
-    return getunbundler(bundler.version, util.chunkbuffer(gengroup), None)
+    return getunbundler(bundler.version, util.chunkbuffer(gengroup), None,
+                        {'clcount': len(outgoing.missing)})
 
 def changegroupsubset(repo, roots, heads, source, version='01'):
     """Compute a changegroup consisting of all the nodes that are

mercurial/cmdutil.py

@@ -83,7 +83,7 b' def filterchunks(ui, originalhunks, usec'
         else:
             recordfn = crecordmod.chunkselector
 
-        return crecordmod.filterpatch(ui, originalhunks, recordfn, operation)
+        return crecordmod.filterpatch(ui, originalhunks, recordfn)
 
     else:
         return patch.filterpatch(ui, originalhunks, operation)
@@ -91,9 +91,9 b' def filterchunks(ui, originalhunks, usec'
 def recordfilter(ui, originalhunks, operation=None):
     """ Prompts the user to filter the originalhunks and return a list of
     selected hunks.
-    *operation* is used for ui purposes to indicate the user
-    what kind of filtering they are doing: reverting, committing, shelving, etc.
-    *operation* has to be a translated string.
+    *operation* is used for to build ui messages to indicate the user what
+    kind of filtering they are doing: reverting, committing, shelving, etc.
+    (see patch.filterpatch).
     """
     usecurses = crecordmod.checkcurses(ui)
     testfile = ui.config('experimental', 'crecordtest', None)
@@ -532,7 +532,7 b' def openrevlog(repo, cmd, file_, opts):'
         msg = _('cannot specify --changelog and --manifest at the same time')
     elif cl and dir:
         msg = _('cannot specify --changelog and --dir at the same time')
-    elif cl or mf:
+    elif cl or mf or dir:
         if file_:
             msg = _('cannot specify filename with --changelog or --manifest')
         elif not repo:
@@ -549,7 +549,7 b' def openrevlog(repo, cmd, file_, opts):'
             if 'treemanifest' not in repo.requirements:
                 raise error.Abort(_("--dir can only be used on repos with "
                                    "treemanifest enabled"))
-            dirlog = repo.dirlog(file_)
+            dirlog = repo.dirlog(dir)
             if len(dirlog):
                 r = dirlog
         elif mf:
@@ -1405,24 +1405,24 b' class jsonchangeset(changeset_printer):'
             self.ui.write(",\n {")
 
         if self.ui.quiet:
-            self.ui.write('\n  "rev": %s' % jrev)
-            self.ui.write(',\n  "node": %s' % jnode)
+            self.ui.write(('\n  "rev": %s') % jrev)
+            self.ui.write((',\n  "node": %s') % jnode)
             self.ui.write('\n }')
             return
 
-        self.ui.write('\n  "rev": %s' % jrev)
-        self.ui.write(',\n  "node": %s' % jnode)
-        self.ui.write(',\n  "branch": "%s"' % j(ctx.branch()))
-        self.ui.write(',\n  "phase": "%s"' % ctx.phasestr())
-        self.ui.write(',\n  "user": "%s"' % j(ctx.user()))
-        self.ui.write(',\n  "date": [%d, %d]' % ctx.date())
-        self.ui.write(',\n  "desc": "%s"' % j(ctx.description()))
-
-        self.ui.write(',\n  "bookmarks": [%s]' %
+        self.ui.write(('\n  "rev": %s') % jrev)
+        self.ui.write((',\n  "node": %s') % jnode)
+        self.ui.write((',\n  "branch": "%s"') % j(ctx.branch()))
+        self.ui.write((',\n  "phase": "%s"') % ctx.phasestr())
+        self.ui.write((',\n  "user": "%s"') % j(ctx.user()))
+        self.ui.write((',\n  "date": [%d, %d]') % ctx.date())
+        self.ui.write((',\n  "desc": "%s"') % j(ctx.description()))
+
+        self.ui.write((',\n  "bookmarks": [%s]') %
                       ", ".join('"%s"' % j(b) for b in ctx.bookmarks()))
-        self.ui.write(',\n  "tags": [%s]' %
+        self.ui.write((',\n  "tags": [%s]') %
                       ", ".join('"%s"' % j(t) for t in ctx.tags()))
-        self.ui.write(',\n  "parents": [%s]' %
+        self.ui.write((',\n  "parents": [%s]') %
                       ", ".join('"%s"' % c.hex() for c in ctx.parents()))
 
         if self.ui.debugflag:
@@ -1430,26 +1430,26 b' class jsonchangeset(changeset_printer):'
                 jmanifestnode = 'null'
             else:
                 jmanifestnode = '"%s"' % hex(ctx.manifestnode())
-            self.ui.write(',\n  "manifest": %s' % jmanifestnode)
-
-            self.ui.write(',\n  "extra": {%s}' %
+            self.ui.write((',\n  "manifest": %s') % jmanifestnode)
+
+            self.ui.write((',\n  "extra": {%s}') %
                           ", ".join('"%s": "%s"' % (j(k), j(v))
                                     for k, v in ctx.extra().items()))
 
             files = ctx.p1().status(ctx)
-            self.ui.write(',\n  "modified": [%s]' %
+            self.ui.write((',\n  "modified": [%s]') %
                           ", ".join('"%s"' % j(f) for f in files[0]))
-            self.ui.write(',\n  "added": [%s]' %
+            self.ui.write((',\n  "added": [%s]') %
                           ", ".join('"%s"' % j(f) for f in files[1]))
-            self.ui.write(',\n  "removed": [%s]' %
+            self.ui.write((',\n  "removed": [%s]') %
                           ", ".join('"%s"' % j(f) for f in files[2]))
 
         elif self.ui.verbose:
-            self.ui.write(',\n  "files": [%s]' %
+            self.ui.write((',\n  "files": [%s]') %
                           ", ".join('"%s"' % j(f) for f in ctx.files()))
 
             if copies:
-                self.ui.write(',\n  "copies": {%s}' %
+                self.ui.write((',\n  "copies": {%s}') %
                               ", ".join('"%s": "%s"' % (j(k), j(v))
                                                         for k, v in copies))
 
@@ -1463,12 +1463,13 b' class jsonchangeset(changeset_printer):'
                 self.ui.pushbuffer()
                 diffordiffstat(self.ui, self.repo, diffopts, prev, node,
                                match=matchfn, stat=True)
-                self.ui.write(',\n  "diffstat": "%s"' % j(self.ui.popbuffer()))
+                self.ui.write((',\n  "diffstat": "%s"')
+                              % j(self.ui.popbuffer()))
             if diff:
                 self.ui.pushbuffer()
                 diffordiffstat(self.ui, self.repo, diffopts, prev, node,
                                match=matchfn, stat=False)
-                self.ui.write(',\n  "diff": "%s"' % j(self.ui.popbuffer()))
+                self.ui.write((',\n  "diff": "%s"') % j(self.ui.popbuffer()))
 
         self.ui.write("\n }")
 
@@ -1998,7 +1999,7 b' def _makelogrevset(repo, pats, opts, rev'
         followfirst = 0
     # --follow with FILE behavior depends on revs...
     it = iter(revs)
-    startrev = it.next()
+    startrev = next(it)
     followdescendants = startrev < next(it, startrev)
 
     # branch and only_branch are really aliases and must be handled at
@@ -2147,7 +2148,8 b' def getgraphlogrevs(repo, pats, opts):'
     if opts.get('rev'):
         # User-specified revs might be unsorted, but don't sort before
         # _makelogrevset because it might depend on the order of revs
-        revs.sort(reverse=True)
+        if not (revs.isdescending() or revs.istopo()):
+            revs.sort(reverse=True)
     if expr:
         # Revset matchers often operate faster on revisions in changelog
         # order, because most filters deal with the changelog.
@@ -3071,7 +3073,7 b' def revert(ui, repo, ctx, parents, *pats'
 
             # tell newly modified apart.
             dsmodified &= modified
-            dsmodified |= modified & dsadded # dirstate added may needs backup
+            dsmodified |= modified & dsadded # dirstate added may need backup
             modified -= dsmodified
 
             # We need to wait for some post-processing to update this set
@@ -3141,11 +3143,17 b' def revert(ui, repo, ctx, parents, *pats'
         # All set to `discard` if `no-backup` is set do avoid checking
         # no_backup lower in the code.
         # These values are ordered for comparison purposes
+        backupinteractive = 3 # do backup if interactively modified
         backup = 2  # unconditionally do backup
         check = 1   # check if the existing file differs from target
         discard = 0 # never do backup
         if opts.get('no_backup'):
-            backup = check = discard
+            backupinteractive = backup = check = discard
+        if interactive:
+            dsmodifiedbackup = backupinteractive
+        else:
+            dsmodifiedbackup = backup
+        tobackup = set()
 
         backupanddel = actions['remove']
         if not opts.get('no_backup'):
@@ -3163,7 +3171,7 b' def revert(ui, repo, ctx, parents, *pats'
             # Modified compared to target, but local file is deleted
             (deleted,       actions['revert'],   discard),
             # Modified compared to target, local change
-            (dsmodified,    actions['revert'],   backup),
+            (dsmodified,    actions['revert'],   dsmodifiedbackup),
             # Added since target
             (added,         actions['remove'],   discard),
             # Added in working directory
@@ -3198,8 +3206,12 b' def revert(ui, repo, ctx, parents, *pats'
                     continue
                 if xlist is not None:
                     xlist.append(abs)
-                    if dobackup and (backup <= dobackup
-                                     or wctx[abs].cmp(ctx[abs])):
+                    if dobackup:
+                        # If in interactive mode, don't automatically create
+                        # .orig files (issue4793)
+                        if dobackup == backupinteractive:
+                            tobackup.add(abs)
+                        elif (backup <= dobackup or wctx[abs].cmp(ctx[abs])):
                             bakname = scmutil.origpath(ui, repo, rel)
                             ui.note(_('saving current version of %s as %s\n') %
                                     (rel, bakname))
@@ -3219,7 +3231,7 b' def revert(ui, repo, ctx, parents, *pats'
         if not opts.get('dry_run'):
             needdata = ('revert', 'add', 'undelete')
             _revertprefetch(repo, ctx, *[actions[name][0] for name in needdata])
-            _performrevert(repo, parents, ctx, actions, interactive)
+            _performrevert(repo, parents, ctx, actions, interactive, tobackup)
 
         if targetsubs:
             # Revert the subrepos on the revert list
@@ -3234,7 +3246,8 b' def _revertprefetch(repo, ctx, *files):'
     """Let extension changing the storage layer prefetch content"""
     pass
 
-def _performrevert(repo, parents, ctx, actions, interactive=False):
+def _performrevert(repo, parents, ctx, actions, interactive=False,
+                   tobackup=None):
     """function that actually perform all the actions computed for revert
 
     This is an independent function to let extension to plug in and react to
@@ -3301,10 +3314,12 b' def _performrevert(repo, parents, ctx, a'
         else:
             diff = patch.diff(repo, None, ctx.node(), m, opts=diffopts)
         originalchunks = patch.parsepatch(diff)
+        operation = 'discard' if node == parent else 'revert'
 
         try:
 
-            chunks, opts = recordfilter(repo.ui, originalchunks)
+            chunks, opts = recordfilter(repo.ui, originalchunks,
+                                        operation=operation)
             if reversehunks:
                 chunks = patch.reversehunks(chunks)
 
@@ -3312,9 +3327,18 b' def _performrevert(repo, parents, ctx, a'
             raise error.Abort(_('error parsing patch: %s') % err)
 
         newlyaddedandmodifiedfiles = newandmodified(chunks, originalchunks)
+        if tobackup is None:
+            tobackup = set()
         # Apply changes
         fp = stringio()
         for c in chunks:
+            # Create a backup file only if this hunk should be backed up
+            if ishunk(c) and c.header.filename() in tobackup:
+                abs = c.header.filename()
+                target = repo.wjoin(abs)
+                bakname = scmutil.origpath(repo.ui, repo, m.rel(abs))
+                util.copyfile(target, bakname)
+                tobackup.remove(abs)
             c.write(fp)
         dopatch = fp.tell()
         fp.seek(0)
@@ -3518,7 +3542,7 b' class dirstateguard(object):'
     def __init__(self, repo, name):
         self._repo = repo
         self._suffix = '.backup.%s.%d' % (name, id(self))
-        repo.dirstate._savebackup(repo.currenttransaction(), self._suffix)
+        repo.dirstate.savebackup(repo.currenttransaction(), self._suffix)
         self._active = True
         self._closed = False
 
@@ -3536,13 +3560,13 b' class dirstateguard(object):'
                    % self._suffix)
             raise error.Abort(msg)
 
-        self._repo.dirstate._clearbackup(self._repo.currenttransaction(),
+        self._repo.dirstate.clearbackup(self._repo.currenttransaction(),
                                          self._suffix)
         self._active = False
         self._closed = True
 
     def _abort(self):
-        self._repo.dirstate._restorebackup(self._repo.currenttransaction(),
+        self._repo.dirstate.restorebackup(self._repo.currenttransaction(),
                                            self._suffix)
         self._active = False
 

mercurial/commands.py

@@ -59,6 +59,7 b' from . import ('
     obsolete,
     patch,
     phases,
+    policy,
     pvec,
     repair,
     revlog,
@@ -215,7 +216,7 b' subrepoopts = ['
 debugrevlogopts = [
     ('c', 'changelog', False, _('open changelog')),
     ('m', 'manifest', False, _('open manifest')),
-    ('', 'dir', False, _('open directory manifest')),
+    ('', 'dir', '', _('open directory manifest')),
 ]
 
 # Commands start here, listed alphabetically
@@ -468,26 +469,27 b' def annotate(ui, repo, *pats, **opts):'
 
         lines = fctx.annotate(follow=follow, linenumber=linenumber,
                               diffopts=diffopts)
+        if not lines:
+            continue
         formats = []
         pieces = []
 
         for f, sep in funcmap:
             l = [f(n) for n, dummy in lines]
-            if l:
-                if fm:
-                    formats.append(['%s' for x in l])
-                else:
-                    sizes = [encoding.colwidth(x) for x in l]
-                    ml = max(sizes)
-                    formats.append([sep + ' ' * (ml - w) + '%s' for w in sizes])
-                pieces.append(l)
+            if fm:
+                formats.append(['%s' for x in l])
+            else:
+                sizes = [encoding.colwidth(x) for x in l]
+                ml = max(sizes)
+                formats.append([sep + ' ' * (ml - w) + '%s' for w in sizes])
+            pieces.append(l)
 
         for f, p, l in zip(zip(*formats), zip(*pieces), lines):
             fm.startitem()
             fm.write(fields, "".join(f), *p)
             fm.write('line', ": %s", l[1])
 
-        if lines and not lines[-1][1].endswith('\n'):
+        if not lines[-1][1].endswith('\n'):
             fm.plain('\n')
 
     fm.end()
@@ -2089,51 +2091,56 b' def debugbundle(ui, bundlepath, all=None'
         gen = exchange.readbundle(ui, f, bundlepath)
         if isinstance(gen, bundle2.unbundle20):
             return _debugbundle2(ui, gen, all=all, **opts)
-        if all:
-            ui.write(("format: id, p1, p2, cset, delta base, len(delta)\n"))
-
-            def showchunks(named):
-                ui.write("\n%s\n" % named)
-                chain = None
-                while True:
-                    chunkdata = gen.deltachunk(chain)
-                    if not chunkdata:
-                        break
-                    node = chunkdata['node']
-                    p1 = chunkdata['p1']
-                    p2 = chunkdata['p2']
-                    cs = chunkdata['cs']
-                    deltabase = chunkdata['deltabase']
-                    delta = chunkdata['delta']
-                    ui.write("%s %s %s %s %s %s\n" %
-                             (hex(node), hex(p1), hex(p2),
-                              hex(cs), hex(deltabase), len(delta)))
-                    chain = node
-
-            chunkdata = gen.changelogheader()
-            showchunks("changelog")
-            chunkdata = gen.manifestheader()
-            showchunks("manifest")
-            while True:
-                chunkdata = gen.filelogheader()
-                if not chunkdata:
-                    break
-                fname = chunkdata['filename']
-                showchunks(fname)
-        else:
-            if isinstance(gen, bundle2.unbundle20):
-                raise error.Abort(_('use debugbundle2 for this file'))
-            chunkdata = gen.changelogheader()
+        _debugchangegroup(ui, gen, all=all, **opts)
+
+def _debugchangegroup(ui, gen, all=None, indent=0, **opts):
+    indent_string = ' ' * indent
+    if all:
+        ui.write(("%sformat: id, p1, p2, cset, delta base, len(delta)\n")
+                 % indent_string)
+
+        def showchunks(named):
+            ui.write("\n%s%s\n" % (indent_string, named))
             chain = None
             while True:
                 chunkdata = gen.deltachunk(chain)
                 if not chunkdata:
                     break
                 node = chunkdata['node']
-                ui.write("%s\n" % hex(node))
+                p1 = chunkdata['p1']
+                p2 = chunkdata['p2']
+                cs = chunkdata['cs']
+                deltabase = chunkdata['deltabase']
+                delta = chunkdata['delta']
+                ui.write("%s%s %s %s %s %s %s\n" %
+                         (indent_string, hex(node), hex(p1), hex(p2),
+                          hex(cs), hex(deltabase), len(delta)))
                 chain = node
 
-def _debugbundle2(ui, gen, **opts):
+        chunkdata = gen.changelogheader()
+        showchunks("changelog")
+        chunkdata = gen.manifestheader()
+        showchunks("manifest")
+        while True:
+            chunkdata = gen.filelogheader()
+            if not chunkdata:
+                break
+            fname = chunkdata['filename']
+            showchunks(fname)
+    else:
+        if isinstance(gen, bundle2.unbundle20):
+            raise error.Abort(_('use debugbundle2 for this file'))
+        chunkdata = gen.changelogheader()
+        chain = None
+        while True:
+            chunkdata = gen.deltachunk(chain)
+            if not chunkdata:
+                break
+            node = chunkdata['node']
+            ui.write("%s%s\n" % (indent_string, hex(node)))
+            chain = node
+
+def _debugbundle2(ui, gen, all=None, **opts):
     """lists the contents of a bundle2"""
     if not isinstance(gen, bundle2.unbundle20):
         raise error.Abort(_('not a bundle2 file'))
@@ -2143,15 +2150,7 b' def _debugbundle2(ui, gen, **opts):'
         if part.type == 'changegroup':
             version = part.params.get('version', '01')
             cg = changegroup.getunbundler(version, part, 'UN')
-            chunkdata = cg.changelogheader()
-            chain = None
-            while True:
-                chunkdata = cg.deltachunk(chain)
-                if not chunkdata:
-                    break
-                node = chunkdata['node']
-                ui.write("    %s\n" % hex(node))
-                chain = node
+            _debugchangegroup(ui, cg, all=all, indent=4, **opts)
 
 @command('debugcreatestreamclonebundle', [], 'FILE')
 def debugcreatestreamclonebundle(ui, repo, fname):
@@ -2301,7 +2300,9 b' def debugdag(ui, repo, file_=None, *revs'
 @command('debugdata', debugrevlogopts, _('-c|-m|FILE REV'))
 def debugdata(ui, repo, file_, rev=None, **opts):
     """dump the contents of a data file revision"""
-    if opts.get('changelog') or opts.get('manifest'):
+    if opts.get('changelog') or opts.get('manifest') or opts.get('dir'):
+        if rev is not None:
+            raise error.CommandError('debugdata', _('invalid arguments'))
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError('debugdata', _('invalid arguments'))
@@ -2524,15 +2525,16 b' def debugignore(ui, repo, *files, **opts'
                             break
             if ignored:
                 if ignored == nf:
-                    ui.write("%s is ignored\n" % f)
+                    ui.write(_("%s is ignored\n") % f)
                 else:
-                    ui.write("%s is ignored because of containing folder %s\n"
+                    ui.write(_("%s is ignored because of "
+                               "containing folder %s\n")
                              % (f, ignored))
                 ignorefile, lineno, line = ignoredata
-                ui.write("(ignore rule in %s, line %d: '%s')\n"
+                ui.write(_("(ignore rule in %s, line %d: '%s')\n")
                          % (ignorefile, lineno, line))
             else:
-                ui.write("%s is not ignored\n" % f)
+                ui.write(_("%s is not ignored\n") % f)
 
 @command('debugindex', debugrevlogopts +
     [('f', 'format', 0, _('revlog format'), _('FORMAT'))],
@@ -2563,12 +2565,12 b' def debugindex(ui, repo, file_=None, **o'
         break
 
     if format == 0:
-        ui.write("   rev    offset  length " + basehdr + " linkrev"
-                 " %s %s p2\n" % ("nodeid".ljust(idlen), "p1".ljust(idlen)))
+        ui.write(("   rev    offset  length " + basehdr + " linkrev"
+                 " %s %s p2\n") % ("nodeid".ljust(idlen), "p1".ljust(idlen)))
     elif format == 1:
-        ui.write("   rev flag   offset   length"
+        ui.write(("   rev flag   offset   length"
                  "     size " + basehdr + "   link     p1     p2"
-                 " %s\n" % "nodeid".rjust(idlen))
+                 " %s\n") % "nodeid".rjust(idlen))
 
     for i in r:
         node = r.node(i)
@@ -2743,7 +2745,16 b' def debuginstall(ui, **opts):'
     fm.write('pythonlib', _("checking Python lib (%s)...\n"),
              os.path.dirname(os.__file__))
 
+    # hg version
+    hgver = util.version()
+    fm.write('hgver', _("checking Mercurial version (%s)\n"),
+             hgver.split('+')[0])
+    fm.write('hgverextra', _("checking Mercurial custom build (%s)\n"),
+             '+'.join(hgver.split('+')[1:]))
+
     # compiled modules
+    fm.write('hgmodulepolicy', _("checking module policy (%s)\n"),
+             policy.policy)
     fm.write('hgmodules', _("checking installed modules (%s)...\n"),
              os.path.dirname(__file__))
 
@@ -3022,13 +3033,13 b' def debuglocks(ui, repo, **opts):'
                     else:
                         locker = 'user %s, process %s, host %s' \
                                  % (user, pid, host)
-                ui.write("%-6s %s (%ds)\n" % (name + ":", locker, age))
+                ui.write(("%-6s %s (%ds)\n") % (name + ":", locker, age))
                 return 1
             except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
 
-        ui.write("%-6s free\n" % (name + ":"))
+        ui.write(("%-6s free\n") % (name + ":"))
         return 0
 
     held += report(repo.svfs, "lock", repo.lock)
@@ -3321,8 +3332,8 b' def debugrevlog(ui, repo, file_=None, **'
 
     if opts.get("dump"):
         numrevs = len(r)
-        ui.write("# rev p1rev p2rev start   end deltastart base   p1   p2"
-                 " rawsize totalsize compression heads chainlen\n")
+        ui.write(("# rev p1rev p2rev start   end deltastart base   p1   p2"
+                 " rawsize totalsize compression heads chainlen\n"))
         ts = 0
         heads = set()
 
@@ -3511,18 +3522,19 b' def debugrevspec(ui, repo, expr, **opts)'
         ui.note(revset.prettyformat(tree), "\n")
         newtree = revset.expandaliases(ui, tree)
         if newtree != tree:
-            ui.note("* expanded:\n", revset.prettyformat(newtree), "\n")
+            ui.note(("* expanded:\n"), revset.prettyformat(newtree), "\n")
         tree = newtree
         newtree = revset.foldconcat(tree)
         if newtree != tree:
-            ui.note("* concatenated:\n", revset.prettyformat(newtree), "\n")
+            ui.note(("* concatenated:\n"), revset.prettyformat(newtree), "\n")
         if opts["optimize"]:
-            weight, optimizedtree = revset.optimize(newtree, True)
-            ui.note("* optimized:\n", revset.prettyformat(optimizedtree), "\n")
+            optimizedtree = revset.optimize(newtree)
+            ui.note(("* optimized:\n"),
+                    revset.prettyformat(optimizedtree), "\n")
     func = revset.match(ui, expr, repo)
     revs = func(repo)
     if ui.verbose:
-        ui.note("* set:\n", revset.prettyformatset(revs), "\n")
+        ui.note(("* set:\n"), revset.prettyformatset(revs), "\n")
     for c in revs:
         ui.write("%s\n" % c)
 
@@ -3677,7 +3689,7 b' def debugtemplate(ui, repo, tmpl, **opts'
         ui.note(templater.prettyformat(tree), '\n')
         newtree = templater.expandaliases(tree, aliases)
         if newtree != tree:
-            ui.note("* expanded:\n", templater.prettyformat(newtree), '\n')
+            ui.note(("* expanded:\n"), templater.prettyformat(newtree), '\n')
 
     mapfile = None
     if revs is None:
@@ -4406,7 +4418,7 b' def grep(ui, repo, pattern, *pats, **opt'
             if not opts.get('files_with_matches'):
                 ui.write(sep, label='grep.sep')
                 if not opts.get('text') and binary():
-                    ui.write(" Binary file matches")
+                    ui.write(_(" Binary file matches"))
                 else:
                     for s, label in l:
                         ui.write(s, label=label)
@@ -4570,7 +4582,10 b' def help_(ui, name=None, **opts):'
     Returns 0 if successful.
     """
 
-    textwidth = min(ui.termwidth(), 80) - 2
+    textwidth = ui.configint('ui', 'textwidth', 78)
+    termwidth = ui.termwidth() - 2
+    if textwidth <= 0 or termwidth < textwidth:
+        textwidth = termwidth
 
     keep = opts.get('system') or []
     if len(keep) == 0:
@@ -5773,6 +5788,9 b' def pull(ui, repo, source="default", **o'
     If SOURCE is omitted, the 'default' path will be used.
     See :hg:`help urls` for more information.
 
+    Specifying bookmark as ``.`` is equivalent to specifying the active
+    bookmark's name.
+
     Returns 0 on success, 1 if an update had unresolved files.
     """
     source, branches = hg.parseurl(ui.expandpath(source), opts.get('branch'))
@@ -5794,6 +5812,7 b' def pull(ui, repo, source="default", **o'
             remotebookmarks = other.listkeys('bookmarks')
             pullopargs['remotebookmarks'] = remotebookmarks
             for b in opts['bookmark']:
+                b = repo._bookmarks.expandname(b)
                 if b not in remotebookmarks:
                     raise error.Abort(_('remote bookmark %s not found!') % b)
                 revs.append(remotebookmarks[b])
@@ -5926,6 +5945,15 b' def push(ui, repo, dest=None, **opts):'
         if not revs:
             raise error.Abort(_("specified revisions evaluate to an empty set"),
                              hint=_("use different revision arguments"))
+    elif path.pushrev:
+        # It doesn't make any sense to specify ancestor revisions. So limit
+        # to DAG heads to make discovery simpler.
+        expr = revset.formatspec('heads(%r)', path.pushrev)
+        revs = scmutil.revrange(repo, [expr])
+        revs = [repo[rev].node() for rev in revs]
+        if not revs:
+            raise error.Abort(_('default push revset for path evaluates to an '
+                                'empty set'))
 
     repo._subtoppath = dest
     try:
@@ -6300,7 +6328,10 b' def revert(ui, repo, *pats, **opts):'
     related method.
 
     Modified files are saved with a .orig suffix before reverting.
-    To disable these backups, use --no-backup.
+    To disable these backups, use --no-backup. It is possible to store
+    the backup files in a custom directory relative to the root of the
+    repository by setting the ``ui.origbackuppath`` configuration
+    option.
 
     See :hg:`help dates` for a list of formats valid for -d/--date.
 
@@ -6380,6 +6411,11 b' def rollback(ui, repo, **opts):'
       commit transaction if it isn't checked out. Use --force to
       override this protection.
 
+      The rollback command can be entirely disabled by setting the
+      ``ui.rollback`` configuration setting to false. If you're here
+      because you want to use rollback and it's disabled, you can
+      re-enable the command by setting ``ui.rollback`` to true.
+
     This command is not intended for use on public repositories. Once
     changes are visible for pull by other users, rolling a transaction
     back locally is ineffective (someone else may already have pulled
@@ -6389,6 +6425,9 b' def rollback(ui, repo, **opts):'
 
     Returns 0 on success, 1 if no rollback data is available.
     """
+    if not ui.configbool('ui', 'rollback', True):
+        raise error.Abort(_('rollback is disabled because it is unsafe'),
+                          hint=('see `hg help -v rollback` for information'))
     return repo.rollback(dryrun=opts.get('dry_run'),
                          force=opts.get('force'))
 

mercurial/commandserver.py

@@ -7,9 +7,13 b''
 
 from __future__ import absolute_import
 
-import SocketServer
 import errno
+import gc
 import os
+import random
+import select
+import signal
+import socket
 import struct
 import sys
 import traceback
@@ -178,6 +182,10 b' class server(object):'
 
         self.client = fin
 
+    def cleanup(self):
+        """release and restore resources taken during server session"""
+        pass
+
     def _read(self, size):
         if not size:
             return ''
@@ -229,12 +237,8 b' class server(object):'
             self.repo.ui = self.repo.dirstate._ui = repoui
             self.repo.invalidateall()
 
-        # reset last-print time of progress bar per command
-        # (progbar is singleton, we don't have to do for all uis)
-        if copiedui._progbar:
-            copiedui._progbar.resetstate()
-
         for ui in uis:
+            ui.resetstate()
             # any kind of interaction must use server channels, but chg may
             # replace channels by fully functional tty files. so nontty is
             # enforced only if cin is a channel.
@@ -278,6 +282,9 b' class server(object):'
         hellomsg += 'encoding: ' + encoding.encoding
         hellomsg += '\n'
         hellomsg += 'pid: %d' % util.getpid()
+        if util.safehasattr(os, 'getpgid'):
+            hellomsg += '\n'
+            hellomsg += 'pgid: %d' % os.getpgid(0)
 
         # write the hello msg in -one- chunk
         self.cout.write(hellomsg)
@@ -332,66 +339,193 b' class pipeservice(object):'
             sv = server(ui, self.repo, fin, fout)
             return sv.serve()
         finally:
+            sv.cleanup()
             _restoreio(ui, fin, fout)
 
-class _requesthandler(SocketServer.StreamRequestHandler):
-    def handle(self):
-        ui = self.server.ui
-        repo = self.server.repo
-        sv = None
+def _initworkerprocess():
+    # use a different process group from the master process, making this
+    # process pass kernel "is_current_pgrp_orphaned" check so signals like
+    # SIGTSTP, SIGTTIN, SIGTTOU are not ignored.
+    os.setpgid(0, 0)
+    # change random state otherwise forked request handlers would have a
+    # same state inherited from parent.
+    random.seed()
+
+def _serverequest(ui, repo, conn, createcmdserver):
+    fin = conn.makefile('rb')
+    fout = conn.makefile('wb')
+    sv = None
+    try:
+        sv = createcmdserver(repo, conn, fin, fout)
+        try:
+            sv.serve()
+        # handle exceptions that may be raised by command server. most of
+        # known exceptions are caught by dispatch.
+        except error.Abort as inst:
+            ui.warn(_('abort: %s\n') % inst)
+        except IOError as inst:
+            if inst.errno != errno.EPIPE:
+                raise
+        except KeyboardInterrupt:
+            pass
+        finally:
+            sv.cleanup()
+    except: # re-raises
+        # also write traceback to error channel. otherwise client cannot
+        # see it because it is written to server's stderr by default.
+        if sv:
+            cerr = sv.cerr
+        else:
+            cerr = channeledoutput(fout, 'e')
+        traceback.print_exc(file=cerr)
+        raise
+    finally:
+        fin.close()
         try:
-            sv = server(ui, repo, self.rfile, self.wfile)
-            try:
-                sv.serve()
-            # handle exceptions that may be raised by command server. most of
-            # known exceptions are caught by dispatch.
-            except error.Abort as inst:
-                ui.warn(_('abort: %s\n') % inst)
-            except IOError as inst:
-                if inst.errno != errno.EPIPE:
-                    raise
-            except KeyboardInterrupt:
-                pass
-        except: # re-raises
-            # also write traceback to error channel. otherwise client cannot
-            # see it because it is written to server's stderr by default.
-            if sv:
-                cerr = sv.cerr
-            else:
-                cerr = channeledoutput(self.wfile, 'e')
-            traceback.print_exc(file=cerr)
-            raise
+            fout.close()  # implicit flush() may cause another EPIPE
+        except IOError as inst:
+            if inst.errno != errno.EPIPE:
+                raise
+
+class unixservicehandler(object):
+    """Set of pluggable operations for unix-mode services
+
+    Almost all methods except for createcmdserver() are called in the main
+    process. You can't pass mutable resource back from createcmdserver().
+    """
+
+    pollinterval = None
+
+    def __init__(self, ui):
+        self.ui = ui
+
+    def bindsocket(self, sock, address):
+        util.bindunixsocket(sock, address)
 
-class unixservice(object):
+    def unlinksocket(self, address):
+        os.unlink(address)
+
+    def printbanner(self, address):
+        self.ui.status(_('listening at %s\n') % address)
+        self.ui.flush()  # avoid buffering of status message
+
+    def shouldexit(self):
+        """True if server should shut down; checked per pollinterval"""
+        return False
+
+    def newconnection(self):
+        """Called when main process notices new connection"""
+        pass
+
+    def createcmdserver(self, repo, conn, fin, fout):
+        """Create new command server instance; called in the process that
+        serves for the current connection"""
+        return server(self.ui, repo, fin, fout)
+
+class unixforkingservice(object):
     """
     Listens on unix domain socket and forks server per connection
     """
-    def __init__(self, ui, repo, opts):
+
+    def __init__(self, ui, repo, opts, handler=None):
         self.ui = ui
         self.repo = repo
         self.address = opts['address']
-        if not util.safehasattr(SocketServer, 'UnixStreamServer'):
+        if not util.safehasattr(socket, 'AF_UNIX'):
             raise error.Abort(_('unsupported platform'))
         if not self.address:
             raise error.Abort(_('no socket path specified with --address'))
+        self._servicehandler = handler or unixservicehandler(ui)
+        self._sock = None
+        self._oldsigchldhandler = None
+        self._workerpids = set()  # updated by signal handler; do not iterate
 
     def init(self):
-        class cls(SocketServer.ForkingMixIn, SocketServer.UnixStreamServer):
-            ui = self.ui
-            repo = self.repo
-        self.server = cls(self.address, _requesthandler)
-        self.ui.status(_('listening at %s\n') % self.address)
-        self.ui.flush()  # avoid buffering of status message
+        self._sock = socket.socket(socket.AF_UNIX)
+        self._servicehandler.bindsocket(self._sock, self.address)
+        self._sock.listen(socket.SOMAXCONN)
+        o = signal.signal(signal.SIGCHLD, self._sigchldhandler)
+        self._oldsigchldhandler = o
+        self._servicehandler.printbanner(self.address)
+
+    def _cleanup(self):
+        signal.signal(signal.SIGCHLD, self._oldsigchldhandler)
+        self._sock.close()
+        self._servicehandler.unlinksocket(self.address)
+        # don't kill child processes as they have active clients, just wait
+        self._reapworkers(0)
 
     def run(self):
         try:
-            self.server.serve_forever()
+            self._mainloop()
         finally:
-            os.unlink(self.address)
+            self._cleanup()
+
+    def _mainloop(self):
+        h = self._servicehandler
+        while not h.shouldexit():
+            try:
+                ready = select.select([self._sock], [], [], h.pollinterval)[0]
+                if not ready:
+                    continue
+                conn, _addr = self._sock.accept()
+            except (select.error, socket.error) as inst:
+                if inst.args[0] == errno.EINTR:
+                    continue
+                raise
+
+            pid = os.fork()
+            if pid:
+                try:
+                    self.ui.debug('forked worker process (pid=%d)\n' % pid)
+                    self._workerpids.add(pid)
+                    h.newconnection()
+                finally:
+                    conn.close()  # release handle in parent process
+            else:
+                try:
+                    self._runworker(conn)
+                    conn.close()
+                    os._exit(0)
+                except:  # never return, hence no re-raises
+                    try:
+                        self.ui.traceback(force=True)
+                    finally:
+                        os._exit(255)
+
+    def _sigchldhandler(self, signal, frame):
+        self._reapworkers(os.WNOHANG)
+
+    def _reapworkers(self, options):
+        while self._workerpids:
+            try:
+                pid, _status = os.waitpid(-1, options)
+            except OSError as inst:
+                if inst.errno == errno.EINTR:
+                    continue
+                if inst.errno != errno.ECHILD:
+                    raise
+                # no child processes at all (reaped by other waitpid()?)
+                self._workerpids.clear()
+                return
+            if pid == 0:
+                # no waitable child processes
+                return
+            self.ui.debug('worker process exited (pid=%d)\n' % pid)
+            self._workerpids.discard(pid)
+
+    def _runworker(self, conn):
+        signal.signal(signal.SIGCHLD, self._oldsigchldhandler)
+        _initworkerprocess()
+        h = self._servicehandler
+        try:
+            _serverequest(self.ui, self.repo, conn, h.createcmdserver)
+        finally:
+            gc.collect()  # trigger __del__ since worker process uses os._exit
 
 _servicemap = {
     'pipe': pipeservice,
-    'unix': unixservice,
+    'unix': unixforkingservice,
     }
 
 def createservice(ui, repo, opts):

mercurial/context.py

@@ -918,28 +918,25 b' class basefilectx(object):'
             return p[1]
         return filectx(self._repo, self._path, fileid=-1, filelog=self._filelog)
 
-    def annotate(self, follow=False, linenumber=None, diffopts=None):
-        '''returns a list of tuples of (ctx, line) for each line
+    def annotate(self, follow=False, linenumber=False, diffopts=None):
+        '''returns a list of tuples of ((ctx, number), line) for each line
         in the file, where ctx is the filectx of the node where
-        that line was last changed.
-        This returns tuples of ((ctx, linenumber), line) for each line,
-        if "linenumber" parameter is NOT "None".
-        In such tuples, linenumber means one at the first appearance
-        in the managed file.
-        To reduce annotation cost,
-        this returns fixed value(False is used) as linenumber,
-        if "linenumber" parameter is "False".'''
+        that line was last changed; if linenumber parameter is true, number is
+        the line number at the first appearance in the managed file, otherwise,
+        number has a fixed value of False.
+        '''
 
-        if linenumber is None:
+        def lines(text):
+            if text.endswith("\n"):
+                return text.count("\n")
+            return text.count("\n") + 1
+
+        if linenumber:
             def decorate(text, rev):
-                return ([rev] * len(text.splitlines()), text)
-        elif linenumber:
-            def decorate(text, rev):
-                size = len(text.splitlines())
-                return ([(rev, i) for i in xrange(1, size + 1)], text)
+                return ([(rev, i) for i in xrange(1, lines(text) + 1)], text)
         else:
             def decorate(text, rev):
-                return ([(rev, False)] * len(text.splitlines()), text)
+                return ([(rev, False)] * lines(text), text)
 
         def pair(parent, child):
             blocks = mdiff.allblocks(parent[1], child[1], opts=diffopts,

mercurial/copies.py

@@ -484,16 +484,16 b' def checkcopies(ctx, f, m1, m2, ca, limi'
             f1r, f2r = f1.linkrev(), f2.linkrev()
 
             if f1r is None:
-                f1 = g1.next()
+                f1 = next(g1)
             if f2r is None:
-                f2 = g2.next()
+                f2 = next(g2)
 
             while True:
                 f1r, f2r = f1.linkrev(), f2.linkrev()
                 if f1r > f2r:
-                    f1 = g1.next()
+                    f1 = next(g1)
                 elif f2r > f1r:
-                    f2 = g2.next()
+                    f2 = next(g2)
                 elif f1 == f2:
                     return f1 # a match
                 elif f1r == f2r or f1r < limit or f2r < limit:

mercurial/crecord.py

@@ -91,6 +91,7 b' class patchnode(object):'
     def allchildren(self):
         "Return a list of all of the direct children of this node"
         raise NotImplementedError("method must be implemented by subclass")
+
     def nextsibling(self):
         """
         Return the closest next item of the same type where there are no items
@@ -110,18 +111,12 b' class patchnode(object):'
     def parentitem(self):
         raise NotImplementedError("method must be implemented by subclass")
 
-
-    def nextitem(self, constrainlevel=True, skipfolded=True):
+    def nextitem(self, skipfolded=True):
         """
-        If constrainLevel == True, return the closest next item
-        of the same type where there are no items of different types between
-        the current item and this closest item.
+        Try to return the next item closest to this item, regardless of item's
+        type (header, hunk, or hunkline).
 
-        If constrainLevel == False, then try to return the next item
-        closest to this item, regardless of item's type (header, hunk, or
-        HunkLine).
-
-        If skipFolded == True, and the current item is folded, then the child
+        If skipfolded == True, and the current item is folded, then the child
         items that are hidden due to folding will be skipped when determining
         the next item.
 
@@ -131,9 +126,7 b' class patchnode(object):'
             itemfolded = self.folded
         except AttributeError:
             itemfolded = False
-        if constrainlevel:
-            return self.nextsibling()
-        elif skipfolded and itemfolded:
+        if skipfolded and itemfolded:
             nextitem = self.nextsibling()
             if nextitem is None:
                 try:
@@ -164,43 +157,31 b' class patchnode(object):'
             except AttributeError: # parent and/or grandparent was None
                 return None
 
-    def previtem(self, constrainlevel=True, skipfolded=True):
+    def previtem(self):
         """
-        If constrainLevel == True, return the closest previous item
-        of the same type where there are no items of different types between
-        the current item and this closest item.
-
-        If constrainLevel == False, then try to return the previous item
-        closest to this item, regardless of item's type (header, hunk, or
-        HunkLine).
-
-        If skipFolded == True, and the current item is folded, then the items
-        that are hidden due to folding will be skipped when determining the
-        next item.
+        Try to return the previous item closest to this item, regardless of
+        item's type (header, hunk, or hunkline).
 
         If it is not possible to get the previous item, return None.
         """
-        if constrainlevel:
-            return self.prevsibling()
-        else:
-            # try previous sibling's last child's last child,
-            # else try previous sibling's last child, else try previous sibling
-            prevsibling = self.prevsibling()
-            if prevsibling is not None:
-                prevsiblinglastchild = prevsibling.lastchild()
-                if ((prevsiblinglastchild is not None) and
-                    not prevsibling.folded):
-                    prevsiblinglclc = prevsiblinglastchild.lastchild()
-                    if ((prevsiblinglclc is not None) and
-                        not prevsiblinglastchild.folded):
-                        return prevsiblinglclc
-                    else:
-                        return prevsiblinglastchild
+        # try previous sibling's last child's last child,
+        # else try previous sibling's last child, else try previous sibling
+        prevsibling = self.prevsibling()
+        if prevsibling is not None:
+            prevsiblinglastchild = prevsibling.lastchild()
+            if ((prevsiblinglastchild is not None) and
+                not prevsibling.folded):
+                prevsiblinglclc = prevsiblinglastchild.lastchild()
+                if ((prevsiblinglclc is not None) and
+                    not prevsiblinglastchild.folded):
+                    return prevsiblinglclc
                 else:
-                    return prevsibling
+                    return prevsiblinglastchild
+            else:
+                return prevsibling
 
-            # try parent (or None)
-            return self.parentitem()
+        # try parent (or None)
+        return self.parentitem()
 
 class patch(patchnode, list): # todo: rename patchroot
     """
@@ -236,7 +217,6 b' class uiheader(patchnode):'
         self.neverunfolded = True
         self.hunks = [uihunk(h, self) for h in self.hunks]
 
-
     def prettystr(self):
         x = stringio()
         self.pretty(x)
@@ -392,6 +372,7 b' class uihunk(patchnode):'
     def allchildren(self):
         "return a list of all of the direct children of this node"
         return self.changedlines
+
     def countchanges(self):
         """changedlines -> (n+,n-)"""
         add = len([l for l in self.changedlines if l.applied
@@ -455,14 +436,12 b' class uihunk(patchnode):'
 
     def __getattr__(self, name):
         return getattr(self._hunk, name)
+
     def __repr__(self):
         return '<hunk %r@%d>' % (self.filename(), self.fromline)
 
-def filterpatch(ui, chunks, chunkselector, operation=None):
+def filterpatch(ui, chunks, chunkselector):
     """interactively filter patch chunks into applied-only chunks"""
-
-    if operation is None:
-        operation = _('confirm')
     chunks = list(chunks)
     # convert chunks list into structure suitable for displaying/modifying
     # with curses.  create a list of headers only.
@@ -603,13 +582,10 b' class curseschunkselector(object):'
         the last hunkline of the hunk prior to the selected hunk.  or, if
         the first hunkline of a hunk is currently selected, then select the
         hunk itself.
-
-        if the currently selected item is already at the top of the screen,
-        scroll the screen down to show the new-selected item.
         """
         currentitem = self.currentselecteditem
 
-        nextitem = currentitem.previtem(constrainlevel=False)
+        nextitem = currentitem.previtem()
 
         if nextitem is None:
             # if no parent item (i.e. currentitem is the first header), then
@@ -623,13 +599,10 b' class curseschunkselector(object):'
         select (if possible) the previous item on the same level as the
         currently selected item.  otherwise, select (if possible) the
         parent-item of the currently selected item.
-
-        if the currently selected item is already at the top of the screen,
-        scroll the screen down to show the new-selected item.
         """
         currentitem = self.currentselecteditem
-        nextitem = currentitem.previtem()
-        # if there's no previous item on this level, try choosing the parent
+        nextitem = currentitem.prevsibling()
+        # if there's no previous sibling, try choosing the parent
         if nextitem is None:
             nextitem = currentitem.parentitem()
         if nextitem is None:
@@ -646,14 +619,11 b' class curseschunkselector(object):'
         the first hunkline of the selected hunk.  or, if the last hunkline of
         a hunk is currently selected, then select the next hunk, if one exists,
         or if not, the next header if one exists.
-
-        if the currently selected item is already at the bottom of the screen,
-        scroll the screen up to show the new-selected item.
         """
         #self.startprintline += 1 #debug
         currentitem = self.currentselecteditem
 
-        nextitem = currentitem.nextitem(constrainlevel=False)
+        nextitem = currentitem.nextitem()
         # if there's no next item, keep the selection as-is
         if nextitem is None:
             nextitem = currentitem
@@ -662,24 +632,21 b' class curseschunkselector(object):'
 
     def downarrowshiftevent(self):
         """
-        if the cursor is already at the bottom chunk, scroll the screen up and
-        move the cursor-position to the subsequent chunk.  otherwise, only move
-        the cursor position down one chunk.
+        select (if possible) the next item on the same level as the currently
+        selected item.  otherwise, select (if possible) the next item on the
+        same level as the parent item of the currently selected item.
         """
-        # todo: update docstring
-
         currentitem = self.currentselecteditem
-        nextitem = currentitem.nextitem()
-        # if there's no previous item on this level, try choosing the parent's
-        # nextitem.
+        nextitem = currentitem.nextsibling()
+        # if there's no next sibling, try choosing the parent's nextsibling
         if nextitem is None:
             try:
-                nextitem = currentitem.parentitem().nextitem()
+                nextitem = currentitem.parentitem().nextsibling()
             except AttributeError:
-                # parentitem returned None, so nextitem() can't be called
+                # parentitem returned None, so nextsibling() can't be called
                 nextitem = None
         if nextitem is None:
-            # if no next item on parent-level, then no change...
+            # if parent has no next sibling, then no change...
             nextitem = currentitem
 
         self.currentselecteditem = nextitem
@@ -766,7 +733,6 b' class curseschunkselector(object):'
             # negative values scroll in pgup direction
             self.scrolllines(selstart - padstartbuffered)
 
-
     def scrolllines(self, numlines):
         "scroll the screen up (down) by numlines when numlines >0 (<0)."
         self.firstlineofpadtoprint += numlines
@@ -894,7 +860,6 b' class curseschunkselector(object):'
         if isinstance(item, (uiheader, uihunk)):
             item.folded = not item.folded
 
-
     def alignstring(self, instr, window):
         """
         add whitespace to the end of a string in order to make it fill
@@ -1133,7 +1098,6 b' class curseschunkselector(object):'
         lineprefix = " "*self.hunkindentnumchars + checkbox
         frtoline = "   " + hunk.getfromtoline().strip("\n")
 
-
         outstr += self.printstring(self.chunkpad, lineprefix, towin=towin,
                                    align=False) # add uncolored checkbox/indent
         outstr += self.printstring(self.chunkpad, frtoline, pair=colorpair,
@@ -1377,7 +1341,7 b' the following are valid keystrokes:'
                       F : fold / unfold parent item and all of its ancestors
                       m : edit / resume editing the commit message
                       e : edit the currently selected hunk
-                      a : toggle amend mode (hg rev >= 2.2), only with commit -i
+                      a : toggle amend mode, only with commit -i
                       c : confirm selected changes
                       r : review/edit and confirm selected changes
                       q : quit without confirming (no changes will be made)

mercurial/demandimport.py

@@ -188,15 +188,23 b' def _demandimport(name, globals=None, lo'
             if globalname and isinstance(symbol, _demandmod):
                 symbol._addref(globalname)
 
+        def chainmodules(rootmod, modname):
+            # recurse down the module chain, and return the leaf module
+            mod = rootmod
+            for comp in modname.split('.')[1:]:
+                if getattr(mod, comp, nothing) is nothing:
+                    setattr(mod, comp,
+                            _demandmod(comp, mod.__dict__, mod.__dict__))
+                mod = getattr(mod, comp)
+            return mod
+
         if level >= 0:
-            # The "from a import b,c,d" or "from .a import b,c,d"
-            # syntax gives errors with some modules for unknown
-            # reasons. Work around the problem.
             if name:
-                return _hgextimport(_origimport, name, globals, locals,
-                                    fromlist, level)
-
-            if _pypy:
+                # "from a import b" or "from .a import b" style
+                rootmod = _hgextimport(_origimport, name, globals, locals,
+                                       level=level)
+                mod = chainmodules(rootmod, name)
+            elif _pypy:
                 # PyPy's __import__ throws an exception if invoked
                 # with an empty name and no fromlist.  Recreate the
                 # desired behaviour by hand.
@@ -220,12 +228,7 b' def _demandimport(name, globals=None, lo'
         # But, we still need to support lazy loading of standard library and 3rd
         # party modules. So handle level == -1.
         mod = _hgextimport(_origimport, name, globals, locals)
-        # recurse down the module chain
-        for comp in name.split('.')[1:]:
-            if getattr(mod, comp, nothing) is nothing:
-                setattr(mod, comp,
-                        _demandmod(comp, mod.__dict__, mod.__dict__))
-            mod = getattr(mod, comp)
+        mod = chainmodules(mod, name)
 
         for x in fromlist:
             processfromitem(mod, x)

mercurial/destutil.py

@@ -95,6 +95,10 b' def _destupdatebranch(repo, clean, check'
     wc = repo[None]
     movemark = node = None
     currentbranch = wc.branch()
+
+    if clean:
+        currentbranch = repo['.'].branch()
+
     if currentbranch in repo.branchmap():
         heads = repo.branchheads(currentbranch)
         if heads:

mercurial/dirstate.py

@@ -74,6 +74,8 b' def _trypending(root, vfs, filename):'
                 raise
     return (vfs(filename), False)
 
+_token = object()
+
 class dirstate(object):
 
     def __init__(self, opener, ui, root, validate):
@@ -365,7 +367,7 b' class dirstate(object):'
 
     def setbranch(self, branch):
         self._branch = encoding.fromlocal(branch)
-        f = self._opener('branch', 'w', atomictemp=True)
+        f = self._opener('branch', 'w', atomictemp=True, checkambig=True)
         try:
             f.write(self._branch + '\n')
             f.close()
@@ -580,6 +582,8 b' class dirstate(object):'
             del self._map[f]
             if f in self._nonnormalset:
                 self._nonnormalset.remove(f)
+            if f in self._copymap:
+                del self._copymap[f]
 
     def _discoverpath(self, path, normed, ignoremissing, exists, storemap):
         if exists is None:
@@ -688,16 +692,15 b' class dirstate(object):'
         self._pl = (parent, nullid)
         self._dirty = True
 
-    def write(self, tr=False):
+    def write(self, tr=_token):
         if not self._dirty:
             return
 
         filename = self._filename
-        if tr is False: # not explicitly specified
-            if (self._ui.configbool('devel', 'all-warnings')
-                or self._ui.configbool('devel', 'check-dirstate-write')):
-                self._ui.develwarn('use dirstate.write with '
-                                   'repo.currenttransaction()')
+        if tr is _token: # not explicitly specified
+            self._ui.deprecwarn('use dirstate.write with '
+                               'repo.currenttransaction()',
+                               '3.9')
 
             if self._opener.lexists(self._pendingfilename):
                 # if pending file already exists, in-memory changes
@@ -727,7 +730,7 b' class dirstate(object):'
                                 self._writedirstate, location='plain')
             return
 
-        st = self._opener(filename, "w", atomictemp=True)
+        st = self._opener(filename, "w", atomictemp=True, checkambig=True)
         self._writedirstate(st)
 
     def _writedirstate(self, st):
@@ -1206,14 +1209,16 b' class dirstate(object):'
         else:
             return self._filename
 
-    def _savebackup(self, tr, suffix):
+    def savebackup(self, tr, suffix='', prefix=''):
         '''Save current dirstate into backup file with suffix'''
+        assert len(suffix) > 0 or len(prefix) > 0
         filename = self._actualfilename(tr)
 
         # use '_writedirstate' instead of 'write' to write changes certainly,
         # because the latter omits writing out if transaction is running.
         # output file will be used to create backup of dirstate at this point.
-        self._writedirstate(self._opener(filename, "w", atomictemp=True))
+        self._writedirstate(self._opener(filename, "w", atomictemp=True,
+                                         checkambig=True))
 
         if tr:
             # ensure that subsequent tr.writepending returns True for
@@ -1227,17 +1232,22 b' class dirstate(object):'
             # end of this transaction
             tr.registertmp(filename, location='plain')
 
-        self._opener.write(filename + suffix, self._opener.tryread(filename))
+        self._opener.write(prefix + self._filename + suffix,
+                           self._opener.tryread(filename))
 
-    def _restorebackup(self, tr, suffix):
+    def restorebackup(self, tr, suffix='', prefix=''):
         '''Restore dirstate by backup file with suffix'''
+        assert len(suffix) > 0 or len(prefix) > 0
         # this "invalidate()" prevents "wlock.release()" from writing
         # changes of dirstate out after restoring from backup file
         self.invalidate()
         filename = self._actualfilename(tr)
-        self._opener.rename(filename + suffix, filename)
+        # using self._filename to avoid having "pending" in the backup filename
+        self._opener.rename(prefix + self._filename + suffix, filename,
+                            checkambig=True)
 
-    def _clearbackup(self, tr, suffix):
+    def clearbackup(self, tr, suffix='', prefix=''):
         '''Clear backup file with suffix'''
-        filename = self._actualfilename(tr)
-        self._opener.unlink(filename + suffix)
+        assert len(suffix) > 0 or len(prefix) > 0
+        # using self._filename to avoid having "pending" in the backup filename
+        self._opener.unlink(prefix + self._filename + suffix)

mercurial/dispatch.py

@@ -384,7 +384,7 b' class cmdalias(object):'
         self.cmdname = ''
         self.definition = definition
         self.fn = None
-        self.args = []
+        self.givenargs = []
         self.opts = []
         self.help = ''
         self.badalias = None
@@ -432,7 +432,7 b' class cmdalias(object):'
                              % (self.name, inst))
             return
         self.cmdname = cmd = args.pop(0)
-        args = map(util.expandpath, args)
+        self.givenargs = args
 
         for invalidarg in ("--cwd", "-R", "--repository", "--repo", "--config"):
             if _earlygetopt([invalidarg], args):
@@ -448,7 +448,6 b' class cmdalias(object):'
             else:
                 self.fn, self.opts = tableentry
 
-            self.args = aliasargs(self.fn, args)
             if self.help.startswith("hg " + cmd):
                 # drop prefix in old-style help lines so hg shows the alias
                 self.help = self.help[4 + len(cmd):]
@@ -462,6 +461,11 b' class cmdalias(object):'
             self.badalias = (_("alias '%s' resolves to ambiguous command '%s'")
                              % (self.name, cmd))
 
+    @property
+    def args(self):
+        args = map(util.expandpath, self.givenargs)
+        return aliasargs(self.fn, args)
+
     def __getattr__(self, name):
         adefaults = {'norepo': True, 'optionalrepo': False, 'inferrepo': False}
         if name not in adefaults:
@@ -629,10 +633,16 b' def runcommand(lui, repo, cmd, fullargs,'
     # run pre-hook, and abort if it fails
     hook.hook(lui, repo, "pre-%s" % cmd, True, args=" ".join(fullargs),
               pats=cmdpats, opts=cmdoptions)
-    ret = _runcommand(ui, options, cmd, d)
-    # run post-hook, passing command result
-    hook.hook(lui, repo, "post-%s" % cmd, False, args=" ".join(fullargs),
-              result=ret, pats=cmdpats, opts=cmdoptions)
+    try:
+        ret = _runcommand(ui, options, cmd, d)
+        # run post-hook, passing command result
+        hook.hook(lui, repo, "post-%s" % cmd, False, args=" ".join(fullargs),
+                  result=ret, pats=cmdpats, opts=cmdoptions)
+    except Exception:
+        # run failure hook and re-raise
+        hook.hook(lui, repo, "fail-%s" % cmd, False, args=" ".join(fullargs),
+                  pats=cmdpats, opts=cmdoptions)
+        raise
     return ret
 
 def _getlocal(ui, rpath, wd=None):
@@ -660,12 +670,8 b' def _getlocal(ui, rpath, wd=None):'
 
     return path, lui
 
-def _checkshellalias(lui, ui, args, precheck=True):
-    """Return the function to run the shell alias, if it is required
-
-    'precheck' is whether this function is invoked before adding
-    aliases or not.
-    """
+def _checkshellalias(lui, ui, args):
+    """Return the function to run the shell alias, if it is required"""
     options = {}
 
     try:
@@ -676,16 +682,11 b' def _checkshellalias(lui, ui, args, prec'
     if not args:
         return
 
-    if precheck:
-        strict = True
-        cmdtable = commands.table.copy()
-        addaliases(lui, cmdtable)
-    else:
-        strict = False
-        cmdtable = commands.table
+    cmdtable = commands.table
 
     cmd = args[0]
     try:
+        strict = ui.configbool("ui", "strict")
         aliases, entry = cmdutil.findcmd(cmd, cmdtable, strict)
     except (error.AmbiguousCommand, error.UnknownCommand):
         return
@@ -735,12 +736,6 b' def _dispatch(req):'
     rpath = _earlygetopt(["-R", "--repository", "--repo"], args)
     path, lui = _getlocal(ui, rpath)
 
-    # Now that we're operating in the right directory/repository with
-    # the right config settings, check for shell aliases
-    shellaliasfn = _checkshellalias(lui, ui, args)
-    if shellaliasfn:
-        return shellaliasfn()
-
     # Configure extensions in phases: uisetup, extsetup, cmdtable, and
     # reposetup. Programs like TortoiseHg will call _dispatch several
     # times so we keep track of configured extensions in _loaded.
@@ -762,13 +757,11 b' def _dispatch(req):'
 
     addaliases(lui, commands.table)
 
-    if not lui.configbool("ui", "strict"):
-        # All aliases and commands are completely defined, now.
-        # Check abbreviation/ambiguity of shell alias again, because shell
-        # alias may cause failure of "_parse" (see issue4355)
-        shellaliasfn = _checkshellalias(lui, ui, args, precheck=False)
-        if shellaliasfn:
-            return shellaliasfn()
+    # All aliases and commands are completely defined, now.
+    # Check abbreviation/ambiguity of shell alias.
+    shellaliasfn = _checkshellalias(lui, ui, args)
+    if shellaliasfn:
+        return shellaliasfn()
 
     # check for fallback encoding
     fallback = lui.config('ui', 'fallbackencoding')
@@ -825,7 +818,7 b' def _dispatch(req):'
 
     if cmdoptions.get('insecure', False):
         for ui_ in uis:
-            ui_.setconfig('web', 'cacerts', '!', '--insecure')
+            ui_.insecureconnections = True
 
     if options['version']:
         return commands.version_(ui)

mercurial/error.py

@@ -15,12 +15,17 b' from __future__ import absolute_import'
 
 # Do not import anything here, please
 
-class HintException(Exception):
+class Hint(object):
+    """Mix-in to provide a hint of an error
+
+    This should come first in the inheritance list to consume a hint and
+    pass remaining arguments to the exception class.
+    """
     def __init__(self, *args, **kw):
-        Exception.__init__(self, *args)
-        self.hint = kw.get('hint')
+        self.hint = kw.pop('hint', None)
+        super(Hint, self).__init__(*args, **kw)
 
-class RevlogError(HintException):
+class RevlogError(Hint, Exception):
     pass
 
 class FilteredIndexError(IndexError):
@@ -50,10 +55,10 b' class ManifestLookupError(LookupError):'
 class CommandError(Exception):
     """Exception raised on errors in parsing the command line."""
 
-class InterventionRequired(HintException):
+class InterventionRequired(Hint, Exception):
     """Exception raised when a command requires human intervention."""
 
-class Abort(HintException):
+class Abort(Hint, Exception):
     """Raised if a command needs to print an error and exit."""
 
 class HookLoadError(Abort):
@@ -87,10 +92,10 b' class ResponseExpected(Abort):'
         from .i18n import _
         Abort.__init__(self, _('response expected'))
 
-class OutOfBandError(HintException):
+class OutOfBandError(Hint, Exception):
     """Exception raised when a remote repo reports failure"""
 
-class ParseError(HintException):
+class ParseError(Hint, Exception):
     """Raised when parsing config files and {rev,file}sets (msg[, pos])"""
 
 class UnknownIdentifier(ParseError):
@@ -102,7 +107,7 b' class UnknownIdentifier(ParseError):'
         self.function = function
         self.symbols = symbols
 
-class RepoError(HintException):
+class RepoError(Hint, Exception):
     pass
 
 class RepoLookupError(RepoError):
@@ -235,3 +240,6 b' class InvalidBundleSpecification(Excepti'
 
 class UnsupportedBundleSpecification(Exception):
     """error raised when a bundle specification is not supported."""
+
+class CorruptedState(Exception):
+    """error raised when a command is not able to read its state from file"""

mercurial/exchange.py

@@ -8,6 +8,7 b''
 from __future__ import absolute_import
 
 import errno
+import hashlib
 
 from .i18n import _
 from .node import (
@@ -857,14 +858,14 b' def _pushbundle2(pushop):'
         try:
             reply = pushop.remote.unbundle(stream, ['force'], 'push')
         except error.BundleValueError as exc:
-            raise error.Abort('missing support for %s' % exc)
+            raise error.Abort(_('missing support for %s') % exc)
         try:
             trgetter = None
             if pushback:
                 trgetter = pushop.trmanager.transaction
             op = bundle2.processbundle(pushop.repo, reply, trgetter)
         except error.BundleValueError as exc:
-            raise error.Abort('missing support for %s' % exc)
+            raise error.Abort(_('missing support for %s') % exc)
         except bundle2.AbortFromPart as exc:
             pushop.ui.status(_('remote: %s\n') % exc)
             raise error.Abort(_('push failed on remote'), hint=exc.hint)
@@ -1055,7 +1056,8 b' class pulloperation(object):'
         # revision we try to pull (None is "all")
         self.heads = heads
         # bookmark pulled explicitly
-        self.explicitbookmarks = bookmarks
+        self.explicitbookmarks = [repo._bookmarks.expandname(bookmark)
+                                  for bookmark in bookmarks]
         # do we force pull?
         self.force = force
         # whether a streaming clone was requested
@@ -1323,7 +1325,7 b' def _pullbundle2(pullop):'
     try:
         op = bundle2.processbundle(pullop.repo, bundle, pullop.gettransaction)
     except error.BundleValueError as exc:
-        raise error.Abort('missing support for %s' % exc)
+        raise error.Abort(_('missing support for %s') % exc)
 
     if pullop.fetch:
         results = [cg['return'] for cg in op.records['changegroup']]
@@ -1646,7 +1648,7 b' def check_heads(repo, their_heads, conte'
     Used by peer for unbundling.
     """
     heads = repo.heads()
-    heads_hash = util.sha1(''.join(sorted(heads))).digest()
+    heads_hash = hashlib.sha1(''.join(sorted(heads))).digest()
     if not (their_heads == ['force'] or their_heads == heads or
             their_heads == ['hashed', heads_hash]):
         # someone else committed/pushed/unbundled while we

mercurial/extensions.py

@@ -25,7 +25,7 b' from . import ('
 _aftercallbacks = {}
 _order = []
 _builtin = set(['hbisect', 'bookmarks', 'parentrevspec', 'progress', 'interhg',
-                'inotify'])
+                'inotify', 'hgcia'])
 
 def extensions(ui=None):
     if ui:
@@ -127,6 +127,21 b' def load(ui, name, path):'
         fn(loaded=True)
     return mod
 
+def _runuisetup(name, ui):
+    uisetup = getattr(_extensions[name], 'uisetup', None)
+    if uisetup:
+        uisetup(ui)
+
+def _runextsetup(name, ui):
+    extsetup = getattr(_extensions[name], 'extsetup', None)
+    if extsetup:
+        try:
+            extsetup(ui)
+        except TypeError:
+            if extsetup.func_code.co_argcount != 0:
+                raise
+            extsetup() # old extsetup with no ui argument
+
 def loadall(ui):
     result = ui.configitems("extensions")
     newindex = len(_order)
@@ -148,19 +163,10 b' def loadall(ui):'
             ui.traceback()
 
     for name in _order[newindex:]:
-        uisetup = getattr(_extensions[name], 'uisetup', None)
-        if uisetup:
-            uisetup(ui)
+        _runuisetup(name, ui)
 
     for name in _order[newindex:]:
-        extsetup = getattr(_extensions[name], 'extsetup', None)
-        if extsetup:
-            try:
-                extsetup(ui)
-            except TypeError:
-                if extsetup.func_code.co_argcount != 0:
-                    raise
-                extsetup() # old extsetup with no ui argument
+        _runextsetup(name, ui)
 
     # Call aftercallbacks that were never met.
     for shortname in _aftercallbacks:

mercurial/formatter.py

@@ -7,7 +7,6 b''
 
 from __future__ import absolute_import
 
-import cPickle
 import os
 
 from .i18n import _
@@ -20,8 +19,11 b' from . import ('
     encoding,
     error,
     templater,
+    util,
 )
 
+pickle = util.pickle
+
 class baseformatter(object):
     def __init__(self, ui, topic, opts):
         self._ui = ui
@@ -107,7 +109,7 b' class pickleformatter(baseformatter):'
         self._data.append(self._item)
     def end(self):
         baseformatter.end(self)
-        self._ui.write(cPickle.dumps(self._data))
+        self._ui.write(pickle.dumps(self._data))
 
 def _jsonifyobj(v):
     if isinstance(v, tuple):

mercurial/graphmod.py

@@ -19,8 +19,6 b' Data depends on type.'
 
 from __future__ import absolute_import
 
-import heapq
-
 from .node import nullrev
 from . import (
     revset,
@@ -32,207 +30,11 b" PARENT = 'P'"
 GRANDPARENT = 'G'
 MISSINGPARENT = 'M'
 # Style of line to draw. None signals a line that ends and is removed at this
-# point.
+# point. A number prefix means only the last N characters of the current block
+# will use that style, the rest will use the PARENT style. Add a - sign
+# (so making N negative) and all but the first N characters use that style.
 EDGES = {PARENT: '|', GRANDPARENT: ':', MISSINGPARENT: None}
 
-def groupbranchiter(revs, parentsfunc, firstbranch=()):
-    """Yield revisions from heads to roots one (topo) branch at a time.
-
-    This function aims to be used by a graph generator that wishes to minimize
-    the number of parallel branches and their interleaving.
-
-    Example iteration order (numbers show the "true" order in a changelog):
-
-      o  4
-      |
-      o  1
-      |
-      | o  3
-      | |
-      | o  2
-      |/
-      o  0
-
-    Note that the ancestors of merges are understood by the current
-    algorithm to be on the same branch. This means no reordering will
-    occur behind a merge.
-    """
-
-    ### Quick summary of the algorithm
-    #
-    # This function is based around a "retention" principle. We keep revisions
-    # in memory until we are ready to emit a whole branch that immediately
-    # "merges" into an existing one. This reduces the number of parallel
-    # branches with interleaved revisions.
-    #
-    # During iteration revs are split into two groups:
-    # A) revision already emitted
-    # B) revision in "retention". They are stored as different subgroups.
-    #
-    # for each REV, we do the following logic:
-    #
-    #   1) if REV is a parent of (A), we will emit it. If there is a
-    #   retention group ((B) above) that is blocked on REV being
-    #   available, we emit all the revisions out of that retention
-    #   group first.
-    #
-    #   2) else, we'll search for a subgroup in (B) awaiting for REV to be
-    #   available, if such subgroup exist, we add REV to it and the subgroup is
-    #   now awaiting for REV.parents() to be available.
-    #
-    #   3) finally if no such group existed in (B), we create a new subgroup.
-    #
-    #
-    # To bootstrap the algorithm, we emit the tipmost revision (which
-    # puts it in group (A) from above).
-
-    revs.sort(reverse=True)
-
-    # Set of parents of revision that have been emitted. They can be considered
-    # unblocked as the graph generator is already aware of them so there is no
-    # need to delay the revisions that reference them.
-    #
-    # If someone wants to prioritize a branch over the others, pre-filling this
-    # set will force all other branches to wait until this branch is ready to be
-    # emitted.
-    unblocked = set(firstbranch)
-
-    # list of groups waiting to be displayed, each group is defined by:
-    #
-    #   (revs:    lists of revs waiting to be displayed,
-    #    blocked: set of that cannot be displayed before those in 'revs')
-    #
-    # The second value ('blocked') correspond to parents of any revision in the
-    # group ('revs') that is not itself contained in the group. The main idea
-    # of this algorithm is to delay as much as possible the emission of any
-    # revision.  This means waiting for the moment we are about to display
-    # these parents to display the revs in a group.
-    #
-    # This first implementation is smart until it encounters a merge: it will
-    # emit revs as soon as any parent is about to be emitted and can grow an
-    # arbitrary number of revs in 'blocked'. In practice this mean we properly
-    # retains new branches but gives up on any special ordering for ancestors
-    # of merges. The implementation can be improved to handle this better.
-    #
-    # The first subgroup is special. It corresponds to all the revision that
-    # were already emitted. The 'revs' lists is expected to be empty and the
-    # 'blocked' set contains the parents revisions of already emitted revision.
-    #
-    # You could pre-seed the <parents> set of groups[0] to a specific
-    # changesets to select what the first emitted branch should be.
-    groups = [([], unblocked)]
-    pendingheap = []
-    pendingset = set()
-
-    heapq.heapify(pendingheap)
-    heappop = heapq.heappop
-    heappush = heapq.heappush
-    for currentrev in revs:
-        # Heap works with smallest element, we want highest so we invert
-        if currentrev not in pendingset:
-            heappush(pendingheap, -currentrev)
-            pendingset.add(currentrev)
-        # iterates on pending rev until after the current rev have been
-        # processed.
-        rev = None
-        while rev != currentrev:
-            rev = -heappop(pendingheap)
-            pendingset.remove(rev)
-
-            # Seek for a subgroup blocked, waiting for the current revision.
-            matching = [i for i, g in enumerate(groups) if rev in g[1]]
-
-            if matching:
-                # The main idea is to gather together all sets that are blocked
-                # on the same revision.
-                #
-                # Groups are merged when a common blocking ancestor is
-                # observed. For example, given two groups:
-                #
-                # revs [5, 4] waiting for 1
-                # revs [3, 2] waiting for 1
-                #
-                # These two groups will be merged when we process
-                # 1. In theory, we could have merged the groups when
-                # we added 2 to the group it is now in (we could have
-                # noticed the groups were both blocked on 1 then), but
-                # the way it works now makes the algorithm simpler.
-                #
-                # We also always keep the oldest subgroup first. We can
-                # probably improve the behavior by having the longest set
-                # first. That way, graph algorithms could minimise the length
-                # of parallel lines their drawing. This is currently not done.
-                targetidx = matching.pop(0)
-                trevs, tparents = groups[targetidx]
-                for i in matching:
-                    gr = groups[i]
-                    trevs.extend(gr[0])
-                    tparents |= gr[1]
-                # delete all merged subgroups (except the one we kept)
-                # (starting from the last subgroup for performance and
-                # sanity reasons)
-                for i in reversed(matching):
-                    del groups[i]
-            else:
-                # This is a new head. We create a new subgroup for it.
-                targetidx = len(groups)
-                groups.append(([], set([rev])))
-
-            gr = groups[targetidx]
-
-            # We now add the current nodes to this subgroups. This is done
-            # after the subgroup merging because all elements from a subgroup
-            # that relied on this rev must precede it.
-            #
-            # we also update the <parents> set to include the parents of the
-            # new nodes.
-            if rev == currentrev: # only display stuff in rev
-                gr[0].append(rev)
-            gr[1].remove(rev)
-            parents = [p for p in parentsfunc(rev) if p > nullrev]
-            gr[1].update(parents)
-            for p in parents:
-                if p not in pendingset:
-                    pendingset.add(p)
-                    heappush(pendingheap, -p)
-
-            # Look for a subgroup to display
-            #
-            # When unblocked is empty (if clause), we were not waiting for any
-            # revisions during the first iteration (if no priority was given) or
-            # if we emitted a whole disconnected set of the graph (reached a
-            # root).  In that case we arbitrarily take the oldest known
-            # subgroup. The heuristic could probably be better.
-            #
-            # Otherwise (elif clause) if the subgroup is blocked on
-            # a revision we just emitted, we can safely emit it as
-            # well.
-            if not unblocked:
-                if len(groups) > 1:  # display other subset
-                    targetidx = 1
-                    gr = groups[1]
-            elif not gr[1] & unblocked:
-                gr = None
-
-            if gr is not None:
-                # update the set of awaited revisions with the one from the
-                # subgroup
-                unblocked |= gr[1]
-                # output all revisions in the subgroup
-                for r in gr[0]:
-                    yield r
-                # delete the subgroup that you just output
-                # unless it is groups[0] in which case you just empty it.
-                if targetidx:
-                    del groups[targetidx]
-                else:
-                    gr[0][:] = []
-    # Check if we have some subgroup waiting for revisions we are not going to
-    # iterate over
-    for g in groups:
-        for r in g[0]:
-            yield r
-
 def dagwalker(repo, revs):
     """cset DAG generator yielding (id, CHANGESET, ctx, [parentinfo]) tuples
 
@@ -250,16 +52,6 b' def dagwalker(repo, revs):'
 
     gpcache = {}
 
-    if repo.ui.configbool('experimental', 'graph-group-branches', False):
-        firstbranch = ()
-        firstbranchrevset = repo.ui.config(
-            'experimental', 'graph-group-branches.firstbranch', '')
-        if firstbranchrevset:
-            firstbranch = repo.revs(firstbranchrevset)
-        parentrevs = repo.changelog.parentrevs
-        revs = groupbranchiter(revs, parentrevs, firstbranch)
-        revs = revset.baseset(revs)
-
     for rev in revs:
         ctx = repo[rev]
         # partition into parents in the rev set and missing parents, then
@@ -653,6 +445,22 b' def ascii(ui, state, type, char, text, c'
     while len(text) < len(lines):
         text.append("")
 
+    if any(len(char) > 1 for char in edgemap.values()):
+        # limit drawing an edge to the first or last N lines of the current
+        # section the rest of the edge is drawn like a parent line.
+        parent = state['styles'][PARENT][-1]
+        def _drawgp(char, i):
+            # should a grandparent character be drawn for this line?
+            if len(char) < 2:
+                return True
+            num = int(char[:-1])
+            # either skip first num lines or take last num lines, based on sign
+            return -num <= i if num < 0 else (len(lines) - i) <= num
+        for i, line in enumerate(lines):
+            line[:] = [c[-1] if _drawgp(c, i) else parent for c in line]
+        edgemap.update(
+            (e, (c if len(c) < 2 else parent)) for e, c in edgemap.items())
+
     # print lines
     indentation_level = max(ncols, ncols + coldiff)
     for (line, logstr) in zip(lines, text):

mercurial/help/config.txt

@@ -811,6 +811,15 b' variables it is passed are listed with n'
   dictionary of options (with unspecified options set to their defaults).
   ``$HG_PATS`` is a list of arguments. Hook failure is ignored.
 
+``fail-<command>``
+  Run after a failed invocation of an associated command. The contents
+  of the command line are passed as ``$HG_ARGS``. Parsed command line
+  arguments are passed as ``$HG_PATS`` and ``$HG_OPTS``. These contain
+  string representations of the python data internally passed to
+  <command>. ``$HG_OPTS`` is a dictionary of options (with unspecified
+  options set to their defaults). ``$HG_PATS`` is a list of arguments.
+  Hook failure is ignored.
+
 ``pre-<command>``
   Run before executing the associated command. The contents of the
   command line are passed as ``$HG_ARGS``. Parsed command line arguments
@@ -967,6 +976,8 b' is treated as a failure.'
 ``hostfingerprints``
 --------------------
 
+(Deprecated. Use ``[hostsecurity]``'s ``fingerprints`` options instead.)
+
 Fingerprints of the certificates of known HTTPS servers.
 
 A HTTPS connection to a server with a fingerprint configured here will
@@ -986,6 +997,114 b' For example::'
     hg.intevation.de = fc:e2:8d:d9:51:cd:cb:c1:4d:18:6b:b7:44:8d:49:72:57:e6:cd:33
     hg.intevation.org = fc:e2:8d:d9:51:cd:cb:c1:4d:18:6b:b7:44:8d:49:72:57:e6:cd:33
 
+``hostsecurity``
+----------------
+
+Used to specify global and per-host security settings for connecting to
+other machines.
+
+The following options control default behavior for all hosts.
+
+``ciphers``
+    Defines the cryptographic ciphers to use for connections.
+
+    Value must be a valid OpenSSL Cipher List Format as documented at
+    https://www.openssl.org/docs/manmaster/apps/ciphers.html#CIPHER-LIST-FORMAT.
+
+    This setting is for advanced users only. Setting to incorrect values
+    can significantly lower connection security or decrease performance.
+    You have been warned.
+
+    This option requires Python 2.7.
+
+``minimumprotocol``
+    Defines the minimum channel encryption protocol to use.
+
+    By default, the highest version of TLS supported by both client and server
+    is used.
+
+    Allowed values are: ``tls1.0``, ``tls1.1``, ``tls1.2``.
+
+    When running on an old Python version, only ``tls1.0`` is allowed since
+    old versions of Python only support up to TLS 1.0.
+
+    When running a Python that supports modern TLS versions, the default is
+    ``tls1.1``. ``tls1.0`` can still be used to allow TLS 1.0. However, this
+    weakens security and should only be used as a feature of last resort if
+    a server does not support TLS 1.1+.
+
+Options in the ``[hostsecurity]`` section can have the form
+``hostname``:``setting``. This allows multiple settings to be defined on a
+per-host basis.
+
+The following per-host settings can be defined.
+
+``ciphers``
+    This behaves like ``ciphers`` as described above except it only applies
+    to the host on which it is defined.
+
+``fingerprints``
+    A list of hashes of the DER encoded peer/remote certificate. Values have
+    the form ``algorithm``:``fingerprint``. e.g.
+    ``sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2``.
+
+    The following algorithms/prefixes are supported: ``sha1``, ``sha256``,
+    ``sha512``.
+
+    Use of ``sha256`` or ``sha512`` is preferred.
+
+    If a fingerprint is specified, the CA chain is not validated for this
+    host and Mercurial will require the remote certificate to match one
+    of the fingerprints specified. This means if the server updates its
+    certificate, Mercurial will abort until a new fingerprint is defined.
+    This can provide stronger security than traditional CA-based validation
+    at the expense of convenience.
+
+    This option takes precedence over ``verifycertsfile``.
+
+``minimumprotocol``
+    This behaves like ``minimumprotocol`` as described above except it
+    only applies to the host on which it is defined.
+
+``verifycertsfile``
+    Path to file a containing a list of PEM encoded certificates used to
+    verify the server certificate. Environment variables and ``~user``
+    constructs are expanded in the filename.
+
+    The server certificate or the certificate's certificate authority (CA)
+    must match a certificate from this file or certificate verification
+    will fail and connections to the server will be refused.
+
+    If defined, only certificates provided by this file will be used:
+    ``web.cacerts`` and any system/default certificates will not be
+    used.
+
+    This option has no effect if the per-host ``fingerprints`` option
+    is set.
+
+    The format of the file is as follows:
+
+        -----BEGIN CERTIFICATE-----
+        ... (certificate in base64 PEM encoding) ...
+        -----END CERTIFICATE-----
+        -----BEGIN CERTIFICATE-----
+        ... (certificate in base64 PEM encoding) ...
+        -----END CERTIFICATE-----
+
+For example::
+
+    [hostsecurity]
+    hg.example.com:fingerprints = sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2
+    hg2.example.com:fingerprints = sha1:914f1aff87249c09b6859b88b1906d30756491ca, sha1:fc:e2:8d:d9:51:cd:cb:c1:4d:18:6b:b7:44:8d:49:72:57:e6:cd:33
+    foo.example.com:verifycertsfile = /etc/ssl/trusted-ca-certs.pem
+
+To change the default minimum protocol version to TLS 1.2 but to allow TLS 1.1
+when connecting to ``hg.example.com``::
+
+    [hostsecurity]
+    minimumprotocol = tls1.2
+    hg.example.com:minimumprotocol = tls1.1
+
 ``http_proxy``
 --------------
 
@@ -1020,8 +1139,8 b' This section specifies behavior during m'
    file in the changeset being merged or updated to, and has different
    contents. Options are ``abort``, ``warn`` and ``ignore``. With ``abort``,
    abort on such files. With ``warn``, warn on such files and back them up as
-   .orig. With ``ignore``, don't print a warning and back them up as
-   .orig. (default: ``abort``)
+   ``.orig``. With ``ignore``, don't print a warning and back them up as
+   ``.orig``. (default: ``abort``)
 
 ``checkunknown``
    Controls behavior when an unknown file that isn't ignored has the same name
@@ -1210,6 +1329,18 b' The following sub-options can be defined'
    The URL to use for push operations. If not defined, the location
    defined by the path's main entry is used.
 
+``pushrev``
+   A revset defining which revisions to push by default.
+
+   When :hg:`push` is executed without a ``-r`` argument, the revset
+   defined by this sub-option is evaluated to determine what to push.
+
+   For example, a value of ``.`` will push the working directory's
+   revision by default.
+
+   Revsets specifying bookmarks will not result in the bookmark being
+   pushed.
+
 The following special named paths exist:
 
 ``default``
@@ -1442,16 +1573,6 b' Configuration for extensions that need t'
     Optional. Method to enable TLS when connecting to mail server: starttls,
     smtps or none. (default: none)
 
-``verifycert``
-    Optional. Verification for the certificate of mail server, when
-    ``tls`` is starttls or smtps. "strict", "loose" or False. For
-    "strict" or "loose", the certificate is verified as same as the
-    verification for HTTPS connections (see ``[hostfingerprints]`` and
-    ``[web] cacerts`` also). For "strict", sending email is also
-    aborted, if there is no configuration for mail server in
-    ``[hostfingerprints]`` and ``[web] cacerts``.  --insecure for
-    :hg:`email` overwrites this as "loose". (default: strict)
-
 ``username``
     Optional. User name for authenticating with the SMTP server.
     (default: None)
@@ -1738,6 +1859,13 b' User interface controls.'
     large organisation with its own Mercurial deployment process and crash
     reports should be addressed to your internal support.
 
+``textwidth``
+    Maximum width of help text. A longer line generated by ``hg help`` or
+    ``hg subcommand --help`` will be broken after white space to get this
+    width or the terminal width, whichever comes first.
+    A non-positive value will disable this and the terminal width will be
+    used. (default: 78)
+
 ``timeout``
     The timeout used when a lock is held (in seconds), a negative value
     means no timeout. (default: 600)
@@ -1945,6 +2073,14 b' The full set of options is:'
 ``ipv6``
     Whether to use IPv6. (default: False)
 
+``labels``
+    List of string *labels* associated with the repository.
+
+    Labels are exposed as a template keyword and can be used to customize
+    output. e.g. the ``index`` template can group or filter repositories
+    by labels and the ``summary`` template can display additional content
+    if a specific label is present.
+
 ``logoimg``
     File name of the logo image that some templates display on each page.
     The file name is relative to ``staticurl``. That is, the full path to

mercurial/help/templates.txt

@@ -81,6 +81,10 b' Some sample command line templates:'
 
    $ hg log -r 0 --template "files: {join(files, ', ')}\n"
 
+- Separate non-empty arguments by a " "::
+
+   $ hg log -r 0 --template "{separate(' ', node, bookmarks, tags}\n"
+
 - Modify each line of a commit description::
 
    $ hg log --template "{splitlines(desc) % '**** {line}\n'}"

mercurial/hg.py

@@ -9,6 +9,7 b''
 from __future__ import absolute_import
 
 import errno
+import hashlib
 import os
 import shutil
 
@@ -43,6 +44,9 b' from . import ('
 
 release = lock.release
 
+# shared features
+sharedbookmarks = 'bookmarks'
+
 def _local(path):
     path = util.expandpath(util.urllocalpath(path))
     return (os.path.isfile(path) and bundlerepo or localrepo)
@@ -257,7 +261,7 b' def postshare(sourcerepo, destrepo, book'
 
     if bookmarks:
         fp = destrepo.vfs('shared', 'w')
-        fp.write('bookmarks\n')
+        fp.write(sharedbookmarks + '\n')
         fp.close()
 
 def _postshareupdate(repo, update, checkout=None):
@@ -480,9 +484,11 b' def clone(ui, peeropts, source, dest=Non'
                 ui.status(_('(not using pooled storage: '
                             'unable to resolve identity of remote)\n'))
         elif sharenamemode == 'remote':
-            sharepath = os.path.join(sharepool, util.sha1(source).hexdigest())
+            sharepath = os.path.join(
+                sharepool, hashlib.sha1(source).hexdigest())
         else:
-            raise error.Abort('unknown share naming mode: %s' % sharenamemode)
+            raise error.Abort(_('unknown share naming mode: %s') %
+                              sharenamemode)
 
         if sharepath:
             return clonewithshare(ui, peeropts, sharepath, source, srcpeer,
@@ -921,9 +927,7 b' def remoteui(src, opts):'
         for key, val in src.configitems(sect):
             dst.setconfig(sect, key, val, 'copied')
     v = src.config('web', 'cacerts')
-    if v == '!':
-        dst.setconfig('web', 'cacerts', v, 'copied')
-    elif v:
+    if v:
         dst.setconfig('web', 'cacerts', util.expandpath(v), 'copied')
 
     return dst

mercurial/hgweb/common.py

@@ -8,11 +8,14 b''
 
 from __future__ import absolute_import
 
-import BaseHTTPServer
 import errno
 import mimetypes
 import os
 
+from .. import util
+
+httpserver = util.httpserver
+
 HTTP_OK = 200
 HTTP_NOT_MODIFIED = 304
 HTTP_BAD_REQUEST = 400
@@ -107,7 +110,7 b' class continuereader(object):'
         raise AttributeError
 
 def _statusmessage(code):
-    responses = BaseHTTPServer.BaseHTTPRequestHandler.responses
+    responses = httpserver.basehttprequesthandler.responses
     return responses.get(code, ('Error', 'Unknown error'))[0]
 
 def statusmessage(code, message=None):
@@ -187,7 +190,7 b' def get_contact(config):'
             os.environ.get("EMAIL") or "")
 
 def caching(web, req):
-    tag = str(web.mtime)
+    tag = 'W/"%s"' % web.mtime
     if req.env.get('HTTP_IF_NONE_MATCH') == tag:
         raise ErrorResponse(HTTP_NOT_MODIFIED)
     req.headers.append(('ETag', tag))

mercurial/hgweb/hgwebdir_mod.py

@@ -366,7 +366,9 b' class hgwebdir(object):'
                            'lastchange': d,
                            'lastchange_sort': d[1]-d[0],
                            'archives': [],
-                           'isdirectory': True}
+                           'isdirectory': True,
+                           'labels': [],
+                           }
 
                     seendirs.add(name)
                     yield row
@@ -416,6 +418,7 b' class hgwebdir(object):'
                        'lastchange_sort': d[1]-d[0],
                        'archives': archivelist(u, "tip", url),
                        'isdirectory': None,
+                       'labels': u.configlist('web', 'labels', untrusted=True),
                        }
 
                 yield row

mercurial/hgweb/server.py

@@ -8,8 +8,6 b''
 
 from __future__ import absolute_import
 
-import BaseHTTPServer
-import SocketServer
 import errno
 import os
 import socket
@@ -23,6 +21,8 b' from .. import ('
     util,
 )
 
+httpservermod = util.httpserver
+socketserver = util.socketserver
 urlerr = util.urlerr
 urlreq = util.urlreq
 
@@ -53,18 +53,18 b' class _error_logger(object):'
         for msg in seq:
             self.handler.log_error("HG error:  %s", msg)
 
-class _httprequesthandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class _httprequesthandler(httpservermod.basehttprequesthandler):
 
     url_scheme = 'http'
 
     @staticmethod
-    def preparehttpserver(httpserver, ssl_cert):
+    def preparehttpserver(httpserver, ui):
         """Prepare .socket of new HTTPServer instance"""
         pass
 
     def __init__(self, *args, **kargs):
         self.protocol_version = 'HTTP/1.1'
-        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kargs)
+        httpservermod.basehttprequesthandler.__init__(self, *args, **kargs)
 
     def _log_any(self, fp, format, *args):
         fp.write("%s - - [%s] %s\n" % (self.client_address[0],
@@ -147,9 +147,9 b' class _httprequesthandler(BaseHTTPServer'
         env['wsgi.input'] = self.rfile
         env['wsgi.errors'] = _error_logger(self)
         env['wsgi.multithread'] = isinstance(self.server,
-                                             SocketServer.ThreadingMixIn)
+                                             socketserver.ThreadingMixIn)
         env['wsgi.multiprocess'] = isinstance(self.server,
-                                              SocketServer.ForkingMixIn)
+                                              socketserver.ForkingMixIn)
         env['wsgi.run_once'] = 0
 
         self.saved_status = None
@@ -222,15 +222,25 b' class _httprequesthandlerssl(_httpreques'
     url_scheme = 'https'
 
     @staticmethod
-    def preparehttpserver(httpserver, ssl_cert):
+    def preparehttpserver(httpserver, ui):
         try:
-            import ssl
-            ssl.wrap_socket
+            from .. import sslutil
+            sslutil.modernssl
         except ImportError:
             raise error.Abort(_("SSL support is unavailable"))
-        httpserver.socket = ssl.wrap_socket(
-            httpserver.socket, server_side=True,
-            certfile=ssl_cert, ssl_version=ssl.PROTOCOL_TLSv1)
+
+        certfile = ui.config('web', 'certificate')
+
+        # These config options are currently only meant for testing. Use
+        # at your own risk.
+        cafile = ui.config('devel', 'servercafile')
+        reqcert = ui.configbool('devel', 'serverrequirecert')
+
+        httpserver.socket = sslutil.wrapserversocket(httpserver.socket,
+                                                     ui,
+                                                     certfile=certfile,
+                                                     cafile=cafile,
+                                                     requireclientcert=reqcert)
 
     def setup(self):
         self.connection = self.request
@@ -240,10 +250,10 b' class _httprequesthandlerssl(_httpreques'
 try:
     import threading
     threading.activeCount() # silence pyflakes and bypass demandimport
-    _mixin = SocketServer.ThreadingMixIn
+    _mixin = socketserver.ThreadingMixIn
 except ImportError:
     if util.safehasattr(os, "fork"):
-        _mixin = SocketServer.ForkingMixIn
+        _mixin = socketserver.ForkingMixIn
     else:
         class _mixin(object):
             pass
@@ -253,18 +263,18 b' def openlog(opt, default):'
         return open(opt, 'a')
     return default
 
-class MercurialHTTPServer(object, _mixin, BaseHTTPServer.HTTPServer):
+class MercurialHTTPServer(object, _mixin, httpservermod.httpserver):
 
     # SO_REUSEADDR has broken semantics on windows
     if os.name == 'nt':
         allow_reuse_address = 0
 
     def __init__(self, ui, app, addr, handler, **kwargs):
-        BaseHTTPServer.HTTPServer.__init__(self, addr, handler, **kwargs)
+        httpservermod.httpserver.__init__(self, addr, handler, **kwargs)
         self.daemon_threads = True
         self.application = app
 
-        handler.preparehttpserver(self, ui.config('web', 'certificate'))
+        handler.preparehttpserver(self, ui)
 
         prefix = ui.config('web', 'prefix', '')
         if prefix:

mercurial/hgweb/webcommands.py

@@ -139,7 +139,7 b' def _filerevision(web, req, tmpl, fctx):'
             yield {"line": t,
                    "lineid": "l%d" % (lineno + 1),
                    "linenumber": "% 6d" % (lineno + 1),
-                   "parity": parity.next()}
+                   "parity": next(parity)}
 
     return tmpl("filerevision",
                 file=f,
@@ -278,7 +278,7 b' def _search(web, req, tmpl):'
             files = webutil.listfilediffs(tmpl, ctx.files(), n, web.maxfiles)
 
             yield tmpl('searchentry',
-                       parity=parity.next(),
+                       parity=next(parity),
                        changelogtag=showtags,
                        files=files,
                        **webutil.commonentry(web.repo, ctx))
@@ -375,7 +375,7 b' def changelog(web, req, tmpl, shortlog=F'
                 break
 
             entry = webutil.changelistentry(web, web.repo[rev], tmpl)
-            entry['parity'] = parity.next()
+            entry['parity'] = next(parity)
             yield entry
 
     if shortlog:
@@ -527,7 +527,7 b' def manifest(web, req, tmpl):'
 
             fctx = ctx.filectx(full)
             yield {"file": full,
-                   "parity": parity.next(),
+                   "parity": next(parity),
                    "basename": f,
                    "date": fctx.date(),
                    "size": fctx.size(),
@@ -545,7 +545,7 b' def manifest(web, req, tmpl):'
                 h = v
 
             path = "%s%s" % (abspath, d)
-            yield {"parity": parity.next(),
+            yield {"parity": next(parity),
                    "path": path,
                    "emptydirs": "/".join(emptydirs),
                    "basename": d}
@@ -554,7 +554,7 b' def manifest(web, req, tmpl):'
                 symrev=symrev,
                 path=abspath,
                 up=webutil.up(abspath),
-                upparity=parity.next(),
+                upparity=next(parity),
                 fentries=filelist,
                 dentries=dirlist,
                 archives=web.archivelist(hex(node)),
@@ -582,7 +582,7 b' def tags(web, req, tmpl):'
         if latestonly:
             t = t[:1]
         for k, n in t:
-            yield {"parity": parity.next(),
+            yield {"parity": next(parity),
                    "tag": k,
                    "date": web.repo[n].date(),
                    "node": hex(n)}
@@ -615,7 +615,7 b' def bookmarks(web, req, tmpl):'
         if latestonly:
             t = i[:1]
         for k, n in t:
-            yield {"parity": parity.next(),
+            yield {"parity": next(parity),
                    "bookmark": k,
                    "date": web.repo[n].date(),
                    "node": hex(n)}
@@ -677,7 +677,7 b' def summary(web, req, tmpl):'
                 break
 
             yield tmpl("tagentry",
-                       parity=parity.next(),
+                       parity=next(parity),
                        tag=k,
                        node=hex(n),
                        date=web.repo[n].date())
@@ -688,7 +688,7 b' def summary(web, req, tmpl):'
         sortkey = lambda b: (web.repo[b[1]].rev(), b[0])
         marks = sorted(marks, key=sortkey, reverse=True)
         for k, n in marks[:10]:  # limit to 10 bookmarks
-            yield {'parity': parity.next(),
+            yield {'parity': next(parity),
                    'bookmark': k,
                    'date': web.repo[n].date(),
                    'node': hex(n)}
@@ -704,11 +704,11 b' def summary(web, req, tmpl):'
 
             l.append(tmpl(
                 'shortlogentry',
-                parity=parity.next(),
+                parity=next(parity),
                 **webutil.commonentry(web.repo, ctx)))
 
-        l.reverse()
-        yield l
+        for entry in reversed(l):
+            yield entry
 
     tip = web.repo['tip']
     count = len(web.repo)
@@ -725,7 +725,8 b' def summary(web, req, tmpl):'
                 shortlog=changelist,
                 node=tip.hex(),
                 symrev='tip',
-                archives=web.archivelist("tip"))
+                archives=web.archivelist("tip"),
+                labels=web.configlist('web', 'labels'))
 
 @webcommand('filediff')
 def filediff(web, req, tmpl):
@@ -863,29 +864,41 b' def annotate(web, req, tmpl):'
     diffopts = patch.difffeatureopts(web.repo.ui, untrusted=True,
                                      section='annotate', whitespace=True)
 
+    def parents(f):
+        for p in f.parents():
+            yield {
+                "node": p.hex(),
+                "rev": p.rev(),
+            }
+
     def annotate(**map):
-        last = None
         if util.binary(fctx.data()):
             mt = (mimetypes.guess_type(fctx.path())[0]
                   or 'application/octet-stream')
-            lines = enumerate([((fctx.filectx(fctx.filerev()), 1),
-                                '(binary:%s)' % mt)])
+            lines = [((fctx.filectx(fctx.filerev()), 1), '(binary:%s)' % mt)]
         else:
-            lines = enumerate(fctx.annotate(follow=True, linenumber=True,
-                                            diffopts=diffopts))
-        for lineno, ((f, targetline), l) in lines:
-            fnode = f.filenode()
-
-            if last != fnode:
-                last = fnode
-
-            yield {"parity": parity.next(),
+            lines = fctx.annotate(follow=True, linenumber=True,
+                                  diffopts=diffopts)
+        previousrev = None
+        blockparitygen = paritygen(1)
+        for lineno, ((f, targetline), l) in enumerate(lines):
+            rev = f.rev()
+            if rev != previousrev:
+                blockhead = True
+                blockparity = next(blockparitygen)
+            else:
+                blockhead = None
+            previousrev = rev
+            yield {"parity": next(parity),
                    "node": f.hex(),
-                   "rev": f.rev(),
+                   "rev": rev,
                    "author": f.user(),
+                   "parents": parents(f),
                    "desc": f.description(),
                    "extra": f.extra(),
                    "file": f.path(),
+                   "blockhead": blockhead,
+                   "blockparity": blockparity,
                    "targetline": targetline,
                    "line": l,
                    "lineno": lineno + 1,
@@ -963,7 +976,7 b' def filelog(web, req, tmpl):'
             iterfctx = fctx.filectx(i)
 
             l.append(dict(
-                parity=parity.next(),
+                parity=next(parity),
                 filerev=i,
                 file=f,
                 rename=webutil.renamelink(iterfctx),

mercurial/hgweb/webutil.py

@@ -75,7 +75,7 b' class revnav(object):'
     def _first(self):
         """return the minimum non-filtered changeset or None"""
         try:
-            return iter(self._revlog).next()
+            return next(iter(self._revlog))
         except StopIteration:
             return None
 
@@ -247,7 +247,7 b' def branchentries(repo, stripecount, lim'
             else:
                 status = 'open'
             yield {
-                'parity': parity.next(),
+                'parity': next(parity),
                 'branch': ctx.branch(),
                 'status': status,
                 'node': ctx.hex(),
@@ -369,7 +369,7 b' def changesetentry(web, req, tmpl, ctx):'
         template = f in ctx and 'filenodelink' or 'filenolink'
         files.append(tmpl(template,
                           node=ctx.hex(), file=f, blockno=blockno + 1,
-                          parity=parity.next()))
+                          parity=next(parity)))
 
     basectx = basechangectx(web.repo, req)
     if basectx is None:
@@ -450,15 +450,15 b' def diffs(repo, tmpl, ctx, basectx, file'
     block = []
     for chunk in patch.diff(repo, node1, node2, m, opts=diffopts):
         if chunk.startswith('diff') and block:
-            blockno = blockcount.next()
-            yield tmpl('diffblock', parity=parity.next(), blockno=blockno,
+            blockno = next(blockcount)
+            yield tmpl('diffblock', parity=next(parity), blockno=blockno,
                        lines=prettyprintlines(''.join(block), blockno))
             block = []
         if chunk.startswith('diff') and style != 'raw':
             chunk = ''.join(chunk.splitlines(True)[1:])
         block.append(chunk)
-    blockno = blockcount.next()
-    yield tmpl('diffblock', parity=parity.next(), blockno=blockno,
+    blockno = next(blockcount)
+    yield tmpl('diffblock', parity=next(parity), blockno=blockno,
                lines=prettyprintlines(''.join(block), blockno))
 
 def compare(tmpl, context, leftlines, rightlines):
@@ -521,14 +521,14 b' def diffstatgen(ctx, basectx):'
 def diffsummary(statgen):
     '''Return a short summary of the diff.'''
 
-    stats, maxname, maxtotal, addtotal, removetotal, binary = statgen.next()
+    stats, maxname, maxtotal, addtotal, removetotal, binary = next(statgen)
     return _(' %d files changed, %d insertions(+), %d deletions(-)\n') % (
              len(stats), addtotal, removetotal)
 
 def diffstat(tmpl, ctx, statgen, parity):
     '''Return a diffstat template for each file in the diff.'''
 
-    stats, maxname, maxtotal, addtotal, removetotal, binary = statgen.next()
+    stats, maxname, maxtotal, addtotal, removetotal, binary = next(statgen)
     files = ctx.files()
 
     def pct(i):
@@ -543,7 +543,7 b' def diffstat(tmpl, ctx, statgen, parity)'
         fileno += 1
         yield tmpl(template, node=ctx.hex(), file=filename, fileno=fileno,
                    total=total, addpct=pct(adds), removepct=pct(removes),
-                   parity=parity.next())
+                   parity=next(parity))
 
 class sessionvars(object):
     def __init__(self, vars, start='?'):

mercurial/httpclient/__init__.py

@@ -40,26 +40,38 b' from __future__ import absolute_import'
 
 # Many functions in this file have too many arguments.
 # pylint: disable=R0913
-
-import cStringIO
+import email
+import email.message
 import errno
-import httplib
+import inspect
 import logging
-import rfc822
 import select
 import socket
+import ssl
+import sys
+
+try:
+    import cStringIO as io
+    io.StringIO
+except ImportError:
+    import io
+
+try:
+    import httplib
+    httplib.HTTPException
+except ImportError:
+    import http.client as httplib
 
 from . import (
     _readers,
-    socketutil,
-    )
+)
 
 logger = logging.getLogger(__name__)
 
 __all__ = ['HTTPConnection', 'HTTPResponse']
 
-HTTP_VER_1_0 = 'HTTP/1.0'
-HTTP_VER_1_1 = 'HTTP/1.1'
+HTTP_VER_1_0 = b'HTTP/1.0'
+HTTP_VER_1_1 = b'HTTP/1.1'
 
 OUTGOING_BUFFER_SIZE = 1 << 15
 INCOMING_BUFFER_SIZE = 1 << 20
@@ -73,7 +85,7 b" XFER_ENCODING_CHUNKED = 'chunked'"
 
 CONNECTION_CLOSE = 'close'
 
-EOL = '\r\n'
+EOL = b'\r\n'
 _END_HEADERS = EOL * 2
 
 # Based on some searching around, 1 second seems like a reasonable
@@ -81,6 +93,57 b" EOL = '\\r\\n'"
 TIMEOUT_ASSUME_CONTINUE = 1
 TIMEOUT_DEFAULT = None
 
+if sys.version_info > (3, 0):
+    _unicode = str
+else:
+    _unicode = unicode
+
+def _ensurebytes(data):
+    if not isinstance(data, (_unicode, bytes)):
+        data = str(data)
+    if not isinstance(data, bytes):
+        try:
+            return data.encode('latin-1')
+        except UnicodeEncodeError as err:
+            raise UnicodeEncodeError(
+                err.encoding,
+                err.object,
+                err.start,
+                err.end,
+                '%r is not valid Latin-1 Use .encode("utf-8") '
+                'if sending as utf-8 is desired.' % (
+                    data[err.start:err.end],))
+    return data
+
+class _CompatMessage(email.message.Message):
+    """Workaround for rfc822.Message and email.message.Message API diffs."""
+
+    @classmethod
+    def from_string(cls, s):
+        if sys.version_info > (3, 0):
+            # Python 3 can't decode headers from bytes, so we have to
+            # trust RFC 2616 and decode the headers as iso-8859-1
+            # bytes.
+            s = s.decode('iso-8859-1')
+        headers = email.message_from_string(s, _class=_CompatMessage)
+        # Fix multi-line headers to match httplib's behavior from
+        # Python 2.x, since email.message.Message handles them in
+        # slightly different ways.
+        if sys.version_info < (3, 0):
+            new = []
+            for h, v in headers._headers:
+                if '\r\n' in v:
+                    v = '\n'.join([' ' + x.lstrip() for x in v.split('\r\n')])[1:]
+                new.append((h, v))
+            headers._headers = new
+        return headers
+
+    def getheaders(self, key):
+        return self.get_all(key)
+
+    def getheader(self, key, default=None):
+        return self.get(key, failobj=default)
+
 
 class HTTPResponse(object):
     """Response from an HTTP server.
@@ -91,11 +154,11 b' class HTTPResponse(object):'
     def __init__(self, sock, timeout, method):
         self.sock = sock
         self.method = method
-        self.raw_response = ''
+        self.raw_response = b''
         self._headers_len = 0
         self.headers = None
         self.will_close = False
-        self.status_line = ''
+        self.status_line = b''
         self.status = None
         self.continued = False
         self.http_version = None
@@ -131,6 +194,10 b' class HTTPResponse(object):'
         return self.headers.getheader(header, default=default)
 
     def getheaders(self):
+        if sys.version_info < (3, 0):
+            return [(k.lower(), v) for k, v in self.headers.items()]
+        # Starting in Python 3, headers aren't lowercased before being
+        # returned here.
         return self.headers.items()
 
     def readline(self):
@@ -141,14 +208,14 b' class HTTPResponse(object):'
         """
         blocks = []
         while True:
-            self._reader.readto('\n', blocks)
+            self._reader.readto(b'\n', blocks)
 
-            if blocks and blocks[-1][-1] == '\n' or self.complete():
+            if blocks and blocks[-1][-1:] == b'\n' or self.complete():
                 break
 
             self._select()
 
-        return ''.join(blocks)
+        return b''.join(blocks)
 
     def read(self, length=None):
         """Read data from the response body."""
@@ -175,8 +242,8 b' class HTTPResponse(object):'
                 raise HTTPTimeoutException('timeout reading data')
         try:
             data = self.sock.recv(INCOMING_BUFFER_SIZE)
-        except socket.sslerror as e:
-            if e.args[0] != socket.SSL_ERROR_WANT_READ:
+        except ssl.SSLError as e:
+            if e.args[0] != ssl.SSL_ERROR_WANT_READ:
                 raise
             logger.debug('SSL_ERROR_WANT_READ in _select, should retry later')
             return True
@@ -203,7 +270,7 b' class HTTPResponse(object):'
         self.raw_response += data
         # This is a bogus server with bad line endings
         if self._eol not in self.raw_response:
-            for bad_eol in ('\n', '\r'):
+            for bad_eol in (b'\n', b'\r'):
                 if (bad_eol in self.raw_response
                     # verify that bad_eol is not the end of the incoming data
                     # as this could be a response line that just got
@@ -220,8 +287,8 b' class HTTPResponse(object):'
 
         # handle 100-continue response
         hdrs, body = self.raw_response.split(self._end_headers, 1)
-        unused_http_ver, status = hdrs.split(' ', 1)
-        if status.startswith('100'):
+        unused_http_ver, status = hdrs.split(b' ', 1)
+        if status.startswith(b'100'):
             self.raw_response = body
             self.continued = True
             logger.debug('continue seen, setting body to %r', body)
@@ -235,14 +302,14 b' class HTTPResponse(object):'
             self.status_line, hdrs = hdrs.split(self._eol, 1)
         else:
             self.status_line = hdrs
-            hdrs = ''
+            hdrs = b''
         # TODO HTTP < 1.0 support
         (self.http_version, self.status,
-         self.reason) = self.status_line.split(' ', 2)
+         self.reason) = self.status_line.split(b' ', 2)
         self.status = int(self.status)
         if self._eol != EOL:
-            hdrs = hdrs.replace(self._eol, '\r\n')
-        headers = rfc822.Message(cStringIO.StringIO(hdrs))
+            hdrs = hdrs.replace(self._eol, b'\r\n')
+        headers = _CompatMessage.from_string(hdrs)
         content_len = None
         if HDR_CONTENT_LENGTH in headers:
             content_len = int(headers[HDR_CONTENT_LENGTH])
@@ -259,8 +326,8 b' class HTTPResponse(object):'
             # HEAD responses are forbidden from returning a body, and
             # it's implausible for a CONNECT response to use
             # close-is-end logic for an OK response.
-            if (self.method == 'HEAD' or
-                (self.method == 'CONNECT' and content_len is None)):
+            if (self.method == b'HEAD' or
+                (self.method == b'CONNECT' and content_len is None)):
                 content_len = 0
             if content_len is not None:
                 logger.debug('using a content-length reader with length %d',
@@ -294,8 +361,48 b' def _foldheaders(headers):'
     >>> _foldheaders({'Accept-Encoding': 'wat'})
     {'accept-encoding': ('Accept-Encoding', 'wat')}
     """
-    return dict((k.lower(), (k, v)) for k, v in headers.iteritems())
+    return dict((k.lower(), (k, v)) for k, v in headers.items())
+
+try:
+    inspect.signature
+    def _handlesarg(func, arg):
+        """ Try to determine if func accepts arg
+
+        If it takes arg, return True
+        If it happens to take **args, then it could do anything:
+            * It could throw a different TypeError, just for fun
+            * It could throw an ArgumentError or anything else
+            * It could choose not to throw an Exception at all
+        ... return 'unknown'
 
+        Otherwise, return False
+        """
+        params = inspect.signature(func).parameters
+        if arg in params:
+            return True
+        for p in params:
+            if params[p].kind == inspect._ParameterKind.VAR_KEYWORD:
+                return 'unknown'
+        return False
+except AttributeError:
+    def _handlesarg(func, arg):
+        """ Try to determine if func accepts arg
+
+        If it takes arg, return True
+        If it happens to take **args, then it could do anything:
+            * It could throw a different TypeError, just for fun
+            * It could throw an ArgumentError or anything else
+            * It could choose not to throw an Exception at all
+        ... return 'unknown'
+
+        Otherwise, return False
+        """
+        spec = inspect.getargspec(func)
+        if arg in spec.args:
+            return True
+        if spec.keywords:
+            return 'unknown'
+        return False
 
 class HTTPConnection(object):
     """Connection to a single http server.
@@ -340,15 +447,38 b' class HTTPConnection(object):'
         Any extra keyword arguments to this function will be provided
         to the ssl_wrap_socket method. If no ssl
         """
-        if port is None and host.count(':') == 1 or ']:' in host:
-            host, port = host.rsplit(':', 1)
+        host = _ensurebytes(host)
+        if port is None and host.count(b':') == 1 or b']:' in host:
+            host, port = host.rsplit(b':', 1)
             port = int(port)
-            if '[' in host:
+            if b'[' in host:
                 host = host[1:-1]
         if ssl_wrap_socket is not None:
-            self._ssl_wrap_socket = ssl_wrap_socket
+            _wrap_socket = ssl_wrap_socket
         else:
-            self._ssl_wrap_socket = socketutil.wrap_socket
+            _wrap_socket = ssl.wrap_socket
+        call_wrap_socket = None
+        handlesubar = _handlesarg(_wrap_socket, 'server_hostname')
+        if handlesubar is True:
+            # supports server_hostname
+            call_wrap_socket = _wrap_socket
+        handlesnobar = _handlesarg(_wrap_socket, 'serverhostname')
+        if handlesnobar is True and handlesubar is not True:
+            # supports serverhostname
+            def call_wrap_socket(sock, server_hostname=None, **ssl_opts):
+                return _wrap_socket(sock, serverhostname=server_hostname,
+                                    **ssl_opts)
+        if handlesubar is False and handlesnobar is False:
+            # does not support either
+            def call_wrap_socket(sock, server_hostname=None, **ssl_opts):
+                return _wrap_socket(sock, **ssl_opts)
+        if call_wrap_socket is None:
+            # we assume it takes **args
+            def call_wrap_socket(sock, **ssl_opts):
+                if 'server_hostname' in ssl_opts:
+                    ssl_opts['serverhostname'] = ssl_opts['server_hostname']
+                return _wrap_socket(sock, **ssl_opts)
+        self._ssl_wrap_socket = call_wrap_socket
         if use_ssl is None and port is None:
             use_ssl = False
             port = 80
@@ -357,8 +487,6 b' class HTTPConnection(object):'
         elif port is None:
             port = (use_ssl and 443 or 80)
         self.port = port
-        if use_ssl and not socketutil.have_ssl:
-            raise Exception('ssl requested but unavailable on this Python')
         self.ssl = use_ssl
         self.ssl_opts = ssl_opts
         self._ssl_validator = ssl_validator
@@ -388,15 +516,15 b' class HTTPConnection(object):'
         if self._proxy_host is not None:
             logger.info('Connecting to http proxy %s:%s',
                         self._proxy_host, self._proxy_port)
-            sock = socketutil.create_connection((self._proxy_host,
-                                                 self._proxy_port))
+            sock = socket.create_connection((self._proxy_host,
+                                             self._proxy_port))
             if self.ssl:
-                data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
-                                                                self.port),
+                data = self._buildheaders(b'CONNECT', b'%s:%d' % (self.host,
+                                                                  self.port),
                                           proxy_headers, HTTP_VER_1_0)
                 sock.send(data)
                 sock.setblocking(0)
-                r = self.response_class(sock, self.timeout, 'CONNECT')
+                r = self.response_class(sock, self.timeout, b'CONNECT')
                 timeout_exc = HTTPTimeoutException(
                     'Timed out waiting for CONNECT response from proxy')
                 while not r.complete():
@@ -421,7 +549,7 b' class HTTPConnection(object):'
                 logger.info('CONNECT (for SSL) to %s:%s via proxy succeeded.',
                             self.host, self.port)
         else:
-            sock = socketutil.create_connection((self.host, self.port))
+            sock = socket.create_connection((self.host, self.port))
         if self.ssl:
             # This is the default, but in the case of proxied SSL
             # requests the proxy logic above will have cleared
@@ -429,7 +557,8 b' class HTTPConnection(object):'
             sock.setblocking(1)
             logger.debug('wrapping socket for ssl with options %r',
                          self.ssl_opts)
-            sock = self._ssl_wrap_socket(sock, **self.ssl_opts)
+            sock = self._ssl_wrap_socket(sock, server_hostname=self.host,
+                                         **self.ssl_opts)
             if self._ssl_validator:
                 self._ssl_validator(sock)
         sock.setblocking(0)
@@ -441,25 +570,26 b' class HTTPConnection(object):'
             hdrhost = self.host
         else:
             # include nonstandard port in header
-            if ':' in self.host:  # must be IPv6
-                hdrhost = '[%s]:%d' % (self.host, self.port)
+            if b':' in self.host:  # must be IPv6
+                hdrhost = b'[%s]:%d' % (self.host, self.port)
             else:
-                hdrhost = '%s:%d' % (self.host, self.port)
+                hdrhost = b'%s:%d' % (self.host, self.port)
         if self._proxy_host and not self.ssl:
             # When talking to a regular http proxy we must send the
             # full URI, but in all other cases we must not (although
             # technically RFC 2616 says servers must accept our
             # request if we screw up, experimentally few do that
             # correctly.)
-            assert path[0] == '/', 'path must start with a /'
-            path = 'http://%s%s' % (hdrhost, path)
-        outgoing = ['%s %s %s%s' % (method, path, http_ver, EOL)]
-        headers['host'] = ('Host', hdrhost)
+            assert path[0:1] == b'/', 'path must start with a /'
+            path = b'http://%s%s' % (hdrhost, path)
+        outgoing = [b'%s %s %s%s' % (method, path, http_ver, EOL)]
+        headers[b'host'] = (b'Host', hdrhost)
         headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
-        for hdr, val in headers.itervalues():
-            outgoing.append('%s: %s%s' % (hdr, val, EOL))
+        for hdr, val in sorted((_ensurebytes(h), _ensurebytes(v))
+                               for h, v in headers.values()):
+            outgoing.append(b'%s: %s%s' % (hdr, val, EOL))
         outgoing.append(EOL)
-        return ''.join(outgoing)
+        return b''.join(outgoing)
 
     def close(self):
         """Close the connection to the server.
@@ -512,6 +642,8 b' class HTTPConnection(object):'
         available. Use the `getresponse()` method to retrieve the
         response.
         """
+        method = _ensurebytes(method)
+        path = _ensurebytes(path)
         if self.busy():
             raise httplib.CannotSendRequest(
                 'Can not send another request before '
@@ -520,11 +652,26 b' class HTTPConnection(object):'
 
         logger.info('sending %s request for %s to %s on port %s',
                     method, path, self.host, self.port)
+
         hdrs = _foldheaders(headers)
-        if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
+        # Figure out headers that have to be computed from the request
+        # body.
+        chunked = False
+        if body and HDR_CONTENT_LENGTH not in hdrs:
+            if getattr(body, '__len__', False):
+                hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH,
+                                            b'%d' % len(body))
+            elif getattr(body, 'read', False):
+                hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
+                                           XFER_ENCODING_CHUNKED)
+                chunked = True
+            else:
+                raise BadRequestData('body has no __len__() nor read()')
+        # Figure out expect-continue header
+        if hdrs.get('expect', ('', ''))[1].lower() == b'100-continue':
             expect_continue = True
         elif expect_continue:
-            hdrs['expect'] = ('Expect', '100-Continue')
+            hdrs['expect'] = (b'Expect', b'100-Continue')
         # httplib compatibility: if the user specified a
         # proxy-authorization header, that's actually intended for a
         # proxy CONNECT action, not the real request, but only if
@@ -534,25 +681,15 b' class HTTPConnection(object):'
             pa = hdrs.pop('proxy-authorization', None)
             if pa is not None:
                 pheaders['proxy-authorization'] = pa
-
-        chunked = False
-        if body and HDR_CONTENT_LENGTH not in hdrs:
-            if getattr(body, '__len__', False):
-                hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
-            elif getattr(body, 'read', False):
-                hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
-                                           XFER_ENCODING_CHUNKED)
-                chunked = True
-            else:
-                raise BadRequestData('body has no __len__() nor read()')
+        # Build header data
+        outgoing_headers = self._buildheaders(
+            method, path, hdrs, self.http_version)
 
         # If we're reusing the underlying socket, there are some
         # conditions where we'll want to retry, so make a note of the
         # state of self.sock
         fresh_socket = self.sock is None
         self._connect(pheaders)
-        outgoing_headers = self._buildheaders(
-            method, path, hdrs, self.http_version)
         response = None
         first = True
 
@@ -592,8 +729,8 b' class HTTPConnection(object):'
                 try:
                     try:
                         data = r[0].recv(INCOMING_BUFFER_SIZE)
-                    except socket.sslerror as e:
-                        if e.args[0] != socket.SSL_ERROR_WANT_READ:
+                    except ssl.SSLError as e:
+                        if e.args[0] != ssl.SSL_ERROR_WANT_READ:
                             raise
                         logger.debug('SSL_ERROR_WANT_READ while sending '
                                      'data, retrying...')
@@ -662,16 +799,20 b' class HTTPConnection(object):'
                             continue
                         if len(data) < OUTGOING_BUFFER_SIZE:
                             if chunked:
-                                body = '0' + EOL + EOL
+                                body = b'0' + EOL + EOL
                             else:
                                 body = None
                         if chunked:
-                            out = hex(len(data))[2:] + EOL + data + EOL
+                            # This encode is okay because we know
+                            # hex() is building us only 0-9 and a-f
+                            # digits.
+                            asciilen = hex(len(data))[2:].encode('ascii')
+                            out = asciilen + EOL + data + EOL
                         else:
                             out = data
                     amt = w[0].send(out)
                 except socket.error as e:
-                    if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
+                    if e[0] == ssl.SSL_ERROR_WANT_WRITE and self.ssl:
                         # This means that SSL hasn't flushed its buffer into
                         # the socket yet.
                         # TODO: find a way to block on ssl flushing its buffer
@@ -690,6 +831,7 b' class HTTPConnection(object):'
                     body = out[amt:]
                 else:
                     outgoing_headers = out[amt:]
+        # End of request-sending loop.
 
         # close if the server response said to or responded before eating
         # the whole request

mercurial/httpclient/_readers.py

@@ -33,7 +33,12 b' have any clients outside of httpplus.'
 """
 from __future__ import absolute_import
 
-import httplib
+try:
+    import httplib
+    httplib.HTTPException
+except ImportError:
+    import http.client as httplib
+
 import logging
 
 logger = logging.getLogger(__name__)
@@ -93,7 +98,7 b' class AbstractReader(object):'
             need -= len(b)
             if need == 0:
                 break
-        result = ''.join(blocks)
+        result = b''.join(blocks)
         assert len(result) == amt or (self._finished and len(result) < amt)
 
         return result

mercurial/httpconnection.py

@@ -280,10 +280,9 b' class http2handler(urlreq.httphandler, u'
         kwargs['keyfile'] = keyfile
         kwargs['certfile'] = certfile
 
-        kwargs.update(sslutil.sslkwargs(self.ui, host))
-
         con = HTTPConnection(host, port, use_ssl=True,
                              ssl_wrap_socket=sslutil.wrapsocket,
-                             ssl_validator=sslutil.validator(self.ui, host),
+                             ssl_validator=sslutil.validatesocket,
+                             ui=self.ui,
                              **kwargs)
         return con

mercurial/httppeer.py

@@ -9,7 +9,6 b''
 from __future__ import absolute_import
 
 import errno
-import httplib
 import os
 import socket
 import tempfile
@@ -27,6 +26,7 b' from . import ('
     wireproto,
 )
 
+httplib = util.httplib
 urlerr = util.urlerr
 urlreq = util.urlreq
 
@@ -302,7 +302,7 b' def instance(ui, path, create):'
     except error.RepoError as httpexception:
         try:
             r = statichttprepo.instance(ui, "static-" + path, create)
-            ui.note('(falling back to static-http)\n')
+            ui.note(_('(falling back to static-http)\n'))
             return r
         except error.RepoError:
             raise httpexception # use the original http RepoError instead

mercurial/i18n.py

@@ -78,7 +78,7 b' def gettext(message):'
             paragraphs = [p.decode("ascii") for p in message.split('\n\n')]
         # Be careful not to translate the empty string -- it holds the
         # meta data of the .po file.
-        u = u'\n\n'.join([p and _ugettext(p) or '' for p in paragraphs])
+        u = u'\n\n'.join([p and _ugettext(p) or u'' for p in paragraphs])
         try:
             # encoding.tolocal cannot be used since it will first try to
             # decode the Unicode string. Calling u.decode(enc) really

mercurial/keepalive.py

@@ -110,15 +110,16 b' EXTRA ATTRIBUTES AND METHODS'
 from __future__ import absolute_import, print_function
 
 import errno
-import httplib
+import hashlib
 import socket
 import sys
-import thread
+import threading
 
 from . import (
     util,
 )
 
+httplib = util.httplib
 urlerr = util.urlerr
 urlreq = util.urlreq
 
@@ -134,7 +135,7 b' class ConnectionManager(object):'
       * keep track of all existing
       """
     def __init__(self):
-        self._lock = thread.allocate_lock()
+        self._lock = threading.Lock()
         self._hostmap = {} # map hosts to a list of connections
         self._connmap = {} # map connections to host
         self._readymap = {} # map connection to ready state
@@ -624,8 +625,7 b' def error_handler(url):'
     keepalive_handler.close_all()
 
 def continuity(url):
-    from . import util
-    md5 = util.md5
+    md5 = hashlib.md5
     format = '%25s: %s'
 
     # first fetch the file with the normal http handler

mercurial/localrepo.py

@@ -8,6 +8,7 b''
 from __future__ import absolute_import
 
 import errno
+import hashlib
 import inspect
 import os
 import random
@@ -57,16 +58,16 b' from . import ('
 )
 
 release = lockmod.release
-propertycache = util.propertycache
 urlerr = util.urlerr
 urlreq = util.urlreq
-filecache = scmutil.filecache
 
-class repofilecache(filecache):
+class repofilecache(scmutil.filecache):
     """All filecache usage on repo are done for logic that should be unfiltered
     """
 
     def __get__(self, repo, type=None):
+        if repo is None:
+            return self
         return super(repofilecache, self).__get__(repo.unfiltered(), type)
     def __set__(self, repo, value):
         return super(repofilecache, self).__set__(repo.unfiltered(), value)
@@ -78,7 +79,7 b' class storecache(repofilecache):'
     def join(self, obj, fname):
         return obj.sjoin(fname)
 
-class unfilteredpropertycache(propertycache):
+class unfilteredpropertycache(util.propertycache):
     """propertycache that apply to unfiltered repo only"""
 
     def __get__(self, repo, type=None):
@@ -87,7 +88,7 b' class unfilteredpropertycache(propertyca'
             return super(unfilteredpropertycache, self).__get__(unfi)
         return getattr(unfi, self.name)
 
-class filteredpropertycache(propertycache):
+class filteredpropertycache(util.propertycache):
     """propertycache that must take filtering in account"""
 
     def cachevalue(self, obj, value):
@@ -553,7 +554,10 b' class localrepository(object):'
         The revset is specified as a string ``expr`` that may contain
         %-formatting to escape certain types. See ``revset.formatspec``.
 
-        Return a revset.abstractsmartset, which is a list-like interface
+        Revset aliases from the configuration are not expanded. To expand
+        user aliases, consider calling ``scmutil.revrange()``.
+
+        Returns a revset.abstractsmartset, which is a list-like interface
         that contains integer revisions.
         '''
         expr = revset.formatspec(expr, *args)
@@ -565,6 +569,9 b' class localrepository(object):'
 
         This is a convenience wrapper around ``revs()`` that iterates the
         result and is a generator of changectx instances.
+
+        Revset aliases from the configuration are not expanded. To expand
+        user aliases, consider calling ``scmutil.revrange()``.
         '''
         for r in self.revs(expr, *args):
             yield self[r]
@@ -881,12 +888,6 b' class localrepository(object):'
             f = f[1:]
         return filelog.filelog(self.svfs, f)
 
-    def parents(self, changeid=None):
-        '''get list of changectxs for parents of changeid'''
-        msg = 'repo.parents() is deprecated, use repo[%r].parents()' % changeid
-        self.ui.deprecwarn(msg, '3.7')
-        return self[changeid].parents()
-
     def changectx(self, changeid):
         return self[changeid]
 
@@ -1008,7 +1009,8 b' class localrepository(object):'
                 or self.ui.configbool('devel', 'check-locks')):
             l = self._lockref and self._lockref()
             if l is None or not l.held:
-                self.ui.develwarn('transaction with no lock')
+                raise RuntimeError('programming error: transaction requires '
+                                   'locking')
         tr = self.currenttransaction()
         if tr is not None:
             return tr.nest()
@@ -1019,11 +1021,8 b' class localrepository(object):'
                 _("abandoned transaction found"),
                 hint=_("run 'hg recover' to clean up transaction"))
 
-        # make journal.dirstate contain in-memory changes at this point
-        self.dirstate.write(None)
-
         idbase = "%.40f#%f" % (random.random(), time.time())
-        txnid = 'TXN:' + util.sha1(idbase).hexdigest()
+        txnid = 'TXN:' + hashlib.sha1(idbase).hexdigest()
         self.hook('pretxnopen', throw=True, txnname=desc, txnid=txnid)
 
         self._writejournal(desc)
@@ -1049,13 +1048,9 b' class localrepository(object):'
                 # transaction running
                 repo.dirstate.write(None)
             else:
-                # prevent in-memory changes from being written out at
-                # the end of outer wlock scope or so
-                repo.dirstate.invalidate()
-
                 # discard all changes (including ones already written
                 # out) in this transaction
-                repo.vfs.rename('journal.dirstate', 'dirstate')
+                repo.dirstate.restorebackup(None, prefix='journal.')
 
                 repo.invalidate(clearfilecache=True)
 
@@ -1110,8 +1105,7 b' class localrepository(object):'
         return [(vfs, undoname(x)) for vfs, x in self._journalfiles()]
 
     def _writejournal(self, desc):
-        self.vfs.write("journal.dirstate",
-                          self.vfs.tryread("dirstate"))
+        self.dirstate.savebackup(None, prefix='journal.')
         self.vfs.write("journal.branch",
                           encoding.fromlocal(self.dirstate.branch()))
         self.vfs.write("journal.desc",
@@ -1186,9 +1180,9 b' class localrepository(object):'
         vfsmap = {'plain': self.vfs, '': self.svfs}
         transaction.rollback(self.svfs, vfsmap, 'undo', ui.warn)
         if self.vfs.exists('undo.bookmarks'):
-            self.vfs.rename('undo.bookmarks', 'bookmarks')
+            self.vfs.rename('undo.bookmarks', 'bookmarks', checkambig=True)
         if self.svfs.exists('undo.phaseroots'):
-            self.svfs.rename('undo.phaseroots', 'phaseroots')
+            self.svfs.rename('undo.phaseroots', 'phaseroots', checkambig=True)
         self.invalidate()
 
         parentgone = (parents[0] not in self.changelog.nodemap or
@@ -1197,7 +1191,7 b' class localrepository(object):'
             # prevent dirstateguard from overwriting already restored one
             dsguard.close()
 
-            self.vfs.rename('undo.dirstate', 'dirstate')
+            self.dirstate.restorebackup(None, prefix='undo.')
             try:
                 branch = self.vfs.read('undo.branch')
                 self.dirstate.setbranch(encoding.tolocal(branch))
@@ -1206,7 +1200,6 b' class localrepository(object):'
                           'current branch is still \'%s\'\n')
                         % self.dirstate.branch())
 
-            self.dirstate.invalidate()
             parents = tuple([p.rev() for p in self[None].parents()])
             if len(parents) > 1:
                 ui.status(_('working directory now based on '

mercurial/mail.py

@@ -41,16 +41,16 b' def _unifiedheaderinit(self, *args, **kw'
     kw['continuation_ws'] = ' '
     _oldheaderinit(self, *args, **kw)
 
-email.Header.Header.__dict__['__init__'] = _unifiedheaderinit
+setattr(email.header.Header, '__init__', _unifiedheaderinit)
 
 class STARTTLS(smtplib.SMTP):
     '''Derived class to verify the peer certificate for STARTTLS.
 
     This class allows to pass any keyword arguments to SSL socket creation.
     '''
-    def __init__(self, sslkwargs, host=None, **kwargs):
+    def __init__(self, ui, host=None, **kwargs):
         smtplib.SMTP.__init__(self, **kwargs)
-        self._sslkwargs = sslkwargs
+        self._ui = ui
         self._host = host
 
     def starttls(self, keyfile=None, certfile=None):
@@ -60,8 +60,8 b' class STARTTLS(smtplib.SMTP):'
         (resp, reply) = self.docmd("STARTTLS")
         if resp == 220:
             self.sock = sslutil.wrapsocket(self.sock, keyfile, certfile,
-                                           serverhostname=self._host,
-                                           **self._sslkwargs)
+                                           ui=self._ui,
+                                           serverhostname=self._host)
             self.file = smtplib.SSLFakeFile(self.sock)
             self.helo_resp = None
             self.ehlo_resp = None
@@ -74,14 +74,14 b' class SMTPS(smtplib.SMTP):'
 
     This class allows to pass any keyword arguments to SSL socket creation.
     '''
-    def __init__(self, sslkwargs, keyfile=None, certfile=None, host=None,
+    def __init__(self, ui, keyfile=None, certfile=None, host=None,
                  **kwargs):
         self.keyfile = keyfile
         self.certfile = certfile
         smtplib.SMTP.__init__(self, **kwargs)
         self._host = host
         self.default_port = smtplib.SMTP_SSL_PORT
-        self._sslkwargs = sslkwargs
+        self._ui = ui
 
     def _get_socket(self, host, port, timeout):
         if self.debuglevel > 0:
@@ -89,8 +89,8 b' class SMTPS(smtplib.SMTP):'
         new_socket = socket.create_connection((host, port), timeout)
         new_socket = sslutil.wrapsocket(new_socket,
                                         self.keyfile, self.certfile,
-                                        serverhostname=self._host,
-                                        **self._sslkwargs)
+                                        ui=self._ui,
+                                        serverhostname=self._host)
         self.file = smtplib.SSLFakeFile(new_socket)
         return new_socket
 
@@ -106,22 +106,11 b' def _smtp(ui):'
     mailhost = ui.config('smtp', 'host')
     if not mailhost:
         raise error.Abort(_('smtp.host not configured - cannot send mail'))
-    verifycert = ui.config('smtp', 'verifycert', 'strict')
-    if verifycert not in ['strict', 'loose']:
-        if util.parsebool(verifycert) is not False:
-            raise error.Abort(_('invalid smtp.verifycert configuration: %s')
-                             % (verifycert))
-        verifycert = False
-    if (starttls or smtps) and verifycert:
-        sslkwargs = sslutil.sslkwargs(ui, mailhost)
-    else:
-        # 'ui' is required by sslutil.wrapsocket() and set by sslkwargs()
-        sslkwargs = {'ui': ui}
     if smtps:
         ui.note(_('(using smtps)\n'))
-        s = SMTPS(sslkwargs, local_hostname=local_hostname, host=mailhost)
+        s = SMTPS(ui, local_hostname=local_hostname, host=mailhost)
     elif starttls:
-        s = STARTTLS(sslkwargs, local_hostname=local_hostname, host=mailhost)
+        s = STARTTLS(ui, local_hostname=local_hostname, host=mailhost)
     else:
         s = smtplib.SMTP(local_hostname=local_hostname)
     if smtps:
@@ -137,9 +126,9 b' def _smtp(ui):'
         s.ehlo()
         s.starttls()
         s.ehlo()
-    if (starttls or smtps) and verifycert:
+    if starttls or smtps:
         ui.note(_('(verifying remote certificate)\n'))
-        sslutil.validator(ui, mailhost)(s.sock, verifycert == 'strict')
+        sslutil.validatesocket(s.sock)
     username = ui.config('smtp', 'username')
     password = ui.config('smtp', 'password')
     if username and not password:

mercurial/manifest.py

@@ -211,8 +211,10 b' class manifestdict(object):'
 
     def filesnotin(self, m2):
         '''Set of files in this manifest that are not in the other'''
-        files = set(self)
-        files.difference_update(m2)
+        diff = self.diff(m2)
+        files = set(filepath
+                    for filepath, hashflags in diff.iteritems()
+                    if hashflags[1][0] is None)
         return files
 
     @propertycache
@@ -966,7 +968,7 b' class manifest(revlog.revlog):'
             return self.readdelta(node)
         if self._usemanifestv2:
             raise error.Abort(
-                "readshallowdelta() not implemented for manifestv2")
+                _("readshallowdelta() not implemented for manifestv2"))
         r = self.rev(node)
         d = mdiff.patchtext(self.revdiff(self.deltaparent(r), r))
         return manifestdict(d)

mercurial/match.py

@@ -38,7 +38,7 b' def _expandsets(kindpats, ctx, listsubre'
     for kind, pat, source in kindpats:
         if kind == 'set':
             if not ctx:
-                raise error.Abort("fileset expression with no context")
+                raise error.Abort(_("fileset expression with no context"))
             s = ctx.getfileset(pat)
             fset.update(s)
 

mercurial/mdiff.py

@@ -58,10 +58,8 b' class diffopts(object):'
         'upgrade': False,
         }
 
-    __slots__ = defaults.keys()
-
     def __init__(self, **opts):
-        for k in self.__slots__:
+        for k in self.defaults.keys():
             v = opts.get(k)
             if v is None:
                 v = self.defaults[k]

mercurial/merge.py

@@ -8,6 +8,7 b''
 from __future__ import absolute_import
 
 import errno
+import hashlib
 import os
 import shutil
 import struct
@@ -373,7 +374,7 b' class mergestate(object):'
         """Write current state on disk in a version 1 file"""
         f = self._repo.vfs(self.statepathv1, 'w')
         irecords = iter(records)
-        lrecords = irecords.next()
+        lrecords = next(irecords)
         assert lrecords[0] == 'L'
         f.write(hex(self._local) + '\n')
         for rtype, data in irecords:
@@ -408,7 +409,7 b' class mergestate(object):'
         if fcl.isabsent():
             hash = nullhex
         else:
-            hash = util.sha1(fcl.path()).hexdigest()
+            hash = hashlib.sha1(fcl.path()).hexdigest()
             self._repo.vfs.write('merge/' + hash, fcl.data())
         self._state[fd] = ['u', hash, fcl.path(),
                            fca.path(), hex(fca.filenode()),
@@ -989,19 +990,19 b' def calculateupdates(repo, wctx, mctx, a'
             if len(bids) == 1: # all bids are the same kind of method
                 m, l = bids.items()[0]
                 if all(a == l[0] for a in l[1:]): # len(bids) is > 1
-                    repo.ui.note(" %s: consensus for %s\n" % (f, m))
+                    repo.ui.note(_(" %s: consensus for %s\n") % (f, m))
                     actions[f] = l[0]
                     continue
             # If keep is an option, just do it.
             if 'k' in bids:
-                repo.ui.note(" %s: picking 'keep' action\n" % f)
+                repo.ui.note(_(" %s: picking 'keep' action\n") % f)
                 actions[f] = bids['k'][0]
                 continue
             # If there are gets and they all agree [how could they not?], do it.
             if 'g' in bids:
                 ga0 = bids['g'][0]
                 if all(a == ga0 for a in bids['g'][1:]):
-                    repo.ui.note(" %s: picking 'get' action\n" % f)
+                    repo.ui.note(_(" %s: picking 'get' action\n") % f)
                     actions[f] = ga0
                     continue
             # TODO: Consider other simple actions such as mode changes
@@ -1075,15 +1076,14 b' def batchget(repo, mctx, actions):'
                 absf = repo.wjoin(f)
                 orig = scmutil.origpath(ui, repo, absf)
                 try:
-                    # TODO Mercurial has always aborted if an untracked
-                    # directory is replaced by a tracked file, or generally
-                    # with file/directory merges. This needs to be sorted out.
                     if repo.wvfs.isfileorlink(f):
                         util.rename(absf, orig)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                         raise
 
+            if repo.wvfs.isdir(f):
+                repo.wvfs.removedirs(f)
             wwrite(f, fctx(f).data(), flags, backgroundclose=True)
             if i == 100:
                 yield i, f
@@ -1442,9 +1442,7 b' def update(repo, node, branchmerge, forc'
             pas = [repo[ancestor]]
 
         if node is None:
-            if (repo.ui.configbool('devel', 'all-warnings')
-                    or repo.ui.configbool('devel', 'oldapi')):
-                repo.ui.develwarn('update with no target')
+            repo.ui.deprecwarn('update with no target', '3.9')
             rev, _mark, _act = destutil.destupdate(repo)
             node = repo[rev].node()
 

mercurial/mpatch.c

@@ -26,6 +26,7 b''
 #include <string.h>
 
 #include "util.h"
+#include "bitmanipulation.h"
 
 static char mpatch_doc[] = "Efficient binary patching.";
 static PyObject *mpatch_Error;

mercurial/obsolete.py

@@ -600,8 +600,8 b' class obsstore(object):'
         Take care of filtering duplicate.
         Return the number of new marker."""
         if self._readonly:
-            raise error.Abort('creating obsolete markers is not enabled on '
-                              'this repo')
+            raise error.Abort(_('creating obsolete markers is not enabled on '
+                              'this repo'))
         known = set(self._all)
         new = []
         for m in markers:
@@ -1171,7 +1171,7 b' def _computebumpedset(repo):'
                                    ignoreflags=bumpedfix):
             prev = torev(pnode) # unfiltered! but so is phasecache
             if (prev is not None) and (phase(repo, prev) <= public):
-                # we have a public precursors
+                # we have a public precursor
                 bumped.add(rev)
                 break # Next draft!
     return bumped
@@ -1234,7 +1234,7 b' def createmarkers(repo, relations, flag='
                 localmetadata.update(rel[2])
 
             if not prec.mutable():
-                raise error.Abort("cannot obsolete public changeset: %s"
+                raise error.Abort(_("cannot obsolete public changeset: %s")
                                  % prec,
                                  hint='see "hg help phases" for details')
             nprec = prec.node()
@@ -1243,7 +1243,8 b' def createmarkers(repo, relations, flag='
             if not nsucs:
                 npare = tuple(p.node() for p in prec.parents())
             if nprec in nsucs:
-                raise error.Abort("changeset %s cannot obsolete itself" % prec)
+                raise error.Abort(_("changeset %s cannot obsolete itself")
+                                  % prec)
 
             # Creating the marker causes the hidden cache to become invalid,
             # which causes recomputation when we ask for prec.parents() above.

mercurial/parser.py

@@ -325,13 +325,13 b' class basealiasrules(object):'
         >>> builddecl('foo')
         ('foo', None, None)
         >>> builddecl('$foo')
-        ('$foo', None, "'$' not for alias arguments")
+        ('$foo', None, "invalid symbol '$foo'")
         >>> builddecl('foo::bar')
         ('foo::bar', None, 'invalid format')
         >>> builddecl('foo()')
         ('foo', [], None)
         >>> builddecl('$foo()')
-        ('$foo()', None, "'$' not for alias arguments")
+        ('$foo()', None, "invalid function '$foo'")
         >>> builddecl('foo($1, $2)')
         ('foo', ['$1', '$2'], None)
         >>> builddecl('foo(bar_bar, baz.baz)')
@@ -358,7 +358,7 b' class basealiasrules(object):'
             # "name = ...." style
             name = tree[1]
             if name.startswith('$'):
-                return (decl, None, _("'$' not for alias arguments"))
+                return (decl, None, _("invalid symbol '%s'") % name)
             return (name, None, None)
 
         func = cls._trygetfunc(tree)
@@ -366,7 +366,7 b' class basealiasrules(object):'
             # "name(arg, ....) = ...." style
             name, args = func
             if name.startswith('$'):
-                return (decl, None, _("'$' not for alias arguments"))
+                return (decl, None, _("invalid function '%s'") % name)
             if any(t[0] != cls._symbolnode for t in args):
                 return (decl, None, _("invalid argument list"))
             if len(args) != len(set(args)):
@@ -389,7 +389,7 b' class basealiasrules(object):'
         if sym in args:
             op = '_aliasarg'
         elif sym.startswith('$'):
-            raise error.ParseError(_("'$' not for alias arguments"))
+            raise error.ParseError(_("invalid symbol '%s'") % sym)
         return (op, sym)
 
     @classmethod
@@ -423,7 +423,7 b' class basealiasrules(object):'
         ...     builddefn('$1 or $bar', args)
         ... except error.ParseError as inst:
         ...     print parseerrordetail(inst)
-        '$' not for alias arguments
+        invalid symbol '$bar'
         >>> args = ['$1', '$10', 'foo']
         >>> pprint(builddefn('$10 or baz', args))
         (or
@@ -447,15 +447,13 b' class basealiasrules(object):'
         repl = efmt = None
         name, args, err = cls._builddecl(decl)
         if err:
-            efmt = _('failed to parse the declaration of %(section)s '
-                     '"%(name)s": %(error)s')
+            efmt = _('bad declaration of %(section)s "%(name)s": %(error)s')
         else:
             try:
                 repl = cls._builddefn(defn, args)
             except error.ParseError as inst:
                 err = parseerrordetail(inst)
-                efmt = _('failed to parse the definition of %(section)s '
-                         '"%(name)s": %(error)s')
+                efmt = _('bad definition of %(section)s "%(name)s": %(error)s')
         if err:
             err = efmt % {'section': cls._section, 'name': name, 'error': err}
         return alias(name, args, err, repl)

mercurial/parsers.c

@@ -13,6 +13,7 b''
 #include <string.h>
 
 #include "util.h"
+#include "bitmanipulation.h"
 
 static char *versionerrortext = "Python minor version mismatch";
 

mercurial/patch.py

@@ -12,6 +12,7 b' import collections'
 import copy
 import email
 import errno
+import hashlib
 import os
 import posixpath
 import re
@@ -978,7 +979,19 b' class recordhunk(object):'
 def filterpatch(ui, headers, operation=None):
     """Interactively filter patch chunks into applied-only chunks"""
     if operation is None:
-        operation = _('record')
+        operation = 'record'
+    messages = {
+        'multiple': {
+            'discard': _("discard change %d/%d to '%s'?"),
+            'record': _("record change %d/%d to '%s'?"),
+            'revert': _("revert change %d/%d to '%s'?"),
+        }[operation],
+        'single': {
+            'discard': _("discard this change to '%s'?"),
+            'record': _("record this change to '%s'?"),
+            'revert': _("revert this change to '%s'?"),
+        }[operation],
+    }
 
     def prompt(skipfile, skipall, query, chunk):
         """prompt query, and process base inputs
@@ -1109,11 +1122,10 b' the hunk is left unchanged.'
             if skipfile is None and skipall is None:
                 chunk.pretty(ui)
             if total == 1:
-                msg = _("record this change to '%s'?") % chunk.filename()
+                msg = messages['single'] % chunk.filename()
             else:
                 idx = pos - len(h.hunks) + i
-                msg = _("record change %d/%d to '%s'?") % (idx, total,
-                                                           chunk.filename())
+                msg = messages['multiple'] % (idx, total, chunk.filename())
             r, skipfile, skipall, newpatches = prompt(skipfile,
                     skipall, msg, chunk)
             if r:
@@ -2172,7 +2184,7 b' def difffeatureopts(ui, opts=None, untru'
     return mdiff.diffopts(**buildopts)
 
 def diff(repo, node1=None, node2=None, match=None, changes=None, opts=None,
-         losedatafn=None, prefix='', relroot=''):
+         losedatafn=None, prefix='', relroot='', copy=None):
     '''yields diff of changes to files between two nodes, or node and
     working directory.
 
@@ -2191,7 +2203,10 b' def diff(repo, node1=None, node2=None, m'
     display (used for subrepos).
 
     relroot, if not empty, must be normalized with a trailing /. Any match
-    patterns that fall outside it will be ignored.'''
+    patterns that fall outside it will be ignored.
+
+    copy, if not empty, should contain mappings {dst@y: src@x} of copy
+    information.'''
 
     if opts is None:
         opts = mdiff.defaultopts
@@ -2238,9 +2253,10 b' def diff(repo, node1=None, node2=None, m'
         hexfunc = short
     revs = [hexfunc(node) for node in [ctx1.node(), ctx2.node()] if node]
 
-    copy = {}
-    if opts.git or opts.upgrade:
-        copy = copies.pathcopies(ctx1, ctx2, match=match)
+    if copy is None:
+        copy = {}
+        if opts.git or opts.upgrade:
+            copy = copies.pathcopies(ctx1, ctx2, match=match)
 
     if relroot is not None:
         if not relfiltered:
@@ -2401,7 +2417,7 b' def trydiff(repo, revs, ctx1, ctx2, modi'
         if not text:
             text = ""
         l = len(text)
-        s = util.sha1('blob %d\0' % l)
+        s = hashlib.sha1('blob %d\0' % l)
         s.update(text)
         return s.hexdigest()
 

mercurial/pathencode.c

@@ -653,24 +653,24 b' static int sha1hash(char hash[20], const'
 	PyObject *shaobj, *hashobj;
 
 	if (shafunc == NULL) {
-		PyObject *util, *name = PyString_FromString("mercurial.util");
+		PyObject *hashlib, *name = PyString_FromString("hashlib");
 
 		if (name == NULL)
 			return -1;
 
-		util = PyImport_Import(name);
+		hashlib = PyImport_Import(name);
 		Py_DECREF(name);
 
-		if (util == NULL) {
-			PyErr_SetString(PyExc_ImportError, "mercurial.util");
+		if (hashlib == NULL) {
+			PyErr_SetString(PyExc_ImportError, "hashlib");
 			return -1;
 		}
-		shafunc = PyObject_GetAttrString(util, "sha1");
-		Py_DECREF(util);
+		shafunc = PyObject_GetAttrString(hashlib, "sha1");
+		Py_DECREF(hashlib);
 
 		if (shafunc == NULL) {
 			PyErr_SetString(PyExc_AttributeError,
-					"module 'mercurial.util' has no "
+					"module 'hashlib' has no "
 					"attribute 'sha1'");
 			return -1;
 		}

mercurial/peer.py

@@ -98,12 +98,12 b' def batchable(f):'
     '''
     def plain(*args, **opts):
         batchable = f(*args, **opts)
-        encargsorres, encresref = batchable.next()
+        encargsorres, encresref = next(batchable)
         if not encresref:
             return encargsorres # a local result in this case
         self = args[0]
         encresref.set(self._submitone(f.func_name, encargsorres))
-        return batchable.next()
+        return next(batchable)
     setattr(plain, 'batchable', f)
     return plain
 

mercurial/phases.py

@@ -251,7 +251,7 b' class phasecache(object):'
     def write(self):
         if not self.dirty:
             return
-        f = self.opener('phaseroots', 'w', atomictemp=True)
+        f = self.opener('phaseroots', 'w', atomictemp=True, checkambig=True)
         try:
             self._write(f)
         finally:

mercurial/posix.py

@@ -598,3 +598,18 b' def readpipe(pipe):'
         return ''.join(chunks)
     finally:
         fcntl.fcntl(pipe, fcntl.F_SETFL, oldflags)
+
+def bindunixsocket(sock, path):
+    """Bind the UNIX domain socket to the specified path"""
+    # use relative path instead of full path at bind() if possible, since
+    # AF_UNIX path has very small length limit (107 chars) on common
+    # platforms (see sys/un.h)
+    dirname, basename = os.path.split(path)
+    bakwdfd = None
+    if dirname:
+        bakwdfd = os.open('.', os.O_DIRECTORY)
+        os.chdir(dirname)
+    sock.bind(basename)
+    if bakwdfd:
+        os.fchdir(bakwdfd)
+        os.close(bakwdfd)

mercurial/pure/osutil.py

@@ -14,6 +14,10 b' import socket'
 import stat as statmod
 import sys
 
+from . import policy
+modulepolicy = policy.policy
+policynocffi = policy.policynocffi
+
 def _mode_to_kind(mode):
     if statmod.S_ISREG(mode):
         return statmod.S_IFREG
@@ -31,7 +35,7 b' def _mode_to_kind(mode):'
         return statmod.S_IFSOCK
     return mode
 
-def listdir(path, stat=False, skip=None):
+def listdirpure(path, stat=False, skip=None):
     '''listdir(path, stat=False) -> list_of_tuples
 
     Return a sorted list containing information about the entries
@@ -61,6 +65,95 b' def listdir(path, stat=False, skip=None)'
             result.append((fn, _mode_to_kind(st.st_mode)))
     return result
 
+ffi = None
+if modulepolicy not in policynocffi and sys.platform == 'darwin':
+    try:
+        from _osutil_cffi import ffi, lib
+    except ImportError:
+        if modulepolicy == 'cffi': # strict cffi import
+            raise
+
+if sys.platform == 'darwin' and ffi is not None:
+    listdir_batch_size = 4096
+    # tweakable number, only affects performance, which chunks
+    # of bytes do we get back from getattrlistbulk
+
+    attrkinds = [None] * 20 # we need the max no for enum VXXX, 20 is plenty
+
+    attrkinds[lib.VREG] = statmod.S_IFREG
+    attrkinds[lib.VDIR] = statmod.S_IFDIR
+    attrkinds[lib.VLNK] = statmod.S_IFLNK
+    attrkinds[lib.VBLK] = statmod.S_IFBLK
+    attrkinds[lib.VCHR] = statmod.S_IFCHR
+    attrkinds[lib.VFIFO] = statmod.S_IFIFO
+    attrkinds[lib.VSOCK] = statmod.S_IFSOCK
+
+    class stat_res(object):
+        def __init__(self, st_mode, st_mtime, st_size):
+            self.st_mode = st_mode
+            self.st_mtime = st_mtime
+            self.st_size = st_size
+
+    tv_sec_ofs = ffi.offsetof("struct timespec", "tv_sec")
+    buf = ffi.new("char[]", listdir_batch_size)
+
+    def listdirinternal(dfd, req, stat, skip):
+        ret = []
+        while True:
+            r = lib.getattrlistbulk(dfd, req, buf, listdir_batch_size, 0)
+            if r == 0:
+                break
+            if r == -1:
+                raise OSError(ffi.errno, os.strerror(ffi.errno))
+            cur = ffi.cast("val_attrs_t*", buf)
+            for i in range(r):
+                lgt = cur.length
+                assert lgt == ffi.cast('uint32_t*', cur)[0]
+                ofs = cur.name_info.attr_dataoffset
+                str_lgt = cur.name_info.attr_length
+                base_ofs = ffi.offsetof('val_attrs_t', 'name_info')
+                name = str(ffi.buffer(ffi.cast("char*", cur) + base_ofs + ofs,
+                           str_lgt - 1))
+                tp = attrkinds[cur.obj_type]
+                if name == "." or name == "..":
+                    continue
+                if skip == name and tp == statmod.S_ISDIR:
+                    return []
+                if stat:
+                    mtime = cur.time.tv_sec
+                    mode = (cur.accessmask & ~lib.S_IFMT)| tp
+                    ret.append((name, tp, stat_res(st_mode=mode, st_mtime=mtime,
+                                st_size=cur.datalength)))
+                else:
+                    ret.append((name, tp))
+                cur += lgt
+        return ret
+
+    def listdir(path, stat=False, skip=None):
+        req = ffi.new("struct attrlist*")
+        req.bitmapcount = lib.ATTR_BIT_MAP_COUNT
+        req.commonattr = (lib.ATTR_CMN_RETURNED_ATTRS |
+                          lib.ATTR_CMN_NAME |
+                          lib.ATTR_CMN_OBJTYPE |
+                          lib.ATTR_CMN_ACCESSMASK |
+                          lib.ATTR_CMN_MODTIME)
+        req.fileattr = lib.ATTR_FILE_DATALENGTH
+        dfd = lib.open(path, lib.O_RDONLY, 0)
+        if dfd == -1:
+            raise OSError(ffi.errno, os.strerror(ffi.errno))
+
+        try:
+            ret = listdirinternal(dfd, req, stat, skip)
+        finally:
+            try:
+                lib.close(dfd)
+            except BaseException:
+                pass # we ignore all the errors from closing, not
+                # much we can do about that
+        return ret
+else:
+    listdir = listdirpure
+
 if os.name != 'nt':
     posixfile = open
 

mercurial/pure/parsers.py

@@ -25,49 +25,111 b' def dirstatetuple(*x):'
     # x is a tuple
     return x
 
-def parse_index2(data, inline):
-    def gettype(q):
-        return int(q & 0xFFFF)
+indexformatng = ">Qiiiiii20s12x"
+indexfirst = struct.calcsize('Q')
+sizeint = struct.calcsize('i')
+indexsize = struct.calcsize(indexformatng)
+
+def gettype(q):
+    return int(q & 0xFFFF)
 
-    def offset_type(offset, type):
-        return long(long(offset) << 16 | type)
+def offset_type(offset, type):
+    return long(long(offset) << 16 | type)
+
+class BaseIndexObject(object):
+    def __len__(self):
+        return self._lgt + len(self._extra) + 1
+
+    def insert(self, i, tup):
+        assert i == -1
+        self._extra.append(tup)
 
-    indexformatng = ">Qiiiiii20s12x"
+    def _fix_index(self, i):
+        if not isinstance(i, int):
+            raise TypeError("expecting int indexes")
+        if i < 0:
+            i = len(self) + i
+        if i < 0 or i >= len(self):
+            raise IndexError
+        return i
 
-    s = struct.calcsize(indexformatng)
-    index = []
-    cache = None
-    off = 0
+    def __getitem__(self, i):
+        i = self._fix_index(i)
+        if i == len(self) - 1:
+            return (0, 0, 0, -1, -1, -1, -1, nullid)
+        if i >= self._lgt:
+            return self._extra[i - self._lgt]
+        index = self._calculate_index(i)
+        r = struct.unpack(indexformatng, self._data[index:index + indexsize])
+        if i == 0:
+            e = list(r)
+            type = gettype(e[0])
+            e[0] = offset_type(0, type)
+            return tuple(e)
+        return r
+
+class IndexObject(BaseIndexObject):
+    def __init__(self, data):
+        assert len(data) % indexsize == 0
+        self._data = data
+        self._lgt = len(data) // indexsize
+        self._extra = []
+
+    def _calculate_index(self, i):
+        return i * indexsize
 
-    l = len(data) - s
-    append = index.append
-    if inline:
-        cache = (0, data)
-        while off <= l:
-            e = _unpack(indexformatng, data[off:off + s])
-            append(e)
-            if e[1] < 0:
-                break
-            off += e[1] + s
-    else:
-        while off <= l:
-            e = _unpack(indexformatng, data[off:off + s])
-            append(e)
-            off += s
+    def __delitem__(self, i):
+        if not isinstance(i, slice) or not i.stop == -1 or not i.step is None:
+            raise ValueError("deleting slices only supports a:-1 with step 1")
+        i = self._fix_index(i.start)
+        if i < self._lgt:
+            self._data = self._data[:i * indexsize]
+            self._lgt = i
+            self._extra = []
+        else:
+            self._extra = self._extra[:i - self._lgt]
+
+class InlinedIndexObject(BaseIndexObject):
+    def __init__(self, data, inline=0):
+        self._data = data
+        self._lgt = self._inline_scan(None)
+        self._inline_scan(self._lgt)
+        self._extra = []
 
-    if off != len(data):
-        raise ValueError('corrupt index file')
+    def _inline_scan(self, lgt):
+        off = 0
+        if lgt is not None:
+            self._offsets = [0] * lgt
+        count = 0
+        while off <= len(self._data) - indexsize:
+            s, = struct.unpack('>i',
+                self._data[off + indexfirst:off + sizeint + indexfirst])
+            if lgt is not None:
+                self._offsets[count] = off
+            count += 1
+            off += indexsize + s
+        if off != len(self._data):
+            raise ValueError("corrupted data")
+        return count
 
-    if index:
-        e = list(index[0])
-        type = gettype(e[0])
-        e[0] = offset_type(0, type)
-        index[0] = tuple(e)
+    def __delitem__(self, i):
+        if not isinstance(i, slice) or not i.stop == -1 or not i.step is None:
+            raise ValueError("deleting slices only supports a:-1 with step 1")
+        i = self._fix_index(i.start)
+        if i < self._lgt:
+            self._offsets = self._offsets[:i]
+            self._lgt = i
+            self._extra = []
+        else:
+            self._extra = self._extra[:i - self._lgt]
 
-    # add the magic null revision at -1
-    index.append((0, 0, 0, -1, -1, -1, -1, nullid))
+    def _calculate_index(self, i):
+        return self._offsets[i]
 
-    return index, cache
+def parse_index2(data, inline):
+    if not inline:
+        return IndexObject(data), None
+    return InlinedIndexObject(data, inline), (0, data)
 
 def parse_dirstate(dmap, copymap, st):
     parents = [st[:20], st[20: 40]]

mercurial/pycompat.py

@@ -10,18 +10,26 b' This contains aliases to hide python ver'
 
 from __future__ import absolute_import
 
-try:
+import sys
+
+if sys.version_info[0] < 3:
+    import cPickle as pickle
     import cStringIO as io
-    stringio = io.StringIO
-except ImportError:
+    import httplib
+    import Queue as _queue
+    import SocketServer as socketserver
+    import urlparse
+    import xmlrpclib
+else:
+    import http.client as httplib
     import io
-    stringio = io.StringIO
+    import pickle
+    import queue as _queue
+    import socketserver
+    import urllib.parse as urlparse
+    import xmlrpc.client as xmlrpclib
 
-try:
-    import Queue as _queue
-    _queue.Queue
-except ImportError:
-    import queue as _queue
+stringio = io.StringIO
 empty = _queue.Empty
 queue = _queue.Queue
 
@@ -41,9 +49,13 b' def _alias(alias, origin, items):'
         except AttributeError:
             pass
 
+httpserver = _pycompatstub()
 urlreq = _pycompatstub()
 urlerr = _pycompatstub()
 try:
+    import BaseHTTPServer
+    import CGIHTTPServer
+    import SimpleHTTPServer
     import urllib2
     import urllib
     _alias(urlreq, urllib, (
@@ -81,6 +93,16 b' try:'
         "HTTPError",
         "URLError",
     ))
+    _alias(httpserver, BaseHTTPServer, (
+        "HTTPServer",
+        "BaseHTTPRequestHandler",
+    ))
+    _alias(httpserver, SimpleHTTPServer, (
+        "SimpleHTTPRequestHandler",
+    ))
+    _alias(httpserver, CGIHTTPServer, (
+        "CGIHTTPRequestHandler",
+    ))
 
 except ImportError:
     import urllib.request
@@ -99,6 +121,7 b' except ImportError:'
         "pathname2url",
         "HTTPBasicAuthHandler",
         "HTTPDigestAuthHandler",
+        "HTTPPasswordMgrWithDefaultRealm",
         "ProxyHandler",
         "quote",
         "Request",
@@ -115,6 +138,13 b' except ImportError:'
         "HTTPError",
         "URLError",
     ))
+    import http.server
+    _alias(httpserver, http.server, (
+        "HTTPServer",
+        "BaseHTTPRequestHandler",
+        "SimpleHTTPRequestHandler",
+        "CGIHTTPRequestHandler",
+    ))
 
 try:
     xrange

mercurial/repair.py

@@ -9,6 +9,7 b''
 from __future__ import absolute_import
 
 import errno
+import hashlib
 
 from .i18n import _
 from .node import short
@@ -35,7 +36,7 b' def _bundle(repo, bases, heads, node, su'
     # Include a hash of all the nodes in the filename for uniqueness
     allcommits = repo.set('%ln::%ln', bases, heads)
     allhashes = sorted(c.hex() for c in allcommits)
-    totalhash = util.sha1(''.join(allhashes)).hexdigest()
+    totalhash = hashlib.sha1(''.join(allhashes)).hexdigest()
     name = "%s/%s-%s-%s.hg" % (backupdir, short(node), totalhash[:8], suffix)
 
     comp = None
@@ -166,6 +167,13 b' def strip(ui, repo, nodelist, backup=Tru'
             tr.startgroup()
             cl.strip(striprev, tr)
             mfst.strip(striprev, tr)
+            if 'treemanifest' in repo.requirements: # safe but unnecessary
+                                                    # otherwise
+                for unencoded, encoded, size in repo.store.datafiles():
+                    if (unencoded.startswith('meta/') and
+                        unencoded.endswith('00manifest.i')):
+                        dir = unencoded[5:-12]
+                        repo.dirlog(dir).strip(striprev, tr)
             for fn in files:
                 repo.file(fn).strip(striprev, tr)
             tr.endgroup()

mercurial/repoview.py

@@ -9,6 +9,7 b''
 from __future__ import absolute_import
 
 import copy
+import hashlib
 import heapq
 import struct
 
@@ -18,7 +19,6 b' from . import ('
     obsolete,
     phases,
     tags as tagsmod,
-    util,
 )
 
 def hideablerevs(repo):
@@ -102,7 +102,7 b' def cachehash(repo, hideable):'
     it to the cache. Upon reading we can easily validate by checking the hash
     against the stored one and discard the cache in case the hashes don't match.
     """
-    h = util.sha1()
+    h = hashlib.sha1()
     h.update(''.join(repo.heads()))
     h.update(str(hash(frozenset(hideable))))
     return h.digest()

mercurial/revlog.py

@@ -15,6 +15,7 b' from __future__ import absolute_import'
 
 import collections
 import errno
+import hashlib
 import os
 import struct
 import zlib
@@ -40,7 +41,6 b' from . import ('
 _unpack = struct.unpack
 _compress = zlib.compress
 _decompress = zlib.decompress
-_sha = util.sha1
 
 # revlog header flags
 REVLOGV0 = 0
@@ -74,7 +74,7 b' def gettype(q):'
 def offset_type(offset, type):
     return long(long(offset) << 16 | type)
 
-_nullhash = _sha(nullid)
+_nullhash = hashlib.sha1(nullid)
 
 def hash(text, p1, p2):
     """generate a hash from the given text and its parent hashes
@@ -92,7 +92,7 b' def hash(text, p1, p2):'
         # none of the parent nodes are nullid
         l = [p1, p2]
         l.sort()
-        s = _sha(l[0])
+        s = hashlib.sha1(l[0])
         s.update(l[1])
     s.update(text)
     return s.digest()
@@ -941,8 +941,11 b' class revlog(object):'
             return None
         except RevlogError:
             # parsers.c radix tree lookup gave multiple matches
+            # fast path: for unfiltered changelog, radix tree is accurate
+            if not getattr(self, 'filteredrevs', None):
+                raise LookupError(id, self.indexfile,
+                                  _('ambiguous identifier'))
             # fall through to slow path that filters hidden revisions
-            pass
         except (AttributeError, ValueError):
             # we are pure python, or key was too short to search radix tree
             pass

mercurial/revset.py

@@ -302,6 +302,11 b' def tokenize(program, lookup=None, symin'
 
 # helpers
 
+def getsymbol(x):
+    if x and x[0] == 'symbol':
+        return x[1]
+    raise error.ParseError(_('not a symbol'))
+
 def getstring(x, err):
     if x and (x[0] == 'string' or x[0] == 'symbol'):
         return x[1]
@@ -330,13 +335,12 b' def getset(repo, subset, x):'
     s = methods[x[0]](repo, subset, *x[1:])
     if util.safehasattr(s, 'isascending'):
         return s
-    if (repo.ui.configbool('devel', 'all-warnings')
-            or repo.ui.configbool('devel', 'old-revset')):
-        # else case should not happen, because all non-func are internal,
-        # ignoring for now.
-        if x[0] == 'func' and x[1][0] == 'symbol' and x[1][1] in symbols:
-            repo.ui.develwarn('revset "%s" use list instead of smartset, '
-                              '(upgrade your code)' % x[1][1])
+    # else case should not happen, because all non-func are internal,
+    # ignoring for now.
+    if x[0] == 'func' and x[1][0] == 'symbol' and x[1][1] in symbols:
+        repo.ui.deprecwarn('revset "%s" uses list instead of smartset'
+                           % x[1][1],
+                           '3.9')
     return baseset(s)
 
 def _getrevsource(repo, r):
@@ -387,9 +391,7 b' def dagrange(repo, subset, x, y):'
     r = fullreposet(repo)
     xs = reachableroots(repo, getset(repo, r, x), getset(repo, r, y),
                          includepath=True)
-    # XXX We should combine with subset first: 'subset & baseset(...)'. This is
-    # necessary to ensure we preserve the order in subset.
-    return xs & subset
+    return subset & xs
 
 def andset(repo, subset, x, y):
     return getset(repo, getset(repo, subset, x), y)
@@ -417,13 +419,14 b' def keyvaluepair(repo, subset, k, v):'
     raise error.ParseError(_("can't use a key-value pair in this context"))
 
 def func(repo, subset, a, b):
-    if a[0] == 'symbol' and a[1] in symbols:
-        return symbols[a[1]](repo, subset, b)
+    f = getsymbol(a)
+    if f in symbols:
+        return symbols[f](repo, subset, b)
 
     keep = lambda fn: getattr(fn, '__doc__', None) is not None
 
     syms = [s for (s, fn) in symbols.items() if keep(fn)]
-    raise error.UnknownIdentifier(a[1], syms)
+    raise error.UnknownIdentifier(f, syms)
 
 # functions
 
@@ -695,20 +698,18 b' def checkstatus(repo, subset, pat, field'
 
     return subset.filter(matches, condrepr=('<status[%r] %r>', field, pat))
 
-def _children(repo, narrow, parentset):
+def _children(repo, subset, parentset):
     if not parentset:
         return baseset()
     cs = set()
     pr = repo.changelog.parentrevs
     minrev = parentset.min()
-    for r in narrow:
+    for r in subset:
         if r <= minrev:
             continue
         for p in pr(r):
             if p in parentset:
                 cs.add(r)
-    # XXX using a set to feed the baseset is wrong. Sets are not ordered.
-    # This does not break because of other fullreposet misbehavior.
     return baseset(cs)
 
 @predicate('children(set)', safe=True)
@@ -1150,13 +1151,9 b' def head(repo, subset, x):'
     getargs(x, 0, 0, _("head takes no arguments"))
     hs = set()
     cl = repo.changelog
-    for b, ls in repo.branchmap().iteritems():
+    for ls in repo.branchmap().itervalues():
         hs.update(cl.rev(h) for h in ls)
-    # XXX using a set to feed the baseset is wrong. Sets are not ordered.
-    # This does not break because of other fullreposet misbehavior.
-    # XXX We should combine with subset first: 'subset & baseset(...)'. This is
-    # necessary to ensure we preserve the order in subset.
-    return baseset(hs) & subset
+    return subset & baseset(hs)
 
 @predicate('heads(set)', safe=True)
 def heads(repo, subset, x):
@@ -1837,7 +1834,54 b' def roots(repo, subset, x):'
         return True
     return subset & s.filter(filter, condrepr='<roots>')
 
-@predicate('sort(set[, [-]key...])', safe=True)
+_sortkeyfuncs = {
+    'rev': lambda c: c.rev(),
+    'branch': lambda c: c.branch(),
+    'desc': lambda c: c.description(),
+    'user': lambda c: c.user(),
+    'author': lambda c: c.user(),
+    'date': lambda c: c.date()[0],
+}
+
+def _getsortargs(x):
+    """Parse sort options into (set, [(key, reverse)], opts)"""
+    args = getargsdict(x, 'sort', 'set keys topo.firstbranch')
+    if 'set' not in args:
+        # i18n: "sort" is a keyword
+        raise error.ParseError(_('sort requires one or two arguments'))
+    keys = "rev"
+    if 'keys' in args:
+        # i18n: "sort" is a keyword
+        keys = getstring(args['keys'], _("sort spec must be a string"))
+
+    keyflags = []
+    for k in keys.split():
+        fk = k
+        reverse = (k[0] == '-')
+        if reverse:
+            k = k[1:]
+        if k not in _sortkeyfuncs and k != 'topo':
+            raise error.ParseError(_("unknown sort key %r") % fk)
+        keyflags.append((k, reverse))
+
+    if len(keyflags) > 1 and any(k == 'topo' for k, reverse in keyflags):
+        # i18n: "topo" is a keyword
+        raise error.ParseError(_(
+            'topo sort order cannot be combined with other sort keys'))
+
+    opts = {}
+    if 'topo.firstbranch' in args:
+        if any(k == 'topo' for k, reverse in keyflags):
+            opts['topo.firstbranch'] = args['topo.firstbranch']
+        else:
+            # i18n: "topo" and "topo.firstbranch" are keywords
+            raise error.ParseError(_(
+                'topo.firstbranch can only be used when using the topo sort '
+                'key'))
+
+    return args['set'], keyflags, opts
+
+@predicate('sort(set[, [-]key... [, ...]])', safe=True)
 def sort(repo, subset, x):
     """Sort set by keys. The default sort order is ascending, specify a key
     as ``-key`` to sort in descending order.
@@ -1849,50 +1893,235 b' def sort(repo, subset, x):'
     - ``desc`` for the commit message (description),
     - ``user`` for user name (``author`` can be used as an alias),
     - ``date`` for the commit date
+    - ``topo`` for a reverse topographical sort
+
+    The ``topo`` sort order cannot be combined with other sort keys. This sort
+    takes one optional argument, ``topo.firstbranch``, which takes a revset that
+    specifies what topographical branches to prioritize in the sort.
+
     """
-    # i18n: "sort" is a keyword
-    l = getargs(x, 1, 2, _("sort requires one or two arguments"))
-    keys = "rev"
-    if len(l) == 2:
-        # i18n: "sort" is a keyword
-        keys = getstring(l[1], _("sort spec must be a string"))
-
-    s = l[0]
-    keys = keys.split()
+    s, keyflags, opts = _getsortargs(x)
     revs = getset(repo, subset, s)
-    if keys == ["rev"]:
-        revs.sort()
+
+    if not keyflags:
+        return revs
+    if len(keyflags) == 1 and keyflags[0][0] == "rev":
+        revs.sort(reverse=keyflags[0][1])
         return revs
-    elif keys == ["-rev"]:
-        revs.sort(reverse=True)
+    elif keyflags[0][0] == "topo":
+        firstbranch = ()
+        if 'topo.firstbranch' in opts:
+            firstbranch = getset(repo, subset, opts['topo.firstbranch'])
+        revs = baseset(_toposort(revs, repo.changelog.parentrevs, firstbranch),
+                       istopo=True)
+        if keyflags[0][1]:
+            revs.reverse()
         return revs
+
     # sort() is guaranteed to be stable
     ctxs = [repo[r] for r in revs]
-    for k in reversed(keys):
-        if k == 'rev':
-            ctxs.sort(key=lambda c: c.rev())
-        elif k == '-rev':
-            ctxs.sort(key=lambda c: c.rev(), reverse=True)
-        elif k == 'branch':
-            ctxs.sort(key=lambda c: c.branch())
-        elif k == '-branch':
-            ctxs.sort(key=lambda c: c.branch(), reverse=True)
-        elif k == 'desc':
-            ctxs.sort(key=lambda c: c.description())
-        elif k == '-desc':
-            ctxs.sort(key=lambda c: c.description(), reverse=True)
-        elif k in 'user author':
-            ctxs.sort(key=lambda c: c.user())
-        elif k in '-user -author':
-            ctxs.sort(key=lambda c: c.user(), reverse=True)
-        elif k == 'date':
-            ctxs.sort(key=lambda c: c.date()[0])
-        elif k == '-date':
-            ctxs.sort(key=lambda c: c.date()[0], reverse=True)
-        else:
-            raise error.ParseError(_("unknown sort key %r") % k)
+    for k, reverse in reversed(keyflags):
+        ctxs.sort(key=_sortkeyfuncs[k], reverse=reverse)
     return baseset([c.rev() for c in ctxs])
 
+def _toposort(revs, parentsfunc, firstbranch=()):
+    """Yield revisions from heads to roots one (topo) branch at a time.
+
+    This function aims to be used by a graph generator that wishes to minimize
+    the number of parallel branches and their interleaving.
+
+    Example iteration order (numbers show the "true" order in a changelog):
+
+      o  4
+      |
+      o  1
+      |
+      | o  3
+      | |
+      | o  2
+      |/
+      o  0
+
+    Note that the ancestors of merges are understood by the current
+    algorithm to be on the same branch. This means no reordering will
+    occur behind a merge.
+    """
+
+    ### Quick summary of the algorithm
+    #
+    # This function is based around a "retention" principle. We keep revisions
+    # in memory until we are ready to emit a whole branch that immediately
+    # "merges" into an existing one. This reduces the number of parallel
+    # branches with interleaved revisions.
+    #
+    # During iteration revs are split into two groups:
+    # A) revision already emitted
+    # B) revision in "retention". They are stored as different subgroups.
+    #
+    # for each REV, we do the following logic:
+    #
+    #   1) if REV is a parent of (A), we will emit it. If there is a
+    #   retention group ((B) above) that is blocked on REV being
+    #   available, we emit all the revisions out of that retention
+    #   group first.
+    #
+    #   2) else, we'll search for a subgroup in (B) awaiting for REV to be
+    #   available, if such subgroup exist, we add REV to it and the subgroup is
+    #   now awaiting for REV.parents() to be available.
+    #
+    #   3) finally if no such group existed in (B), we create a new subgroup.
+    #
+    #
+    # To bootstrap the algorithm, we emit the tipmost revision (which
+    # puts it in group (A) from above).
+
+    revs.sort(reverse=True)
+
+    # Set of parents of revision that have been emitted. They can be considered
+    # unblocked as the graph generator is already aware of them so there is no
+    # need to delay the revisions that reference them.
+    #
+    # If someone wants to prioritize a branch over the others, pre-filling this
+    # set will force all other branches to wait until this branch is ready to be
+    # emitted.
+    unblocked = set(firstbranch)
+
+    # list of groups waiting to be displayed, each group is defined by:
+    #
+    #   (revs:    lists of revs waiting to be displayed,
+    #    blocked: set of that cannot be displayed before those in 'revs')
+    #
+    # The second value ('blocked') correspond to parents of any revision in the
+    # group ('revs') that is not itself contained in the group. The main idea
+    # of this algorithm is to delay as much as possible the emission of any
+    # revision.  This means waiting for the moment we are about to display
+    # these parents to display the revs in a group.
+    #
+    # This first implementation is smart until it encounters a merge: it will
+    # emit revs as soon as any parent is about to be emitted and can grow an
+    # arbitrary number of revs in 'blocked'. In practice this mean we properly
+    # retains new branches but gives up on any special ordering for ancestors
+    # of merges. The implementation can be improved to handle this better.
+    #
+    # The first subgroup is special. It corresponds to all the revision that
+    # were already emitted. The 'revs' lists is expected to be empty and the
+    # 'blocked' set contains the parents revisions of already emitted revision.
+    #
+    # You could pre-seed the <parents> set of groups[0] to a specific
+    # changesets to select what the first emitted branch should be.
+    groups = [([], unblocked)]
+    pendingheap = []
+    pendingset = set()
+
+    heapq.heapify(pendingheap)
+    heappop = heapq.heappop
+    heappush = heapq.heappush
+    for currentrev in revs:
+        # Heap works with smallest element, we want highest so we invert
+        if currentrev not in pendingset:
+            heappush(pendingheap, -currentrev)
+            pendingset.add(currentrev)
+        # iterates on pending rev until after the current rev have been
+        # processed.
+        rev = None
+        while rev != currentrev:
+            rev = -heappop(pendingheap)
+            pendingset.remove(rev)
+
+            # Seek for a subgroup blocked, waiting for the current revision.
+            matching = [i for i, g in enumerate(groups) if rev in g[1]]
+
+            if matching:
+                # The main idea is to gather together all sets that are blocked
+                # on the same revision.
+                #
+                # Groups are merged when a common blocking ancestor is
+                # observed. For example, given two groups:
+                #
+                # revs [5, 4] waiting for 1
+                # revs [3, 2] waiting for 1
+                #
+                # These two groups will be merged when we process
+                # 1. In theory, we could have merged the groups when
+                # we added 2 to the group it is now in (we could have
+                # noticed the groups were both blocked on 1 then), but
+                # the way it works now makes the algorithm simpler.
+                #
+                # We also always keep the oldest subgroup first. We can
+                # probably improve the behavior by having the longest set
+                # first. That way, graph algorithms could minimise the length
+                # of parallel lines their drawing. This is currently not done.
+                targetidx = matching.pop(0)
+                trevs, tparents = groups[targetidx]
+                for i in matching:
+                    gr = groups[i]
+                    trevs.extend(gr[0])
+                    tparents |= gr[1]
+                # delete all merged subgroups (except the one we kept)
+                # (starting from the last subgroup for performance and
+                # sanity reasons)
+                for i in reversed(matching):
+                    del groups[i]
+            else:
+                # This is a new head. We create a new subgroup for it.
+                targetidx = len(groups)
+                groups.append(([], set([rev])))
+
+            gr = groups[targetidx]
+
+            # We now add the current nodes to this subgroups. This is done
+            # after the subgroup merging because all elements from a subgroup
+            # that relied on this rev must precede it.
+            #
+            # we also update the <parents> set to include the parents of the
+            # new nodes.
+            if rev == currentrev: # only display stuff in rev
+                gr[0].append(rev)
+            gr[1].remove(rev)
+            parents = [p for p in parentsfunc(rev) if p > node.nullrev]
+            gr[1].update(parents)
+            for p in parents:
+                if p not in pendingset:
+                    pendingset.add(p)
+                    heappush(pendingheap, -p)
+
+            # Look for a subgroup to display
+            #
+            # When unblocked is empty (if clause), we were not waiting for any
+            # revisions during the first iteration (if no priority was given) or
+            # if we emitted a whole disconnected set of the graph (reached a
+            # root).  In that case we arbitrarily take the oldest known
+            # subgroup. The heuristic could probably be better.
+            #
+            # Otherwise (elif clause) if the subgroup is blocked on
+            # a revision we just emitted, we can safely emit it as
+            # well.
+            if not unblocked:
+                if len(groups) > 1:  # display other subset
+                    targetidx = 1
+                    gr = groups[1]
+            elif not gr[1] & unblocked:
+                gr = None
+
+            if gr is not None:
+                # update the set of awaited revisions with the one from the
+                # subgroup
+                unblocked |= gr[1]
+                # output all revisions in the subgroup
+                for r in gr[0]:
+                    yield r
+                # delete the subgroup that you just output
+                # unless it is groups[0] in which case you just empty it.
+                if targetidx:
+                    del groups[targetidx]
+                else:
+                    gr[0][:] = []
+    # Check if we have some subgroup waiting for revisions we are not going to
+    # iterate over
+    for g in groups:
+        for r in g[0]:
+            yield r
+
 @predicate('subrepo([pattern])')
 def subrepo(repo, subset, x):
     """Changesets that add, modify or remove the given subrepo.  If no subrepo
@@ -2073,7 +2302,22 b' methods = {'
     "parentpost": p1,
 }
 
-def optimize(x, small):
+def _matchonly(revs, bases):
+    """
+    >>> f = lambda *args: _matchonly(*map(parse, args))
+    >>> f('ancestors(A)', 'not ancestors(B)')
+    ('list', ('symbol', 'A'), ('symbol', 'B'))
+    """
+    if (revs is not None
+        and revs[0] == 'func'
+        and getsymbol(revs[1]) == 'ancestors'
+        and bases is not None
+        and bases[0] == 'not'
+        and bases[1][0] == 'func'
+        and getsymbol(bases[1][1]) == 'ancestors'):
+        return ('list', revs[2], bases[1][2])
+
+def _optimize(x, small):
     if x is None:
         return 0, x
 
@@ -2083,47 +2327,36 b' def optimize(x, small):'
 
     op = x[0]
     if op == 'minus':
-        return optimize(('and', x[1], ('not', x[2])), small)
+        return _optimize(('and', x[1], ('not', x[2])), small)
     elif op == 'only':
-        return optimize(('func', ('symbol', 'only'),
-                         ('list', x[1], x[2])), small)
+        t = ('func', ('symbol', 'only'), ('list', x[1], x[2]))
+        return _optimize(t, small)
     elif op == 'onlypost':
-        return optimize(('func', ('symbol', 'only'), x[1]), small)
+        return _optimize(('func', ('symbol', 'only'), x[1]), small)
     elif op == 'dagrangepre':
-        return optimize(('func', ('symbol', 'ancestors'), x[1]), small)
+        return _optimize(('func', ('symbol', 'ancestors'), x[1]), small)
     elif op == 'dagrangepost':
-        return optimize(('func', ('symbol', 'descendants'), x[1]), small)
+        return _optimize(('func', ('symbol', 'descendants'), x[1]), small)
     elif op == 'rangeall':
-        return optimize(('range', ('string', '0'), ('string', 'tip')), small)
+        return _optimize(('range', ('string', '0'), ('string', 'tip')), small)
     elif op == 'rangepre':
-        return optimize(('range', ('string', '0'), x[1]), small)
+        return _optimize(('range', ('string', '0'), x[1]), small)
     elif op == 'rangepost':
-        return optimize(('range', x[1], ('string', 'tip')), small)
+        return _optimize(('range', x[1], ('string', 'tip')), small)
     elif op == 'negate':
-        return optimize(('string',
-                         '-' + getstring(x[1], _("can't negate that"))), small)
+        s = getstring(x[1], _("can't negate that"))
+        return _optimize(('string', '-' + s), small)
     elif op in 'string symbol negate':
         return smallbonus, x # single revisions are small
     elif op == 'and':
-        wa, ta = optimize(x[1], True)
-        wb, tb = optimize(x[2], True)
+        wa, ta = _optimize(x[1], True)
+        wb, tb = _optimize(x[2], True)
+        w = min(wa, wb)
 
         # (::x and not ::y)/(not ::y and ::x) have a fast path
-        def isonly(revs, bases):
-            return (
-                revs is not None
-                and revs[0] == 'func'
-                and getstring(revs[1], _('not a symbol')) == 'ancestors'
-                and bases is not None
-                and bases[0] == 'not'
-                and bases[1][0] == 'func'
-                and getstring(bases[1][1], _('not a symbol')) == 'ancestors')
-
-        w = min(wa, wb)
-        if isonly(ta, tb):
-            return w, ('func', ('symbol', 'only'), ('list', ta[2], tb[1][2]))
-        if isonly(tb, ta):
-            return w, ('func', ('symbol', 'only'), ('list', tb[2], ta[1][2]))
+        tm = _matchonly(ta, tb) or _matchonly(tb, ta)
+        if tm:
+            return w, ('func', ('symbol', 'only'), tm)
 
         if tb is not None and tb[0] == 'not':
             return wa, ('difference', ta, tb[1])
@@ -2143,12 +2376,12 b' def optimize(x, small):'
             else:
                 s = '\0'.join(t[1] for w, t in ss)
                 y = ('func', ('symbol', '_list'), ('string', s))
-                w, t = optimize(y, False)
+                w, t = _optimize(y, False)
             ws.append(w)
             ts.append(t)
             del ss[:]
         for y in x[1:]:
-            w, t = optimize(y, False)
+            w, t = _optimize(y, False)
             if t is not None and (t[0] == 'string' or t[0] == 'symbol'):
                 ss.append((w, t))
                 continue
@@ -2166,34 +2399,34 b' def optimize(x, small):'
         # Optimize not public() to _notpublic() because we have a fast version
         if x[1] == ('func', ('symbol', 'public'), None):
             newsym = ('func', ('symbol', '_notpublic'), None)
-            o = optimize(newsym, not small)
+            o = _optimize(newsym, not small)
             return o[0], o[1]
         else:
-            o = optimize(x[1], not small)
+            o = _optimize(x[1], not small)
             return o[0], (op, o[1])
     elif op == 'parentpost':
-        o = optimize(x[1], small)
+        o = _optimize(x[1], small)
         return o[0], (op, o[1])
     elif op == 'group':
-        return optimize(x[1], small)
+        return _optimize(x[1], small)
     elif op in 'dagrange range parent ancestorspec':
         if op == 'parent':
             # x^:y means (x^) : y, not x ^ (:y)
             post = ('parentpost', x[1])
             if x[2][0] == 'dagrangepre':
-                return optimize(('dagrange', post, x[2][1]), small)
+                return _optimize(('dagrange', post, x[2][1]), small)
             elif x[2][0] == 'rangepre':
-                return optimize(('range', post, x[2][1]), small)
-
-        wa, ta = optimize(x[1], small)
-        wb, tb = optimize(x[2], small)
+                return _optimize(('range', post, x[2][1]), small)
+
+        wa, ta = _optimize(x[1], small)
+        wb, tb = _optimize(x[2], small)
         return wa + wb, (op, ta, tb)
     elif op == 'list':
-        ws, ts = zip(*(optimize(y, small) for y in x[1:]))
+        ws, ts = zip(*(_optimize(y, small) for y in x[1:]))
         return sum(ws), (op,) + ts
     elif op == 'func':
-        f = getstring(x[1], _("not a symbol"))
-        wa, ta = optimize(x[2], small)
+        f = getsymbol(x[1])
+        wa, ta = _optimize(x[2], small)
         if f in ("author branch closed date desc file grep keyword "
                  "outgoing user"):
             w = 10 # slow
@@ -2212,33 +2445,32 b' def optimize(x, small):'
         return w + wa, (op, x[1], ta)
     return 1, x
 
+def optimize(tree):
+    _weight, newtree = _optimize(tree, small=True)
+    return newtree
+
 # the set of valid characters for the initial letter of symbols in
 # alias declarations and definitions
 _aliassyminitletters = set(c for c in [chr(i) for i in xrange(256)]
                            if c.isalnum() or c in '._@$' or ord(c) > 127)
 
-def _tokenizealias(program, lookup=None):
-    """Parse alias declaration/definition into a stream of tokens
-
-    This allows symbol names to use also ``$`` as an initial letter
-    (for backward compatibility), and callers of this function should
-    examine whether ``$`` is used also for unexpected symbols or not.
-    """
-    return tokenize(program, lookup=lookup,
-                    syminitletters=_aliassyminitletters)
-
-def _parsealias(spec):
-    """Parse alias declaration/definition ``spec``
-
-    >>> _parsealias('foo($1)')
+def _parsewith(spec, lookup=None, syminitletters=None):
+    """Generate a parse tree of given spec with given tokenizing options
+
+    >>> _parsewith('foo($1)', syminitletters=_aliassyminitletters)
     ('func', ('symbol', 'foo'), ('symbol', '$1'))
-    >>> _parsealias('foo bar')
+    >>> _parsewith('$1')
+    Traceback (most recent call last):
+      ...
+    ParseError: ("syntax error in revset '$1'", 0)
+    >>> _parsewith('foo bar')
     Traceback (most recent call last):
       ...
     ParseError: ('invalid token', 4)
     """
     p = parser.parser(elements)
-    tree, pos = p.parse(_tokenizealias(spec))
+    tree, pos = p.parse(tokenize(spec, lookup=lookup,
+                                 syminitletters=syminitletters))
     if pos != len(spec):
         raise error.ParseError(_('invalid token'), pos)
     return parser.simplifyinfixops(tree, ('list', 'or'))
@@ -2246,7 +2478,16 b' def _parsealias(spec):'
 class _aliasrules(parser.basealiasrules):
     """Parsing and expansion rule set of revset aliases"""
     _section = _('revset alias')
-    _parse = staticmethod(_parsealias)
+
+    @staticmethod
+    def _parse(spec):
+        """Parse alias declaration/definition ``spec``
+
+        This allows symbol names to use also ``$`` as an initial letter
+        (for backward compatibility), and callers of this function should
+        examine whether ``$`` is used also for unexpected symbols or not.
+        """
+        return _parsewith(spec, syminitletters=_aliassyminitletters)
 
     @staticmethod
     def _trygetfunc(tree):
@@ -2286,24 +2527,15 b' def foldconcat(tree):'
         return tuple(foldconcat(t) for t in tree)
 
 def parse(spec, lookup=None):
-    p = parser.parser(elements)
-    tree, pos = p.parse(tokenize(spec, lookup=lookup))
-    if pos != len(spec):
-        raise error.ParseError(_("invalid token"), pos)
-    return parser.simplifyinfixops(tree, ('list', 'or'))
+    return _parsewith(spec, lookup=lookup)
 
 def posttreebuilthook(tree, repo):
     # hook for extensions to execute code on the optimized tree
     pass
 
 def match(ui, spec, repo=None):
-    if not spec:
-        raise error.ParseError(_("empty query"))
-    lookup = None
-    if repo:
-        lookup = repo.__contains__
-    tree = parse(spec, lookup)
-    return _makematcher(ui, tree, repo)
+    """Create a matcher for a single revision spec."""
+    return matchany(ui, [spec], repo=repo)
 
 def matchany(ui, specs, repo=None):
     """Create a matcher that will include any revisions matching one of the
@@ -2327,7 +2559,7 b' def _makematcher(ui, tree, repo):'
     if ui:
         tree = expandaliases(ui, tree, showwarning=ui.warn)
     tree = foldconcat(tree)
-    weight, tree = optimize(tree, True)
+    tree = optimize(tree)
     posttreebuilthook(tree, repo)
     def mfunc(repo, subset=None):
         if subset is None:
@@ -2426,7 +2658,8 b' def formatspec(expr, *args):'
                 ret += listexp(list(args[arg]), d)
                 arg += 1
             else:
-                raise error.Abort('unexpected revspec format character %s' % d)
+                raise error.Abort(_('unexpected revspec format character %s')
+                                  % d)
         else:
             ret += c
         pos += 1
@@ -2506,6 +2739,10 b' class abstractsmartset(object):'
         """True if the set will iterate in descending order"""
         raise NotImplementedError()
 
+    def istopo(self):
+        """True if the set will iterate in topographical order"""
+        raise NotImplementedError()
+
     @util.cachefunc
     def min(self):
         """return the minimum element in the set"""
@@ -2591,12 +2828,13 b' class baseset(abstractsmartset):'
 
     Every method in this class should be implemented by any smartset class.
     """
-    def __init__(self, data=(), datarepr=None):
+    def __init__(self, data=(), datarepr=None, istopo=False):
         """
         datarepr: a tuple of (format, obj, ...), a function or an object that
                   provides a printable representation of the given data.
         """
         self._ascending = None
+        self._istopo = istopo
         if not isinstance(data, list):
             if isinstance(data, set):
                 self._set = data
@@ -2639,12 +2877,14 b' class baseset(abstractsmartset):'
 
     def sort(self, reverse=False):
         self._ascending = not bool(reverse)
+        self._istopo = False
 
     def reverse(self):
         if self._ascending is None:
             self._list.reverse()
         else:
             self._ascending = not self._ascending
+        self._istopo = False
 
     def __len__(self):
         return len(self._list)
@@ -2665,6 +2905,14 b' class baseset(abstractsmartset):'
             return True
         return self._ascending is not None and not self._ascending
 
+    def istopo(self):
+        """Is the collection is in topographical order or not.
+
+        This is part of the mandatory API for smartset."""
+        if len(self) <= 1:
+            return True
+        return self._istopo
+
     def first(self):
         if self:
             if self._ascending is None:
@@ -2741,9 +2989,16 b' class filteredset(abstractsmartset):'
         return lambda: self._iterfilter(it())
 
     def __nonzero__(self):
-        fast = self.fastasc
-        if fast is None:
-            fast = self.fastdesc
+        fast = None
+        candidates = [self.fastasc if self.isascending() else None,
+                      self.fastdesc if self.isdescending() else None,
+                      self.fastasc,
+                      self.fastdesc]
+        for candidate in candidates:
+            if candidate is not None:
+                fast = candidate
+                break
+
         if fast is not None:
             it = fast()
         else:
@@ -2773,6 +3028,9 b' class filteredset(abstractsmartset):'
     def isdescending(self):
         return self._subset.isdescending()
 
+    def istopo(self):
+        return self._subset.istopo()
+
     def first(self):
         for x in self:
             return x
@@ -2816,14 +3074,14 b' def _iterordered(ascending, iter1, iter2'
         # Consume both iterators in an ordered way until one is empty
         while True:
             if val1 is None:
-                val1 = iter1.next()
+                val1 = next(iter1)
             if val2 is None:
-                val2 = iter2.next()
-            next = choice(val1, val2)
-            yield next
-            if val1 == next:
+                val2 = next(iter2)
+            n = choice(val1, val2)
+            yield n
+            if val1 == n:
                 val1 = None
-            if val2 == next:
+            if val2 == n:
                 val2 = None
     except StopIteration:
         # Flush any remaining values and consume the other one
@@ -3019,6 +3277,12 b' class addset(abstractsmartset):'
     def isdescending(self):
         return self._ascending is not None and not self._ascending
 
+    def istopo(self):
+        # not worth the trouble asserting if the two sets combined are still
+        # in topographical order. Use the sort() predicate to explicitly sort
+        # again instead.
+        return False
+
     def reverse(self):
         if self._ascending is None:
             self._list.reverse()
@@ -3186,6 +3450,12 b' class generatorset(abstractsmartset):'
     def isdescending(self):
         return not self._ascending
 
+    def istopo(self):
+        # not worth the trouble asserting if the two sets combined are still
+        # in topographical order. Use the sort() predicate to explicitly sort
+        # again instead.
+        return False
+
     def first(self):
         if self._ascending:
             it = self.fastasc
@@ -3248,6 +3518,12 b' class spanset(abstractsmartset):'
     def reverse(self):
         self._ascending = not self._ascending
 
+    def istopo(self):
+        # not worth the trouble asserting if the two sets combined are still
+        # in topographical order. Use the sort() predicate to explicitly sort
+        # again instead.
+        return False
+
     def _iterfilter(self, iterrange):
         s = self._hiddenrevs
         for r in iterrange:

mercurial/scmutil.py

@@ -10,6 +10,7 b' from __future__ import absolute_import'
 import contextlib
 import errno
 import glob
+import hashlib
 import os
 import re
 import shutil
@@ -224,7 +225,7 b' def filteredhash(repo, maxrev):'
     key = None
     revs = sorted(r for r in cl.filteredrevs if r <= maxrev)
     if revs:
-        s = util.sha1()
+        s = hashlib.sha1()
         for rev in revs:
             s.update('%s;' % rev)
         key = s.digest()
@@ -377,8 +378,24 b' class abstractvfs(object):'
     def readlock(self, path):
         return util.readlock(self.join(path))
 
-    def rename(self, src, dst):
-        return util.rename(self.join(src), self.join(dst))
+    def rename(self, src, dst, checkambig=False):
+        """Rename from src to dst
+
+        checkambig argument is used with util.filestat, and is useful
+        only if destination file is guarded by any lock
+        (e.g. repo.lock or repo.wlock).
+        """
+        dstpath = self.join(dst)
+        oldstat = checkambig and util.filestat(dstpath)
+        if oldstat and oldstat.stat:
+            ret = util.rename(self.join(src), dstpath)
+            newstat = util.filestat(dstpath)
+            if newstat.isambig(oldstat):
+                # stat of renamed file is ambiguous to original one
+                advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
+                os.utime(dstpath, (advanced, advanced))
+            return ret
+        return util.rename(self.join(src), dstpath)
 
     def readlink(self, path):
         return os.readlink(self.join(path))
@@ -451,7 +468,8 b' class abstractvfs(object):'
         # have a use case.
         vfs = getattr(self, 'vfs', self)
         if getattr(vfs, '_backgroundfilecloser', None):
-            raise error.Abort('can only have 1 active background file closer')
+            raise error.Abort(
+                _('can only have 1 active background file closer'))
 
         with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
             try:
@@ -502,7 +520,7 b' class vfs(abstractvfs):'
         os.chmod(name, self.createmode & 0o666)
 
     def __call__(self, path, mode="r", text=False, atomictemp=False,
-                 notindexed=False, backgroundclose=False):
+                 notindexed=False, backgroundclose=False, checkambig=False):
         '''Open ``path`` file, which is relative to vfs root.
 
         Newly created directories are marked as "not to be indexed by
@@ -521,6 +539,10 b' class vfs(abstractvfs):'
            closing a file on a background thread and reopening it. (If the
            file were opened multiple times, there could be unflushed data
            because the original file handle hasn't been flushed/closed yet.)
+
+        ``checkambig`` argument is passed to atomictemplfile (valid
+        only for writing), and is useful only if target file is
+        guarded by any lock (e.g. repo.lock or repo.wlock).
         '''
         if self._audit:
             r = util.checkosfilename(path)
@@ -540,7 +562,8 b' class vfs(abstractvfs):'
             if basename:
                 if atomictemp:
                     util.makedirs(dirname, self.createmode, notindexed)
-                    return util.atomictempfile(f, mode, self.createmode)
+                    return util.atomictempfile(f, mode, self.createmode,
+                                               checkambig=checkambig)
                 try:
                     if 'w' in mode:
                         util.unlink(f)
@@ -568,8 +591,9 b' class vfs(abstractvfs):'
 
         if backgroundclose:
             if not self._backgroundfilecloser:
-                raise error.Abort('backgroundclose can only be used when a '
+                raise error.Abort(_('backgroundclose can only be used when a '
                                   'backgroundclosing context manager is active')
+                                  )
 
             fp = delayclosedfile(fp, self._backgroundfilecloser)
 
@@ -640,7 +664,7 b' class readonlyvfs(abstractvfs, auditvfs)'
 
     def __call__(self, path, mode='r', *args, **kw):
         if mode not in ('r', 'rb'):
-            raise error.Abort('this vfs is read only')
+            raise error.Abort(_('this vfs is read only'))
         return self.vfs(path, mode, *args, **kw)
 
     def join(self, path, *insidef):
@@ -751,7 +775,7 b" def revsingle(repo, revspec, default='.'"
 
 def _pairspec(revspec):
     tree = revset.parse(revspec)
-    tree = revset.optimize(tree, True)[1]  # fix up "x^:y" -> "(x^):y"
+    tree = revset.optimize(tree)  # fix up "x^:y" -> "(x^):y"
     return tree and tree[0] in ('range', 'rangepre', 'rangepost', 'rangeall')
 
 def revpair(repo, revs):
@@ -784,10 +808,29 b' def revpair(repo, revs):'
 
     return repo.lookup(first), repo.lookup(second)
 
-def revrange(repo, revs):
-    """Yield revision as strings from a list of revision specifications."""
+def revrange(repo, specs):
+    """Execute 1 to many revsets and return the union.
+
+    This is the preferred mechanism for executing revsets using user-specified
+    config options, such as revset aliases.
+
+    The revsets specified by ``specs`` will be executed via a chained ``OR``
+    expression. If ``specs`` is empty, an empty result is returned.
+
+    ``specs`` can contain integers, in which case they are assumed to be
+    revision numbers.
+
+    It is assumed the revsets are already formatted. If you have arguments
+    that need to be expanded in the revset, call ``revset.formatspec()``
+    and pass the result as an element of ``specs``.
+
+    Specifying a single revset is allowed.
+
+    Returns a ``revset.abstractsmartset`` which is a list-like interface over
+    integer revisions.
+    """
     allspecs = []
-    for spec in revs:
+    for spec in specs:
         if isinstance(spec, int):
             spec = revset.formatspec('rev(%d)', spec)
         allspecs.append(spec)
@@ -1183,6 +1226,9 b' class filecache(object):'
         return self
 
     def __get__(self, obj, type=None):
+        # if accessed on the class, return the descriptor itself.
+        if obj is None:
+            return self
         # do we need to check if the file changed?
         if self.name in obj.__dict__:
             assert self.name in obj._filecache, self.name
@@ -1358,8 +1404,8 b' class backgroundfilecloser(object):'
     def close(self, fh):
         """Schedule a file for closing."""
         if not self._entered:
-            raise error.Abort('can only call close() when context manager '
-                              'active')
+            raise error.Abort(_('can only call close() when context manager '
+                              'active'))
 
         # If a background thread encountered an exception, raise now so we fail
         # fast. Otherwise we may potentially go on for minutes until the error
@@ -1375,4 +1421,3 b' class backgroundfilecloser(object):'
             return
 
         self._queue.put(fh, block=True, timeout=None)
-

mercurial/similar.py

@@ -7,6 +7,8 b''
 
 from __future__ import absolute_import
 
+import hashlib
+
 from .i18n import _
 from . import (
     bdiff,
@@ -27,14 +29,14 b' def _findexactmatches(repo, added, remov'
     for i, fctx in enumerate(removed):
         repo.ui.progress(_('searching for exact renames'), i, total=numfiles,
                          unit=_('files'))
-        h = util.sha1(fctx.data()).digest()
+        h = hashlib.sha1(fctx.data()).digest()
         hashes[h] = fctx
 
     # For each added file, see if it corresponds to a removed file.
     for i, fctx in enumerate(added):
         repo.ui.progress(_('searching for exact renames'), i + len(removed),
                 total=numfiles, unit=_('files'))
-        h = util.sha1(fctx.data()).digest()
+        h = hashlib.sha1(fctx.data()).digest()
         if h in hashes:
             yield (hashes[h], fctx)
 
@@ -106,4 +108,3 b' def findrenames(repo, added, removed, th'
         for (a, b, score) in _findsimilarmatches(repo,
                 sorted(addedfiles), sorted(removedfiles), threshold):
             yield (a.path(), b.path(), score)
-

mercurial/sshpeer.py

@@ -307,7 +307,7 b' class sshpeer(wireproto.wirepeer):'
         r = self._call(cmd, **args)
         if r:
             # XXX needs to be made better
-            raise error.Abort('unexpected remote reply: %s' % r)
+            raise error.Abort(_('unexpected remote reply: %s') % r)
         while True:
             d = fp.read(4096)
             if not d:

mercurial/sshserver.py

@@ -11,6 +11,7 b' from __future__ import absolute_import'
 import os
 import sys
 
+from .i18n import _
 from . import (
     error,
     hook,
@@ -40,7 +41,7 b' class sshserver(wireproto.abstractserver'
             argline = self.fin.readline()[:-1]
             arg, l = argline.split()
             if arg not in keys:
-                raise error.Abort("unexpected parameter %r" % arg)
+                raise error.Abort(_("unexpected parameter %r") % arg)
             if arg == '*':
                 star = {}
                 for k in xrange(int(l)):

mercurial/sslutil.py

@@ -9,6 +9,7 b''
 
 from __future__ import absolute_import
 
+import hashlib
 import os
 import re
 import ssl
@@ -28,14 +29,21 b' from . import ('
 # modern/secure or legacy/insecure. Many operations in this module have
 # separate code paths depending on support in Python.
 
+configprotocols = set([
+    'tls1.0',
+    'tls1.1',
+    'tls1.2',
+])
+
 hassni = getattr(ssl, 'HAS_SNI', False)
 
-try:
-    OP_NO_SSLv2 = ssl.OP_NO_SSLv2
-    OP_NO_SSLv3 = ssl.OP_NO_SSLv3
-except AttributeError:
-    OP_NO_SSLv2 = 0x1000000
-    OP_NO_SSLv3 = 0x2000000
+# TLS 1.1 and 1.2 may not be supported if the OpenSSL Python is compiled
+# against doesn't support them.
+supportedprotocols = set(['tls1.0'])
+if util.safehasattr(ssl, 'PROTOCOL_TLSv1_1'):
+    supportedprotocols.add('tls1.1')
+if util.safehasattr(ssl, 'PROTOCOL_TLSv1_2'):
+    supportedprotocols.add('tls1.2')
 
 try:
     # ssl.SSLContext was added in 2.7.9 and presence indicates modern
@@ -76,15 +84,19 b' except AttributeError:'
 
         def load_verify_locations(self, cafile=None, capath=None, cadata=None):
             if capath:
-                raise error.Abort('capath not supported')
+                raise error.Abort(_('capath not supported'))
             if cadata:
-                raise error.Abort('cadata not supported')
+                raise error.Abort(_('cadata not supported'))
 
             self._cacerts = cafile
 
         def set_ciphers(self, ciphers):
             if not self._supportsciphers:
-                raise error.Abort('setting ciphers not supported')
+                raise error.Abort(_('setting ciphers in [hostsecurity] is not '
+                                    'supported by this version of Python'),
+                                  hint=_('remove the config option or run '
+                                         'Mercurial with a modern Python '
+                                         'version (preferred)'))
 
             self._ciphers = ciphers
 
@@ -107,8 +119,213 b' except AttributeError:'
 
             return ssl.wrap_socket(socket, **args)
 
-def wrapsocket(sock, keyfile, certfile, ui, cert_reqs=ssl.CERT_NONE,
-               ca_certs=None, serverhostname=None):
+def _hostsettings(ui, hostname):
+    """Obtain security settings for a hostname.
+
+    Returns a dict of settings relevant to that hostname.
+    """
+    s = {
+        # Whether we should attempt to load default/available CA certs
+        # if an explicit ``cafile`` is not defined.
+        'allowloaddefaultcerts': True,
+        # List of 2-tuple of (hash algorithm, hash).
+        'certfingerprints': [],
+        # Path to file containing concatenated CA certs. Used by
+        # SSLContext.load_verify_locations().
+        'cafile': None,
+        # Whether certificate verification should be disabled.
+        'disablecertverification': False,
+        # Whether the legacy [hostfingerprints] section has data for this host.
+        'legacyfingerprint': False,
+        # PROTOCOL_* constant to use for SSLContext.__init__.
+        'protocol': None,
+        # ssl.CERT_* constant used by SSLContext.verify_mode.
+        'verifymode': None,
+        # Defines extra ssl.OP* bitwise options to set.
+        'ctxoptions': None,
+        # OpenSSL Cipher List to use (instead of default).
+        'ciphers': None,
+    }
+
+    # Allow minimum TLS protocol to be specified in the config.
+    def validateprotocol(protocol, key):
+        if protocol not in configprotocols:
+            raise error.Abort(
+                _('unsupported protocol from hostsecurity.%s: %s') %
+                (key, protocol),
+                hint=_('valid protocols: %s') %
+                     ' '.join(sorted(configprotocols)))
+
+    # We default to TLS 1.1+ where we can because TLS 1.0 has known
+    # vulnerabilities (like BEAST and POODLE). We allow users to downgrade to
+    # TLS 1.0+ via config options in case a legacy server is encountered.
+    if 'tls1.1' in supportedprotocols:
+        defaultprotocol = 'tls1.1'
+    else:
+        # Let people know they are borderline secure.
+        # We don't document this config option because we want people to see
+        # the bold warnings on the web site.
+        # internal config: hostsecurity.disabletls10warning
+        if not ui.configbool('hostsecurity', 'disabletls10warning'):
+            ui.warn(_('warning: connecting to %s using legacy security '
+                      'technology (TLS 1.0); see '
+                      'https://mercurial-scm.org/wiki/SecureConnections for '
+                      'more info\n') % hostname)
+        defaultprotocol = 'tls1.0'
+
+    key = 'minimumprotocol'
+    protocol = ui.config('hostsecurity', key, defaultprotocol)
+    validateprotocol(protocol, key)
+
+    key = '%s:minimumprotocol' % hostname
+    protocol = ui.config('hostsecurity', key, protocol)
+    validateprotocol(protocol, key)
+
+    s['protocol'], s['ctxoptions'] = protocolsettings(protocol)
+
+    ciphers = ui.config('hostsecurity', 'ciphers')
+    ciphers = ui.config('hostsecurity', '%s:ciphers' % hostname, ciphers)
+    s['ciphers'] = ciphers
+
+    # Look for fingerprints in [hostsecurity] section. Value is a list
+    # of <alg>:<fingerprint> strings.
+    fingerprints = ui.configlist('hostsecurity', '%s:fingerprints' % hostname,
+                                 [])
+    for fingerprint in fingerprints:
+        if not (fingerprint.startswith(('sha1:', 'sha256:', 'sha512:'))):
+            raise error.Abort(_('invalid fingerprint for %s: %s') % (
+                                hostname, fingerprint),
+                              hint=_('must begin with "sha1:", "sha256:", '
+                                     'or "sha512:"'))
+
+        alg, fingerprint = fingerprint.split(':', 1)
+        fingerprint = fingerprint.replace(':', '').lower()
+        s['certfingerprints'].append((alg, fingerprint))
+
+    # Fingerprints from [hostfingerprints] are always SHA-1.
+    for fingerprint in ui.configlist('hostfingerprints', hostname, []):
+        fingerprint = fingerprint.replace(':', '').lower()
+        s['certfingerprints'].append(('sha1', fingerprint))
+        s['legacyfingerprint'] = True
+
+    # If a host cert fingerprint is defined, it is the only thing that
+    # matters. No need to validate CA certs.
+    if s['certfingerprints']:
+        s['verifymode'] = ssl.CERT_NONE
+        s['allowloaddefaultcerts'] = False
+
+    # If --insecure is used, don't take CAs into consideration.
+    elif ui.insecureconnections:
+        s['disablecertverification'] = True
+        s['verifymode'] = ssl.CERT_NONE
+        s['allowloaddefaultcerts'] = False
+
+    if ui.configbool('devel', 'disableloaddefaultcerts'):
+        s['allowloaddefaultcerts'] = False
+
+    # If both fingerprints and a per-host ca file are specified, issue a warning
+    # because users should not be surprised about what security is or isn't
+    # being performed.
+    cafile = ui.config('hostsecurity', '%s:verifycertsfile' % hostname)
+    if s['certfingerprints'] and cafile:
+        ui.warn(_('(hostsecurity.%s:verifycertsfile ignored when host '
+                  'fingerprints defined; using host fingerprints for '
+                  'verification)\n') % hostname)
+
+    # Try to hook up CA certificate validation unless something above
+    # makes it not necessary.
+    if s['verifymode'] is None:
+        # Look at per-host ca file first.
+        if cafile:
+            cafile = util.expandpath(cafile)
+            if not os.path.exists(cafile):
+                raise error.Abort(_('path specified by %s does not exist: %s') %
+                                  ('hostsecurity.%s:verifycertsfile' % hostname,
+                                   cafile))
+            s['cafile'] = cafile
+        else:
+            # Find global certificates file in config.
+            cafile = ui.config('web', 'cacerts')
+
+            if cafile:
+                cafile = util.expandpath(cafile)
+                if not os.path.exists(cafile):
+                    raise error.Abort(_('could not find web.cacerts: %s') %
+                                      cafile)
+            elif s['allowloaddefaultcerts']:
+                # CAs not defined in config. Try to find system bundles.
+                cafile = _defaultcacerts(ui)
+                if cafile:
+                    ui.debug('using %s for CA file\n' % cafile)
+
+            s['cafile'] = cafile
+
+        # Require certificate validation if CA certs are being loaded and
+        # verification hasn't been disabled above.
+        if cafile or (_canloaddefaultcerts and s['allowloaddefaultcerts']):
+            s['verifymode'] = ssl.CERT_REQUIRED
+        else:
+            # At this point we don't have a fingerprint, aren't being
+            # explicitly insecure, and can't load CA certs. Connecting
+            # is insecure. We allow the connection and abort during
+            # validation (once we have the fingerprint to print to the
+            # user).
+            s['verifymode'] = ssl.CERT_NONE
+
+    assert s['protocol'] is not None
+    assert s['ctxoptions'] is not None
+    assert s['verifymode'] is not None
+
+    return s
+
+def protocolsettings(protocol):
+    """Resolve the protocol and context options for a config value."""
+    if protocol not in configprotocols:
+        raise ValueError('protocol value not supported: %s' % protocol)
+
+    # Despite its name, PROTOCOL_SSLv23 selects the highest protocol
+    # that both ends support, including TLS protocols. On legacy stacks,
+    # the highest it likely goes is TLS 1.0. On modern stacks, it can
+    # support TLS 1.2.
+    #
+    # The PROTOCOL_TLSv* constants select a specific TLS version
+    # only (as opposed to multiple versions). So the method for
+    # supporting multiple TLS versions is to use PROTOCOL_SSLv23 and
+    # disable protocols via SSLContext.options and OP_NO_* constants.
+    # However, SSLContext.options doesn't work unless we have the
+    # full/real SSLContext available to us.
+    if supportedprotocols == set(['tls1.0']):
+        if protocol != 'tls1.0':
+            raise error.Abort(_('current Python does not support protocol '
+                                'setting %s') % protocol,
+                              hint=_('upgrade Python or disable setting since '
+                                     'only TLS 1.0 is supported'))
+
+        return ssl.PROTOCOL_TLSv1, 0
+
+    # WARNING: returned options don't work unless the modern ssl module
+    # is available. Be careful when adding options here.
+
+    # SSLv2 and SSLv3 are broken. We ban them outright.
+    options = ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3
+
+    if protocol == 'tls1.0':
+        # Defaults above are to use TLS 1.0+
+        pass
+    elif protocol == 'tls1.1':
+        options |= ssl.OP_NO_TLSv1
+    elif protocol == 'tls1.2':
+        options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
+    else:
+        raise error.Abort(_('this should not happen'))
+
+    # Prevent CRIME.
+    # There is no guarantee this attribute is defined on the module.
+    options |= getattr(ssl, 'OP_NO_COMPRESSION', 0)
+
+    return ssl.PROTOCOL_SSLv23, options
+
+def wrapsocket(sock, keyfile, certfile, ui, serverhostname=None):
     """Add SSL/TLS to a socket.
 
     This is a glorified wrapper for ``ssl.wrap_socket()``. It makes sane
@@ -121,32 +338,33 b' def wrapsocket(sock, keyfile, certfile, '
       server (and client) support SNI, this tells the server which certificate
       to use.
     """
-    # Despite its name, PROTOCOL_SSLv23 selects the highest protocol
-    # that both ends support, including TLS protocols. On legacy stacks,
-    # the highest it likely goes in TLS 1.0. On modern stacks, it can
-    # support TLS 1.2.
-    #
-    # The PROTOCOL_TLSv* constants select a specific TLS version
-    # only (as opposed to multiple versions). So the method for
-    # supporting multiple TLS versions is to use PROTOCOL_SSLv23 and
-    # disable protocols via SSLContext.options and OP_NO_* constants.
-    # However, SSLContext.options doesn't work unless we have the
-    # full/real SSLContext available to us.
-    #
-    # SSLv2 and SSLv3 are broken. We ban them outright.
-    if modernssl:
-        protocol = ssl.PROTOCOL_SSLv23
-    else:
-        protocol = ssl.PROTOCOL_TLSv1
+    if not serverhostname:
+        raise error.Abort(_('serverhostname argument is required'))
+
+    settings = _hostsettings(ui, serverhostname)
 
-    # TODO use ssl.create_default_context() on modernssl.
-    sslcontext = SSLContext(protocol)
+    # We can't use ssl.create_default_context() because it calls
+    # load_default_certs() unless CA arguments are passed to it. We want to
+    # have explicit control over CA loading because implicitly loading
+    # CAs may undermine the user's intent. For example, a user may define a CA
+    # bundle with a specific CA cert removed. If the system/default CA bundle
+    # is loaded and contains that removed CA, you've just undone the user's
+    # choice.
+    sslcontext = SSLContext(settings['protocol'])
 
-    # This is a no-op on old Python.
-    sslcontext.options |= OP_NO_SSLv2 | OP_NO_SSLv3
+    # This is a no-op unless using modern ssl.
+    sslcontext.options |= settings['ctxoptions']
 
     # This still works on our fake SSLContext.
-    sslcontext.verify_mode = cert_reqs
+    sslcontext.verify_mode = settings['verifymode']
+
+    if settings['ciphers']:
+        try:
+            sslcontext.set_ciphers(settings['ciphers'])
+        except ssl.SSLError as e:
+            raise error.Abort(_('could not set ciphers: %s') % e.args[0],
+                              hint=_('change cipher string (%s) in config') %
+                                   settings['ciphers'])
 
     if certfile is not None:
         def password():
@@ -154,20 +372,123 b' def wrapsocket(sock, keyfile, certfile, '
             return ui.getpass(_('passphrase for %s: ') % f, '')
         sslcontext.load_cert_chain(certfile, keyfile, password)
 
-    if ca_certs is not None:
-        sslcontext.load_verify_locations(cafile=ca_certs)
-    else:
+    if settings['cafile'] is not None:
+        try:
+            sslcontext.load_verify_locations(cafile=settings['cafile'])
+        except ssl.SSLError as e:
+            raise error.Abort(_('error loading CA file %s: %s') % (
+                              settings['cafile'], e.args[1]),
+                              hint=_('file is empty or malformed?'))
+        caloaded = True
+    elif settings['allowloaddefaultcerts']:
         # This is a no-op on old Python.
         sslcontext.load_default_certs()
+        caloaded = True
+    else:
+        caloaded = False
 
-    sslsocket = sslcontext.wrap_socket(sock, server_hostname=serverhostname)
+    try:
+        sslsocket = sslcontext.wrap_socket(sock, server_hostname=serverhostname)
+    except ssl.SSLError as e:
+        # If we're doing certificate verification and no CA certs are loaded,
+        # that is almost certainly the reason why verification failed. Provide
+        # a hint to the user.
+        # Only modern ssl module exposes SSLContext.get_ca_certs() so we can
+        # only show this warning if modern ssl is available.
+        if (caloaded and settings['verifymode'] == ssl.CERT_REQUIRED and
+            modernssl and not sslcontext.get_ca_certs()):
+            ui.warn(_('(an attempt was made to load CA certificates but none '
+                      'were loaded; see '
+                      'https://mercurial-scm.org/wiki/SecureConnections for '
+                      'how to configure Mercurial to avoid this error)\n'))
+        # Try to print more helpful error messages for known failures.
+        if util.safehasattr(e, 'reason'):
+            if e.reason == 'UNSUPPORTED_PROTOCOL':
+                ui.warn(_('(could not negotiate a common protocol; see '
+                          'https://mercurial-scm.org/wiki/SecureConnections '
+                          'for how to configure Mercurial to avoid this '
+                          'error)\n'))
+        raise
+
     # check if wrap_socket failed silently because socket had been
     # closed
     # - see http://bugs.python.org/issue13721
     if not sslsocket.cipher():
         raise error.Abort(_('ssl connection failed'))
+
+    sslsocket._hgstate = {
+        'caloaded': caloaded,
+        'hostname': serverhostname,
+        'settings': settings,
+        'ui': ui,
+    }
+
     return sslsocket
 
+def wrapserversocket(sock, ui, certfile=None, keyfile=None, cafile=None,
+                     requireclientcert=False):
+    """Wrap a socket for use by servers.
+
+    ``certfile`` and ``keyfile`` specify the files containing the certificate's
+    public and private keys, respectively. Both keys can be defined in the same
+    file via ``certfile`` (the private key must come first in the file).
+
+    ``cafile`` defines the path to certificate authorities.
+
+    ``requireclientcert`` specifies whether to require client certificates.
+
+    Typically ``cafile`` is only defined if ``requireclientcert`` is true.
+    """
+    protocol, options = protocolsettings('tls1.0')
+
+    # This config option is intended for use in tests only. It is a giant
+    # footgun to kill security. Don't define it.
+    exactprotocol = ui.config('devel', 'serverexactprotocol')
+    if exactprotocol == 'tls1.0':
+        protocol = ssl.PROTOCOL_TLSv1
+    elif exactprotocol == 'tls1.1':
+        if 'tls1.1' not in supportedprotocols:
+            raise error.Abort(_('TLS 1.1 not supported by this Python'))
+        protocol = ssl.PROTOCOL_TLSv1_1
+    elif exactprotocol == 'tls1.2':
+        if 'tls1.2' not in supportedprotocols:
+            raise error.Abort(_('TLS 1.2 not supported by this Python'))
+        protocol = ssl.PROTOCOL_TLSv1_2
+    elif exactprotocol:
+        raise error.Abort(_('invalid value for serverexactprotocol: %s') %
+                          exactprotocol)
+
+    if modernssl:
+        # We /could/ use create_default_context() here since it doesn't load
+        # CAs when configured for client auth. However, it is hard-coded to
+        # use ssl.PROTOCOL_SSLv23 which may not be appropriate here.
+        sslcontext = SSLContext(protocol)
+        sslcontext.options |= options
+
+        # Improve forward secrecy.
+        sslcontext.options |= getattr(ssl, 'OP_SINGLE_DH_USE', 0)
+        sslcontext.options |= getattr(ssl, 'OP_SINGLE_ECDH_USE', 0)
+
+        # Use the list of more secure ciphers if found in the ssl module.
+        if util.safehasattr(ssl, '_RESTRICTED_SERVER_CIPHERS'):
+            sslcontext.options |= getattr(ssl, 'OP_CIPHER_SERVER_PREFERENCE', 0)
+            sslcontext.set_ciphers(ssl._RESTRICTED_SERVER_CIPHERS)
+    else:
+        sslcontext = SSLContext(ssl.PROTOCOL_TLSv1)
+
+    if requireclientcert:
+        sslcontext.verify_mode = ssl.CERT_REQUIRED
+    else:
+        sslcontext.verify_mode = ssl.CERT_NONE
+
+    if certfile or keyfile:
+        sslcontext.load_cert_chain(certfile=certfile, keyfile=keyfile)
+
+    if cafile:
+        sslcontext.load_verify_locations(cafile=cafile)
+
+    return sslcontext.wrap_socket(sock, server_side=True)
+
 class wildcarderror(Exception):
     """Represents an error parsing wildcards in DNS name."""
 
@@ -268,10 +589,6 b' def _verifycert(cert, hostname):'
     else:
         return _('no commonName or subjectAltName found in certificate')
 
-
-# CERT_REQUIRED means fetch the cert from the server all the time AND
-# validate it against the CA store provided in web.cacerts.
-
 def _plainapplepython():
     """return true if this seems to be a pure Apple Python that
     * is unfrozen and presumably has the whole mercurial module in the file
@@ -286,97 +603,172 b' def _plainapplepython():'
     return (exe.startswith('/usr/bin/python') or
             exe.startswith('/system/library/frameworks/python.framework/'))
 
-def _defaultcacerts():
-    """return path to CA certificates; None for system's store; ! to disable"""
+_systemcacertpaths = [
+    # RHEL, CentOS, and Fedora
+    '/etc/pki/tls/certs/ca-bundle.trust.crt',
+    # Debian, Ubuntu, Gentoo
+    '/etc/ssl/certs/ca-certificates.crt',
+]
+
+def _defaultcacerts(ui):
+    """return path to default CA certificates or None.
+
+    It is assumed this function is called when the returned certificates
+    file will actually be used to validate connections. Therefore this
+    function may print warnings or debug messages assuming this usage.
+
+    We don't print a message when the Python is able to load default
+    CA certs because this scenario is detected at socket connect time.
+    """
+    # The "certifi" Python package provides certificates. If it is installed,
+    # assume the user intends it to be used and use it.
+    try:
+        import certifi
+        certs = certifi.where()
+        ui.debug('using ca certificates from certifi\n')
+        return certs
+    except ImportError:
+        pass
+
+    # On Windows, only the modern ssl module is capable of loading the system
+    # CA certificates. If we're not capable of doing that, emit a warning
+    # because we'll get a certificate verification error later and the lack
+    # of loaded CA certificates will be the reason why.
+    # Assertion: this code is only called if certificates are being verified.
+    if os.name == 'nt':
+        if not _canloaddefaultcerts:
+            ui.warn(_('(unable to load Windows CA certificates; see '
+                      'https://mercurial-scm.org/wiki/SecureConnections for '
+                      'how to configure Mercurial to avoid this message)\n'))
+
+        return None
+
+    # Apple's OpenSSL has patches that allow a specially constructed certificate
+    # to load the system CA store. If we're running on Apple Python, use this
+    # trick.
     if _plainapplepython():
         dummycert = os.path.join(os.path.dirname(__file__), 'dummycert.pem')
         if os.path.exists(dummycert):
             return dummycert
-    if _canloaddefaultcerts:
+
+    # The Apple OpenSSL trick isn't available to us. If Python isn't able to
+    # load system certs, we're out of luck.
+    if sys.platform == 'darwin':
+        # FUTURE Consider looking for Homebrew or MacPorts installed certs
+        # files. Also consider exporting the keychain certs to a file during
+        # Mercurial install.
+        if not _canloaddefaultcerts:
+            ui.warn(_('(unable to load CA certificates; see '
+                      'https://mercurial-scm.org/wiki/SecureConnections for '
+                      'how to configure Mercurial to avoid this message)\n'))
         return None
-    return '!'
+
+    # / is writable on Windows. Out of an abundance of caution make sure
+    # we're not on Windows because paths from _systemcacerts could be installed
+    # by non-admin users.
+    assert os.name != 'nt'
 
-def sslkwargs(ui, host):
-    kws = {'ui': ui}
-    hostfingerprint = ui.config('hostfingerprints', host)
-    if hostfingerprint:
-        return kws
-    cacerts = ui.config('web', 'cacerts')
-    if cacerts == '!':
-        pass
-    elif cacerts:
-        cacerts = util.expandpath(cacerts)
-        if not os.path.exists(cacerts):
-            raise error.Abort(_('could not find web.cacerts: %s') % cacerts)
-    else:
-        cacerts = _defaultcacerts()
-        if cacerts and cacerts != '!':
-            ui.debug('using %s to enable OS X system CA\n' % cacerts)
-        ui.setconfig('web', 'cacerts', cacerts, 'defaultcacerts')
-    if cacerts != '!':
-        kws.update({'ca_certs': cacerts,
-                    'cert_reqs': ssl.CERT_REQUIRED,
-                    })
-    return kws
+    # Try to find CA certificates in well-known locations. We print a warning
+    # when using a found file because we don't want too much silent magic
+    # for security settings. The expectation is that proper Mercurial
+    # installs will have the CA certs path defined at install time and the
+    # installer/packager will make an appropriate decision on the user's
+    # behalf. We only get here and perform this setting as a feature of
+    # last resort.
+    if not _canloaddefaultcerts:
+        for path in _systemcacertpaths:
+            if os.path.isfile(path):
+                ui.warn(_('(using CA certificates from %s; if you see this '
+                          'message, your Mercurial install is not properly '
+                          'configured; see '
+                          'https://mercurial-scm.org/wiki/SecureConnections '
+                          'for how to configure Mercurial to avoid this '
+                          'message)\n') % path)
+                return path
 
-class validator(object):
-    def __init__(self, ui, host):
-        self.ui = ui
-        self.host = host
+        ui.warn(_('(unable to load CA certificates; see '
+                  'https://mercurial-scm.org/wiki/SecureConnections for '
+                  'how to configure Mercurial to avoid this message)\n'))
 
-    def __call__(self, sock, strict=False):
-        host = self.host
+    return None
+
+def validatesocket(sock):
+    """Validate a socket meets security requiremnets.
 
-        if not sock.cipher(): # work around http://bugs.python.org/issue13721
-            raise error.Abort(_('%s ssl connection error') % host)
-        try:
-            peercert = sock.getpeercert(True)
-            peercert2 = sock.getpeercert()
-        except AttributeError:
-            raise error.Abort(_('%s ssl connection error') % host)
+    The passed socket must have been created with ``wrapsocket()``.
+    """
+    host = sock._hgstate['hostname']
+    ui = sock._hgstate['ui']
+    settings = sock._hgstate['settings']
+
+    try:
+        peercert = sock.getpeercert(True)
+        peercert2 = sock.getpeercert()
+    except AttributeError:
+        raise error.Abort(_('%s ssl connection error') % host)
+
+    if not peercert:
+        raise error.Abort(_('%s certificate error: '
+                           'no certificate received') % host)
 
-        if not peercert:
-            raise error.Abort(_('%s certificate error: '
-                               'no certificate received') % host)
+    if settings['disablecertverification']:
+        # We don't print the certificate fingerprint because it shouldn't
+        # be necessary: if the user requested certificate verification be
+        # disabled, they presumably already saw a message about the inability
+        # to verify the certificate and this message would have printed the
+        # fingerprint. So printing the fingerprint here adds little to no
+        # value.
+        ui.warn(_('warning: connection security to %s is disabled per current '
+                  'settings; communication is susceptible to eavesdropping '
+                  'and tampering\n') % host)
+        return
+
+    # If a certificate fingerprint is pinned, use it and only it to
+    # validate the remote cert.
+    peerfingerprints = {
+        'sha1': hashlib.sha1(peercert).hexdigest(),
+        'sha256': hashlib.sha256(peercert).hexdigest(),
+        'sha512': hashlib.sha512(peercert).hexdigest(),
+    }
+
+    def fmtfingerprint(s):
+        return ':'.join([s[x:x + 2] for x in range(0, len(s), 2)])
+
+    nicefingerprint = 'sha256:%s' % fmtfingerprint(peerfingerprints['sha256'])
 
-        # If a certificate fingerprint is pinned, use it and only it to
-        # validate the remote cert.
-        hostfingerprints = self.ui.configlist('hostfingerprints', host)
-        peerfingerprint = util.sha1(peercert).hexdigest()
-        nicefingerprint = ":".join([peerfingerprint[x:x + 2]
-            for x in xrange(0, len(peerfingerprint), 2)])
-        if hostfingerprints:
-            fingerprintmatch = False
-            for hostfingerprint in hostfingerprints:
-                if peerfingerprint.lower() == \
-                        hostfingerprint.replace(':', '').lower():
-                    fingerprintmatch = True
-                    break
-            if not fingerprintmatch:
-                raise error.Abort(_('certificate for %s has unexpected '
-                                   'fingerprint %s') % (host, nicefingerprint),
-                                 hint=_('check hostfingerprint configuration'))
-            self.ui.debug('%s certificate matched fingerprint %s\n' %
-                          (host, nicefingerprint))
-            return
+    if settings['certfingerprints']:
+        for hash, fingerprint in settings['certfingerprints']:
+            if peerfingerprints[hash].lower() == fingerprint:
+                ui.debug('%s certificate matched fingerprint %s:%s\n' %
+                         (host, hash, fmtfingerprint(fingerprint)))
+                return
+
+        # Pinned fingerprint didn't match. This is a fatal error.
+        if settings['legacyfingerprint']:
+            section = 'hostfingerprint'
+            nice = fmtfingerprint(peerfingerprints['sha1'])
+        else:
+            section = 'hostsecurity'
+            nice = '%s:%s' % (hash, fmtfingerprint(peerfingerprints[hash]))
+        raise error.Abort(_('certificate for %s has unexpected '
+                            'fingerprint %s') % (host, nice),
+                          hint=_('check %s configuration') % section)
 
-        # No pinned fingerprint. Establish trust by looking at the CAs.
-        cacerts = self.ui.config('web', 'cacerts')
-        if cacerts != '!':
-            msg = _verifycert(peercert2, host)
-            if msg:
-                raise error.Abort(_('%s certificate error: %s') % (host, msg),
-                                 hint=_('configure hostfingerprint %s or use '
-                                        '--insecure to connect insecurely') %
-                                      nicefingerprint)
-            self.ui.debug('%s certificate successfully verified\n' % host)
-        elif strict:
-            raise error.Abort(_('%s certificate with fingerprint %s not '
-                               'verified') % (host, nicefingerprint),
-                             hint=_('check hostfingerprints or web.cacerts '
-                                     'config setting'))
-        else:
-            self.ui.warn(_('warning: %s certificate with fingerprint %s not '
-                           'verified (check hostfingerprints or web.cacerts '
-                           'config setting)\n') %
-                         (host, nicefingerprint))
+    # Security is enabled but no CAs are loaded. We can't establish trust
+    # for the cert so abort.
+    if not sock._hgstate['caloaded']:
+        raise error.Abort(
+            _('unable to verify security of %s (no loaded CA certificates); '
+              'refusing to connect') % host,
+            hint=_('see https://mercurial-scm.org/wiki/SecureConnections for '
+                   'how to configure Mercurial to avoid this error or set '
+                   'hostsecurity.%s:fingerprints=%s to trust this server') %
+                   (host, nicefingerprint))
+
+    msg = _verifycert(peercert2, host)
+    if msg:
+        raise error.Abort(_('%s certificate error: %s') % (host, msg),
+                         hint=_('set hostsecurity.%s:certfingerprints=%s '
+                                'config setting or use --insecure to connect '
+                                'insecurely') %
+                              (host, nicefingerprint))

mercurial/store.py

@@ -8,6 +8,7 b''
 from __future__ import absolute_import
 
 import errno
+import hashlib
 import os
 import stat
 
@@ -19,8 +20,6 b' from . import ('
     util,
 )
 
-_sha = util.sha1
-
 # This avoids a collision between a file named foo and a dir named
 # foo.i or foo.d
 def _encodedir(path):
@@ -57,6 +56,23 b' def decodedir(path):'
             .replace(".i.hg/", ".i/")
             .replace(".hg.hg/", ".hg/"))
 
+def _reserved():
+    ''' characters that are problematic for filesystems
+
+    * ascii escapes (0..31)
+    * ascii hi (126..255)
+    * windows specials
+
+    these characters will be escaped by encodefunctions
+    '''
+    winreserved = [ord(x) for x in '\\:*?"<>|']
+    for x in range(32):
+        yield x
+    for x in range(126, 256):
+        yield x
+    for x in winreserved:
+        yield x
+
 def _buildencodefun():
     '''
     >>> enc, dec = _buildencodefun()
@@ -82,11 +98,10 b' def _buildencodefun():'
     'the\\x07quick\\xadshot'
     '''
     e = '_'
-    winreserved = [ord(x) for x in '\\:*?"<>|']
     cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
-    for x in (range(32) + range(126, 256) + winreserved):
+    for x in _reserved():
         cmap[chr(x)] = "~%02x" % x
-    for x in range(ord("A"), ord("Z") + 1) + [ord(e)]:
+    for x in list(range(ord("A"), ord("Z") + 1)) + [ord(e)]:
         cmap[chr(x)] = e + chr(x).lower()
     dmap = {}
     for k, v in cmap.iteritems():
@@ -134,9 +149,8 b' def _buildlowerencodefun():'
     >>> f('the\x07quick\xADshot')
     'the~07quick~adshot'
     '''
-    winreserved = [ord(x) for x in '\\:*?"<>|']
     cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
-    for x in (range(32) + range(126, 256) + winreserved):
+    for x in _reserved():
         cmap[chr(x)] = "~%02x" % x
     for x in range(ord("A"), ord("Z") + 1):
         cmap[chr(x)] = chr(x).lower()
@@ -196,7 +210,7 b' def _auxencode(path, dotencode):'
 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
 
 def _hashencode(path, dotencode):
-    digest = _sha(path).hexdigest()
+    digest = hashlib.sha1(path).hexdigest()
     le = lowerencode(path[5:]).split('/') # skips prefix 'data/' or 'meta/'
     parts = _auxencode(le, dotencode)
     basename = parts[-1]

mercurial/subrepo.py

@@ -9,6 +9,7 b' from __future__ import absolute_import'
 
 import copy
 import errno
+import hashlib
 import os
 import posixpath
 import re
@@ -50,14 +51,14 b' def _expandedabspath(path):'
 
 def _getstorehashcachename(remotepath):
     '''get a unique filename for the store hash cache of a remote repository'''
-    return util.sha1(_expandedabspath(remotepath)).hexdigest()[0:12]
+    return hashlib.sha1(_expandedabspath(remotepath)).hexdigest()[0:12]
 
 class SubrepoAbort(error.Abort):
     """Exception class used to avoid handling a subrepo error more than once"""
     def __init__(self, *args, **kw):
+        self.subrepo = kw.pop('subrepo', None)
+        self.cause = kw.pop('cause', None)
         error.Abort.__init__(self, *args, **kw)
-        self.subrepo = kw.get('subrepo')
-        self.cause = kw.get('cause')
 
 def annotatesubrepoerror(func):
     def decoratedmethod(self, *args, **kargs):
@@ -585,7 +586,7 b' class abstractsubrepo(object):'
         return 1
 
     def revert(self, substate, *pats, **opts):
-        self.ui.warn('%s: reverting %s subrepos is unsupported\n' \
+        self.ui.warn(_('%s: reverting %s subrepos is unsupported\n') \
             % (substate[0], substate[2]))
         return []
 
@@ -659,7 +660,7 b' class hgsubrepo(abstractsubrepo):'
         yield '# %s\n' % _expandedabspath(remotepath)
         vfs = self._repo.vfs
         for relname in filelist:
-            filehash = util.sha1(vfs.tryread(relname)).hexdigest()
+            filehash = hashlib.sha1(vfs.tryread(relname)).hexdigest()
             yield '%s = %s\n' % (relname, filehash)
 
     @propertycache
@@ -1413,7 +1414,7 b' class gitsubrepo(abstractsubrepo):'
             if command in ('cat-file', 'symbolic-ref'):
                 return retdata, p.returncode
             # for all others, abort
-            raise error.Abort('git %s error %d in %s' %
+            raise error.Abort(_('git %s error %d in %s') %
                              (command, p.returncode, self._relpath))
 
         return retdata, p.returncode

mercurial/tags.py

@@ -292,7 +292,7 b' def _readtagcache(ui, repo):'
     cachehash = None
     if cachefile:
         try:
-            validline = cachelines.next()
+            validline = next(cachelines)
             validline = validline.split()
             cacherev = int(validline[0])
             cachenode = bin(validline[1])

mercurial/templater.py

@@ -724,6 +724,25 b' def rstdoc(context, mapping, args):'
 
     return minirst.format(text, style=style, keep=['verbose'])
 
+@templatefunc('separate(sep, args)')
+def separate(context, mapping, args):
+    """Add a separator between non-empty arguments."""
+    if not args:
+        # i18n: "separate" is a keyword
+        raise error.ParseError(_("separate expects at least one argument"))
+
+    sep = evalstring(context, mapping, args[0])
+    first = True
+    for arg in args[1:]:
+        argstr = evalstring(context, mapping, arg)
+        if not argstr:
+            continue
+        if first:
+            first = False
+        else:
+            yield sep
+        yield argstr
+
 @templatefunc('shortest(node, minlength=4)')
 def shortest(context, mapping, args):
     """Obtain the shortest representation of

mercurial/templates/atom/branchentry.tmpl

@@ -4,5 +4,5 b''
   <id>{urlbase}{url|urlescape}#branch-{node}</id>
   <updated>{date|rfc3339date}</updated>
   <published>{date|rfc3339date}</published>
-  <content type="text"><![CDATA[{branch|strip|escape|addbreaks}]]></content>
+  <content type="text">{branch|strip|escape}</content>
  </entry>

mercurial/templates/atom/changelogentry.tmpl

@@ -9,35 +9,35 b''
   <updated>{date|rfc3339date}</updated>
   <published>{date|rfc3339date}</published>
   <content type="xhtml">
-	<table xmlns="http://www.w3.org/1999/xhtml">
-	<tr>
-		<th style="text-align:left;">changeset</th>
-		<td>{node|short}</td>
-              </tr>
-              <tr>
-                              <th style="text-align:left;">branch</th>
-                              <td>{inbranch%"{name|escape}"}{branches%"{name|escape}"}</td>
-              </tr>
-              <tr>
-                              <th style="text-align:left;">bookmark</th>
-		<td>{bookmarks%"{name|escape}"}</td>
-	</tr>
-	<tr>
-		<th style="text-align:left;">tag</th>
-		<td>{tags%"{name|escape}"}</td>
-	</tr>
-	<tr>
-		<th style="text-align:left;">user</th>
-		<td>{author|obfuscate}</td>
-	</tr>
-	<tr>
-		<th style="text-align:left;vertical-align:top;">description</th>
-		<td>{desc|strip|escape|websub|addbreaks|nonempty}</td>
-	</tr>
-	<tr>
-		<th style="text-align:left;vertical-align:top;">files</th>
-		<td>{files}</td>
-	</tr>
-	</table>
+   <table xmlns="http://www.w3.org/1999/xhtml">
+    <tr>
+     <th style="text-align:left;">changeset</th>
+     <td>{node|short}</td>
+    </tr>
+    <tr>
+     <th style="text-align:left;">branch</th>
+     <td>{inbranch%"{name|escape}"}{branches%"{name|escape}"}</td>
+    </tr>
+    <tr>
+     <th style="text-align:left;">bookmark</th>
+     <td>{bookmarks%"{name|escape}"}</td>
+    </tr>
+    <tr>
+     <th style="text-align:left;">tag</th>
+     <td>{tags%"{name|escape}"}</td>
+    </tr>
+    <tr>
+     <th style="text-align:left;">user</th>
+     <td>{author|obfuscate}</td>
+    </tr>
+    <tr>
+     <th style="text-align:left;vertical-align:top;">description</th>
+     <td>{desc|strip|escape|websub|addbreaks|nonempty}</td>
+    </tr>
+    <tr>
+     <th style="text-align:left;vertical-align:top;">files</th>
+     <td>{files}</td>
+    </tr>
+   </table>
   </content>
  </entry>

mercurial/templates/atom/map

@@ -5,7 +5,6 b' header = header.tmpl'
 changelog = changelog.tmpl
 changelogentry = changelogentry.tmpl
 filelog = filelog.tmpl
-filelogentry = filelogentry.tmpl
 tags = tags.tmpl
 tagentry = tagentry.tmpl
 bookmarks = bookmarks.tmpl

mercurial/templates/gitweb/map

@@ -95,14 +95,29 b' filelog = filelog.tmpl'
 fileline = '
   <a href="#{lineid}"></a><span id="{lineid}">{strip(line|escape, '\r\n')}</span>'
 annotateline = '
-  <tr id="{lineid}" style="font-family:monospace" class="parity{parity}">
-    <td class="linenr" style="text-align: right;">
-      <a href="{url|urlescape}annotate/{node|short}/{file|urlescape}{sessionvars%urlparameter}#l{targetline}"
-         title="{node|short}: {desc|escape|firstline}">{author|user}@{rev}</a>
+  <tr id="{lineid}" style="font-family:monospace" class="parity{parity}{ifeq(node, originalnode, ' thisrev')}">
+    <td class="annotate linenr parity{blockparity}" style="text-align: right;">
+      {if(blockhead,
+          '<a href="{url|urlescape}annotate/{node|short}/{file|urlescape}{sessionvars%urlparameter}#l{targetline}">
+             {rev}
+           </a>')}
+      <div class="annotate-info">
+        <div>
+          <a href="{url|urlescape}annotate/{node|short}/{file|urlescape}{sessionvars%urlparameter}#l{targetline}">
+            {node|short}</a>
+          {desc|escape|firstline}
+        </div>
+        <div><em>{author|obfuscate}</em></div>
+        <div>parents: {parents%annotateparent}</div>
+        <a href="{url|urlescape}diff/{node|short}/{file|urlescape}{sessionvars%urlparameter}">diff</a>
+        <a href="{url|urlescape}rev/{node|short}{sessionvars%urlparameter}">changeset</a>
+      </div>
     </td>
     <td><pre><a class="linenr" href="#{lineid}">{linenumber}</a></pre></td>
     <td><pre>{line|escape}</pre></td>
   </tr>'
+annotateparent = '
+  <a href="{url|urlescape}annotate/{node|short}/{file|urlescape}{sessionvars%urlparameter}">{rev}</a>'
 difflineplus = '
   <a href="#{lineid}"></a><span id="{lineid}" class="difflineplus">{strip(line|escape, '\r\n')}</span>'
 difflineminus = '