merge default into stable for 5.4 release

contrib/packaging/pyoxidizer.bzl

@@ -0,0 +1,60 b''
+# Instructions:
+#
+# 1. cargo install --version 0.5.0 pyoxidizer
+# 2. cd /path/to/hg
+# 3. pyoxidizer build --path contrib/packaging [--release]
+# 4. Run build/pyoxidizer/<arch>/<debug|release>/app/hg
+#
+# If you need to build again, you need to remove the build/lib.* and
+# build/temp.* directories, otherwise PyOxidizer fails to pick up C
+# extensions. This is a bug in PyOxidizer.
+
+ROOT = CWD + "/../.."
+
+set_build_path(ROOT + "/build/pyoxidizer")
+
+def make_exe():
+    dist = default_python_distribution()
+
+    code = "import hgdemandimport; hgdemandimport.enable(); from mercurial import dispatch; dispatch.run()"
+
+    config = PythonInterpreterConfig(
+        raw_allocator = "system",
+        run_eval = code,
+        # We want to let the user load extensions from the file system
+        filesystem_importer = True,
+        # We need this to make resourceutil happy, since it looks for sys.frozen.
+        sys_frozen = True,
+        legacy_windows_stdio = True,
+    )
+
+    exe = dist.to_python_executable(
+        name = "hg",
+        config = config,
+    )
+
+    # Use setup.py install to build Mercurial and collect Python resources to
+    # embed in the executable.
+    resources = dist.setup_py_install(ROOT)
+    exe.add_python_resources(resources)
+
+    return exe
+
+def make_install(exe):
+    m = FileManifest()
+
+    # `hg` goes in root directory.
+    m.add_python_resource(".", exe)
+
+    templates = glob(
+        include=[ROOT + "/mercurial/templates/**/*"],
+        strip_prefix = ROOT + "/mercurial/",
+    )
+    m.add_manifest(templates)
+
+    return m
+
+register_target("exe", make_exe)
+register_target("app", make_install, depends = ["exe"], default = True)
+
+resolve_targets()

contrib/phab-clean.py

@@ -0,0 +1,93 b''
+#!/usr/bin/env python
+#
+# A small script to automatically reject idle Diffs
+#
+# you need to set the PHABBOT_USER and PHABBOT_TOKEN environment variable for authentication
+from __future__ import absolute_import, print_function
+
+import datetime
+import os
+import sys
+
+import phabricator
+
+MESSAGE = """There seems to have been no activities on this Diff for the past 3 Months.
+
+By policy, we are automatically moving it out of the `need-review` state.
+
+Please, move it back to `need-review` without hesitation if this diff should still be discussed.
+
+:baymax:need-review-idle:
+"""
+
+
+PHAB_URL = "https://phab.mercurial-scm.org/api/"
+USER = os.environ.get("PHABBOT_USER", "baymax")
+TOKEN = os.environ.get("PHABBOT_TOKEN")
+
+
+NOW = datetime.datetime.now()
+
+# 3 months in seconds
+DELAY = 60 * 60 * 24 * 30 * 3
+
+
+def get_all_diff(phab):
+    """Fetch all the diff that the need review"""
+    return phab.differential.query(
+        status="status-needs-review",
+        order="order-modified",
+        paths=[('HG', None)],
+    )
+
+
+def filter_diffs(diffs, older_than):
+    """filter diffs to only keep the one unmodified sin <older_than> seconds"""
+    olds = []
+    for d in diffs:
+        modified = int(d['dateModified'])
+        modified = datetime.datetime.fromtimestamp(modified)
+        d["idleFor"] = idle_for = NOW - modified
+        if idle_for.total_seconds() > older_than:
+            olds.append(d)
+    return olds
+
+
+def nudge_diff(phab, diff):
+    """Comment on the idle diff and reject it"""
+    diff_id = int(d['id'])
+    phab.differential.createcomment(
+        revision_id=diff_id, message=MESSAGE, action="reject"
+    )
+
+
+if not USER:
+    print(
+        "not user specified please set PHABBOT_USER and PHABBOT_TOKEN",
+        file=sys.stderr,
+    )
+elif not TOKEN:
+    print(
+        "not api-token specified please set PHABBOT_USER and PHABBOT_TOKEN",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+phab = phabricator.Phabricator(USER, host=PHAB_URL, token=TOKEN)
+phab.connect()
+phab.update_interfaces()
+print('Hello "%s".' % phab.user.whoami()['realName'])
+
+diffs = get_all_diff(phab)
+print("Found %d Diffs" % len(diffs))
+olds = filter_diffs(diffs, DELAY)
+print("Found %d old Diffs" % len(olds))
+for d in olds:
+    diff_id = d['id']
+    status = d['statusName']
+    modified = int(d['dateModified'])
+    idle_for = d["idleFor"]
+    msg = 'nudging D%s in "%s" state for %s'
+    print(msg % (diff_id, status, idle_for))
+    # uncomment to actually affect phab
+    nudge_diff(phab, d)

hgext/fastexport.py

@@ -0,0 +1,219 b''
+# Copyright 2020 Joerg Sonnenberger <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+"""export repositories as git fast-import stream"""
+
+# The format specification for fast-import streams can be found at
+# https://git-scm.com/docs/git-fast-import#_input_format
+
+from __future__ import absolute_import
+import re
+
+from mercurial.i18n import _
+from mercurial.node import hex, nullrev
+from mercurial.utils import stringutil
+from mercurial import (
+    error,
+    pycompat,
+    registrar,
+    scmutil,
+)
+from .convert import convcmd
+
+# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
+# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
+# be specifying the version(s) of Mercurial they are tested with, or
+# leave the attribute unspecified.
+testedwith = b"ships-with-hg-core"
+
+cmdtable = {}
+command = registrar.command(cmdtable)
+
+GIT_PERSON_PROHIBITED = re.compile(b'[<>\n"]')
+GIT_EMAIL_PROHIBITED = re.compile(b"[<> \n]")
+
+
+def convert_to_git_user(authormap, user, rev):
+    mapped_user = authormap.get(user, user)
+    user_person = stringutil.person(mapped_user)
+    user_email = stringutil.email(mapped_user)
+    if GIT_EMAIL_PROHIBITED.match(user_email) or GIT_PERSON_PROHIBITED.match(
+        user_person
+    ):
+        raise error.Abort(
+            _(b"Unable to parse user into person and email for revision %s")
+            % rev
+        )
+    if user_person:
+        return b'"' + user_person + b'" <' + user_email + b'>'
+    else:
+        return b"<" + user_email + b">"
+
+
+def convert_to_git_date(date):
+    timestamp, utcoff = date
+    tzsign = b"+" if utcoff < 0 else b"-"
+    if utcoff % 60 != 0:
+        raise error.Abort(
+            _(b"UTC offset in %b is not an integer number of seconds") % (date,)
+        )
+    utcoff = abs(utcoff) // 60
+    tzh = utcoff // 60
+    tzmin = utcoff % 60
+    return b"%d " % int(timestamp) + tzsign + b"%02d%02d" % (tzh, tzmin)
+
+
+def convert_to_git_ref(branch):
+    # XXX filter/map depending on git restrictions
+    return b"refs/heads/" + branch
+
+
+def write_data(buf, data, skip_newline):
+    buf.append(b"data %d\n" % len(data))
+    buf.append(data)
+    if not skip_newline or data[-1:] != b"\n":
+        buf.append(b"\n")
+
+
+def export_commit(ui, repo, rev, marks, authormap):
+    ctx = repo[rev]
+    revid = ctx.hex()
+    if revid in marks:
+        ui.warn(_(b"warning: revision %s already exported, skipped\n") % revid)
+        return
+    parents = [p for p in ctx.parents() if p.rev() != nullrev]
+    for p in parents:
+        if p.hex() not in marks:
+            ui.warn(
+                _(b"warning: parent %s of %s has not been exported, skipped\n")
+                % (p, revid)
+            )
+            return
+
+    # For all files modified by the commit, check if they have already
+    # been exported and otherwise dump the blob with the new mark.
+    for fname in ctx.files():
+        if fname not in ctx:
+            continue
+        filectx = ctx.filectx(fname)
+        filerev = hex(filectx.filenode())
+        if filerev not in marks:
+            mark = len(marks) + 1
+            marks[filerev] = mark
+            data = filectx.data()
+            buf = [b"blob\n", b"mark :%d\n" % mark]
+            write_data(buf, data, False)
+            ui.write(*buf, keepprogressbar=True)
+            del buf
+
+    # Assign a mark for the current revision for references by
+    # latter merge commits.
+    mark = len(marks) + 1
+    marks[revid] = mark
+
+    ref = convert_to_git_ref(ctx.branch())
+    buf = [
+        b"commit %s\n" % ref,
+        b"mark :%d\n" % mark,
+        b"committer %s %s\n"
+        % (
+            convert_to_git_user(authormap, ctx.user(), revid),
+            convert_to_git_date(ctx.date()),
+        ),
+    ]
+    write_data(buf, ctx.description(), True)
+    if parents:
+        buf.append(b"from :%d\n" % marks[parents[0].hex()])
+    if len(parents) == 2:
+        buf.append(b"merge :%d\n" % marks[parents[1].hex()])
+        p0ctx = repo[parents[0]]
+        files = ctx.manifest().diff(p0ctx.manifest())
+    else:
+        files = ctx.files()
+    filebuf = []
+    for fname in files:
+        if fname not in ctx:
+            filebuf.append((fname, b"D %s\n" % fname))
+        else:
+            filectx = ctx.filectx(fname)
+            filerev = filectx.filenode()
+            fileperm = b"755" if filectx.isexec() else b"644"
+            changed = b"M %s :%d %s\n" % (fileperm, marks[hex(filerev)], fname)
+            filebuf.append((fname, changed))
+    filebuf.sort()
+    buf.extend(changed for (fname, changed) in filebuf)
+    del filebuf
+    buf.append(b"\n")
+    ui.write(*buf, keepprogressbar=True)
+    del buf
+
+
+isrev = re.compile(b"^[0-9a-f]{40}$")
+
+
+@command(
+    b"fastexport",
+    [
+        (b"r", b"rev", [], _(b"revisions to export"), _(b"REV")),
+        (b"i", b"import-marks", b"", _(b"old marks file to read"), _(b"FILE")),
+        (b"e", b"export-marks", b"", _(b"new marks file to write"), _(b"FILE")),
+        (
+            b"A",
+            b"authormap",
+            b"",
+            _(b"remap usernames using this file"),
+            _(b"FILE"),
+        ),
+    ],
+    _(b"[OPTION]... [REV]..."),
+    helpcategory=command.CATEGORY_IMPORT_EXPORT,
+)
+def fastexport(ui, repo, *revs, **opts):
+    """export repository as git fast-import stream
+
+    This command lets you dump a repository as a human-readable text stream.
+    It can be piped into corresponding import routines like "git fast-import".
+    Incremental dumps can be created by using marks files.
+    """
+    opts = pycompat.byteskwargs(opts)
+
+    revs += tuple(opts.get(b"rev", []))
+    if not revs:
+        revs = scmutil.revrange(repo, [b":"])
+    else:
+        revs = scmutil.revrange(repo, revs)
+    if not revs:
+        raise error.Abort(_(b"no revisions matched"))
+    authorfile = opts.get(b"authormap")
+    if authorfile:
+        authormap = convcmd.readauthormap(ui, authorfile)
+    else:
+        authormap = {}
+
+    import_marks = opts.get(b"import_marks")
+    marks = {}
+    if import_marks:
+        with open(import_marks, "rb") as import_marks_file:
+            for line in import_marks_file:
+                line = line.strip()
+                if not isrev.match(line) or line in marks:
+                    raise error.Abort(_(b"Corrupted marks file"))
+                marks[line] = len(marks) + 1
+
+    revs.sort()
+    with ui.makeprogress(
+        _(b"exporting"), unit=_(b"revisions"), total=len(revs)
+    ) as progress:
+        for rev in revs:
+            export_commit(ui, repo, rev, marks, authormap)
+            progress.increment()
+
+    export_marks = opts.get(b"export_marks")
+    if export_marks:
+        with open(export_marks, "wb") as export_marks_file:
+            output_marks = [None] * len(marks)
+            for k, v in marks.items():
+                output_marks[v - 1] = k
+            for k in output_marks:
+                export_marks_file.write(k + b"\n")

hgext/git/TODO.md

@@ -0,0 +1,30 b''
+Octopus Merge Support
+=====================
+
+This will be moderately complicated, as we'll need to synthesize phony
+changeset entries to explode the octopus into "revisions" that only
+have two parents each. For today, we can probably just do something like
+
+    aaaaaaaaaaaaaaaaaaXX{20 bytes of exploded node's hex sha}
+
+where XX is a counter (so we could have as many as 255 parents in a
+git commit - more than I think we'd ever see.) That means that we can
+install some check in this extension to disallow checking out or
+otherwise interacting with the `aaaaaaaaaaaaaaaaaa` revisions.
+
+
+Interface Creation
+====================
+
+We at least need an interface definition for `changelog` in core that
+this extension can satisfy, and again for `basicstore`.
+
+
+Reason About Locking
+====================
+
+We should spend some time thinking hard about locking, especially on
+.git/index etc. We're probably adequately locking the _git_
+repository, but may not have enough locking correctness in places
+where hg does locking that git isn't aware of (notably the working
+copy, which I believe Git does not lock.)

hgext/git/__init__.py

@@ -0,0 +1,318 b''
+"""grant Mercurial the ability to operate on Git repositories. (EXPERIMENTAL)
+
+This is currently super experimental. It probably will consume your
+firstborn a la Rumpelstiltskin, etc.
+"""
+
+from __future__ import absolute_import
+
+import os
+
+from mercurial.i18n import _
+
+from mercurial import (
+    commands,
+    error,
+    extensions,
+    localrepo,
+    pycompat,
+    scmutil,
+    store,
+    util,
+)
+
+from . import (
+    dirstate,
+    gitlog,
+    gitutil,
+    index,
+)
+
+# TODO: extract an interface for this in core
+class gitstore(object):  # store.basicstore):
+    def __init__(self, path, vfstype):
+        self.vfs = vfstype(path)
+        self.path = self.vfs.base
+        self.createmode = store._calcmode(self.vfs)
+        # above lines should go away in favor of:
+        # super(gitstore, self).__init__(path, vfstype)
+
+        self.git = gitutil.get_pygit2().Repository(
+            os.path.normpath(os.path.join(path, b'..', b'.git'))
+        )
+        self._progress_factory = lambda *args, **kwargs: None
+
+    @util.propertycache
+    def _db(self):
+        # We lazy-create the database because we want to thread a
+        # progress callback down to the indexing process if it's
+        # required, and we don't have a ui handle in makestore().
+        return index.get_index(self.git, self._progress_factory)
+
+    def join(self, f):
+        """Fake store.join method for git repositories.
+
+        For the most part, store.join is used for @storecache
+        decorators to invalidate caches when various files
+        change. We'll map the ones we care about, and ignore the rest.
+        """
+        if f in (b'00changelog.i', b'00manifest.i'):
+            # This is close enough: in order for the changelog cache
+            # to be invalidated, HEAD will have to change.
+            return os.path.join(self.path, b'HEAD')
+        elif f == b'lock':
+            # TODO: we probably want to map this to a git lock, I
+            # suspect index.lock. We should figure out what the
+            # most-alike file is in git-land. For now we're risking
+            # bad concurrency errors if another git client is used.
+            return os.path.join(self.path, b'hgit-bogus-lock')
+        elif f in (b'obsstore', b'phaseroots', b'narrowspec', b'bookmarks'):
+            return os.path.join(self.path, b'..', b'.hg', f)
+        raise NotImplementedError(b'Need to pick file for %s.' % f)
+
+    def changelog(self, trypending):
+        # TODO we don't have a plan for trypending in hg's git support yet
+        return gitlog.changelog(self.git, self._db)
+
+    def manifestlog(self, repo, storenarrowmatch):
+        # TODO handle storenarrowmatch and figure out if we need the repo arg
+        return gitlog.manifestlog(self.git, self._db)
+
+    def invalidatecaches(self):
+        pass
+
+    def write(self, tr=None):
+        # normally this handles things like fncache writes, which we don't have
+        pass
+
+
+def _makestore(orig, requirements, storebasepath, vfstype):
+    if b'git' in requirements:
+        if not os.path.exists(os.path.join(storebasepath, b'..', b'.git')):
+            raise error.Abort(
+                _(
+                    b'repository specified git format in '
+                    b'.hg/requires but has no .git directory'
+                )
+            )
+        # Check for presence of pygit2 only here. The assumption is that we'll
+        # run this code iff we'll later need pygit2.
+        if gitutil.get_pygit2() is None:
+            raise error.Abort(
+                _(
+                    b'the git extension requires the Python '
+                    b'pygit2 library to be installed'
+                )
+            )
+
+        return gitstore(storebasepath, vfstype)
+    return orig(requirements, storebasepath, vfstype)
+
+
+class gitfilestorage(object):
+    def file(self, path):
+        if path[0:1] == b'/':
+            path = path[1:]
+        return gitlog.filelog(self.store.git, self.store._db, path)
+
+
+def _makefilestorage(orig, requirements, features, **kwargs):
+    store = kwargs['store']
+    if isinstance(store, gitstore):
+        return gitfilestorage
+    return orig(requirements, features, **kwargs)
+
+
+def _setupdothg(ui, path):
+    dothg = os.path.join(path, b'.hg')
+    if os.path.exists(dothg):
+        ui.warn(_(b'git repo already initialized for hg\n'))
+    else:
+        os.mkdir(os.path.join(path, b'.hg'))
+        # TODO is it ok to extend .git/info/exclude like this?
+        with open(
+            os.path.join(path, b'.git', b'info', b'exclude'), 'ab'
+        ) as exclude:
+            exclude.write(b'\n.hg\n')
+    with open(os.path.join(dothg, b'requires'), 'wb') as f:
+        f.write(b'git\n')
+
+
+_BMS_PREFIX = 'refs/heads/'
+
+
+class gitbmstore(object):
+    def __init__(self, gitrepo):
+        self.gitrepo = gitrepo
+        self._aclean = True
+        self._active = gitrepo.references['HEAD']  # git head, not mark
+
+    def __contains__(self, name):
+        return (
+            _BMS_PREFIX + pycompat.fsdecode(name)
+        ) in self.gitrepo.references
+
+    def __iter__(self):
+        for r in self.gitrepo.listall_references():
+            if r.startswith(_BMS_PREFIX):
+                yield pycompat.fsencode(r[len(_BMS_PREFIX) :])
+
+    def __getitem__(self, k):
+        return (
+            self.gitrepo.references[_BMS_PREFIX + pycompat.fsdecode(k)]
+            .peel()
+            .id.raw
+        )
+
+    def get(self, k, default=None):
+        try:
+            if k in self:
+                return self[k]
+            return default
+        except gitutil.get_pygit2().InvalidSpecError:
+            return default
+
+    @property
+    def active(self):
+        h = self.gitrepo.references['HEAD']
+        if not isinstance(h.target, str) or not h.target.startswith(
+            _BMS_PREFIX
+        ):
+            return None
+        return pycompat.fsencode(h.target[len(_BMS_PREFIX) :])
+
+    @active.setter
+    def active(self, mark):
+        githead = mark is not None and (_BMS_PREFIX + mark) or None
+        if githead is not None and githead not in self.gitrepo.references:
+            raise AssertionError(b'bookmark %s does not exist!' % mark)
+
+        self._active = githead
+        self._aclean = False
+
+    def _writeactive(self):
+        if self._aclean:
+            return
+        self.gitrepo.references.create('HEAD', self._active, True)
+        self._aclean = True
+
+    def names(self, node):
+        r = []
+        for ref in self.gitrepo.listall_references():
+            if not ref.startswith(_BMS_PREFIX):
+                continue
+            if self.gitrepo.references[ref].peel().id.raw != node:
+                continue
+            r.append(pycompat.fsencode(ref[len(_BMS_PREFIX) :]))
+        return r
+
+    # Cleanup opportunity: this is *identical* to core's bookmarks store.
+    def expandname(self, bname):
+        if bname == b'.':
+            if self.active:
+                return self.active
+            raise error.RepoLookupError(_(b"no active bookmark"))
+        return bname
+
+    def applychanges(self, repo, tr, changes):
+        """Apply a list of changes to bookmarks
+        """
+        # TODO: this should respect transactions, but that's going to
+        # require enlarging the gitbmstore to know how to do in-memory
+        # temporary writes and read those back prior to transaction
+        # finalization.
+        for name, node in changes:
+            if node is None:
+                self.gitrepo.references.delete(
+                    _BMS_PREFIX + pycompat.fsdecode(name)
+                )
+            else:
+                self.gitrepo.references.create(
+                    _BMS_PREFIX + pycompat.fsdecode(name),
+                    gitutil.togitnode(node),
+                    force=True,
+                )
+
+    def checkconflict(self, mark, force=False, target=None):
+        githead = _BMS_PREFIX + mark
+        cur = self.gitrepo.references['HEAD']
+        if githead in self.gitrepo.references and not force:
+            if target:
+                if self.gitrepo.references[githead] == target and target == cur:
+                    # re-activating a bookmark
+                    return []
+                # moving a bookmark - forward?
+                raise NotImplementedError
+            raise error.Abort(
+                _(b"bookmark '%s' already exists (use -f to force)") % mark
+            )
+        if len(mark) > 3 and not force:
+            try:
+                shadowhash = scmutil.isrevsymbol(self._repo, mark)
+            except error.LookupError:  # ambiguous identifier
+                shadowhash = False
+            if shadowhash:
+                self._repo.ui.warn(
+                    _(
+                        b"bookmark %s matches a changeset hash\n"
+                        b"(did you leave a -r out of an 'hg bookmark' "
+                        b"command?)\n"
+                    )
+                    % mark
+                )
+        return []
+
+
+def init(orig, ui, dest=b'.', **opts):
+    if opts.get('git', False):
+        path = os.path.abspath(dest)
+        # TODO: walk up looking for the git repo
+        _setupdothg(ui, path)
+        return 0
+    return orig(ui, dest=dest, **opts)
+
+
+def reposetup(ui, repo):
+    if repo.local() and isinstance(repo.store, gitstore):
+        orig = repo.__class__
+        repo.store._progress_factory = repo.ui.makeprogress
+
+        class gitlocalrepo(orig):
+            def _makedirstate(self):
+                # TODO narrow support here
+                return dirstate.gitdirstate(
+                    self.ui, self.vfs.base, self.store.git
+                )
+
+            def commit(self, *args, **kwargs):
+                ret = orig.commit(self, *args, **kwargs)
+                tid = self.store.git[gitutil.togitnode(ret)].tree.id
+                # DANGER! This will flush any writes staged to the
+                # index in Git, but we're sidestepping the index in a
+                # way that confuses git when we commit. Alas.
+                self.store.git.index.read_tree(tid)
+                self.store.git.index.write()
+                return ret
+
+            @property
+            def _bookmarks(self):
+                return gitbmstore(self.store.git)
+
+        repo.__class__ = gitlocalrepo
+    return repo
+
+
+def _featuresetup(ui, supported):
+    # don't die on seeing a repo with the git requirement
+    supported |= {b'git'}
+
+
+def extsetup(ui):
+    extensions.wrapfunction(localrepo, b'makestore', _makestore)
+    extensions.wrapfunction(localrepo, b'makefilestorage', _makefilestorage)
+    # Inject --git flag for `hg init`
+    entry = extensions.wrapcommand(commands.table, b'init', init)
+    entry[1].extend(
+        [(b'', b'git', None, b'setup up a git repository instead of hg')]
+    )
+    localrepo.featuresetupfuncs.add(_featuresetup)

hgext/git/dirstate.py

@@ -0,0 +1,307 b''
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import os
+
+from mercurial import (
+    error,
+    extensions,
+    match as matchmod,
+    node as nodemod,
+    pycompat,
+    scmutil,
+    util,
+)
+from mercurial.interfaces import (
+    dirstate as intdirstate,
+    util as interfaceutil,
+)
+
+from . import gitutil
+
+pygit2 = gitutil.get_pygit2()
+
+
+def readpatternfile(orig, filepath, warn, sourceinfo=False):
+    if not (b'info/exclude' in filepath or filepath.endswith(b'.gitignore')):
+        return orig(filepath, warn, sourceinfo=False)
+    result = []
+    warnings = []
+    with open(filepath, b'rb') as fp:
+        for l in fp:
+            l = l.strip()
+            if not l or l.startswith(b'#'):
+                continue
+            if l.startswith(b'!'):
+                warnings.append(b'unsupported ignore pattern %s' % l)
+                continue
+            if l.startswith(b'/'):
+                result.append(b'rootglob:' + l[1:])
+            else:
+                result.append(b'relglob:' + l)
+    return result, warnings
+
+
+extensions.wrapfunction(matchmod, b'readpatternfile', readpatternfile)
+
+
+_STATUS_MAP = {}
+if pygit2:
+    _STATUS_MAP = {
+        pygit2.GIT_STATUS_CONFLICTED: b'm',
+        pygit2.GIT_STATUS_CURRENT: b'n',
+        pygit2.GIT_STATUS_IGNORED: b'?',
+        pygit2.GIT_STATUS_INDEX_DELETED: b'r',
+        pygit2.GIT_STATUS_INDEX_MODIFIED: b'n',
+        pygit2.GIT_STATUS_INDEX_NEW: b'a',
+        pygit2.GIT_STATUS_INDEX_RENAMED: b'a',
+        pygit2.GIT_STATUS_INDEX_TYPECHANGE: b'n',
+        pygit2.GIT_STATUS_WT_DELETED: b'r',
+        pygit2.GIT_STATUS_WT_MODIFIED: b'n',
+        pygit2.GIT_STATUS_WT_NEW: b'?',
+        pygit2.GIT_STATUS_WT_RENAMED: b'a',
+        pygit2.GIT_STATUS_WT_TYPECHANGE: b'n',
+        pygit2.GIT_STATUS_WT_UNREADABLE: b'?',
+        pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_WT_MODIFIED: 'm',
+    }
+
+
[e-mail](intdirstate.idirstate)
+class gitdirstate(object):
+    def __init__(self, ui, root, gitrepo):
+        self._ui = ui
+        self._root = os.path.dirname(root)
+        self.git = gitrepo
+        self._plchangecallbacks = {}
+
+    def p1(self):
+        try:
+            return self.git.head.peel().id.raw
+        except pygit2.GitError:
+            # Typically happens when peeling HEAD fails, as in an
+            # empty repository.
+            return nodemod.nullid
+
+    def p2(self):
+        # TODO: MERGE_HEAD? something like that, right?
+        return nodemod.nullid
+
+    def setparents(self, p1, p2=nodemod.nullid):
+        assert p2 == nodemod.nullid, b'TODO merging support'
+        self.git.head.set_target(gitutil.togitnode(p1))
+
+    @util.propertycache
+    def identity(self):
+        return util.filestat.frompath(
+            os.path.join(self._root, b'.git', b'index')
+        )
+
+    def branch(self):
+        return b'default'
+
+    def parents(self):
+        # TODO how on earth do we find p2 if a merge is in flight?
+        return self.p1(), nodemod.nullid
+
+    def __iter__(self):
+        return (pycompat.fsencode(f.path) for f in self.git.index)
+
+    def items(self):
+        for ie in self.git.index:
+            yield ie.path, None  # value should be a dirstatetuple
+
+    # py2,3 compat forward
+    iteritems = items
+
+    def __getitem__(self, filename):
+        try:
+            gs = self.git.status_file(filename)
+        except KeyError:
+            return b'?'
+        return _STATUS_MAP[gs]
+
+    def __contains__(self, filename):
+        try:
+            gs = self.git.status_file(filename)
+            return _STATUS_MAP[gs] != b'?'
+        except KeyError:
+            return False
+
+    def status(self, match, subrepos, ignored, clean, unknown):
+        # TODO handling of clean files - can we get that from git.status()?
+        modified, added, removed, deleted, unknown, ignored, clean = (
+            [],
+            [],
+            [],
+            [],
+            [],
+            [],
+            [],
+        )
+        gstatus = self.git.status()
+        for path, status in gstatus.items():
+            path = pycompat.fsencode(path)
+            if status == pygit2.GIT_STATUS_IGNORED:
+                if path.endswith(b'/'):
+                    continue
+                ignored.append(path)
+            elif status in (
+                pygit2.GIT_STATUS_WT_MODIFIED,
+                pygit2.GIT_STATUS_INDEX_MODIFIED,
+                pygit2.GIT_STATUS_WT_MODIFIED
+                | pygit2.GIT_STATUS_INDEX_MODIFIED,
+            ):
+                modified.append(path)
+            elif status == pygit2.GIT_STATUS_INDEX_NEW:
+                added.append(path)
+            elif status == pygit2.GIT_STATUS_WT_NEW:
+                unknown.append(path)
+            elif status == pygit2.GIT_STATUS_WT_DELETED:
+                deleted.append(path)
+            elif status == pygit2.GIT_STATUS_INDEX_DELETED:
+                removed.append(path)
+            else:
+                raise error.Abort(
+                    b'unhandled case: status for %r is %r' % (path, status)
+                )
+
+        # TODO are we really always sure of status here?
+        return (
+            False,
+            scmutil.status(
+                modified, added, removed, deleted, unknown, ignored, clean
+            ),
+        )
+
+    def flagfunc(self, buildfallback):
+        # TODO we can do better
+        return buildfallback()
+
+    def getcwd(self):
+        # TODO is this a good way to do this?
+        return os.path.dirname(
+            os.path.dirname(pycompat.fsencode(self.git.path))
+        )
+
+    def normalize(self, path):
+        normed = util.normcase(path)
+        assert normed == path, b"TODO handling of case folding: %s != %s" % (
+            normed,
+            path,
+        )
+        return path
+
+    @property
+    def _checklink(self):
+        return util.checklink(os.path.dirname(pycompat.fsencode(self.git.path)))
+
+    def copies(self):
+        # TODO support copies?
+        return {}
+
+    # # TODO what the heck is this
+    _filecache = set()
+
+    def pendingparentchange(self):
+        # TODO: we need to implement the context manager bits and
+        # correctly stage/revert index edits.
+        return False
+
+    def write(self, tr):
+        # TODO: call parent change callbacks
+
+        if tr:
+
+            def writeinner(category):
+                self.git.index.write()
+
+            tr.addpending(b'gitdirstate', writeinner)
+        else:
+            self.git.index.write()
+
+    def pathto(self, f, cwd=None):
+        if cwd is None:
+            cwd = self.getcwd()
+        # TODO core dirstate does something about slashes here
+        assert isinstance(f, bytes)
+        r = util.pathto(self._root, cwd, f)
+        return r
+
+    def matches(self, match):
+        for x in self.git.index:
+            p = pycompat.fsencode(x.path)
+            if match(p):
+                yield p
+
+    def normal(self, f, parentfiledata=None):
+        """Mark a file normal and clean."""
+        # TODO: for now we just let libgit2 re-stat the file. We can
+        # clearly do better.
+
+    def normallookup(self, f):
+        """Mark a file normal, but possibly dirty."""
+        # TODO: for now we just let libgit2 re-stat the file. We can
+        # clearly do better.
+
+    def walk(self, match, subrepos, unknown, ignored, full=True):
+        # TODO: we need to use .status() and not iterate the index,
+        # because the index doesn't force a re-walk and so `hg add` of
+        # a new file without an intervening call to status will
+        # silently do nothing.
+        r = {}
+        cwd = self.getcwd()
+        for path, status in self.git.status().items():
+            if path.startswith('.hg/'):
+                continue
+            path = pycompat.fsencode(path)
+            if not match(path):
+                continue
+            # TODO construct the stat info from the status object?
+            try:
+                s = os.stat(os.path.join(cwd, path))
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                continue
+            r[path] = s
+        return r
+
+    def savebackup(self, tr, backupname):
+        # TODO: figure out a strategy for saving index backups.
+        pass
+
+    def restorebackup(self, tr, backupname):
+        # TODO: figure out a strategy for saving index backups.
+        pass
+
+    def add(self, f):
+        self.git.index.add(pycompat.fsdecode(f))
+
+    def drop(self, f):
+        self.git.index.remove(pycompat.fsdecode(f))
+
+    def remove(self, f):
+        self.git.index.remove(pycompat.fsdecode(f))
+
+    def copied(self, path):
+        # TODO: track copies?
+        return None
+
+    @contextlib.contextmanager
+    def parentchange(self):
+        # TODO: track this maybe?
+        yield
+
+    def addparentchangecallback(self, category, callback):
+        # TODO: should this be added to the dirstate interface?
+        self._plchangecallbacks[category] = callback
+
+    def clearbackup(self, tr, backupname):
+        # TODO
+        pass
+
+    def setbranch(self, branch):
+        raise error.Abort(
+            b'git repos do not support branches. try using bookmarks'
+        )

hgext/git/gitlog.py

@@ -0,0 +1,466 b''
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+
+from mercurial import (
+    ancestor,
+    changelog as hgchangelog,
+    dagop,
+    encoding,
+    error,
+    manifest,
+    node as nodemod,
+    pycompat,
+)
+from mercurial.interfaces import (
+    repository,
+    util as interfaceutil,
+)
+from mercurial.utils import stringutil
+from . import (
+    gitutil,
+    index,
+    manifest as gitmanifest,
+)
+
+pygit2 = gitutil.get_pygit2()
+
+
+class baselog(object):  # revlog.revlog):
+    """Common implementations between changelog and manifestlog."""
+
+    def __init__(self, gr, db):
+        self.gitrepo = gr
+        self._db = db
+
+    def __len__(self):
+        return int(
+            self._db.execute('SELECT COUNT(*) FROM changelog').fetchone()[0]
+        )
+
+    def rev(self, n):
+        if n == nodemod.nullid:
+            return -1
+        t = self._db.execute(
+            'SELECT rev FROM changelog WHERE node = ?', (gitutil.togitnode(n),)
+        ).fetchone()
+        if t is None:
+            raise error.LookupError(n, b'00changelog.i', _(b'no node %d'))
+        return t[0]
+
+    def node(self, r):
+        if r == nodemod.nullrev:
+            return nodemod.nullid
+        t = self._db.execute(
+            'SELECT node FROM changelog WHERE rev = ?', (r,)
+        ).fetchone()
+        if t is None:
+            raise error.LookupError(r, b'00changelog.i', _(b'no node'))
+        return nodemod.bin(t[0])
+
+    def hasnode(self, n):
+        t = self._db.execute(
+            'SELECT node FROM changelog WHERE node = ?', (n,)
+        ).fetchone()
+        return t is not None
+
+
+class baselogindex(object):
+    def __init__(self, log):
+        self._log = log
+
+    def has_node(self, n):
+        return self._log.rev(n) != -1
+
+    def __len__(self):
+        return len(self._log)
+
+    def __getitem__(self, idx):
+        p1rev, p2rev = self._log.parentrevs(idx)
+        # TODO: it's messy that the index leaks so far out of the
+        # storage layer that we have to implement things like reading
+        # this raw tuple, which exposes revlog internals.
+        return (
+            # Pretend offset is just the index, since we don't really care.
+            idx,
+            # Same with lengths
+            idx,  # length
+            idx,  # rawsize
+            -1,  # delta base
+            idx,  # linkrev TODO is this right?
+            p1rev,
+            p2rev,
+            self._log.node(idx),
+        )
+
+
+# TODO: an interface for the changelog type?
+class changelog(baselog):
+    def __contains__(self, rev):
+        try:
+            self.node(rev)
+            return True
+        except error.LookupError:
+            return False
+
+    def __iter__(self):
+        return iter(pycompat.xrange(len(self)))
+
+    @property
+    def filteredrevs(self):
+        # TODO: we should probably add a refs/hg/ namespace for hidden
+        # heads etc, but that's an idea for later.
+        return set()
+
+    @property
+    def index(self):
+        return baselogindex(self)
+
+    @property
+    def nodemap(self):
+        r = {
+            nodemod.bin(v[0]): v[1]
+            for v in self._db.execute('SELECT node, rev FROM changelog')
+        }
+        r[nodemod.nullid] = nodemod.nullrev
+        return r
+
+    def tip(self):
+        t = self._db.execute(
+            'SELECT node FROM changelog ORDER BY rev DESC LIMIT 1'
+        ).fetchone()
+        if t:
+            return nodemod.bin(t[0])
+        return nodemod.nullid
+
+    def revs(self, start=0, stop=None):
+        if stop is None:
+            stop = self.tip()
+        t = self._db.execute(
+            'SELECT rev FROM changelog '
+            'WHERE rev >= ? AND rev <= ? '
+            'ORDER BY REV ASC',
+            (start, stop),
+        )
+        return (int(r[0]) for r in t)
+
+    def _partialmatch(self, id):
+        if nodemod.wdirhex.startswith(id):
+            raise error.WdirUnsupported
+        candidates = [
+            nodemod.bin(x[0])
+            for x in self._db.execute(
+                'SELECT node FROM changelog WHERE node LIKE ?', (id + b'%',)
+            )
+        ]
+        if nodemod.nullhex.startswith(id):
+            candidates.append(nodemod.nullid)
+        if len(candidates) > 1:
+            raise error.AmbiguousPrefixLookupError(
+                id, b'00changelog.i', _(b'ambiguous identifier')
+            )
+        if candidates:
+            return candidates[0]
+        return None
+
+    def flags(self, rev):
+        return 0
+
+    def shortest(self, node, minlength=1):
+        nodehex = nodemod.hex(node)
+        for attempt in pycompat.xrange(minlength, len(nodehex) + 1):
+            candidate = nodehex[:attempt]
+            matches = int(
+                self._db.execute(
+                    'SELECT COUNT(*) FROM changelog WHERE node LIKE ?',
+                    (pycompat.sysstr(candidate + b'%'),),
+                ).fetchone()[0]
+            )
+            if matches == 1:
+                return candidate
+        return nodehex
+
+    def headrevs(self, revs=None):
+        realheads = [
+            int(x[0])
+            for x in self._db.execute(
+                'SELECT rev FROM changelog '
+                'INNER JOIN heads ON changelog.node = heads.node'
+            )
+        ]
+        if revs:
+            return sorted([r for r in revs if r in realheads])
+        return sorted(realheads)
+
+    def changelogrevision(self, nodeorrev):
+        # Ensure we have a node id
+        if isinstance(nodeorrev, int):
+            n = self.node(nodeorrev)
+        else:
+            n = nodeorrev
+        # handle looking up nullid
+        if n == nodemod.nullid:
+            return hgchangelog._changelogrevision(extra={})
+        hn = gitutil.togitnode(n)
+        # We've got a real commit!
+        files = [
+            r[0]
+            for r in self._db.execute(
+                'SELECT filename FROM changedfiles '
+                'WHERE node = ? and filenode != ?',
+                (hn, gitutil.nullgit),
+            )
+        ]
+        filesremoved = [
+            r[0]
+            for r in self._db.execute(
+                'SELECT filename FROM changedfiles '
+                'WHERE node = ? and filenode = ?',
+                (hn, nodemod.nullhex),
+            )
+        ]
+        c = self.gitrepo[hn]
+        return hgchangelog._changelogrevision(
+            manifest=n,  # pretend manifest the same as the commit node
+            user=b'%s <%s>'
+            % (c.author.name.encode('utf8'), c.author.email.encode('utf8')),
+            date=(c.author.time, -c.author.offset * 60),
+            files=files,
+            # TODO filesadded in the index
+            filesremoved=filesremoved,
+            description=c.message.encode('utf8'),
+            # TODO do we want to handle extra? how?
+            extra={b'branch': b'default'},
+        )
+
+    def ancestors(self, revs, stoprev=0, inclusive=False):
+        revs = list(revs)
+        tip = self.rev(self.tip())
+        for r in revs:
+            if r > tip:
+                raise IndexError(b'Invalid rev %r' % r)
+        return ancestor.lazyancestors(
+            self.parentrevs, revs, stoprev=stoprev, inclusive=inclusive
+        )
+
+    # Cleanup opportunity: this is *identical* to the revlog.py version
+    def descendants(self, revs):
+        return dagop.descendantrevs(revs, self.revs, self.parentrevs)
+
+    def reachableroots(self, minroot, heads, roots, includepath=False):
+        return dagop._reachablerootspure(
+            self.parentrevs, minroot, roots, heads, includepath
+        )
+
+    # Cleanup opportunity: this is *identical* to the revlog.py version
+    def isancestor(self, a, b):
+        a, b = self.rev(a), self.rev(b)
+        return self.isancestorrev(a, b)
+
+    # Cleanup opportunity: this is *identical* to the revlog.py version
+    def isancestorrev(self, a, b):
+        if a == nodemod.nullrev:
+            return True
+        elif a == b:
+            return True
+        elif a > b:
+            return False
+        return bool(self.reachableroots(a, [b], [a], includepath=False))
+
+    def parentrevs(self, rev):
+        n = self.node(rev)
+        hn = gitutil.togitnode(n)
+        c = self.gitrepo[hn]
+        p1 = p2 = nodemod.nullrev
+        if c.parents:
+            p1 = self.rev(c.parents[0].id.raw)
+            if len(c.parents) > 2:
+                raise error.Abort(b'TODO octopus merge handling')
+            if len(c.parents) == 2:
+                p2 = self.rev(c.parents[1].id.raw)
+        return p1, p2
+
+    # Private method is used at least by the tags code.
+    _uncheckedparentrevs = parentrevs
+
+    def commonancestorsheads(self, a, b):
+        # TODO the revlog verson of this has a C path, so we probably
+        # need to optimize this...
+        a, b = self.rev(a), self.rev(b)
+        return [
+            self.node(n)
+            for n in ancestor.commonancestorsheads(self.parentrevs, a, b)
+        ]
+
+    def branchinfo(self, rev):
+        """Git doesn't do named branches, so just put everything on default."""
+        return b'default', False
+
+    def delayupdate(self, tr):
+        # TODO: I think we can elide this because we're just dropping
+        # an object in the git repo?
+        pass
+
+    def add(
+        self,
+        manifest,
+        files,
+        desc,
+        transaction,
+        p1,
+        p2,
+        user,
+        date=None,
+        extra=None,
+        p1copies=None,
+        p2copies=None,
+        filesadded=None,
+        filesremoved=None,
+    ):
+        parents = []
+        hp1, hp2 = gitutil.togitnode(p1), gitutil.togitnode(p2)
+        if p1 != nodemod.nullid:
+            parents.append(hp1)
+        if p2 and p2 != nodemod.nullid:
+            parents.append(hp2)
+        assert date is not None
+        timestamp, tz = date
+        sig = pygit2.Signature(
+            encoding.unifromlocal(stringutil.person(user)),
+            encoding.unifromlocal(stringutil.email(user)),
+            timestamp,
+            -(tz // 60),
+        )
+        oid = self.gitrepo.create_commit(
+            None, sig, sig, desc, gitutil.togitnode(manifest), parents
+        )
+        # Set up an internal reference to force the commit into the
+        # changelog. Hypothetically, we could even use this refs/hg/
+        # namespace to allow for anonymous heads on git repos, which
+        # would be neat.
+        self.gitrepo.references.create(
+            'refs/hg/internal/latest-commit', oid, force=True
+        )
+        # Reindex now to pick up changes. We omit the progress
+        # callback because this will be very quick.
+        index._index_repo(self.gitrepo, self._db)
+        return oid.raw
+
+
+class manifestlog(baselog):
+    def __getitem__(self, node):
+        return self.get(b'', node)
+
+    def get(self, relpath, node):
+        if node == nodemod.nullid:
+            # TODO: this should almost certainly be a memgittreemanifestctx
+            return manifest.memtreemanifestctx(self, relpath)
+        commit = self.gitrepo[gitutil.togitnode(node)]
+        t = commit.tree
+        if relpath:
+            parts = relpath.split(b'/')
+            for p in parts:
+                te = t[p]
+                t = self.gitrepo[te.id]
+        return gitmanifest.gittreemanifestctx(self.gitrepo, t)
+
+
[e-mail](repository.ifilestorage)
+class filelog(baselog):
+    def __init__(self, gr, db, path):
+        super(filelog, self).__init__(gr, db)
+        assert isinstance(path, bytes)
+        self.path = path
+
+    def read(self, node):
+        if node == nodemod.nullid:
+            return b''
+        return self.gitrepo[gitutil.togitnode(node)].data
+
+    def lookup(self, node):
+        if len(node) not in (20, 40):
+            node = int(node)
+        if isinstance(node, int):
+            assert False, b'todo revnums for nodes'
+        if len(node) == 40:
+            node = nodemod.bin(node)
+        hnode = gitutil.togitnode(node)
+        if hnode in self.gitrepo:
+            return node
+        raise error.LookupError(self.path, node, _(b'no match found'))
+
+    def cmp(self, node, text):
+        """Returns True if text is different than content at `node`."""
+        return self.read(node) != text
+
+    def add(self, text, meta, transaction, link, p1=None, p2=None):
+        assert not meta  # Should we even try to handle this?
+        return self.gitrepo.create_blob(text).raw
+
+    def __iter__(self):
+        for clrev in self._db.execute(
+            '''
+SELECT rev FROM changelog
+INNER JOIN changedfiles ON changelog.node = changedfiles.node
+WHERE changedfiles.filename = ? AND changedfiles.filenode != ?
+        ''',
+            (pycompat.fsdecode(self.path), gitutil.nullgit),
+        ):
+            yield clrev[0]
+
+    def linkrev(self, fr):
+        return fr
+
+    def rev(self, node):
+        row = self._db.execute(
+            '''
+SELECT rev FROM changelog
+INNER JOIN changedfiles ON changelog.node = changedfiles.node
+WHERE changedfiles.filename = ? AND changedfiles.filenode = ?''',
+            (pycompat.fsdecode(self.path), gitutil.togitnode(node)),
+        ).fetchone()
+        if row is None:
+            raise error.LookupError(self.path, node, _(b'no such node'))
+        return int(row[0])
+
+    def node(self, rev):
+        maybe = self._db.execute(
+            '''SELECT filenode FROM changedfiles
+INNER JOIN changelog ON changelog.node = changedfiles.node
+WHERE changelog.rev = ? AND filename = ?
+''',
+            (rev, pycompat.fsdecode(self.path)),
+        ).fetchone()
+        if maybe is None:
+            raise IndexError('gitlog %r out of range %d' % (self.path, rev))
+        return nodemod.bin(maybe[0])
+
+    def parents(self, node):
+        gn = gitutil.togitnode(node)
+        gp = pycompat.fsdecode(self.path)
+        ps = []
+        for p in self._db.execute(
+            '''SELECT p1filenode, p2filenode FROM changedfiles
+WHERE filenode = ? AND filename = ?
+''',
+            (gn, gp),
+        ).fetchone():
+            if p is None:
+                commit = self._db.execute(
+                    "SELECT node FROM changedfiles "
+                    "WHERE filenode = ? AND filename = ?",
+                    (gn, gp),
+                ).fetchone()[0]
+                # This filelog is missing some data. Build the
+                # filelog, then recurse (which will always find data).
+                if pycompat.ispy3:
+                    commit = commit.decode('ascii')
+                index.fill_in_filelog(self.gitrepo, self._db, commit, gp, gn)
+                return self.parents(node)
+            else:
+                ps.append(nodemod.bin(p))
+        return ps
+
+    def renamed(self, node):
+        # TODO: renames/copies
+        return False

hgext/git/gitutil.py

@@ -0,0 +1,40 b''
+"""utilities to assist in working with pygit2"""
+from __future__ import absolute_import
+
+from mercurial.node import bin, hex, nullid
+
+from mercurial import pycompat
+
+pygit2_module = None
+
+
+def get_pygit2():
+    global pygit2_module
+    if pygit2_module is None:
+        try:
+            import pygit2 as pygit2_module
+
+            pygit2_module.InvalidSpecError
+        except (ImportError, AttributeError):
+            pass
+    return pygit2_module
+
+
+def togitnode(n):
+    """Wrapper to convert a Mercurial binary node to a unicode hexlified node.
+
+    pygit2 and sqlite both need nodes as strings, not bytes.
+    """
+    assert len(n) == 20
+    return pycompat.sysstr(hex(n))
+
+
+def fromgitnode(n):
+    """Opposite of togitnode."""
+    assert len(n) == 40
+    if pycompat.ispy3:
+        return bin(n.encode('ascii'))
+    return bin(n)
+
+
+nullgit = togitnode(nullid)

hgext/git/index.py

@@ -0,0 +1,350 b''
+from __future__ import absolute_import
+
+import collections
+import os
+import sqlite3
+
+from mercurial.i18n import _
+
+from mercurial import (
+    encoding,
+    error,
+    node as nodemod,
+    pycompat,
+)
+
+from . import gitutil
+
+
+pygit2 = gitutil.get_pygit2()
+
+_CURRENT_SCHEMA_VERSION = 1
+_SCHEMA = (
+    """
+CREATE TABLE refs (
+  -- node and name are unique together. There may be more than one name for
+  -- a given node, and there may be no name at all for a given node (in the
+  -- case of an anonymous hg head).
+  node TEXT NOT NULL,
+  name TEXT
+);
+
+-- The "possible heads" of the repository, which we use to figure out
+-- if we need to re-walk the changelog.
+CREATE TABLE possible_heads (
+  node TEXT NOT NULL
+);
+
+-- The topological heads of the changelog, which hg depends on.
+CREATE TABLE heads (
+  node TEXT NOT NULL
+);
+
+-- A total ordering of the changelog
+CREATE TABLE changelog (
+  rev INTEGER NOT NULL PRIMARY KEY,
+  node TEXT NOT NULL,
+  p1 TEXT,
+  p2 TEXT
+);
+
+CREATE UNIQUE INDEX changelog_node_idx ON changelog(node);
+CREATE UNIQUE INDEX changelog_node_rev_idx ON changelog(rev, node);
+
+-- Changed files for each commit, which lets us dynamically build
+-- filelogs.
+CREATE TABLE changedfiles (
+  node TEXT NOT NULL,
+  filename TEXT NOT NULL,
+  -- 40 zeroes for deletions
+  filenode TEXT NOT NULL,
+-- to handle filelog parentage:
+  p1node TEXT,
+  p1filenode TEXT,
+  p2node TEXT,
+  p2filenode TEXT
+);
+
+CREATE INDEX changedfiles_nodes_idx
+  ON changedfiles(node);
+
+PRAGMA user_version=%d
+"""
+    % _CURRENT_SCHEMA_VERSION
+)
+
+
+def _createdb(path):
+    # print('open db', path)
+    # import traceback
+    # traceback.print_stack()
+    db = sqlite3.connect(encoding.strfromlocal(path))
+    db.text_factory = bytes
+
+    res = db.execute('PRAGMA user_version').fetchone()[0]
+
+    # New database.
+    if res == 0:
+        for statement in _SCHEMA.split(';'):
+            db.execute(statement.strip())
+
+        db.commit()
+
+    elif res == _CURRENT_SCHEMA_VERSION:
+        pass
+
+    else:
+        raise error.Abort(_(b'sqlite database has unrecognized version'))
+
+    db.execute('PRAGMA journal_mode=WAL')
+
+    return db
+
+
+_OUR_ORDER = ()
+if pygit2:
+    _OUR_ORDER = (
+        pygit2.GIT_SORT_TOPOLOGICAL
+        | pygit2.GIT_SORT_TIME
+        | pygit2.GIT_SORT_REVERSE
+    )
+
+_DIFF_FLAGS = 1 << 21  # GIT_DIFF_FORCE_BINARY, which isn't exposed by pygit2
+
+
+def _find_nearest_ancestor_introducing_node(
+    db, gitrepo, file_path, walk_start, filenode
+):
+    """Find the nearest ancestor that introduces a file node.
+
+    Args:
+      db: a handle to our sqlite database.
+      gitrepo: A pygit2.Repository instance.
+      file_path: the path of a file in the repo
+      walk_start: a pygit2.Oid that is a commit where we should start walking
+                  for our nearest ancestor.
+
+    Returns:
+      A hexlified SHA that is the commit ID of the next-nearest parent.
+    """
+    assert isinstance(file_path, str), 'file_path must be str, got %r' % type(
+        file_path
+    )
+    assert isinstance(filenode, str), 'filenode must be str, got %r' % type(
+        filenode
+    )
+    parent_options = {
+        row[0].decode('ascii')
+        for row in db.execute(
+            'SELECT node FROM changedfiles '
+            'WHERE filename = ? AND filenode = ?',
+            (file_path, filenode),
+        )
+    }
+    inner_walker = gitrepo.walk(walk_start, _OUR_ORDER)
+    for w in inner_walker:
+        if w.id.hex in parent_options:
+            return w.id.hex
+    raise error.ProgrammingError(
+        'Unable to find introducing commit for %s node %s from %s',
+        (file_path, filenode, walk_start),
+    )
+
+
+def fill_in_filelog(gitrepo, db, startcommit, path, startfilenode):
+    """Given a starting commit and path, fill in a filelog's parent pointers.
+
+    Args:
+      gitrepo: a pygit2.Repository
+      db: a handle to our sqlite database
+      startcommit: a hexlified node id for the commit to start at
+      path: the path of the file whose parent pointers we should fill in.
+      filenode: the hexlified node id of the file at startcommit
+
+    TODO: make filenode optional
+    """
+    assert isinstance(
+        startcommit, str
+    ), 'startcommit must be str, got %r' % type(startcommit)
+    assert isinstance(
+        startfilenode, str
+    ), 'startfilenode must be str, got %r' % type(startfilenode)
+    visit = collections.deque([(startcommit, startfilenode)])
+    while visit:
+        cnode, filenode = visit.popleft()
+        commit = gitrepo[cnode]
+        parents = []
+        for parent in commit.parents:
+            t = parent.tree
+            for comp in path.split('/'):
+                try:
+                    t = gitrepo[t[comp].id]
+                except KeyError:
+                    break
+            else:
+                introducer = _find_nearest_ancestor_introducing_node(
+                    db, gitrepo, path, parent.id, t.id.hex
+                )
+                parents.append((introducer, t.id.hex))
+        p1node = p1fnode = p2node = p2fnode = gitutil.nullgit
+        for par, parfnode in parents:
+            found = int(
+                db.execute(
+                    'SELECT COUNT(*) FROM changedfiles WHERE '
+                    'node = ? AND filename = ? AND filenode = ? AND '
+                    'p1node NOT NULL',
+                    (par, path, parfnode),
+                ).fetchone()[0]
+            )
+            if found == 0:
+                assert par is not None
+                visit.append((par, parfnode))
+        if parents:
+            p1node, p1fnode = parents[0]
+        if len(parents) == 2:
+            p2node, p2fnode = parents[1]
+        if len(parents) > 2:
+            raise error.ProgrammingError(
+                b"git support can't handle octopus merges"
+            )
+        db.execute(
+            'UPDATE changedfiles SET '
+            'p1node = ?, p1filenode = ?, p2node = ?, p2filenode = ? '
+            'WHERE node = ? AND filename = ? AND filenode = ?',
+            (p1node, p1fnode, p2node, p2fnode, commit.id.hex, path, filenode),
+        )
+    db.commit()
+
+
+def _index_repo(gitrepo, db, progress_factory=lambda *args, **kwargs: None):
+    # Identify all references so we can tell the walker to visit all of them.
+    all_refs = gitrepo.listall_references()
+    possible_heads = set()
+    prog = progress_factory(b'refs')
+    for pos, ref in enumerate(all_refs):
+        if prog is not None:
+            prog.update(pos)
+        if not (
+            ref.startswith('refs/heads/')  # local branch
+            or ref.startswith('refs/tags/')  # tag
+            or ref.startswith('refs/remotes/')  # remote branch
+            or ref.startswith('refs/hg/')  # from this extension
+        ):
+            continue
+        try:
+            start = gitrepo.lookup_reference(ref).peel(pygit2.GIT_OBJ_COMMIT)
+        except ValueError:
+            # No commit to be found, so we don't care for hg's purposes.
+            continue
+        possible_heads.add(start.id)
+    # Optimization: if the list of heads hasn't changed, don't
+    # reindex, the changelog. This doesn't matter on small
+    # repositories, but on even moderately deep histories (eg cpython)
+    # this is a very important performance win.
+    #
+    # TODO: we should figure out how to incrementally index history
+    # (preferably by detecting rewinds!) so that we don't have to do a
+    # full changelog walk every time a new commit is created.
+    cache_heads = {x[0] for x in db.execute('SELECT node FROM possible_heads')}
+    walker = None
+    cur_cache_heads = {h.hex for h in possible_heads}
+    if cur_cache_heads == cache_heads:
+        return
+    for start in possible_heads:
+        if walker is None:
+            walker = gitrepo.walk(start, _OUR_ORDER)
+        else:
+            walker.push(start)
+
+    # Empty out the existing changelog. Even for large-ish histories
+    # we can do the top-level "walk all the commits" dance very
+    # quickly as long as we don't need to figure out the changed files
+    # list.
+    db.execute('DELETE FROM changelog')
+    if prog is not None:
+        prog.complete()
+    prog = progress_factory(b'commits')
+    # This walker is sure to visit all the revisions in history, but
+    # only once.
+    for pos, commit in enumerate(walker):
+        if prog is not None:
+            prog.update(pos)
+        p1 = p2 = nodemod.nullhex
+        if len(commit.parents) > 2:
+            raise error.ProgrammingError(
+                (
+                    b"git support can't handle octopus merges, "
+                    b"found a commit with %d parents :("
+                )
+                % len(commit.parents)
+            )
+        if commit.parents:
+            p1 = commit.parents[0].id.hex
+        if len(commit.parents) == 2:
+            p2 = commit.parents[1].id.hex
+        db.execute(
+            'INSERT INTO changelog (rev, node, p1, p2) VALUES(?, ?, ?, ?)',
+            (pos, commit.id.hex, p1, p2),
+        )
+
+        num_changedfiles = db.execute(
+            "SELECT COUNT(*) from changedfiles WHERE node = ?",
+            (commit.id.hex,),
+        ).fetchone()[0]
+        if not num_changedfiles:
+            files = {}
+            # I *think* we only need to check p1 for changed files
+            # (and therefore linkrevs), because any node that would
+            # actually have this commit as a linkrev would be
+            # completely new in this rev.
+            p1 = commit.parents[0].id.hex if commit.parents else None
+            if p1 is not None:
+                patchgen = gitrepo.diff(p1, commit.id.hex, flags=_DIFF_FLAGS)
+            else:
+                patchgen = commit.tree.diff_to_tree(
+                    swap=True, flags=_DIFF_FLAGS
+                )
+            new_files = (p.delta.new_file for p in patchgen)
+            files = {
+                nf.path: nf.id.hex
+                for nf in new_files
+                if nf.id.raw != nodemod.nullid
+            }
+            for p, n in files.items():
+                # We intentionally set NULLs for any file parentage
+                # information so it'll get demand-computed later. We
+                # used to do it right here, and it was _very_ slow.
+                db.execute(
+                    'INSERT INTO changedfiles ('
+                    'node, filename, filenode, p1node, p1filenode, p2node, '
+                    'p2filenode) VALUES(?, ?, ?, ?, ?, ?, ?)',
+                    (commit.id.hex, p, n, None, None, None, None),
+                )
+    db.execute('DELETE FROM heads')
+    db.execute('DELETE FROM possible_heads')
+    for hid in possible_heads:
+        h = hid.hex
+        db.execute('INSERT INTO possible_heads (node) VALUES(?)', (h,))
+        haschild = db.execute(
+            'SELECT COUNT(*) FROM changelog WHERE p1 = ? OR p2 = ?', (h, h)
+        ).fetchone()[0]
+        if not haschild:
+            db.execute('INSERT INTO heads (node) VALUES(?)', (h,))
+
+    db.commit()
+    if prog is not None:
+        prog.complete()
+
+
+def get_index(gitrepo, progress_factory=lambda *args, **kwargs: None):
+    cachepath = os.path.join(
+        pycompat.fsencode(gitrepo.path), b'..', b'.hg', b'cache'
+    )
+    if not os.path.exists(cachepath):
+        os.makedirs(cachepath)
+    dbpath = os.path.join(cachepath, b'git-commits.sqlite')
+    db = _createdb(dbpath)
+    # TODO check against gitrepo heads before doing a full index
+    # TODO thread a ui.progress call into this layer
+    _index_repo(gitrepo, db, progress_factory)
+    return db

hgext/git/manifest.py

@@ -0,0 +1,297 b''
+from __future__ import absolute_import
+
+from mercurial import (
+    match as matchmod,
+    pathutil,
+    pycompat,
+    util,
+)
+from mercurial.interfaces import (
+    repository,
+    util as interfaceutil,
+)
+from . import gitutil
+
+
+pygit2 = gitutil.get_pygit2()
+
+
[e-mail](repository.imanifestdict)
+class gittreemanifest(object):
+    """Expose git trees (and optionally a builder's overlay) as a manifestdict.
+
+    Very similar to mercurial.manifest.treemanifest.
+    """
+
+    def __init__(self, git_repo, root_tree, pending_changes):
+        """Initializer.
+
+        Args:
+          git_repo: The git_repo we're walking (required to look up child
+              trees).
+          root_tree: The root Git tree object for this manifest.
+          pending_changes: A dict in which pending changes will be
+              tracked. The enclosing memgittreemanifestctx will use this to
+              construct any required Tree objects in Git during it's
+              `write()` method.
+        """
+        self._git_repo = git_repo
+        self._tree = root_tree
+        if pending_changes is None:
+            pending_changes = {}
+        # dict of path: Optional[Tuple(node, flags)]
+        self._pending_changes = pending_changes
+
+    def _resolve_entry(self, path):
+        """Given a path, load its node and flags, or raise KeyError if missing.
+
+        This takes into account any pending writes in the builder.
+        """
+        upath = pycompat.fsdecode(path)
+        ent = None
+        if path in self._pending_changes:
+            val = self._pending_changes[path]
+            if val is None:
+                raise KeyError
+            return val
+        t = self._tree
+        comps = upath.split('/')
+        for comp in comps[:-1]:
+            te = self._tree[comp]
+            t = self._git_repo[te.id]
+        ent = t[comps[-1]]
+        if ent.filemode == pygit2.GIT_FILEMODE_BLOB:
+            flags = b''
+        elif ent.filemode == pygit2.GIT_FILEMODE_BLOB_EXECUTABLE:
+            flags = b'x'
+        elif ent.filemode == pygit2.GIT_FILEMODE_LINK:
+            flags = b'l'
+        else:
+            raise ValueError('unsupported mode %s' % oct(ent.filemode))
+        return ent.id.raw, flags
+
+    def __getitem__(self, path):
+        return self._resolve_entry(path)[0]
+
+    def find(self, path):
+        return self._resolve_entry(path)
+
+    def __len__(self):
+        return len(list(self.walk(matchmod.always())))
+
+    def __nonzero__(self):
+        try:
+            next(iter(self))
+            return True
+        except StopIteration:
+            return False
+
+    __bool__ = __nonzero__
+
+    def __contains__(self, path):
+        try:
+            self._resolve_entry(path)
+            return True
+        except KeyError:
+            return False
+
+    def iterkeys(self):
+        return self.walk(matchmod.always())
+
+    def keys(self):
+        return list(self.iterkeys())
+
+    def __iter__(self):
+        return self.iterkeys()
+
+    def __setitem__(self, path, node):
+        self._pending_changes[path] = node, self.flags(path)
+
+    def __delitem__(self, path):
+        # TODO: should probably KeyError for already-deleted  files?
+        self._pending_changes[path] = None
+
+    def filesnotin(self, other, match=None):
+        if match is not None:
+            match = matchmod.badmatch(match, lambda path, msg: None)
+            sm2 = set(other.walk(match))
+            return {f for f in self.walk(match) if f not in sm2}
+        return {f for f in self if f not in other}
+
+    @util.propertycache
+    def _dirs(self):
+        return pathutil.dirs(self)
+
+    def hasdir(self, dir):
+        return dir in self._dirs
+
+    def diff(self, other, match=None, clean=False):
+        # TODO
+        assert False
+
+    def setflag(self, path, flag):
+        node, unused_flag = self._resolve_entry(path)
+        self._pending_changes[path] = node, flag
+
+    def get(self, path, default=None):
+        try:
+            return self._resolve_entry(path)[0]
+        except KeyError:
+            return default
+
+    def flags(self, path):
+        try:
+            return self._resolve_entry(path)[1]
+        except KeyError:
+            return b''
+
+    def copy(self):
+        pass
+
+    def items(self):
+        for f in self:
+            # TODO: build a proper iterator version of this
+            yield self[f]
+
+    def iteritems(self):
+        return self.items()
+
+    def iterentries(self):
+        for f in self:
+            # TODO: build a proper iterator version of this
+            yield self._resolve_entry(f)
+
+    def text(self):
+        assert False  # TODO can this method move out of the manifest iface?
+
+    def _walkonetree(self, tree, match, subdir):
+        for te in tree:
+            # TODO: can we prune dir walks with the matcher?
+            realname = subdir + pycompat.fsencode(te.name)
+            if te.type == r'tree':
+                for inner in self._walkonetree(
+                    self._git_repo[te.id], match, realname + b'/'
+                ):
+                    yield inner
+            if not match(realname):
+                continue
+            yield pycompat.fsencode(realname)
+
+    def walk(self, match):
+        # TODO: this is a very lazy way to merge in the pending
+        # changes. There is absolutely room for optimization here by
+        # being clever about walking over the sets...
+        baseline = set(self._walkonetree(self._tree, match, b''))
+        deleted = {p for p, v in self._pending_changes.items() if v is None}
+        pend = {p for p in self._pending_changes if match(p)}
+        return iter(sorted((baseline | pend) - deleted))
+
+
[e-mail](repository.imanifestrevisionstored)
+class gittreemanifestctx(object):
+    def __init__(self, repo, gittree):
+        self._repo = repo
+        self._tree = gittree
+
+    def read(self):
+        return gittreemanifest(self._repo, self._tree, None)
+
+    def readfast(self, shallow=False):
+        return self.read()
+
+    def copy(self):
+        # NB: it's important that we return a memgittreemanifestctx
+        # because the caller expects a mutable manifest.
+        return memgittreemanifestctx(self._repo, self._tree)
+
+    def find(self, path):
+        self.read()[path]
+
+
[e-mail](repository.imanifestrevisionwritable)
+class memgittreemanifestctx(object):
+    def __init__(self, repo, tree):
+        self._repo = repo
+        self._tree = tree
+        # dict of path: Optional[Tuple(node, flags)]
+        self._pending_changes = {}
+
+    def read(self):
+        return gittreemanifest(self._repo, self._tree, self._pending_changes)
+
+    def copy(self):
+        # TODO: if we have a builder in play, what should happen here?
+        # Maybe we can shuffle copy() into the immutable interface.
+        return memgittreemanifestctx(self._repo, self._tree)
+
+    def write(self, transaction, link, p1, p2, added, removed, match=None):
+        # We're not (for now, anyway) going to audit filenames, so we
+        # can ignore added and removed.
+
+        # TODO what does this match argument get used for? hopefully
+        # just narrow?
+        assert not match or isinstance(match, matchmod.alwaysmatcher)
+
+        touched_dirs = pathutil.dirs(list(self._pending_changes))
+        trees = {
+            b'': self._tree,
+        }
+        # path: treebuilder
+        builders = {
+            b'': self._repo.TreeBuilder(self._tree),
+        }
+        # get a TreeBuilder for every tree in the touched_dirs set
+        for d in sorted(touched_dirs, key=lambda x: (len(x), x)):
+            if d == b'':
+                # loaded root tree above
+                continue
+            comps = d.split(b'/')
+            full = b''
+            for part in comps:
+                parent = trees[full]
+                try:
+                    new = self._repo[parent[pycompat.fsdecode(part)]]
+                except KeyError:
+                    # new directory
+                    new = None
+                full += b'/' + part
+                if new is not None:
+                    # existing directory
+                    trees[full] = new
+                    builders[full] = self._repo.TreeBuilder(new)
+                else:
+                    # new directory, use an empty dict to easily
+                    # generate KeyError as any nested new dirs get
+                    # created.
+                    trees[full] = {}
+                    builders[full] = self._repo.TreeBuilder()
+        for f, info in self._pending_changes.items():
+            if b'/' not in f:
+                dirname = b''
+                basename = f
+            else:
+                dirname, basename = f.rsplit(b'/', 1)
+                dirname = b'/' + dirname
+            if info is None:
+                builders[dirname].remove(pycompat.fsdecode(basename))
+            else:
+                n, fl = info
+                mode = {
+                    b'': pygit2.GIT_FILEMODE_BLOB,
+                    b'x': pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
+                    b'l': pygit2.GIT_FILEMODE_LINK,
+                }[fl]
+                builders[dirname].insert(
+                    pycompat.fsdecode(basename), gitutil.togitnode(n), mode
+                )
+        # This visits the buffered TreeBuilders in deepest-first
+        # order, bubbling up the edits.
+        for b in sorted(builders, key=len, reverse=True):
+            if b == b'':
+                break
+            cb = builders[b]
+            dn, bn = b.rsplit(b'/', 1)
+            builders[dn].insert(
+                pycompat.fsdecode(bn), cb.write(), pygit2.GIT_FILEMODE_TREE
+            )
+        return builders[b''].write().raw

hgext/hooklib/__init__.py

@@ -0,0 +1,26 b''
+"""collection of simple hooks for common tasks (EXPERIMENTAL)
+
+This extension provides a number of simple hooks to handle issues
+commonly found in repositories with many contributors:
+- email notification when changesets move from draft to public phase
+- email notification when changesets are obsoleted
+- enforcement of draft phase for all incoming changesets
+- enforcement of a no-branch-merge policy
+- enforcement of a no-multiple-heads policy
+
+The implementation of the hooks is subject to change, e.g. whether to
+implement them as individual hooks or merge them into the notify
+extension as option. The functionality itself is planned to be supported
+long-term.
+"""
+from __future__ import absolute_import
+from . import (
+    changeset_obsoleted,
+    changeset_published,
+)
+
+# configtable is only picked up from the "top-level" module of the extension,
+# so expand it here to ensure all items are properly loaded
+configtable = {}
+configtable.update(changeset_published.configtable)
+configtable.update(changeset_obsoleted.configtable)

hgext/hooklib/changeset_obsoleted.py

@@ -0,0 +1,131 b''
+# Copyright 2020 Joerg Sonnenberger <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+"""changeset_obsoleted is a hook to send a mail when an
+existing draft changeset is obsoleted by an obsmarker without successor.
+
+Correct message threading requires the same messageidseed to be used for both
+the original notification and the new mail.
+
+Usage:
+  [notify]
+  messageidseed = myseed
+
+  [hooks]
+  pretxnclose.changeset_obsoleted = \
+    python:hgext.hooklib.changeset_obsoleted.hook
+"""
+
+from __future__ import absolute_import
+
+import email.errors as emailerrors
+import email.utils as emailutils
+
+from mercurial.i18n import _
+from mercurial import (
+    encoding,
+    error,
+    logcmdutil,
+    mail,
+    obsutil,
+    pycompat,
+    registrar,
+)
+from mercurial.utils import dateutil
+from .. import notify
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem(
+    b'notify_obsoleted', b'domain', default=None,
+)
+configitem(
+    b'notify_obsoleted', b'messageidseed', default=None,
+)
+configitem(
+    b'notify_obsoleted',
+    b'template',
+    default=b'''Subject: changeset abandoned
+
+This changeset has been abandoned.
+''',
+)
+
+
+def _report_commit(ui, repo, ctx):
+    domain = ui.config(b'notify_obsoleted', b'domain') or ui.config(
+        b'notify', b'domain'
+    )
+    messageidseed = ui.config(
+        b'notify_obsoleted', b'messageidseed'
+    ) or ui.config(b'notify', b'messageidseed')
+    template = ui.config(b'notify_obsoleted', b'template')
+    spec = logcmdutil.templatespec(template, None)
+    templater = logcmdutil.changesettemplater(ui, repo, spec)
+    ui.pushbuffer()
+    n = notify.notifier(ui, repo, b'incoming')
+
+    subs = set()
+    for sub, spec in n.subs:
+        if spec is None:
+            subs.add(sub)
+            continue
+        revs = repo.revs(b'%r and %d:', spec, ctx.rev())
+        if len(revs):
+            subs.add(sub)
+            continue
+    if len(subs) == 0:
+        ui.debug(
+            b'notify_obsoleted: no subscribers to selected repo and revset\n'
+        )
+        return
+
+    templater.show(
+        ctx,
+        changes=ctx.changeset(),
+        baseurl=ui.config(b'web', b'baseurl'),
+        root=repo.root,
+        webroot=n.root,
+    )
+    data = ui.popbuffer()
+
+    try:
+        msg = mail.parsebytes(data)
+    except emailerrors.MessageParseError as inst:
+        raise error.Abort(inst)
+
+    msg['In-reply-to'] = notify.messageid(ctx, domain, messageidseed)
+    msg['Message-Id'] = notify.messageid(
+        ctx, domain, messageidseed + b'-obsoleted'
+    )
+    msg['Date'] = encoding.strfromlocal(
+        dateutil.datestr(format=b"%a, %d %b %Y %H:%M:%S %1%2")
+    )
+    if not msg['From']:
+        sender = ui.config(b'email', b'from') or ui.username()
+        if b'@' not in sender or b'@localhost' in sender:
+            sender = n.fixmail(sender)
+        msg['From'] = mail.addressencode(ui, sender, n.charsets, n.test)
+    msg['To'] = ', '.join(sorted(subs))
+
+    msgtext = msg.as_bytes() if pycompat.ispy3 else msg.as_string()
+    if ui.configbool(b'notify', b'test'):
+        ui.write(msgtext)
+        if not msgtext.endswith(b'\n'):
+            ui.write(b'\n')
+    else:
+        ui.status(_(b'notify_obsoleted: sending mail for %d\n') % ctx.rev())
+        mail.sendmail(
+            ui, emailutils.parseaddr(msg['From'])[1], subs, msgtext, mbox=n.mbox
+        )
+
+
+def hook(ui, repo, hooktype, node=None, **kwargs):
+    if hooktype != b"pretxnclose":
+        raise error.Abort(
+            _(b'Unsupported hook type %r') % pycompat.bytestr(hooktype)
+        )
+    for rev in obsutil.getobsoleted(repo, repo.currenttransaction()):
+        _report_commit(ui, repo, repo.unfiltered()[rev])

hgext/hooklib/changeset_published.py

@@ -0,0 +1,131 b''
+# Copyright 2020 Joerg Sonnenberger <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+"""changeset_published is a hook to send a mail when an
+existing draft changeset is moved to the public phase.
+
+Correct message threading requires the same messageidseed to be used for both
+the original notification and the new mail.
+
+Usage:
+  [notify]
+  messageidseed = myseed
+
+  [hooks]
+  txnclose-phase.changeset_published = \
+    python:hgext.hooklib.changeset_published.hook
+"""
+
+from __future__ import absolute_import
+
+import email.errors as emailerrors
+import email.utils as emailutils
+
+from mercurial.i18n import _
+from mercurial import (
+    encoding,
+    error,
+    logcmdutil,
+    mail,
+    pycompat,
+    registrar,
+)
+from mercurial.utils import dateutil
+from .. import notify
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem(
+    b'notify_published', b'domain', default=None,
+)
+configitem(
+    b'notify_published', b'messageidseed', default=None,
+)
+configitem(
+    b'notify_published',
+    b'template',
+    default=b'''Subject: changeset published
+
+This changeset has been published.
+''',
+)
+
+
+def _report_commit(ui, repo, ctx):
+    domain = ui.config(b'notify_published', b'domain') or ui.config(
+        b'notify', b'domain'
+    )
+    messageidseed = ui.config(
+        b'notify_published', b'messageidseed'
+    ) or ui.config(b'notify', b'messageidseed')
+    template = ui.config(b'notify_published', b'template')
+    spec = logcmdutil.templatespec(template, None)
+    templater = logcmdutil.changesettemplater(ui, repo, spec)
+    ui.pushbuffer()
+    n = notify.notifier(ui, repo, b'incoming')
+
+    subs = set()
+    for sub, spec in n.subs:
+        if spec is None:
+            subs.add(sub)
+            continue
+        revs = repo.revs(b'%r and %d:', spec, ctx.rev())
+        if len(revs):
+            subs.add(sub)
+            continue
+    if len(subs) == 0:
+        ui.debug(
+            b'notify_published: no subscribers to selected repo and revset\n'
+        )
+        return
+
+    templater.show(
+        ctx,
+        changes=ctx.changeset(),
+        baseurl=ui.config(b'web', b'baseurl'),
+        root=repo.root,
+        webroot=n.root,
+    )
+    data = ui.popbuffer()
+
+    try:
+        msg = mail.parsebytes(data)
+    except emailerrors.MessageParseError as inst:
+        raise error.Abort(inst)
+
+    msg['In-reply-to'] = notify.messageid(ctx, domain, messageidseed)
+    msg['Message-Id'] = notify.messageid(
+        ctx, domain, messageidseed + b'-published'
+    )
+    msg['Date'] = encoding.strfromlocal(
+        dateutil.datestr(format=b"%a, %d %b %Y %H:%M:%S %1%2")
+    )
+    if not msg['From']:
+        sender = ui.config(b'email', b'from') or ui.username()
+        if b'@' not in sender or b'@localhost' in sender:
+            sender = n.fixmail(sender)
+        msg['From'] = mail.addressencode(ui, sender, n.charsets, n.test)
+    msg['To'] = ', '.join(sorted(subs))
+
+    msgtext = msg.as_bytes() if pycompat.ispy3 else msg.as_string()
+    if ui.configbool(b'notify', b'test'):
+        ui.write(msgtext)
+        if not msgtext.endswith(b'\n'):
+            ui.write(b'\n')
+    else:
+        ui.status(_(b'notify_published: sending mail for %d\n') % ctx.rev())
+        mail.sendmail(
+            ui, emailutils.parseaddr(msg['From'])[1], subs, msgtext, mbox=n.mbox
+        )
+
+
+def hook(ui, repo, hooktype, node=None, **kwargs):
+    if hooktype != b"txnclose-phase":
+        raise error.Abort(
+            _(b'Unsupported hook type %r') % pycompat.bytestr(hooktype)
+        )
+    ctx = repo.unfiltered()[node]
+    if kwargs['oldphase'] == b'draft' and kwargs['phase'] == b'public':
+        _report_commit(ui, repo, ctx)

hgext/hooklib/enforce_draft_commits.py

@@ -0,0 +1,45 b''
+# Copyright 2020 Joerg Sonnenberger <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+"""enforce_draft_commits us a hook to ensure that all new changesets are
+in the draft phase. This allows enforcing policies for work-in-progress
+changes in overlay repositories, i.e. a shared hidden repositories with
+different views for work-in-progress code and public history.
+
+Usage:
+  [hooks]
+  pretxnclose-phase.enforce_draft_commits = \
+    python:hgext.hooklib.enforce_draft_commits.hook
+"""
+
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+from mercurial import (
+    error,
+    pycompat,
+)
+
+
+def hook(ui, repo, hooktype, node=None, **kwargs):
+    if hooktype != b"pretxnclose-phase":
+        raise error.Abort(
+            _(b'Unsupported hook type %r') % pycompat.bytestr(hooktype)
+        )
+    ctx = repo.unfiltered()[node]
+    if kwargs['oldphase']:
+        raise error.Abort(
+            _(b'Phase change from %r to %r for %s rejected')
+            % (
+                pycompat.bytestr(kwargs['oldphase']),
+                pycompat.bytestr(kwargs['phase']),
+                ctx,
+            )
+        )
+    elif kwargs['phase'] != b'draft':
+        raise error.Abort(
+            _(b'New changeset %s in phase %r rejected')
+            % (ctx, pycompat.bytestr(kwargs['phase']))
+        )

hgext/hooklib/reject_merge_commits.py

@@ -0,0 +1,45 b''
+# Copyright 2020 Joerg Sonnenberger <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+"""reject_merge_commits is a hook to check new changesets for merge commits.
+Merge commits are allowed only between different branches, i.e. merging
+a feature branch into the main development branch. This can be used to
+enforce policies for linear commit histories.
+
+Usage:
+  [hooks]
+  pretxnchangegroup.reject_merge_commits = \
+    python:hgext.hooklib.reject_merge_commits.hook
+"""
+
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+from mercurial import (
+    error,
+    pycompat,
+)
+
+
+def hook(ui, repo, hooktype, node=None, **kwargs):
+    if hooktype != b"pretxnchangegroup":
+        raise error.Abort(
+            _(b'Unsupported hook type %r') % pycompat.bytestr(hooktype)
+        )
+
+    ctx = repo.unfiltered()[node]
+    for rev in repo.changelog.revs(start=ctx.rev()):
+        rev = repo[rev]
+        parents = rev.parents()
+        if len(parents) < 2:
+            continue
+        if all(repo[p].branch() == rev.branch() for p in parents):
+            raise error.Abort(
+                _(
+                    b'%s rejected as merge on the same branch. '
+                    b'Please consider rebase.'
+                )
+                % rev
+            )

hgext/hooklib/reject_new_heads.py

@@ -0,0 +1,41 b''
+# Copyright 2020 Joerg Sonnenberger <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+"""reject_new_heads is a hook to check that branches touched by new changesets
+have at most one open head. It can be used to enforce policies for
+merge-before-push or rebase-before-push. It does not handle pre-existing
+hydras.
+
+Usage:
+  [hooks]
+  pretxnclose.reject_new_heads = \
+    python:hgext.hooklib.reject_new_heads.hook
+"""
+
+from __future__ import absolute_import
+
+from mercurial.i18n import _
+from mercurial import (
+    error,
+    pycompat,
+)
+
+
+def hook(ui, repo, hooktype, node=None, **kwargs):
+    if hooktype != b"pretxnclose":
+        raise error.Abort(
+            _(b'Unsupported hook type %r') % pycompat.bytestr(hooktype)
+        )
+    ctx = repo.unfiltered()[node]
+    branches = set()
+    for rev in repo.changelog.revs(start=ctx.rev()):
+        rev = repo[rev]
+        branches.add(rev.branch())
+    for branch in branches:
+        if len(repo.revs("head() and not closed() and branch(%s)", branch)) > 1:
+            raise error.Abort(
+                _(b'Changes on branch %r resulted in multiple heads')
+                % pycompat.bytestr(branch)
+            )

Makefile

@@ -64,6 +64,7 b' doc:'
 	$(MAKE) -C doc
 
 cleanbutpackages:
+	rm -f hg.exe
 	-$(PYTHON) setup.py clean --all # ignore errors from this command
 	find contrib doc hgext hgext3rd i18n mercurial tests hgdemandimport \
 		\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'

black.toml

@@ -9,7 +9,6 b' build/'
 | \.mypy_cache/
 | \.venv/
 | mercurial/thirdparty/
-| contrib/python-zstandard/
 '''
 skip-string-normalization = true
 quiet = true

contrib/benchmarks/__init__.py

@@ -81,7 +81,7 b' def runperfcommand(reponame, command, *a'
     output = ui.popbuffer()
     match = outputre.search(output)
     if not match:
-        raise ValueError("Invalid output {0}".format(output))
+        raise ValueError("Invalid output {}".format(output))
     return float(match.group(1))
 
 

contrib/check-py3-compat.py

@@ -32,7 +32,7 b' def check_compat_py2(f):'
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
-                futures |= set(n.name for n in node.names)
+                futures |= {n.name for n in node.names}
         elif isinstance(node, ast.Print):
             haveprint = True
 

contrib/chg/chg.c

@@ -226,6 +226,16 b' static void execcmdserver(const struct c'
 	}
 	argv[argsize - 1] = NULL;
 
+	const char *lc_ctype_env = getenv("LC_CTYPE");
+	if (lc_ctype_env == NULL) {
+		if (putenv("CHG_CLEAR_LC_CTYPE=") != 0)
+			abortmsgerrno("failed to putenv CHG_CLEAR_LC_CTYPE");
+	} else {
+		if (setenv("CHGORIG_LC_CTYPE", lc_ctype_env, 1) != 0) {
+			abortmsgerrno("failed to setenv CHGORIG_LC_CTYYPE");
+		}
+	}
+
 	if (putenv("CHGINTERNALMARK=") != 0)
 		abortmsgerrno("failed to putenv");
 	if (execvp(hgcmd, (char **)argv) < 0)
@@ -364,8 +374,7 b' static int runinstructions(struct cmdser'
 
 /*
  * Test whether the command is unsupported or not. This is not designed to
- * cover all cases. But it's fast, does not depend on the server and does
- * not return false positives.
+ * cover all cases. But it's fast, does not depend on the server.
  */
 static int isunsupported(int argc, const char *argv[])
 {
@@ -378,7 +387,12 b' static int isunsupported(int argc, const'
 	for (i = 0; i < argc; ++i) {
 		if (strcmp(argv[i], "--") == 0)
 			break;
-		if (i == 0 && strcmp("serve", argv[i]) == 0)
+		/*
+		 * there can be false positives but no false negative
+		 * we cannot assume `serve` will always be first argument
+		 * because global options can be passed before the command name
+		 */
+		if (strcmp("serve", argv[i]) == 0)
 			state |= SERVE;
 		else if (strcmp("-d", argv[i]) == 0 ||
 		         strcmp("--daemon", argv[i]) == 0)

contrib/examples/fix.hgrc

@@ -6,7 +6,7 b' rustfmt:command = rustfmt +nightly'
 rustfmt:pattern = set:**.rs
 
 black:command = black --config=black.toml -
-black:pattern = set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"
+black:pattern = set:**.py - mercurial/thirdparty/**
 
 # Mercurial doesn't have any Go code, but if we did this is how we
 # would configure `hg fix` for Go:

contrib/heptapod-ci.yml

@@ -42,13 +42,13 b' rust-cargo-test-py3:'
 test-py2:
     <<: *runtests
     variables:
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "c"
 
 test-py3:
     <<: *runtests
     variables:
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
 
@@ -69,13 +69,13 b' test-py2-rust:'
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py3-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"

contrib/import-checker.py

@@ -392,9 +392,10 b' def imported_modules(source, modulename,'
                     modnotfound = True
                     continue
                 yield found[1]
-            if modnotfound:
+            if modnotfound and dottedpath != modulename:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
+                # specifically allow "from . import foo" from __init__.py
                 yield dottedpath
 
 

contrib/perf.py

@@ -1536,6 +1536,7 b' def perfindex(ui, repo, **opts):'
     matters.
 
     Example of useful set to test:
+
     * tip
     * 0
     * -10:
@@ -2522,7 +2523,7 b' def perfdiffwd(ui, repo, **opts):'
     }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
-        opts = dict((options[c], b'1') for c in diffopt)
+        opts = {options[c]: b'1' for c in diffopt}
 
         def d():
             ui.pushbuffer()
@@ -3047,7 +3048,7 b' def perfrevlogchunks(ui, repo, file_=Non'
 
     # Verify engines argument.
     if engines:
-        engines = set(e.strip() for e in engines.split(b','))
+        engines = {e.strip() for e in engines.split(b',')}
         for engine in engines:
             try:
                 util.compressionengines[engine]

contrib/python-zstandard/make_cffi.py

@@ -52,7 +52,8 b' SOURCES = ['
 
 # Headers whose preprocessed output will be fed into cdef().
 HEADERS = [
-    os.path.join(HERE, "zstd", *p) for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
+    os.path.join(HERE, "zstd", *p)
+    for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
 ]
 
 INCLUDE_DIRS = [
@@ -139,7 +140,9 b' def preprocess(path):'
         env = dict(os.environ)
         if getattr(compiler, "_paths", None):
             env["PATH"] = compiler._paths
-        process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE, env=env)
+        process = subprocess.Popen(
+            args + [input_file], stdout=subprocess.PIPE, env=env
+        )
         output = process.communicate()[0]
         ret = process.poll()
         if ret:

contrib/python-zstandard/setup.py

@@ -87,7 +87,9 b' with open("c-ext/python-zstandard.h", "r'
         break
 
 if not version:
-    raise Exception("could not resolve package version; " "this should never happen")
+    raise Exception(
+        "could not resolve package version; " "this should never happen"
+    )
 
 setup(
     name="zstandard",

contrib/python-zstandard/setup_zstd.py

@@ -138,12 +138,16 b' def get_c_extension('
     if not system_zstd:
         sources.update([os.path.join(actual_root, p) for p in zstd_sources])
         if support_legacy:
-            sources.update([os.path.join(actual_root, p) for p in zstd_sources_legacy])
+            sources.update(
+                [os.path.join(actual_root, p) for p in zstd_sources_legacy]
+            )
     sources = list(sources)
 
     include_dirs = set([os.path.join(actual_root, d) for d in ext_includes])
     if not system_zstd:
-        include_dirs.update([os.path.join(actual_root, d) for d in zstd_includes])
+        include_dirs.update(
+            [os.path.join(actual_root, d) for d in zstd_includes]
+        )
         if support_legacy:
             include_dirs.update(
                 [os.path.join(actual_root, d) for d in zstd_includes_legacy]

contrib/python-zstandard/tests/common.py

@@ -50,7 +50,9 b' def make_cffi(cls):'
         os.environ.update(old_env)
 
     if mod.backend != "cffi":
-        raise Exception("got the zstandard %s backend instead of cffi" % mod.backend)
+        raise Exception(
+            "got the zstandard %s backend instead of cffi" % mod.backend
+        )
 
     # If CFFI version is available, dynamically construct test methods
     # that use it.
@@ -84,7 +86,9 b' def make_cffi(cls):'
                 fn.__func__.func_defaults,
                 fn.__func__.func_closure,
             )
-            new_method = types.UnboundMethodType(new_fn, fn.im_self, fn.im_class)
+            new_method = types.UnboundMethodType(
+                new_fn, fn.im_self, fn.im_class
+            )
 
         setattr(cls, name, new_method)
 
@@ -194,4 +198,6 b' if hypothesis:'
     expensive_settings = hypothesis.settings(deadline=None, max_examples=10000)
     hypothesis.settings.register_profile("expensive", expensive_settings)
 
-    hypothesis.settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "default"))
+    hypothesis.settings.load_profile(
+        os.environ.get("HYPOTHESIS_PROFILE", "default")
+    )

contrib/python-zstandard/tests/test_buffer_util.py

@@ -67,7 +67,8 b' class TestBufferWithSegments(TestCase):'
             self.skipTest("BufferWithSegments not available")
 
         b = zstd.BufferWithSegments(
-            b"foofooxfooxy", b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)])
+            b"foofooxfooxy",
+            b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)]),
         )
         self.assertEqual(len(b), 3)
         self.assertEqual(b.size, 12)
@@ -83,17 +84,23 b' class TestBufferWithSegmentsCollection(T'
         if not hasattr(zstd, "BufferWithSegmentsCollection"):
             self.skipTest("BufferWithSegmentsCollection not available")
 
-        with self.assertRaisesRegex(ValueError, "must pass at least 1 argument"):
+        with self.assertRaisesRegex(
+            ValueError, "must pass at least 1 argument"
+        ):
             zstd.BufferWithSegmentsCollection()
 
     def test_argument_validation(self):
         if not hasattr(zstd, "BufferWithSegmentsCollection"):
             self.skipTest("BufferWithSegmentsCollection not available")
 
-        with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
+        with self.assertRaisesRegex(
+            TypeError, "arguments must be BufferWithSegments"
+        ):
             zstd.BufferWithSegmentsCollection(None)
 
-        with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
+        with self.assertRaisesRegex(
+            TypeError, "arguments must be BufferWithSegments"
+        ):
             zstd.BufferWithSegmentsCollection(
                 zstd.BufferWithSegments(b"foo", ss.pack(0, 3)), None
             )

contrib/python-zstandard/tests/test_compressor.py

@@ -24,7 +24,9 b' else:'
 
 
 def multithreaded_chunk_size(level, source_size=0):
-    params = zstd.ZstdCompressionParameters.from_level(level, source_size=source_size)
+    params = zstd.ZstdCompressionParameters.from_level(
+        level, source_size=source_size
+    )
 
     return 1 << (params.window_log + 2)
 
@@ -86,7 +88,9 b' class TestCompressor_compress(TestCase):'
 
         # This matches the test for read_to_iter() below.
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
-        result = cctx.compress(b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b"o")
+        result = cctx.compress(
+            b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b"o"
+        )
         self.assertEqual(
             result,
             b"\x28\xb5\x2f\xfd\x00\x40\x54\x00\x00"
@@ -99,7 +103,9 b' class TestCompressor_compress(TestCase):'
         result = cctx.compress(b"foo" * 256)
 
     def test_no_magic(self):
-        params = zstd.ZstdCompressionParameters.from_level(1, format=zstd.FORMAT_ZSTD1)
+        params = zstd.ZstdCompressionParameters.from_level(
+            1, format=zstd.FORMAT_ZSTD1
+        )
         cctx = zstd.ZstdCompressor(compression_params=params)
         magic = cctx.compress(b"foobar")
 
@@ -223,7 +229,8 b' class TestCompressor_compress(TestCase):'
 
         self.assertEqual(
             result,
-            b"\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00" b"\x66\x6f\x6f",
+            b"\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00"
+            b"\x66\x6f\x6f",
         )
 
     def test_multithreaded_compression_params(self):
@@ -234,7 +241,9 b' class TestCompressor_compress(TestCase):'
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, 3)
 
-        self.assertEqual(result, b"\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f")
+        self.assertEqual(
+            result, b"\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f"
+        )
 
 
 @make_cffi
@@ -347,7 +356,9 b' class TestCompressor_compressobj(TestCas'
         )
         self.assertEqual(cobj.compress(b"bar"), b"")
         # 3 byte header plus content.
-        self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"\x18\x00\x00bar")
+        self.assertEqual(
+            cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"\x18\x00\x00bar"
+        )
         self.assertEqual(cobj.flush(), b"\x01\x00\x00")
 
     def test_flush_empty_block(self):
@@ -445,7 +456,9 b' class TestCompressor_copy_stream(TestCas'
         self.assertEqual(int(r), 0)
         self.assertEqual(w, 9)
 
-        self.assertEqual(dest.getvalue(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
+        self.assertEqual(
+            dest.getvalue(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00"
+        )
 
     def test_large_data(self):
         source = io.BytesIO()
@@ -478,7 +491,9 b' class TestCompressor_copy_stream(TestCas'
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         cctx.copy_stream(source, with_checksum)
 
-        self.assertEqual(len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4)
+        self.assertEqual(
+            len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4
+        )
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
@@ -585,7 +600,9 b' class TestCompressor_stream_reader(TestC'
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b"foo") as reader:
-            with self.assertRaisesRegex(ValueError, "cannot __enter__ multiple times"):
+            with self.assertRaisesRegex(
+                ValueError, "cannot __enter__ multiple times"
+            ):
                 with reader as reader2:
                     pass
 
@@ -744,7 +761,9 b' class TestCompressor_stream_reader(TestC'
         source = io.BytesIO(b"foobar")
 
         with cctx.stream_reader(source, size=2) as reader:
-            with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
+            with self.assertRaisesRegex(
+                zstd.ZstdError, "Src size is incorrect"
+            ):
                 reader.read(10)
 
         # Try another compression operation.
@@ -1126,7 +1145,9 b' class TestCompressor_stream_writer(TestC'
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
-        self.assertEqual(len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4)
+        self.assertEqual(
+            len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4
+        )
 
     def test_write_content_size(self):
         no_size = NonClosingBytesIO()
@@ -1145,7 +1166,9 b' class TestCompressor_stream_writer(TestC'
 
         # Declaring size will write the header.
         with_size = NonClosingBytesIO()
-        with cctx.stream_writer(with_size, size=len(b"foobar" * 256)) as compressor:
+        with cctx.stream_writer(
+            with_size, size=len(b"foobar" * 256)
+        ) as compressor:
             self.assertEqual(compressor.write(b"foobar" * 256), 0)
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
@@ -1191,7 +1214,9 b' class TestCompressor_stream_writer(TestC'
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
-        self.assertEqual(len(with_dict_id.getvalue()), len(no_dict_id.getvalue()) + 4)
+        self.assertEqual(
+            len(with_dict_id.getvalue()), len(no_dict_id.getvalue()) + 4
+        )
 
     def test_memory_size(self):
         cctx = zstd.ZstdCompressor(level=3)
@@ -1337,7 +1362,9 b' class TestCompressor_read_to_iter(TestCa'
         for chunk in cctx.read_to_iter(b"foobar"):
             pass
 
-        with self.assertRaisesRegex(ValueError, "must pass an object with a read"):
+        with self.assertRaisesRegex(
+            ValueError, "must pass an object with a read"
+        ):
             for chunk in cctx.read_to_iter(True):
                 pass
 
@@ -1513,7 +1540,9 b' class TestCompressor_chunker(TestCase):'
 
         dctx = zstd.ZstdDecompressor()
 
-        self.assertEqual(dctx.decompress(b"".join(chunks)), (b"x" * 1000) + (b"y" * 24))
+        self.assertEqual(
+            dctx.decompress(b"".join(chunks)), (b"x" * 1000) + (b"y" * 24)
+        )
 
     def test_small_chunk_size(self):
         cctx = zstd.ZstdCompressor()
@@ -1533,7 +1562,8 b' class TestCompressor_chunker(TestCase):'
 
         dctx = zstd.ZstdDecompressor()
         self.assertEqual(
-            dctx.decompress(b"".join(chunks), max_output_size=10000), b"foo" * 1024
+            dctx.decompress(b"".join(chunks), max_output_size=10000),
+            b"foo" * 1024,
         )
 
     def test_input_types(self):
@@ -1602,7 +1632,8 b' class TestCompressor_chunker(TestCase):'
         list(chunker.finish())
 
         with self.assertRaisesRegex(
-            zstd.ZstdError, r"cannot call compress\(\) after compression finished"
+            zstd.ZstdError,
+            r"cannot call compress\(\) after compression finished",
         ):
             list(chunker.compress(b"foo"))
 
@@ -1644,7 +1675,9 b' class TestCompressor_multi_compress_to_b'
         with self.assertRaises(TypeError):
             cctx.multi_compress_to_buffer((1, 2))
 
-        with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
+        with self.assertRaisesRegex(
+            TypeError, "item 0 not a bytes like object"
+        ):
             cctx.multi_compress_to_buffer([u"foo"])
 
     def test_empty_input(self):

contrib/python-zstandard/tests/test_compressor_fuzzing.py

@@ -28,9 +28,13 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_stream_source_read(self, original, level, source_read_size, read_size):
+    def test_stream_source_read(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
@@ -58,9 +62,13 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_buffer_source_read(self, original, level, source_read_size, read_size):
+    def test_buffer_source_read(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
@@ -155,9 +163,13 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_stream_source_readinto(self, original, level, source_read_size, read_size):
+    def test_stream_source_readinto(
+        self, original, level, source_read_size, read_size
+    ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
@@ -184,9 +196,13 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_buffer_source_readinto(self, original, level, source_read_size, read_size):
+    def test_buffer_source_readinto(
+        self, original, level, source_read_size, read_size
+    ):
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
@@ -285,9 +301,13 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_stream_source_read1(self, original, level, source_read_size, read_size):
+    def test_stream_source_read1(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
@@ -315,9 +335,13 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_buffer_source_read1(self, original, level, source_read_size, read_size):
+    def test_buffer_source_read1(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
@@ -412,7 +436,9 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
     def test_stream_source_readinto1(
         self, original, level, source_read_size, read_size
@@ -446,7 +472,9 b' class TestCompressor_stream_reader_fuzzi'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
     def test_buffer_source_readinto1(
         self, original, level, source_read_size, read_size
@@ -576,7 +604,9 b' class TestCompressor_copy_stream_fuzzing'
         read_size=strategies.integers(min_value=1, max_value=1048576),
         write_size=strategies.integers(min_value=1, max_value=1048576),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
@@ -585,7 +615,11 b' class TestCompressor_copy_stream_fuzzing'
         dest = io.BytesIO()
 
         cctx.copy_stream(
-            source, dest, size=len(original), read_size=read_size, write_size=write_size
+            source,
+            dest,
+            size=len(original),
+            read_size=read_size,
+            write_size=write_size,
         )
 
         self.assertEqual(dest.getvalue(), ref_frame)
@@ -675,7 +709,9 b' class TestCompressor_compressobj_fuzzing'
         decompressed_chunks.append(dobj.decompress(chunk))
 
         self.assertEqual(
-            dctx.decompress(b"".join(compressed_chunks), max_output_size=len(original)),
+            dctx.decompress(
+                b"".join(compressed_chunks), max_output_size=len(original)
+            ),
             original,
         )
         self.assertEqual(b"".join(decompressed_chunks), original)
@@ -690,7 +726,9 b' class TestCompressor_read_to_iter_fuzzin'
         read_size=strategies.integers(min_value=1, max_value=4096),
         write_size=strategies.integers(min_value=1, max_value=4096),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         refcctx = zstd.ZstdCompressor(level=level)
         ref_frame = refcctx.compress(original)
 
@@ -699,7 +737,10 b' class TestCompressor_read_to_iter_fuzzin'
         cctx = zstd.ZstdCompressor(level=level)
         chunks = list(
             cctx.read_to_iter(
-                source, size=len(original), read_size=read_size, write_size=write_size
+                source,
+                size=len(original),
+                read_size=read_size,
+                write_size=write_size,
             )
         )
 
@@ -710,7 +751,9 b' class TestCompressor_read_to_iter_fuzzin'
 class TestCompressor_multi_compress_to_buffer_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.lists(
-            strategies.sampled_from(random_input_data()), min_size=1, max_size=1024
+            strategies.sampled_from(random_input_data()),
+            min_size=1,
+            max_size=1024,
         ),
         threads=strategies.integers(min_value=1, max_value=8),
         use_dict=strategies.booleans(),
@@ -776,7 +819,8 b' class TestCompressor_chunker_fuzzing(Tes'
         dctx = zstd.ZstdDecompressor()
 
         self.assertEqual(
-            dctx.decompress(b"".join(chunks), max_output_size=len(original)), original
+            dctx.decompress(b"".join(chunks), max_output_size=len(original)),
+            original,
         )
 
         self.assertTrue(all(len(chunk) == chunk_size for chunk in chunks[:-1]))
@@ -794,7 +838,9 b' class TestCompressor_chunker_fuzzing(Tes'
         input_sizes=strategies.data(),
         flushes=strategies.data(),
     )
-    def test_flush_block(self, original, level, chunk_size, input_sizes, flushes):
+    def test_flush_block(
+        self, original, level, chunk_size, input_sizes, flushes
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         chunker = cctx.chunker(chunk_size=chunk_size)
 
@@ -830,7 +876,9 b' class TestCompressor_chunker_fuzzing(Tes'
         decompressed_chunks.append(dobj.decompress(b"".join(chunks)))
 
         self.assertEqual(
-            dctx.decompress(b"".join(compressed_chunks), max_output_size=len(original)),
+            dctx.decompress(
+                b"".join(compressed_chunks), max_output_size=len(original)
+            ),
             original,
         )
         self.assertEqual(b"".join(decompressed_chunks), original)

contrib/python-zstandard/tests/test_data_structures.py

@@ -65,7 +65,9 b' class TestCompressionParameters(TestCase'
         p = zstd.ZstdCompressionParameters(threads=4)
         self.assertEqual(p.threads, 4)
 
-        p = zstd.ZstdCompressionParameters(threads=2, job_size=1048576, overlap_log=6)
+        p = zstd.ZstdCompressionParameters(
+            threads=2, job_size=1048576, overlap_log=6
+        )
         self.assertEqual(p.threads, 2)
         self.assertEqual(p.job_size, 1048576)
         self.assertEqual(p.overlap_log, 6)
@@ -128,7 +130,9 b' class TestCompressionParameters(TestCase'
         with self.assertRaisesRegex(
             ValueError, "cannot specify both ldm_hash_rate_log"
         ):
-            zstd.ZstdCompressionParameters(ldm_hash_rate_log=8, ldm_hash_every_log=4)
+            zstd.ZstdCompressionParameters(
+                ldm_hash_rate_log=8, ldm_hash_every_log=4
+            )
 
         p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
         self.assertEqual(p.ldm_hash_every_log, 8)
@@ -137,7 +141,9 b' class TestCompressionParameters(TestCase'
         self.assertEqual(p.ldm_hash_every_log, 16)
 
     def test_overlap_log(self):
-        with self.assertRaisesRegex(ValueError, "cannot specify both overlap_log"):
+        with self.assertRaisesRegex(
+            ValueError, "cannot specify both overlap_log"
+        ):
             zstd.ZstdCompressionParameters(overlap_log=1, overlap_size_log=9)
 
         p = zstd.ZstdCompressionParameters(overlap_log=2)
@@ -169,10 +175,14 b' class TestFrameParameters(TestCase):'
                     zstd.get_frame_parameters(u"foobarbaz")
 
     def test_invalid_input_sizes(self):
-        with self.assertRaisesRegex(zstd.ZstdError, "not enough data for frame"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "not enough data for frame"
+        ):
             zstd.get_frame_parameters(b"")
 
-        with self.assertRaisesRegex(zstd.ZstdError, "not enough data for frame"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "not enough data for frame"
+        ):
             zstd.get_frame_parameters(zstd.FRAME_HEADER)
 
     def test_invalid_frame(self):
@@ -201,7 +211,9 b' class TestFrameParameters(TestCase):'
         self.assertTrue(params.has_checksum)
 
         # Upper 2 bits indicate content size.
-        params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x40\x00\xff\x00")
+        params = zstd.get_frame_parameters(
+            zstd.FRAME_HEADER + b"\x40\x00\xff\x00"
+        )
         self.assertEqual(params.content_size, 511)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
@@ -215,7 +227,9 b' class TestFrameParameters(TestCase):'
         self.assertFalse(params.has_checksum)
 
         # Set multiple things.
-        params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00")
+        params = zstd.get_frame_parameters(
+            zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00"
+        )
         self.assertEqual(params.content_size, 272)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 15)

contrib/python-zstandard/tests/test_data_structures_fuzzing.py

@@ -23,7 +23,9 b' s_windowlog = strategies.integers('
 s_chainlog = strategies.integers(
     min_value=zstd.CHAINLOG_MIN, max_value=zstd.CHAINLOG_MAX
 )
-s_hashlog = strategies.integers(min_value=zstd.HASHLOG_MIN, max_value=zstd.HASHLOG_MAX)
+s_hashlog = strategies.integers(
+    min_value=zstd.HASHLOG_MIN, max_value=zstd.HASHLOG_MAX
+)
 s_searchlog = strategies.integers(
     min_value=zstd.SEARCHLOG_MIN, max_value=zstd.SEARCHLOG_MAX
 )
@@ -61,7 +63,14 b' class TestCompressionParametersHypothesi'
         s_strategy,
     )
     def test_valid_init(
-        self, windowlog, chainlog, hashlog, searchlog, minmatch, targetlength, strategy
+        self,
+        windowlog,
+        chainlog,
+        hashlog,
+        searchlog,
+        minmatch,
+        targetlength,
+        strategy,
     ):
         zstd.ZstdCompressionParameters(
             window_log=windowlog,
@@ -83,7 +92,14 b' class TestCompressionParametersHypothesi'
         s_strategy,
     )
     def test_estimated_compression_context_size(
-        self, windowlog, chainlog, hashlog, searchlog, minmatch, targetlength, strategy
+        self,
+        windowlog,
+        chainlog,
+        hashlog,
+        searchlog,
+        minmatch,
+        targetlength,
+        strategy,
     ):
         if minmatch == zstd.MINMATCH_MIN and strategy in (
             zstd.STRATEGY_FAST,

contrib/python-zstandard/tests/test_decompressor.py

@@ -170,11 +170,15 b' class TestDecompressor_decompress(TestCa'
             dctx.decompress(compressed, max_output_size=len(source) - 1)
 
         # Input size + 1 works
-        decompressed = dctx.decompress(compressed, max_output_size=len(source) + 1)
+        decompressed = dctx.decompress(
+            compressed, max_output_size=len(source) + 1
+        )
         self.assertEqual(decompressed, source)
 
         # A much larger buffer works.
-        decompressed = dctx.decompress(compressed, max_output_size=len(source) * 64)
+        decompressed = dctx.decompress(
+            compressed, max_output_size=len(source) * 64
+        )
         self.assertEqual(decompressed, source)
 
     def test_stupidly_large_output_buffer(self):
@@ -237,7 +241,8 b' class TestDecompressor_decompress(TestCa'
         dctx = zstd.ZstdDecompressor(max_window_size=2 ** zstd.WINDOWLOG_MIN)
 
         with self.assertRaisesRegex(
-            zstd.ZstdError, "decompression error: Frame requires too much memory"
+            zstd.ZstdError,
+            "decompression error: Frame requires too much memory",
         ):
             dctx.decompress(frame, max_output_size=len(source))
 
@@ -291,7 +296,9 b' class TestDecompressor_copy_stream(TestC'
         self.assertEqual(w, len(source.getvalue()))
 
     def test_read_write_size(self):
-        source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b"foobarfoobar"))
+        source = OpCountingBytesIO(
+            zstd.ZstdCompressor().compress(b"foobarfoobar")
+        )
 
         dest = OpCountingBytesIO()
         dctx = zstd.ZstdDecompressor()
@@ -309,7 +316,9 b' class TestDecompressor_stream_reader(Tes'
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(b"foo") as reader:
-            with self.assertRaisesRegex(ValueError, "cannot __enter__ multiple times"):
+            with self.assertRaisesRegex(
+                ValueError, "cannot __enter__ multiple times"
+            ):
                 with reader as reader2:
                     pass
 
@@ -474,7 +483,9 b' class TestDecompressor_stream_reader(Tes'
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(frame) as reader:
-            with self.assertRaisesRegex(ValueError, "cannot seek to negative position"):
+            with self.assertRaisesRegex(
+                ValueError, "cannot seek to negative position"
+            ):
                 reader.seek(-1, os.SEEK_SET)
 
             reader.read(1)
@@ -490,7 +501,8 b' class TestDecompressor_stream_reader(Tes'
                 reader.seek(-1, os.SEEK_CUR)
 
             with self.assertRaisesRegex(
-                ValueError, "zstd decompression streams cannot be seeked with SEEK_END"
+                ValueError,
+                "zstd decompression streams cannot be seeked with SEEK_END",
             ):
                 reader.seek(0, os.SEEK_END)
 
@@ -743,7 +755,9 b' class TestDecompressor_stream_reader(Tes'
 
     def test_read_lines(self):
         cctx = zstd.ZstdCompressor()
-        source = b"\n".join(("line %d" % i).encode("ascii") for i in range(1024))
+        source = b"\n".join(
+            ("line %d" % i).encode("ascii") for i in range(1024)
+        )
 
         frame = cctx.compress(source)
 
@@ -821,7 +835,9 b' class TestDecompressor_decompressobj(Tes'
         dobj = dctx.decompressobj()
         dobj.decompress(data)
 
-        with self.assertRaisesRegex(zstd.ZstdError, "cannot use a decompressobj"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "cannot use a decompressobj"
+        ):
             dobj.decompress(data)
             self.assertIsNone(dobj.flush())
 
@@ -1124,7 +1140,9 b' class TestDecompressor_read_to_iter(Test'
         # Buffer protocol works.
         dctx.read_to_iter(b"foobar")
 
-        with self.assertRaisesRegex(ValueError, "must pass an object with a read"):
+        with self.assertRaisesRegex(
+            ValueError, "must pass an object with a read"
+        ):
             b"".join(dctx.read_to_iter(True))
 
     def test_empty_input(self):
@@ -1226,7 +1244,9 b' class TestDecompressor_read_to_iter(Test'
         decompressed = b"".join(chunks)
         self.assertEqual(decompressed, source.getvalue())
 
-    @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
+    @unittest.skipUnless(
+        "ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set"
+    )
     def test_large_input(self):
         bytes = list(struct.Struct(">B").pack(i) for i in range(256))
         compressed = NonClosingBytesIO()
@@ -1241,13 +1261,16 b' class TestDecompressor_read_to_iter(Test'
                     len(compressed.getvalue())
                     > zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
                 )
-                have_raw = input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2
+                have_raw = (
+                    input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2
+                )
                 if have_compressed and have_raw:
                     break
 
         compressed = io.BytesIO(compressed.getvalue())
         self.assertGreater(
-            len(compressed.getvalue()), zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
+            len(compressed.getvalue()),
+            zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         )
 
         dctx = zstd.ZstdDecompressor()
@@ -1303,7 +1326,9 b' class TestDecompressor_read_to_iter(Test'
         self.assertEqual(streamed, source.getvalue())
 
     def test_read_write_size(self):
-        source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b"foobarfoobar"))
+        source = OpCountingBytesIO(
+            zstd.ZstdCompressor().compress(b"foobarfoobar")
+        )
         dctx = zstd.ZstdDecompressor()
         for chunk in dctx.read_to_iter(source, read_size=1, write_size=1):
             self.assertEqual(len(chunk), 1)
@@ -1355,10 +1380,14 b' class TestDecompressor_content_dict_chai'
         ):
             dctx.decompress_content_dict_chain([zstd.FRAME_HEADER])
 
-        with self.assertRaisesRegex(ValueError, "chunk 0 is not a valid zstd frame"):
+        with self.assertRaisesRegex(
+            ValueError, "chunk 0 is not a valid zstd frame"
+        ):
             dctx.decompress_content_dict_chain([b"foo" * 8])
 
-        no_size = zstd.ZstdCompressor(write_content_size=False).compress(b"foo" * 64)
+        no_size = zstd.ZstdCompressor(write_content_size=False).compress(
+            b"foo" * 64
+        )
 
         with self.assertRaisesRegex(
             ValueError, "chunk 0 missing content size in frame"
@@ -1389,10 +1418,14 b' class TestDecompressor_content_dict_chai'
         ):
             dctx.decompress_content_dict_chain([initial, zstd.FRAME_HEADER])
 
-        with self.assertRaisesRegex(ValueError, "chunk 1 is not a valid zstd frame"):
+        with self.assertRaisesRegex(
+            ValueError, "chunk 1 is not a valid zstd frame"
+        ):
             dctx.decompress_content_dict_chain([initial, b"foo" * 8])
 
-        no_size = zstd.ZstdCompressor(write_content_size=False).compress(b"foo" * 64)
+        no_size = zstd.ZstdCompressor(write_content_size=False).compress(
+            b"foo" * 64
+        )
 
         with self.assertRaisesRegex(
             ValueError, "chunk 1 missing content size in frame"
@@ -1400,7 +1433,9 b' class TestDecompressor_content_dict_chai'
             dctx.decompress_content_dict_chain([initial, no_size])
 
         # Corrupt second frame.
-        cctx = zstd.ZstdCompressor(dict_data=zstd.ZstdCompressionDict(b"foo" * 64))
+        cctx = zstd.ZstdCompressor(
+            dict_data=zstd.ZstdCompressionDict(b"foo" * 64)
+        )
         frame = cctx.compress(b"bar" * 64)
         frame = frame[0:12] + frame[15:]
 
@@ -1447,7 +1482,9 b' class TestDecompressor_multi_decompress_'
         with self.assertRaises(TypeError):
             dctx.multi_decompress_to_buffer((1, 2))
 
-        with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
+        with self.assertRaisesRegex(
+            TypeError, "item 0 not a bytes like object"
+        ):
             dctx.multi_decompress_to_buffer([u"foo"])
 
         with self.assertRaisesRegex(
@@ -1491,7 +1528,9 b' class TestDecompressor_multi_decompress_'
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
-        result = dctx.multi_decompress_to_buffer(frames, decompressed_sizes=sizes)
+        result = dctx.multi_decompress_to_buffer(
+            frames, decompressed_sizes=sizes
+        )
 
         self.assertEqual(len(result), len(frames))
         self.assertEqual(result.size(), sum(map(len, original)))
@@ -1582,10 +1621,15 b' class TestDecompressor_multi_decompress_'
         # And a manual mode.
         b = b"".join([frames[0].tobytes(), frames[1].tobytes()])
         b1 = zstd.BufferWithSegments(
-            b, struct.pack("=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1]))
+            b,
+            struct.pack(
+                "=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1])
+            ),
         )
 
-        b = b"".join([frames[2].tobytes(), frames[3].tobytes(), frames[4].tobytes()])
+        b = b"".join(
+            [frames[2].tobytes(), frames[3].tobytes(), frames[4].tobytes()]
+        )
         b2 = zstd.BufferWithSegments(
             b,
             struct.pack(

contrib/python-zstandard/tests/test_decompressor_fuzzing.py

@@ -196,7 +196,9 b' class TestDecompressor_stream_reader_fuz'
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
     )
-    def test_stream_source_readall(self, original, level, streaming, source_read_size):
+    def test_stream_source_readall(
+        self, original, level, streaming, source_read_size
+    ):
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
@@ -398,7 +400,9 b' class TestDecompressor_stream_writer_fuz'
         write_size=strategies.integers(min_value=1, max_value=8192),
         input_sizes=strategies.data(),
     )
-    def test_write_size_variance(self, original, level, write_size, input_sizes):
+    def test_write_size_variance(
+        self, original, level, write_size, input_sizes
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
@@ -433,7 +437,9 b' class TestDecompressor_copy_stream_fuzzi'
         read_size=strategies.integers(min_value=1, max_value=8192),
         write_size=strategies.integers(min_value=1, max_value=8192),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
@@ -441,7 +447,9 b' class TestDecompressor_copy_stream_fuzzi'
         dest = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor()
-        dctx.copy_stream(source, dest, read_size=read_size, write_size=write_size)
+        dctx.copy_stream(
+            source, dest, read_size=read_size, write_size=write_size
+        )
 
         self.assertEqual(dest.getvalue(), original)
 
@@ -490,11 +498,14 b' class TestDecompressor_decompressobj_fuz'
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         write_size=strategies.integers(
-            min_value=1, max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
+            min_value=1,
+            max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         ),
         chunk_sizes=strategies.data(),
     )
-    def test_random_output_sizes(self, original, level, write_size, chunk_sizes):
+    def test_random_output_sizes(
+        self, original, level, write_size, chunk_sizes
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
@@ -524,7 +535,9 b' class TestDecompressor_read_to_iter_fuzz'
         read_size=strategies.integers(min_value=1, max_value=4096),
         write_size=strategies.integers(min_value=1, max_value=4096),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
@@ -532,7 +545,9 b' class TestDecompressor_read_to_iter_fuzz'
 
         dctx = zstd.ZstdDecompressor()
         chunks = list(
-            dctx.read_to_iter(source, read_size=read_size, write_size=write_size)
+            dctx.read_to_iter(
+                source, read_size=read_size, write_size=write_size
+            )
         )
 
         self.assertEqual(b"".join(chunks), original)
@@ -542,7 +557,9 b' class TestDecompressor_read_to_iter_fuzz'
 class TestDecompressor_multi_decompress_to_buffer_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.lists(
-            strategies.sampled_from(random_input_data()), min_size=1, max_size=1024
+            strategies.sampled_from(random_input_data()),
+            min_size=1,
+            max_size=1024,
         ),
         threads=strategies.integers(min_value=1, max_value=8),
         use_dict=strategies.booleans(),

contrib/python-zstandard/tests/test_train_dictionary.py

@@ -51,11 +51,15 b' class TestTrainDictionary(TestCase):'
         self.assertEqual(d.d, 16)
 
     def test_set_dict_id(self):
-        d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16, dict_id=42)
+        d = zstd.train_dictionary(
+            8192, generate_samples(), k=64, d=16, dict_id=42
+        )
         self.assertEqual(d.dict_id(), 42)
 
     def test_optimize(self):
-        d = zstd.train_dictionary(8192, generate_samples(), threads=-1, steps=1, d=16)
+        d = zstd.train_dictionary(
+            8192, generate_samples(), threads=-1, steps=1, d=16
+        )
 
         # This varies by platform.
         self.assertIn(d.k, (50, 2000))
@@ -71,10 +75,14 b' class TestCompressionDict(TestCase):'
     def test_bad_precompute_compress(self):
         d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
 
-        with self.assertRaisesRegex(ValueError, "must specify one of level or "):
+        with self.assertRaisesRegex(
+            ValueError, "must specify one of level or "
+        ):
             d.precompute_compress()
 
-        with self.assertRaisesRegex(ValueError, "must only specify one of level or "):
+        with self.assertRaisesRegex(
+            ValueError, "must only specify one of level or "
+        ):
             d.precompute_compress(
                 level=3, compression_params=zstd.CompressionParameters()
             )
@@ -88,5 +96,7 b' class TestCompressionDict(TestCase):'
         d = zstd.ZstdCompressionDict(
             b"dictcontent" * 64, dict_type=zstd.DICT_TYPE_FULLDICT
         )
-        with self.assertRaisesRegex(zstd.ZstdError, "unable to precompute dictionary"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "unable to precompute dictionary"
+        ):
             d.precompute_compress(level=1)

contrib/python-zstandard/zstandard/cffi.py

@@ -299,10 +299,14 b' class ZstdCompressionParameters(object):'
         _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log)
         _set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log)
         _set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match)
-        _set_compression_parameter(params, lib.ZSTD_c_targetLength, target_length)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_targetLength, target_length
+        )
 
         if strategy != -1 and compression_strategy != -1:
-            raise ValueError("cannot specify both compression_strategy and strategy")
+            raise ValueError(
+                "cannot specify both compression_strategy and strategy"
+            )
 
         if compression_strategy != -1:
             strategy = compression_strategy
@@ -313,12 +317,16 b' class ZstdCompressionParameters(object):'
         _set_compression_parameter(
             params, lib.ZSTD_c_contentSizeFlag, write_content_size
         )
-        _set_compression_parameter(params, lib.ZSTD_c_checksumFlag, write_checksum)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_checksumFlag, write_checksum
+        )
         _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id)
         _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size)
 
         if overlap_log != -1 and overlap_size_log != -1:
-            raise ValueError("cannot specify both overlap_log and overlap_size_log")
+            raise ValueError(
+                "cannot specify both overlap_log and overlap_size_log"
+            )
 
         if overlap_size_log != -1:
             overlap_log = overlap_size_log
@@ -326,12 +334,16 b' class ZstdCompressionParameters(object):'
             overlap_log = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log)
-        _set_compression_parameter(params, lib.ZSTD_c_forceMaxWindow, force_max_window)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_forceMaxWindow, force_max_window
+        )
         _set_compression_parameter(
             params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm
         )
         _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log)
-        _set_compression_parameter(params, lib.ZSTD_c_ldmMinMatch, ldm_min_match)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_ldmMinMatch, ldm_min_match
+        )
         _set_compression_parameter(
             params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log
         )
@@ -346,7 +358,9 b' class ZstdCompressionParameters(object):'
         elif ldm_hash_rate_log == -1:
             ldm_hash_rate_log = 0
 
-        _set_compression_parameter(params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log
+        )
 
     @property
     def format(self):
@@ -354,7 +368,9 b' class ZstdCompressionParameters(object):'
 
     @property
     def compression_level(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_compressionLevel)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_compressionLevel
+        )
 
     @property
     def window_log(self):
@@ -386,7 +402,9 b' class ZstdCompressionParameters(object):'
 
     @property
     def write_content_size(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_contentSizeFlag)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_contentSizeFlag
+        )
 
     @property
     def write_checksum(self):
@@ -410,7 +428,9 b' class ZstdCompressionParameters(object):'
 
     @property
     def force_max_window(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_forceMaxWindow)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_forceMaxWindow
+        )
 
     @property
     def enable_ldm(self):
@@ -428,11 +448,15 b' class ZstdCompressionParameters(object):'
 
     @property
     def ldm_bucket_size_log(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmBucketSizeLog)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_ldmBucketSizeLog
+        )
 
     @property
     def ldm_hash_rate_log(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashRateLog)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_ldmHashRateLog
+        )
 
     @property
     def ldm_hash_every_log(self):
@@ -457,7 +481,8 b' def _set_compression_parameter(params, p'
     zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value)
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
-            "unable to set compression context parameter: %s" % _zstd_error(zresult)
+            "unable to set compression context parameter: %s"
+            % _zstd_error(zresult)
         )
 
 
@@ -467,14 +492,17 b' def _get_compression_parameter(params, p'
     zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result)
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
-            "unable to get compression context parameter: %s" % _zstd_error(zresult)
+            "unable to get compression context parameter: %s"
+            % _zstd_error(zresult)
         )
 
     return result[0]
 
 
 class ZstdCompressionWriter(object):
-    def __init__(self, compressor, writer, source_size, write_size, write_return_read):
+    def __init__(
+        self, compressor, writer, source_size, write_size, write_return_read
+    ):
         self._compressor = compressor
         self._writer = writer
         self._write_size = write_size
@@ -491,7 +519,9 b' class ZstdCompressionWriter(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, source_size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
     def __enter__(self):
         if self._closed:
@@ -595,13 +625,20 b' class ZstdCompressionWriter(object):'
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_compressStream2(
-                self._compressor._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
+                self._compressor._cctx,
+                out_buffer,
+                in_buffer,
+                lib.ZSTD_e_continue,
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if out_buffer.pos:
-                self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+                self._writer.write(
+                    ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+                )
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
@@ -637,10 +674,14 b' class ZstdCompressionWriter(object):'
                 self._compressor._cctx, out_buffer, in_buffer, flush
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if out_buffer.pos:
-                self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+                self._writer.write(
+                    ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+                )
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
@@ -672,7 +713,9 b' class ZstdCompressionObj(object):'
                 self._compressor._cctx, self._out, source, lib.ZSTD_e_continue
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
@@ -681,7 +724,10 b' class ZstdCompressionObj(object):'
         return b"".join(chunks)
 
     def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH):
-        if flush_mode not in (COMPRESSOBJ_FLUSH_FINISH, COMPRESSOBJ_FLUSH_BLOCK):
+        if flush_mode not in (
+            COMPRESSOBJ_FLUSH_FINISH,
+            COMPRESSOBJ_FLUSH_BLOCK,
+        ):
             raise ValueError("flush mode not recognized")
 
         if self._finished:
@@ -768,7 +814,9 b' class ZstdCompressionChunker(object):'
                 self._in.pos = 0
 
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos == self._out.size:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
@@ -780,7 +828,8 b' class ZstdCompressionChunker(object):'
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
-                "cannot call flush() before consuming output from " "previous operation"
+                "cannot call flush() before consuming output from "
+                "previous operation"
             )
 
         while True:
@@ -788,7 +837,9 b' class ZstdCompressionChunker(object):'
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_flush
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
@@ -812,7 +863,9 b' class ZstdCompressionChunker(object):'
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
@@ -939,7 +992,10 b' class ZstdCompressionReader(object):'
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
-            self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_continue
+            self._compressor._cctx,
+            out_buffer,
+            self._in_buffer,
+            lib.ZSTD_e_continue,
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
@@ -997,7 +1053,9 b' class ZstdCompressionReader(object):'
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error ending compression stream: %s", _zstd_error(zresult))
+            raise ZstdError(
+                "error ending compression stream: %s", _zstd_error(zresult)
+            )
 
         if zresult == 0:
             self._finished_output = True
@@ -1102,7 +1160,9 b' class ZstdCompressionReader(object):'
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error ending compression stream: %s", _zstd_error(zresult))
+            raise ZstdError(
+                "error ending compression stream: %s", _zstd_error(zresult)
+            )
 
         if zresult == 0:
             self._finished_output = True
@@ -1170,13 +1230,17 b' class ZstdCompressor(object):'
         threads=0,
     ):
         if level > lib.ZSTD_maxCLevel():
-            raise ValueError("level must be less than %d" % lib.ZSTD_maxCLevel())
+            raise ValueError(
+                "level must be less than %d" % lib.ZSTD_maxCLevel()
+            )
 
         if threads < 0:
             threads = _cpu_count()
 
         if compression_params and write_checksum is not None:
-            raise ValueError("cannot define compression_params and " "write_checksum")
+            raise ValueError(
+                "cannot define compression_params and " "write_checksum"
+            )
 
         if compression_params and write_content_size is not None:
             raise ValueError(
@@ -1184,7 +1248,9 b' class ZstdCompressor(object):'
             )
 
         if compression_params and write_dict_id is not None:
-            raise ValueError("cannot define compression_params and " "write_dict_id")
+            raise ValueError(
+                "cannot define compression_params and " "write_dict_id"
+            )
 
         if compression_params and threads:
             raise ValueError("cannot define compression_params and threads")
@@ -1201,7 +1267,9 b' class ZstdCompressor(object):'
 
             self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
-            _set_compression_parameter(self._params, lib.ZSTD_c_compressionLevel, level)
+            _set_compression_parameter(
+                self._params, lib.ZSTD_c_compressionLevel, level
+            )
 
             _set_compression_parameter(
                 self._params,
@@ -1210,7 +1278,9 b' class ZstdCompressor(object):'
             )
 
             _set_compression_parameter(
-                self._params, lib.ZSTD_c_checksumFlag, 1 if write_checksum else 0
+                self._params,
+                lib.ZSTD_c_checksumFlag,
+                1 if write_checksum else 0,
             )
 
             _set_compression_parameter(
@@ -1218,7 +1288,9 b' class ZstdCompressor(object):'
             )
 
             if threads:
-                _set_compression_parameter(self._params, lib.ZSTD_c_nbWorkers, threads)
+                _set_compression_parameter(
+                    self._params, lib.ZSTD_c_nbWorkers, threads
+                )
 
         cctx = lib.ZSTD_createCCtx()
         if cctx == ffi.NULL:
@@ -1237,10 +1309,13 b' class ZstdCompressor(object):'
             )
 
     def _setup_cctx(self):
-        zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(self._cctx, self._params)
+        zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(
+            self._cctx, self._params
+        )
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
-                "could not set compression parameters: %s" % _zstd_error(zresult)
+                "could not set compression parameters: %s"
+                % _zstd_error(zresult)
             )
 
         dict_data = self._dict_data
@@ -1259,7 +1334,8 b' class ZstdCompressor(object):'
 
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
-                    "could not load compression dictionary: %s" % _zstd_error(zresult)
+                    "could not load compression dictionary: %s"
+                    % _zstd_error(zresult)
                 )
 
     def memory_size(self):
@@ -1275,7 +1351,9 b' class ZstdCompressor(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer))
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         in_buffer = ffi.new("ZSTD_inBuffer *")
@@ -1307,11 +1385,15 b' class ZstdCompressor(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         cobj = ZstdCompressionObj()
         cobj._out = ffi.new("ZSTD_outBuffer *")
-        cobj._dst_buffer = ffi.new("char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+        cobj._dst_buffer = ffi.new(
+            "char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        )
         cobj._out.dst = cobj._dst_buffer
         cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
         cobj._out.pos = 0
@@ -1328,7 +1410,9 b' class ZstdCompressor(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         return ZstdCompressionChunker(self, chunk_size=chunk_size)
 
@@ -1353,7 +1437,9 b' class ZstdCompressor(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
@@ -1381,7 +1467,9 b' class ZstdCompressor(object):'
                     self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
                 )
                 if lib.ZSTD_isError(zresult):
-                    raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                    raise ZstdError(
+                        "zstd compress error: %s" % _zstd_error(zresult)
+                    )
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
@@ -1423,7 +1511,9 b' class ZstdCompressor(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         return ZstdCompressionReader(self, source, read_size)
 
@@ -1443,7 +1533,9 b' class ZstdCompressor(object):'
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
-        return ZstdCompressionWriter(self, writer, size, write_size, write_return_read)
+        return ZstdCompressionWriter(
+            self, writer, size, write_size, write_return_read
+        )
 
     write_to = stream_writer
 
@@ -1473,7 +1565,9 b' class ZstdCompressor(object):'
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
@@ -1517,7 +1611,9 b' class ZstdCompressor(object):'
                     self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
                 )
                 if lib.ZSTD_isError(zresult):
-                    raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                    raise ZstdError(
+                        "zstd compress error: %s" % _zstd_error(zresult)
+                    )
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
@@ -1596,10 +1692,14 b' def get_frame_parameters(data):'
     data_buffer = ffi.from_buffer(data)
     zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
-        raise ZstdError("cannot get frame parameters: %s" % _zstd_error(zresult))
+        raise ZstdError(
+            "cannot get frame parameters: %s" % _zstd_error(zresult)
+        )
 
     if zresult:
-        raise ZstdError("not enough data for frame parameters; need %d bytes" % zresult)
+        raise ZstdError(
+            "not enough data for frame parameters; need %d bytes" % zresult
+        )
 
     return FrameParameters(params[0])
 
@@ -1611,9 +1711,14 b' class ZstdCompressionDict(object):'
         self.k = k
         self.d = d
 
-        if dict_type not in (DICT_TYPE_AUTO, DICT_TYPE_RAWCONTENT, DICT_TYPE_FULLDICT):
+        if dict_type not in (
+            DICT_TYPE_AUTO,
+            DICT_TYPE_RAWCONTENT,
+            DICT_TYPE_FULLDICT,
+        ):
             raise ValueError(
-                "invalid dictionary load mode: %d; must use " "DICT_TYPE_* constants"
+                "invalid dictionary load mode: %d; must use "
+                "DICT_TYPE_* constants"
             )
 
         self._dict_type = dict_type
@@ -1630,7 +1735,9 b' class ZstdCompressionDict(object):'
 
     def precompute_compress(self, level=0, compression_params=None):
         if level and compression_params:
-            raise ValueError("must only specify one of level or " "compression_params")
+            raise ValueError(
+                "must only specify one of level or " "compression_params"
+            )
 
         if not level and not compression_params:
             raise ValueError("must specify one of level or compression_params")
@@ -1675,7 +1782,9 b' class ZstdCompressionDict(object):'
         if ddict == ffi.NULL:
             raise ZstdError("could not create decompression dict")
 
-        ddict = ffi.gc(ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict))
+        ddict = ffi.gc(
+            ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict)
+        )
         self.__dict__["_ddict"] = ddict
 
         return ddict
@@ -1805,7 +1914,9 b' class ZstdDecompressionObj(object):'
                 self._decompressor._dctx, out_buffer, in_buffer
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd decompressor error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd decompressor error: %s" % _zstd_error(zresult)
+                )
 
             if zresult == 0:
                 self._finished = True
@@ -2105,16 +2216,22 b' class ZstdDecompressionReader(object):'
 
         if whence == os.SEEK_SET:
             if pos < 0:
-                raise ValueError("cannot seek to negative position with SEEK_SET")
+                raise ValueError(
+                    "cannot seek to negative position with SEEK_SET"
+                )
 
             if pos < self._bytes_decompressed:
-                raise ValueError("cannot seek zstd decompression stream " "backwards")
+                raise ValueError(
+                    "cannot seek zstd decompression stream " "backwards"
+                )
 
             read_amount = pos - self._bytes_decompressed
 
         elif whence == os.SEEK_CUR:
             if pos < 0:
-                raise ValueError("cannot seek zstd decompression stream " "backwards")
+                raise ValueError(
+                    "cannot seek zstd decompression stream " "backwards"
+                )
 
             read_amount = pos
         elif whence == os.SEEK_END:
@@ -2123,7 +2240,9 b' class ZstdDecompressionReader(object):'
             )
 
         while read_amount:
-            result = self.read(min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+            result = self.read(
+                min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+            )
 
             if not result:
                 break
@@ -2257,10 +2376,14 b' class ZstdDecompressionWriter(object):'
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd decompress error: %s" % _zstd_error(zresult)
+                )
 
             if out_buffer.pos:
-                self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+                self._writer.write(
+                    ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+                )
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
@@ -2299,7 +2422,9 b' class ZstdDecompressor(object):'
 
         data_buffer = ffi.from_buffer(data)
 
-        output_size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
+        output_size = lib.ZSTD_getFrameContentSize(
+            data_buffer, len(data_buffer)
+        )
 
         if output_size == lib.ZSTD_CONTENTSIZE_ERROR:
             raise ZstdError("error determining content size from frame header")
@@ -2307,7 +2432,9 b' class ZstdDecompressor(object):'
             return b""
         elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             if not max_output_size:
-                raise ZstdError("could not determine content size in frame header")
+                raise ZstdError(
+                    "could not determine content size in frame header"
+                )
 
             result_buffer = ffi.new("char[]", max_output_size)
             result_size = max_output_size
@@ -2330,7 +2457,9 b' class ZstdDecompressor(object):'
         if lib.ZSTD_isError(zresult):
             raise ZstdError("decompression error: %s" % _zstd_error(zresult))
         elif zresult:
-            raise ZstdError("decompression error: did not decompress full frame")
+            raise ZstdError(
+                "decompression error: did not decompress full frame"
+            )
         elif output_size and out_buffer.pos != output_size:
             raise ZstdError(
                 "decompression error: decompressed %d bytes; expected %d"
@@ -2346,7 +2475,9 b' class ZstdDecompressor(object):'
         read_across_frames=False,
     ):
         self._ensure_dctx()
-        return ZstdDecompressionReader(self, source, read_size, read_across_frames)
+        return ZstdDecompressionReader(
+            self, source, read_size, read_across_frames
+        )
 
     def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if write_size < 1:
@@ -2421,9 +2552,13 b' class ZstdDecompressor(object):'
             while in_buffer.pos < in_buffer.size:
                 assert out_buffer.pos == 0
 
-                zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+                zresult = lib.ZSTD_decompressStream(
+                    self._dctx, out_buffer, in_buffer
+                )
                 if lib.ZSTD_isError(zresult):
-                    raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
+                    raise ZstdError(
+                        "zstd decompress error: %s" % _zstd_error(zresult)
+                    )
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
@@ -2449,7 +2584,9 b' class ZstdDecompressor(object):'
         if not hasattr(writer, "write"):
             raise ValueError("must pass an object with a write() method")
 
-        return ZstdDecompressionWriter(self, writer, write_size, write_return_read)
+        return ZstdDecompressionWriter(
+            self, writer, write_size, write_return_read
+        )
 
     write_to = stream_writer
 
@@ -2491,7 +2628,9 b' class ZstdDecompressor(object):'
 
             # Flush all read data to output.
             while in_buffer.pos < in_buffer.size:
-                zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+                zresult = lib.ZSTD_decompressStream(
+                    self._dctx, out_buffer, in_buffer
+                )
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError(
                         "zstd decompressor error: %s" % _zstd_error(zresult)
@@ -2521,7 +2660,9 b' class ZstdDecompressor(object):'
         # All chunks should be zstd frames and should have content size set.
         chunk_buffer = ffi.from_buffer(chunk)
         params = ffi.new("ZSTD_frameHeader *")
-        zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
+        zresult = lib.ZSTD_getFrameHeader(
+            params, chunk_buffer, len(chunk_buffer)
+        )
         if lib.ZSTD_isError(zresult):
             raise ValueError("chunk 0 is not a valid zstd frame")
         elif zresult:
@@ -2546,7 +2687,9 b' class ZstdDecompressor(object):'
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("could not decompress chunk 0: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "could not decompress chunk 0: %s" % _zstd_error(zresult)
+            )
         elif zresult:
             raise ZstdError("chunk 0 did not decompress full frame")
 
@@ -2561,11 +2704,15 b' class ZstdDecompressor(object):'
                 raise ValueError("chunk %d must be bytes" % i)
 
             chunk_buffer = ffi.from_buffer(chunk)
-            zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
+            zresult = lib.ZSTD_getFrameHeader(
+                params, chunk_buffer, len(chunk_buffer)
+            )
             if lib.ZSTD_isError(zresult):
                 raise ValueError("chunk %d is not a valid zstd frame" % i)
             elif zresult:
-                raise ValueError("chunk %d is too small to contain a zstd frame" % i)
+                raise ValueError(
+                    "chunk %d is too small to contain a zstd frame" % i
+                )
 
             if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
                 raise ValueError("chunk %d missing content size in frame" % i)
@@ -2580,7 +2727,9 b' class ZstdDecompressor(object):'
             in_buffer.size = len(chunk_buffer)
             in_buffer.pos = 0
 
-            zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+            zresult = lib.ZSTD_decompressStream(
+                self._dctx, out_buffer, in_buffer
+            )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "could not decompress chunk %d: %s" % _zstd_error(zresult)
@@ -2597,7 +2746,9 b' class ZstdDecompressor(object):'
         lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only)
 
         if self._max_window_size:
-            zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx, self._max_window_size)
+            zresult = lib.ZSTD_DCtx_setMaxWindowSize(
+                self._dctx, self._max_window_size
+            )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "unable to set max window size: %s" % _zstd_error(zresult)
@@ -2605,11 +2756,14 b' class ZstdDecompressor(object):'
 
         zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("unable to set decoding format: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "unable to set decoding format: %s" % _zstd_error(zresult)
+            )
 
         if self._dict_data and load_dict:
             zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
-                    "unable to reference prepared dictionary: %s" % _zstd_error(zresult)
+                    "unable to reference prepared dictionary: %s"
+                    % _zstd_error(zresult)
                 )

doc/Makefile

@@ -5,7 +5,7 b' GENDOC=gendoc.py ../mercurial/commands.p'
 	../mercurial/helptext/*.txt ../hgext/*.py ../hgext/*/__init__.py
 PREFIX=/usr/local
 MANDIR=$(PREFIX)/share/man
-INSTALL=install -c -m 644
+INSTALL=install -m 644
 PYTHON?=python
 RSTARGS=
 

hgext/absorb.py

@@ -407,7 +407,7 b' class filefixupstate(object):'
             involved = [
                 annotated[i] for i in nearbylinenums if annotated[i][0] != 1
             ]
-        involvedrevs = list(set(r for r, l in involved))
+        involvedrevs = list({r for r, l in involved})
         newfixups = []
         if len(involvedrevs) == 1 and self._iscontinuous(a1, a2 - 1, True):
             # chunk belongs to a single revision
@@ -734,10 +734,10 b' class fixupstate(object):'
     @property
     def chunkstats(self):
         """-> {path: chunkstats}. collect chunkstats from filefixupstates"""
-        return dict(
-            (path, state.chunkstats)
+        return {
+            path: state.chunkstats
             for path, state in pycompat.iteritems(self.fixupmap)
-        )
+        }
 
     def commit(self):
         """commit changes. update self.finalnode, self.replacemap"""
@@ -1077,7 +1077,7 b' def absorb(ui, repo, stack=None, targetc'
             b'i',
             b'interactive',
             None,
-            _(b'interactively select which chunks to apply (EXPERIMENTAL)'),
+            _(b'interactively select which chunks to apply'),
         ),
         (
             b'e',

hgext/beautifygraph.py

@@ -71,6 +71,8 b' def getprettygraphnode(orig, *args, **kw'
         return b'\xE2\x97\x8B'  # U+25CB ○
     if node == b'@':
         return b'\xE2\x97\x8D'  # U+25CD ◍
+    if node == b'%':
+        return b'\xE2\x97\x8D'  # U+25CE ◎
     if node == b'*':
         return b'\xE2\x88\x97'  # U+2217 ∗
     if node == b'x':

hgext/closehead.py

@@ -76,7 +76,7 b' def close_branch(ui, repo, *revs, **opts'
     heads = []
     for branch in repo.branchmap():
         heads.extend(repo.branchheads(branch))
-    heads = set(repo[h].rev() for h in heads)
+    heads = {repo[h].rev() for h in heads}
     for rev in revs:
         if rev not in heads:
             raise error.Abort(_(b'revision is not an open head: %d') % rev)

hgext/convert/hg.py

@@ -677,13 +677,9 b' class mercurial_source(common.converter_'
             for t in self.repo.tagslist()
             if self.repo.tagtype(t[0]) == b'global'
         ]
-        return dict(
-            [
-                (name, nodemod.hex(node))
-                for name, node in tags
-                if self.keep(node)
-            ]
-        )
+        return {
+            name: nodemod.hex(node) for name, node in tags if self.keep(node)
+        }
 
     def getchangedfiles(self, rev, i):
         ctx = self._changectx(rev)

hgext/convert/subversion.py

@@ -710,11 +710,11 b' class svn_source(converter_source):'
                 # Here/tags/tag.1 discarded as well as its children.
                 # It happens with tools like cvs2svn. Such tags cannot
                 # be represented in mercurial.
-                addeds = dict(
-                    (p, e.copyfrom_path)
+                addeds = {
+                    p: e.copyfrom_path
                     for p, e in pycompat.iteritems(origpaths)
                     if e.action == b'A' and e.copyfrom_path
-                )
+                }
                 badroots = set()
                 for destroot in addeds:
                     for source, sourcerev, dest in pendings:

hgext/eol.py

@@ -221,7 +221,7 b' class eolfile(object):'
         self.match = match.match(root, b'', [], include, exclude)
 
     def copytoui(self, ui):
-        newpatterns = set(pattern for pattern, key, m in self.patterns)
+        newpatterns = {pattern for pattern, key, m in self.patterns}
         for section in (b'decode', b'encode'):
             for oldpattern, _filter in ui.configitems(section):
                 if oldpattern not in newpatterns:

hgext/fastannotate/commands.py

@@ -233,7 +233,7 b' def fastannotate(ui, repo, *pats, **opts'
                         showlines=(showlines and not showdeleted),
                     )
                     if showdeleted:
-                        existinglines = set((l[0], l[1]) for l in result)
+                        existinglines = {(l[0], l[1]) for l in result}
                         result = a.annotatealllines(
                             rev, showpath=showpath, showlines=showlines
                         )

hgext/fetch.py

@@ -171,11 +171,11 b" def fetch(ui, repo, source=b'default', *"
                     % (repo.changelog.rev(firstparent), short(firstparent))
                 )
             hg.clean(repo, firstparent)
+            p2ctx = repo[secondparent]
             ui.status(
-                _(b'merging with %d:%s\n')
-                % (repo.changelog.rev(secondparent), short(secondparent))
+                _(b'merging with %d:%s\n') % (p2ctx.rev(), short(secondparent))
             )
-            err = hg.merge(repo, secondparent, remind=False)
+            err = hg.merge(p2ctx, remind=False)
 
         if not err:
             # we don't translate commit messages

hgext/fix.py

@@ -213,7 +213,14 b' baseopt = ('
     ),
     _(b'REV'),
 )
-revopt = (b'r', b'rev', [], _(b'revisions to fix'), _(b'REV'))
+revopt = (b'r', b'rev', [], _(b'revisions to fix (ADVANCED)'), _(b'REV'))
+sourceopt = (
+    b's',
+    b'source',
+    [],
+    _(b'fix the specified revisions and their descendants'),
+    _(b'REV'),
+)
 wdiropt = (b'w', b'working-dir', False, _(b'fix the working directory'))
 wholeopt = (b'', b'whole', False, _(b'always fix every line of a file'))
 usage = _(b'[OPTION]... [FILE]...')
@@ -221,7 +228,7 b" usage = _(b'[OPTION]... [FILE]...')"
 
 @command(
     b'fix',
-    [allopt, baseopt, revopt, wdiropt, wholeopt],
+    [allopt, baseopt, revopt, sourceopt, wdiropt, wholeopt],
     usage,
     helpcategory=command.CATEGORY_FILE_CONTENTS,
 )
@@ -249,10 +256,11 b' def fix(ui, repo, *pats, **opts):'
     override this default behavior, though it is not usually desirable to do so.
     """
     opts = pycompat.byteskwargs(opts)
-    cmdutil.check_at_most_one_arg(opts, b'all', b'rev')
-    if opts[b'all']:
-        opts[b'rev'] = [b'not public() and not obsolete()']
-        opts[b'working_dir'] = True
+    cmdutil.check_at_most_one_arg(opts, b'all', b'source', b'rev')
+    cmdutil.check_incompatible_arguments(
+        opts, b'working_dir', [b'all', b'source']
+    )
+
     with repo.wlock(), repo.lock(), repo.transaction(b'fix'):
         revstofix = getrevstofix(ui, repo, opts)
         basectxs = getbasectxs(repo, opts, revstofix)
@@ -398,16 +406,28 b' def getworkqueue(ui, repo, pats, opts, r'
 
 def getrevstofix(ui, repo, opts):
     """Returns the set of revision numbers that should be fixed"""
-    revs = set(scmutil.revrange(repo, opts[b'rev']))
+    if opts[b'all']:
+        revs = repo.revs(b'(not public() and not obsolete()) or wdir()')
+    elif opts[b'source']:
+        source_revs = scmutil.revrange(repo, opts[b'source'])
+        revs = set(repo.revs(b'%ld::', source_revs))
+        if wdirrev in source_revs:
+            # `wdir()::` is currently empty, so manually add wdir
+            revs.add(wdirrev)
+        if repo[b'.'].rev() in revs:
+            revs.add(wdirrev)
+    else:
+        revs = set(scmutil.revrange(repo, opts[b'rev']))
+        if opts.get(b'working_dir'):
+            revs.add(wdirrev)
     for rev in revs:
         checkfixablectx(ui, repo, repo[rev])
-    if revs:
+    # Allow fixing only wdir() even if there's an unfinished operation
+    if not (len(revs) == 1 and wdirrev in revs):
         cmdutil.checkunfinished(repo)
         rewriteutil.precheck(repo, revs, b'fix')
-    if opts.get(b'working_dir'):
-        revs.add(wdirrev)
-        if list(merge.mergestate.read(repo).unresolved()):
-            raise error.Abort(b'unresolved conflicts', hint=b"use 'hg resolve'")
+    if wdirrev in revs and list(merge.mergestate.read(repo).unresolved()):
+        raise error.Abort(b'unresolved conflicts', hint=b"use 'hg resolve'")
     if not revs:
         raise error.Abort(
             b'no changesets specified', hint=b'use --rev or --working-dir'
@@ -735,15 +755,7 b' def replacerev(ui, repo, ctx, filedata, '
 
     wctx = context.overlayworkingctx(repo)
     wctx.setbase(repo[newp1node])
-    merge.update(
-        repo,
-        ctx.rev(),
-        branchmerge=False,
-        force=True,
-        ancestor=p1rev,
-        mergeancestor=False,
-        wc=wctx,
-    )
+    merge.revert_to(ctx, wc=wctx)
     copies.graftcopies(wctx, ctx, ctx.p1())
 
     for path in filedata.keys():

hgext/fsmonitor/__init__.py

@@ -397,7 +397,7 b' def overridewalk(orig, self, match, subr'
     # for file paths which require normalization and we encounter a case
     # collision, we store our own foldmap
     if normalize:
-        foldmap = dict((normcase(k), k) for k in results)
+        foldmap = {normcase(k): k for k in results}
 
     switch_slashes = pycompat.ossep == b'\\'
     # The order of the results is, strictly speaking, undefined.
@@ -459,22 +459,16 b' def overridewalk(orig, self, match, subr'
     if normalize:
         # any notable files that have changed case will already be handled
         # above, so just check membership in the foldmap
-        notefiles = set(
-            (
-                normalize(f, True, True)
-                for f in notefiles
-                if normcase(f) not in foldmap
-            )
-        )
-    visit = set(
-        (
-            f
+        notefiles = {
+            normalize(f, True, True)
             for f in notefiles
-            if (
-                f not in results and matchfn(f) and (f in dmap or not ignore(f))
-            )
-        )
-    )
+            if normcase(f) not in foldmap
+        }
+    visit = {
+        f
+        for f in notefiles
+        if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
+    }
 
     if not fresh_instance:
         if matchalways:

hgext/hgk.py

@@ -358,7 +358,7 b' def revtree(ui, args, repo, full=b"tree"'
 )
 def revlist(ui, repo, *revs, **opts):
     """print revisions"""
-    if opts[b'header']:
+    if opts['header']:
         full = b"commit"
     else:
         full = None

hgext/histedit.py

@@ -649,7 +649,7 b' def applychanges(ui, repo, ctx, opts):'
             repo.ui.setconfig(
                 b'ui', b'forcemerge', opts.get(b'tool', b''), b'histedit'
             )
-            stats = mergemod.graft(repo, ctx, ctx.p1(), [b'local', b'histedit'])
+            stats = mergemod.graft(repo, ctx, labels=[b'local', b'histedit'])
         finally:
             repo.ui.setconfig(b'ui', b'forcemerge', b'', b'histedit')
     return stats
@@ -835,10 +835,10 b' class fold(histeditaction):'
             return ctx, [(self.node, (parentctxnode,))]
 
         parentctx = repo[parentctxnode]
-        newcommits = set(
+        newcommits = {
             c.node()
             for c in repo.set(b'(%d::. - %d)', parentctx.rev(), parentctx.rev())
-        )
+        }
         if not newcommits:
             repo.ui.warn(
                 _(
@@ -945,7 +945,7 b' class fold(histeditaction):'
 class base(histeditaction):
     def run(self):
         if self.repo[b'.'].node() != self.node:
-            mergemod.update(self.repo, self.node, branchmerge=False, force=True)
+            mergemod.clean_update(self.repo[self.node])
         return self.continueclean()
 
     def continuedirty(self):
@@ -1113,7 +1113,8 b' def screen_size():'
 
 
 class histeditrule(object):
-    def __init__(self, ctx, pos, action=b'pick'):
+    def __init__(self, ui, ctx, pos, action=b'pick'):
+        self.ui = ui
         self.ctx = ctx
         self.action = action
         self.origpos = pos
@@ -1153,6 +1154,14 b' class histeditrule(object):'
 
     @property
     def desc(self):
+        summary = (
+            cmdutil.rendertemplate(
+                self.ctx, self.ui.config(b'histedit', b'summary-template')
+            )
+            or b''
+        )
+        if summary:
+            return summary
         # This is split off from the prefix property so that we can
         # separately make the description for 'roll' red (since it
         # will get discarded).
@@ -1258,7 +1267,7 b' def changeview(state, delta, unit):'
     num_lines = len(mode_state[b'patchcontents'])
     page_height = state[b'page_height']
     unit = page_height if unit == b'page' else 1
-    num_pages = 1 + (num_lines - 1) / page_height
+    num_pages = 1 + (num_lines - 1) // page_height
     max_offset = (num_pages - 1) * page_height
     newline = mode_state[b'line_offset'] + delta * unit
     mode_state[b'line_offset'] = max(0, min(max_offset, newline))
@@ -1700,7 +1709,7 b' def _chistedit(ui, repo, freeargs, opts)'
 
         ctxs = []
         for i, r in enumerate(revs):
-            ctxs.append(histeditrule(repo[r], i))
+            ctxs.append(histeditrule(ui, repo[r], i))
         # Curses requires setting the locale or it will default to the C
         # locale. This sets the locale to the user's default system
         # locale.
@@ -2412,7 +2421,7 b' def verifyactions(actions, state, ctxs):'
     Will abort if there are to many or too few rules, a malformed rule,
     or a rule on a changeset outside of the user-given range.
     """
-    expected = set(c.node() for c in ctxs)
+    expected = {c.node() for c in ctxs}
     seen = set()
     prev = None
 

hgext/largefiles/basestore.py

@@ -67,7 +67,7 b' class basestore(object):'
         ui = self.ui
 
         at = 0
-        available = self.exists(set(hash for (_filename, hash) in files))
+        available = self.exists({hash for (_filename, hash) in files})
         with ui.makeprogress(
             _(b'getting largefiles'), unit=_(b'files'), total=len(files)
         ) as progress:

hgext/largefiles/lfutil.py

@@ -92,16 +92,30 b' def _usercachedir(ui, name=longname):'
     path = ui.configpath(name, b'usercache')
     if path:
         return path
+
+    hint = None
+
     if pycompat.iswindows:
         appdata = encoding.environ.get(
             b'LOCALAPPDATA', encoding.environ.get(b'APPDATA')
         )
         if appdata:
             return os.path.join(appdata, name)
+
+        hint = _(b"define %s or %s in the environment, or set %s.usercache") % (
+            b"LOCALAPPDATA",
+            b"APPDATA",
+            name,
+        )
     elif pycompat.isdarwin:
         home = encoding.environ.get(b'HOME')
         if home:
             return os.path.join(home, b'Library', b'Caches', name)
+
+        hint = _(b"define %s in the environment, or set %s.usercache") % (
+            b"HOME",
+            name,
+        )
     elif pycompat.isposix:
         path = encoding.environ.get(b'XDG_CACHE_HOME')
         if path:
@@ -109,11 +123,18 b' def _usercachedir(ui, name=longname):'
         home = encoding.environ.get(b'HOME')
         if home:
             return os.path.join(home, b'.cache', name)
+
+        hint = _(b"define %s or %s in the environment, or set %s.usercache") % (
+            b"XDG_CACHE_HOME",
+            b"HOME",
+            name,
+        )
     else:
         raise error.Abort(
             _(b'unknown operating system: %s\n') % pycompat.osname
         )
-    raise error.Abort(_(b'unknown %s usercache location') % name)
+
+    raise error.Abort(_(b'unknown %s usercache location') % name, hint=hint)
 
 
 def inusercache(ui, hash):

hgext/largefiles/overrides.py

@@ -1564,11 +1564,11 b' def overridepurge(orig, ui, repo, *dirs,'
 def overriderollback(orig, ui, repo, **opts):
     with repo.wlock():
         before = repo.dirstate.parents()
-        orphans = set(
+        orphans = {
             f
             for f in repo.dirstate
             if lfutil.isstandin(f) and repo.dirstate[f] != b'r'
-        )
+        }
         result = orig(ui, repo, **opts)
         after = repo.dirstate.parents()
         if before == after:

hgext/largefiles/remotestore.py

@@ -48,12 +48,12 b' class remotestore(basestore.basestore):'
         )
 
     def exists(self, hashes):
-        return dict(
-            (h, s == 0)
+        return {
+            h: s == 0
             for (h, s) in pycompat.iteritems(
                 self._stat(hashes)
             )  # dict-from-generator
-        )
+        }
 
     def sendfile(self, filename, hash):
         self.ui.debug(b'remotestore: sendfile(%s, %s)\n' % (filename, hash))

hgext/lfs/TODO.rst

@@ -38,9 +38,6 b' ruled out) prior to taking off the exper'
 
    * `hg diff` is similar, and probably shouldn't see the pointer file
 
-#. `Fix https multiplexing, and re-enable workers
-   <https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-January/109916.html>`_.
-
 #. Show to-be-applied rules with `hg files -r 'wdir()' 'set:lfs()'`
 
    * `debugignore` can show file + line number, so a dedicated command could be

hgext/lfs/__init__.py

@@ -181,7 +181,7 b' eh.configitem('
     b'experimental', b'lfs.disableusercache', default=False,
 )
 eh.configitem(
-    b'experimental', b'lfs.worker-enable', default=False,
+    b'experimental', b'lfs.worker-enable', default=True,
 )
 
 eh.configitem(

hgext/lfs/blobstore.py

@@ -21,6 +21,7 b' from mercurial.pycompat import getattr'
 from mercurial import (
     encoding,
     error,
+    httpconnection as httpconnectionmod,
     node,
     pathutil,
     pycompat,
@@ -94,33 +95,16 b' class nullvfs(lfsvfs):'
         pass
 
 
-class filewithprogress(object):
-    """a file-like object that supports __len__ and read.
-
-    Useful to provide progress information for how many bytes are read.
+class lfsuploadfile(httpconnectionmod.httpsendfile):
+    """a file-like object that supports keepalive.
     """
 
-    def __init__(self, fp, callback):
-        self._fp = fp
-        self._callback = callback  # func(readsize)
-        fp.seek(0, os.SEEK_END)
-        self._len = fp.tell()
-        fp.seek(0)
-
-    def __len__(self):
-        return self._len
+    def __init__(self, ui, filename):
+        super(lfsuploadfile, self).__init__(ui, filename, b'rb')
+        self.read = self._data.read
 
-    def read(self, size):
-        if self._fp is None:
-            return b''
-        data = self._fp.read(size)
-        if data:
-            if self._callback:
-                self._callback(len(data))
-        else:
-            self._fp.close()
-            self._fp = None
-        return data
+    def _makeprogress(self):
+        return None  # progress is handled by the worker client
 
 
 class local(object):
@@ -144,6 +128,17 b' class local(object):'
     def open(self, oid):
         """Open a read-only file descriptor to the named blob, in either the
         usercache or the local store."""
+        return open(self.path(oid), 'rb')
+
+    def path(self, oid):
+        """Build the path for the given blob ``oid``.
+
+        If the blob exists locally, the path may point to either the usercache
+        or the local store.  If it doesn't, it will point to the local store.
+        This is meant for situations where existing code that isn't LFS aware
+        needs to open a blob.  Generally, prefer the ``open`` method on this
+        class.
+        """
         # The usercache is the most likely place to hold the file.  Commit will
         # write to both it and the local store, as will anything that downloads
         # the blobs.  However, things like clone without an update won't
@@ -151,9 +146,9 b' class local(object):'
         # the usercache is the only place it _could_ be.  If not present, the
         # missing file msg here will indicate the local repo, not the usercache.
         if self.cachevfs.exists(oid):
-            return self.cachevfs(oid, b'rb')
+            return self.cachevfs.join(oid)
 
-        return self.vfs(oid, b'rb')
+        return self.vfs.join(oid)
 
     def download(self, oid, src, content_length):
         """Read the blob from the remote source in chunks, verify the content,
@@ -495,15 +490,17 b' class _gitlfsremote(object):'
                     _(b'detected corrupt lfs object: %s') % oid,
                     hint=_(b'run hg verify'),
                 )
-            request.data = filewithprogress(localstore.open(oid), None)
-            request.get_method = lambda: r'PUT'
-            request.add_header('Content-Type', 'application/octet-stream')
-            request.add_header('Content-Length', len(request.data))
 
         for k, v in headers:
             request.add_header(pycompat.strurl(k), pycompat.strurl(v))
 
         try:
+            if action == b'upload':
+                request.data = lfsuploadfile(self.ui, localstore.path(oid))
+                request.get_method = lambda: 'PUT'
+                request.add_header('Content-Type', 'application/octet-stream')
+                request.add_header('Content-Length', request.data.length)
+
             with contextlib.closing(self.urlopener.open(request)) as res:
                 contentlength = res.info().get(b"content-length")
                 ui = self.ui  # Shorten debug lines
@@ -545,6 +542,9 b' class _gitlfsremote(object):'
             raise LfsRemoteError(
                 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
             )
+        finally:
+            if request.data:
+                request.data.close()
 
     def _batch(self, pointers, localstore, action):
         if action not in [b'upload', b'download']:

hgext/logtoprocess.py

@@ -59,6 +59,13 b' class processlogger(object):'
 
     def log(self, ui, event, msg, opts):
         script = self._scripts[event]
+        maxmsg = 100000
+        if len(msg) > maxmsg:
+            # Each env var has a 128KiB limit on linux. msg can be long, in
+            # particular for command event, where it's the full command line.
+            # Prefer truncating the message than raising "Argument list too
+            # long" error.
+            msg = msg[:maxmsg] + b' (truncated)'
         env = {
             b'EVENT': event,
             b'HGPID': os.getpid(),

hgext/mq.py

@@ -858,7 +858,7 b' class queue(object):'
         strip(self.ui, repo, [n], update=False, backup=False)
 
         ctx = repo[rev]
-        ret = hg.merge(repo, rev)
+        ret = hg.merge(ctx, remind=False)
         if ret:
             raise error.Abort(_(b"update returned %d") % ret)
         n = newcommit(repo, None, ctx.description(), ctx.user(), force=True)
@@ -1162,7 +1162,7 b' class queue(object):'
 
         if unknown:
             if numrevs:
-                rev = dict((entry.name, entry.node) for entry in qfinished)
+                rev = {entry.name: entry.node for entry in qfinished}
                 for p in unknown:
                     msg = _(b'revision %s refers to unknown patches: %s\n')
                     self.ui.warn(msg % (short(rev[p]), p))
@@ -3361,7 +3361,7 b' def guard(ui, repo, *args, **opts):'
         ui.write(b'\n')
 
     q = repo.mq
-    applied = set(p.name for p in q.applied)
+    applied = {p.name for p in q.applied}
     patch = None
     args = list(args)
     if opts.get('list'):

hgext/notify.py

@@ -133,6 +133,15 b' notify.fromauthor'
   the "From" field of the notification mail. If not set, take the user
   from the pushing repo.  Default: False.
 
+notify.reply-to-predecessor (EXPERIMENTAL)
+  If set and the changeset has a predecessor in the repository, try to thread
+  the notification mail with the predecessor. This adds the "In-Reply-To" header
+  to the notification mail with a reference to the predecessor with the smallest
+  revision number. Mail threads can still be torn, especially when changesets
+  are folded.
+
+  This option must  be used in combination with ``notify.messageidseed``.
+
 If set, the following entries will also be used to customize the
 notifications:
 
@@ -160,6 +169,7 b' from mercurial import ('
     error,
     logcmdutil,
     mail,
+    obsutil,
     patch,
     pycompat,
     registrar,
@@ -219,6 +229,9 b' configitem('
     b'notify', b'outgoing', default=None,
 )
 configitem(
+    b'notify', b'reply-to-predecessor', default=False,
+)
+configitem(
     b'notify', b'sources', default=b'serve',
 )
 configitem(
@@ -281,6 +294,16 b' class notifier(object):'
         self.merge = self.ui.configbool(b'notify', b'merge')
         self.showfunc = self.ui.configbool(b'notify', b'showfunc')
         self.messageidseed = self.ui.config(b'notify', b'messageidseed')
+        self.reply = self.ui.configbool(b'notify', b'reply-to-predecessor')
+
+        if self.reply and not self.messageidseed:
+            raise error.Abort(
+                _(
+                    b'notify.reply-to-predecessor used without '
+                    b'notify.messageidseed'
+                )
+            )
+
         if self.showfunc is None:
             self.showfunc = self.ui.configbool(b'diff', b'showfunc')
 
@@ -437,6 +460,26 b' class notifier(object):'
         msg['X-Hg-Notification'] = 'changeset %s' % ctx
         if not msg['Message-Id']:
             msg['Message-Id'] = messageid(ctx, self.domain, self.messageidseed)
+        if self.reply:
+            unfi = self.repo.unfiltered()
+            has_node = unfi.changelog.index.has_node
+            predecessors = [
+                unfi[ctx2]
+                for ctx2 in obsutil.allpredecessors(unfi.obsstore, [ctx.node()])
+                if ctx2 != ctx.node() and has_node(ctx2)
+            ]
+            if predecessors:
+                # There is at least one predecessor, so which to pick?
+                # Ideally, there is a unique root because changesets have
+                # been evolved/rebased one step at a time. In this case,
+                # just picking the oldest known changeset provides a stable
+                # base. It doesn't help when changesets are folded. Any
+                # better solution would require storing more information
+                # in the repository.
+                pred = min(predecessors, key=lambda ctx: ctx.rev())
+                msg['In-Reply-To'] = messageid(
+                    pred, self.domain, self.messageidseed
+                )
         msg['To'] = ', '.join(sorted(subs))
 
         msgtext = msg.as_bytes() if pycompat.ispy3 else msg.as_string()

hgext/phabricator.py

@@ -54,13 +54,14 b' import mimetypes'
 import operator
 import re
 
-from mercurial.node import bin, nullid
+from mercurial.node import bin, nullid, short
 from mercurial.i18n import _
 from mercurial.pycompat import getattr
 from mercurial.thirdparty import attr
 from mercurial import (
     cmdutil,
     context,
+    copies,
     encoding,
     error,
     exthelper,
@@ -114,6 +115,10 b' eh.configitem('
 eh.configitem(
     b'phabricator', b'curlcmd', default=None,
 )
+# developer config: phabricator.debug
+eh.configitem(
+    b'phabricator', b'debug', default=False,
+)
 # developer config: phabricator.repophid
 eh.configitem(
     b'phabricator', b'repophid', default=None,
@@ -124,6 +129,12 b' eh.configitem('
 eh.configitem(
     b'phabsend', b'confirm', default=False,
 )
+eh.configitem(
+    b'phabimport', b'secret', default=False,
+)
+eh.configitem(
+    b'phabimport', b'obsolete', default=False,
+)
 
 colortable = {
     b'phabricator.action.created': b'green',
@@ -257,19 +268,36 b' def vcrcommand(name, flags, spec, helpca'
                         return fn(*args, **kwargs)
             return fn(*args, **kwargs)
 
-        inner.__name__ = fn.__name__
-        inner.__doc__ = fn.__doc__
+        cmd = util.checksignature(inner, depth=2)
+        cmd.__name__ = fn.__name__
+        cmd.__doc__ = fn.__doc__
+
         return command(
             name,
             fullflags,
             spec,
             helpcategory=helpcategory,
             optionalrepo=optionalrepo,
-        )(inner)
+        )(cmd)
 
     return decorate
 
 
+def _debug(ui, *msg, **opts):
+    """write debug output for Phabricator if ``phabricator.debug`` is set
+
+    Specifically, this avoids dumping Conduit and HTTP auth chatter that is
+    printed with the --debug argument.
+    """
+    if ui.configbool(b"phabricator", b"debug"):
+        flag = ui.debugflag
+        try:
+            ui.debugflag = True
+            ui.write(*msg, **opts)
+        finally:
+            ui.debugflag = flag
+
+
 def urlencodenested(params):
     """like urlencode, but works with nested parameters.
 
@@ -446,7 +474,8 b' def getoldnodedrevmap(repo, nodelist):'
     has_node = unfi.changelog.index.has_node
 
     result = {}  # {node: (oldnode?, lastdiff?, drev)}
-    toconfirm = {}  # {node: (force, {precnode}, drev)}
+    # ordered for test stability when printing new -> old mapping below
+    toconfirm = util.sortdict()  # {node: (force, {precnode}, drev)}
     for node in nodelist:
         ctx = unfi[node]
         # For tags like "D123", put them into "toconfirm" to verify later
@@ -474,18 +503,23 b' def getoldnodedrevmap(repo, nodelist):'
         alldiffs = callconduit(
             unfi.ui, b'differential.querydiffs', {b'revisionIDs': drevs}
         )
-        getnode = lambda d: bin(getdiffmeta(d).get(b'node', b'')) or None
+
+        def getnodes(d, precset):
+            # Ignore other nodes that were combined into the Differential
+            # that aren't predecessors of the current local node.
+            return [n for n in getlocalcommits(d) if n in precset]
+
         for newnode, (force, precset, drev) in toconfirm.items():
             diffs = [
                 d for d in alldiffs.values() if int(d[b'revisionID']) == drev
             ]
 
-            # "precursors" as known by Phabricator
-            phprecset = set(getnode(d) for d in diffs)
+            # local predecessors known by Phabricator
+            phprecset = {n for d in diffs for n in getnodes(d, precset)}
 
             # Ignore if precursors (Phabricator and local repo) do not overlap,
             # and force is not set (when commit message says nothing)
-            if not force and not bool(phprecset & precset):
+            if not force and not phprecset:
                 tagname = b'D%d' % drev
                 tags.tag(
                     repo,
@@ -510,7 +544,33 b' def getoldnodedrevmap(repo, nodelist):'
             oldnode = lastdiff = None
             if diffs:
                 lastdiff = max(diffs, key=lambda d: int(d[b'id']))
-                oldnode = getnode(lastdiff)
+                oldnodes = getnodes(lastdiff, precset)
+
+                _debug(
+                    unfi.ui,
+                    b"%s mapped to old nodes %s\n"
+                    % (
+                        short(newnode),
+                        stringutil.pprint([short(n) for n in sorted(oldnodes)]),
+                    ),
+                )
+
+                # If this commit was the result of `hg fold` after submission,
+                # and now resubmitted with --fold, the easiest thing to do is
+                # to leave the node clear.  This only results in creating a new
+                # diff for the _same_ Differential Revision if this commit is
+                # the first or last in the selected range.  If we picked a node
+                # from the list instead, it would have to be the lowest if at
+                # the beginning of the --fold range, or the highest at the end.
+                # Otherwise, one or more of the nodes wouldn't be considered in
+                # the diff, and the Differential wouldn't be properly updated.
+                # If this commit is the result of `hg split` in the same
+                # scenario, there is a single oldnode here (and multiple
+                # newnodes mapped to it).  That makes it the same as the normal
+                # case, as the edges of the newnode range cleanly maps to one
+                # oldnode each.
+                if len(oldnodes) == 1:
+                    oldnode = oldnodes[0]
                 if oldnode and not has_node(oldnode):
                     oldnode = None
 
@@ -542,11 +602,11 b' def getdrevmap(repo, revs):'
     return result
 
 
-def getdiff(ctx, diffopts):
+def getdiff(basectx, ctx, diffopts):
     """plain-text diff without header (user, commit message, etc)"""
     output = util.stringio()
     for chunk, _label in patch.diffui(
-        ctx.repo(), ctx.p1().node(), ctx.node(), None, opts=diffopts
+        ctx.repo(), basectx.p1().node(), ctx.node(), None, opts=diffopts
     ):
         output.write(chunk)
     return output.getvalue()
@@ -653,13 +713,13 b' class phabdiff(object):'
         )
 
 
-def maketext(pchange, ctx, fname):
+def maketext(pchange, basectx, ctx, fname):
     """populate the phabchange for a text file"""
     repo = ctx.repo()
     fmatcher = match.exact([fname])
     diffopts = mdiff.diffopts(git=True, context=32767)
     _pfctx, _fctx, header, fhunks = next(
-        patch.diffhunks(repo, ctx.p1(), ctx, fmatcher, opts=diffopts)
+        patch.diffhunks(repo, basectx.p1(), ctx, fmatcher, opts=diffopts)
     )
 
     for fhunk in fhunks:
@@ -747,12 +807,14 b' def uploadfile(fctx):'
     return fphid
 
 
-def addoldbinary(pchange, fctx):
+def addoldbinary(pchange, oldfctx, fctx):
     """add the metadata for the previous version of a binary file to the
     phabchange for the new version
+
+    ``oldfctx`` is the previous version of the file; ``fctx`` is the new
+    version of the file, or None if the file is being removed.
     """
-    oldfctx = fctx.p1()
-    if fctx.cmp(oldfctx):
+    if not fctx or fctx.cmp(oldfctx):
         # Files differ, add the old one
         pchange.metadata[b'old:file:size'] = oldfctx.size()
         mimeguess, _enc = mimetypes.guess_type(
@@ -794,8 +856,6 b' def notutf8(fctx):'
     """
     try:
         fctx.data().decode('utf-8')
-        if fctx.parents():
-            fctx.p1().data().decode('utf-8')
         return False
     except UnicodeDecodeError:
         fctx.repo().ui.write(
@@ -805,56 +865,76 b' def notutf8(fctx):'
         return True
 
 
-def addremoved(pdiff, ctx, removed):
+def addremoved(pdiff, basectx, ctx, removed):
     """add removed files to the phabdiff. Shouldn't include moves"""
     for fname in removed:
         pchange = phabchange(
             currentPath=fname, oldPath=fname, type=DiffChangeType.DELETE
         )
-        pchange.addoldmode(gitmode[ctx.p1()[fname].flags()])
-        fctx = ctx.p1()[fname]
-        if not (fctx.isbinary() or notutf8(fctx)):
-            maketext(pchange, ctx, fname)
+        oldfctx = basectx.p1()[fname]
+        pchange.addoldmode(gitmode[oldfctx.flags()])
+        if not (oldfctx.isbinary() or notutf8(oldfctx)):
+            maketext(pchange, basectx, ctx, fname)
 
         pdiff.addchange(pchange)
 
 
-def addmodified(pdiff, ctx, modified):
+def addmodified(pdiff, basectx, ctx, modified):
     """add modified files to the phabdiff"""
     for fname in modified:
         fctx = ctx[fname]
+        oldfctx = basectx.p1()[fname]
         pchange = phabchange(currentPath=fname, oldPath=fname)
-        filemode = gitmode[ctx[fname].flags()]
-        originalmode = gitmode[ctx.p1()[fname].flags()]
+        filemode = gitmode[fctx.flags()]
+        originalmode = gitmode[oldfctx.flags()]
         if filemode != originalmode:
             pchange.addoldmode(originalmode)
             pchange.addnewmode(filemode)
 
-        if fctx.isbinary() or notutf8(fctx):
+        if (
+            fctx.isbinary()
+            or notutf8(fctx)
+            or oldfctx.isbinary()
+            or notutf8(oldfctx)
+        ):
             makebinary(pchange, fctx)
-            addoldbinary(pchange, fctx)
+            addoldbinary(pchange, oldfctx, fctx)
         else:
-            maketext(pchange, ctx, fname)
+            maketext(pchange, basectx, ctx, fname)
 
         pdiff.addchange(pchange)
 
 
-def addadded(pdiff, ctx, added, removed):
+def addadded(pdiff, basectx, ctx, added, removed):
     """add file adds to the phabdiff, both new files and copies/moves"""
     # Keep track of files that've been recorded as moved/copied, so if there are
     # additional copies we can mark them (moves get removed from removed)
     copiedchanges = {}
     movedchanges = {}
+
+    copy = {}
+    if basectx != ctx:
+        copy = copies.pathcopies(basectx.p1(), ctx)
+
     for fname in added:
         fctx = ctx[fname]
+        oldfctx = None
         pchange = phabchange(currentPath=fname)
 
-        filemode = gitmode[ctx[fname].flags()]
-        renamed = fctx.renamed()
+        filemode = gitmode[fctx.flags()]
+
+        if copy:
+            originalfname = copy.get(fname, fname)
+        else:
+            originalfname = fname
+            if fctx.renamed():
+                originalfname = fctx.renamed()[0]
+
+        renamed = fname != originalfname
 
         if renamed:
-            originalfname = renamed[0]
-            originalmode = gitmode[ctx.p1()[originalfname].flags()]
+            oldfctx = basectx.p1()[originalfname]
+            originalmode = gitmode[oldfctx.flags()]
             pchange.oldPath = originalfname
 
             if originalfname in removed:
@@ -889,12 +969,16 b' def addadded(pdiff, ctx, added, removed)'
             pchange.addnewmode(gitmode[fctx.flags()])
             pchange.type = DiffChangeType.ADD
 
-        if fctx.isbinary() or notutf8(fctx):
+        if (
+            fctx.isbinary()
+            or notutf8(fctx)
+            or (oldfctx and (oldfctx.isbinary() or notutf8(oldfctx)))
+        ):
             makebinary(pchange, fctx)
             if renamed:
-                addoldbinary(pchange, fctx)
+                addoldbinary(pchange, oldfctx, fctx)
         else:
-            maketext(pchange, ctx, fname)
+            maketext(pchange, basectx, ctx, fname)
 
         pdiff.addchange(pchange)
 
@@ -904,21 +988,21 b' def addadded(pdiff, ctx, added, removed)'
         pdiff.addchange(movedchange)
 
 
-def creatediff(ctx):
+def creatediff(basectx, ctx):
     """create a Differential Diff"""
     repo = ctx.repo()
     repophid = getrepophid(repo)
     # Create a "Differential Diff" via "differential.creatediff" API
     pdiff = phabdiff(
-        sourceControlBaseRevision=b'%s' % ctx.p1().hex(),
+        sourceControlBaseRevision=b'%s' % basectx.p1().hex(),
         branch=b'%s' % ctx.branch(),
     )
-    modified, added, removed, _d, _u, _i, _c = ctx.p1().status(ctx)
+    modified, added, removed, _d, _u, _i, _c = basectx.p1().status(ctx)
     # addadded will remove moved files from removed, so addremoved won't get
     # them
-    addadded(pdiff, ctx, added, removed)
-    addmodified(pdiff, ctx, modified)
-    addremoved(pdiff, ctx, removed)
+    addadded(pdiff, basectx, ctx, added, removed)
+    addmodified(pdiff, basectx, ctx, modified)
+    addremoved(pdiff, basectx, ctx, removed)
     if repophid:
         pdiff.repositoryPHID = repophid
     diff = callconduit(
@@ -927,52 +1011,64 b' def creatediff(ctx):'
         pycompat.byteskwargs(attr.asdict(pdiff)),
     )
     if not diff:
-        raise error.Abort(_(b'cannot create diff for %s') % ctx)
+        if basectx != ctx:
+            msg = _(b'cannot create diff for %s::%s') % (basectx, ctx)
+        else:
+            msg = _(b'cannot create diff for %s') % ctx
+        raise error.Abort(msg)
     return diff
 
 
-def writediffproperties(ctx, diff):
-    """write metadata to diff so patches could be applied losslessly"""
+def writediffproperties(ctxs, diff):
+    """write metadata to diff so patches could be applied losslessly
+
+    ``ctxs`` is the list of commits that created the diff, in ascending order.
+    The list is generally a single commit, but may be several when using
+    ``phabsend --fold``.
+    """
     # creatediff returns with a diffid but query returns with an id
     diffid = diff.get(b'diffid', diff.get(b'id'))
+    basectx = ctxs[0]
+    tipctx = ctxs[-1]
+
     params = {
         b'diff_id': diffid,
         b'name': b'hg:meta',
         b'data': templatefilters.json(
             {
-                b'user': ctx.user(),
-                b'date': b'%d %d' % ctx.date(),
-                b'branch': ctx.branch(),
-                b'node': ctx.hex(),
-                b'parent': ctx.p1().hex(),
+                b'user': tipctx.user(),
+                b'date': b'%d %d' % tipctx.date(),
+                b'branch': tipctx.branch(),
+                b'node': tipctx.hex(),
+                b'parent': basectx.p1().hex(),
             }
         ),
     }
-    callconduit(ctx.repo().ui, b'differential.setdiffproperty', params)
+    callconduit(basectx.repo().ui, b'differential.setdiffproperty', params)
 
+    commits = {}
+    for ctx in ctxs:
+        commits[ctx.hex()] = {
+            b'author': stringutil.person(ctx.user()),
+            b'authorEmail': stringutil.email(ctx.user()),
+            b'time': int(ctx.date()[0]),
+            b'commit': ctx.hex(),
+            b'parents': [ctx.p1().hex()],
+            b'branch': ctx.branch(),
+        }
     params = {
         b'diff_id': diffid,
         b'name': b'local:commits',
-        b'data': templatefilters.json(
-            {
-                ctx.hex(): {
-                    b'author': stringutil.person(ctx.user()),
-                    b'authorEmail': stringutil.email(ctx.user()),
-                    b'time': int(ctx.date()[0]),
-                    b'commit': ctx.hex(),
-                    b'parents': [ctx.p1().hex()],
-                    b'branch': ctx.branch(),
-                },
-            }
-        ),
+        b'data': templatefilters.json(commits),
     }
-    callconduit(ctx.repo().ui, b'differential.setdiffproperty', params)
+    callconduit(basectx.repo().ui, b'differential.setdiffproperty', params)
 
 
 def createdifferentialrevision(
-    ctx,
+    ctxs,
     revid=None,
     parentrevphid=None,
+    oldbasenode=None,
     oldnode=None,
     olddiff=None,
     actions=None,
@@ -983,22 +1079,38 b' def createdifferentialrevision('
     If revid is None, create a new Differential Revision, otherwise update
     revid. If parentrevphid is not None, set it as a dependency.
 
+    If there is a single commit for the new Differential Revision, ``ctxs`` will
+    be a list of that single context.  Otherwise, it is a list that covers the
+    range of changes for the differential, where ``ctxs[0]`` is the first change
+    to include and ``ctxs[-1]`` is the last.
+
     If oldnode is not None, check if the patch content (without commit message
-    and metadata) has changed before creating another diff.
+    and metadata) has changed before creating another diff.  For a Revision with
+    a single commit, ``oldbasenode`` and ``oldnode`` have the same value.  For a
+    Revision covering multiple commits, ``oldbasenode`` corresponds to
+    ``ctxs[0]`` the previous time this Revision was posted, and ``oldnode``
+    corresponds to ``ctxs[-1]``.
 
     If actions is not None, they will be appended to the transaction.
     """
+    ctx = ctxs[-1]
+    basectx = ctxs[0]
+
     repo = ctx.repo()
     if oldnode:
         diffopts = mdiff.diffopts(git=True, context=32767)
-        oldctx = repo.unfiltered()[oldnode]
-        neednewdiff = getdiff(ctx, diffopts) != getdiff(oldctx, diffopts)
+        unfi = repo.unfiltered()
+        oldctx = unfi[oldnode]
+        oldbasectx = unfi[oldbasenode]
+        neednewdiff = getdiff(basectx, ctx, diffopts) != getdiff(
+            oldbasectx, oldctx, diffopts
+        )
     else:
         neednewdiff = True
 
     transactions = []
     if neednewdiff:
-        diff = creatediff(ctx)
+        diff = creatediff(basectx, ctx)
         transactions.append({b'type': b'update', b'value': diff[b'phid']})
         if comment:
             transactions.append({b'type': b'comment', b'value': comment})
@@ -1008,7 +1120,7 b' def createdifferentialrevision('
         # pushers could know the correct node metadata.
         assert olddiff
         diff = olddiff
-    writediffproperties(ctx, diff)
+    writediffproperties(ctxs, diff)
 
     # Set the parent Revision every time, so commit re-ordering is picked-up
     if parentrevphid:
@@ -1019,14 +1131,42 b' def createdifferentialrevision('
     if actions:
         transactions += actions
 
-    # Parse commit message and update related fields.
-    desc = ctx.description()
-    info = callconduit(
-        repo.ui, b'differential.parsecommitmessage', {b'corpus': desc}
-    )
-    for k, v in info[b'fields'].items():
-        if k in [b'title', b'summary', b'testPlan']:
-            transactions.append({b'type': k, b'value': v})
+    # When folding multiple local commits into a single review, arcanist will
+    # take the summary line of the first commit as the title, and then
+    # concatenate the rest of the remaining messages (including each of their
+    # first lines) to the rest of the first commit message (each separated by
+    # an empty line), and use that as the summary field.  Do the same here.
+    # For commits with only a one line message, there is no summary field, as
+    # this gets assigned to the title.
+    fields = util.sortdict()  # sorted for stable wire protocol in tests
+
+    for i, _ctx in enumerate(ctxs):
+        # Parse commit message and update related fields.
+        desc = _ctx.description()
+        info = callconduit(
+            repo.ui, b'differential.parsecommitmessage', {b'corpus': desc}
+        )
+
+        for k in [b'title', b'summary', b'testPlan']:
+            v = info[b'fields'].get(k)
+            if not v:
+                continue
+
+            if i == 0:
+                # Title, summary and test plan (if present) are taken verbatim
+                # for the first commit.
+                fields[k] = v.rstrip()
+                continue
+            elif k == b'title':
+                # Add subsequent titles (i.e. the first line of the commit
+                # message) back to the summary.
+                k = b'summary'
+
+            # Append any current field to the existing composite field
+            fields[k] = b'\n\n'.join(filter(None, [fields.get(k), v.rstrip()]))
+
+    for k, v in fields.items():
+        transactions.append({b'type': k, b'value': v})
 
     params = {b'transactions': transactions}
     if revid is not None:
@@ -1035,20 +1175,24 b' def createdifferentialrevision('
 
     revision = callconduit(repo.ui, b'differential.revision.edit', params)
     if not revision:
-        raise error.Abort(_(b'cannot create revision for %s') % ctx)
+        if len(ctxs) == 1:
+            msg = _(b'cannot create revision for %s') % ctx
+        else:
+            msg = _(b'cannot create revision for %s::%s') % (basectx, ctx)
+        raise error.Abort(msg)
 
     return revision, diff
 
 
-def userphids(repo, names):
+def userphids(ui, names):
     """convert user names to PHIDs"""
     names = [name.lower() for name in names]
     query = {b'constraints': {b'usernames': names}}
-    result = callconduit(repo.ui, b'user.search', query)
+    result = callconduit(ui, b'user.search', query)
     # username not found is not an error of the API. So check if we have missed
     # some names here.
     data = result[b'data']
-    resolved = set(entry[b'fields'][b'username'].lower() for entry in data)
+    resolved = {entry[b'fields'][b'username'].lower() for entry in data}
     unresolved = set(names) - resolved
     if unresolved:
         raise error.Abort(
@@ -1057,6 +1201,45 b' def userphids(repo, names):'
     return [entry[b'phid'] for entry in data]
 
 
+def _print_phabsend_action(ui, ctx, newrevid, action):
+    """print the ``action`` that occurred when posting ``ctx`` for review
+
+    This is a utility function for the sending phase of ``phabsend``, which
+    makes it easier to show a status for all local commits with `--fold``.
+    """
+    actiondesc = ui.label(
+        {
+            b'created': _(b'created'),
+            b'skipped': _(b'skipped'),
+            b'updated': _(b'updated'),
+        }[action],
+        b'phabricator.action.%s' % action,
+    )
+    drevdesc = ui.label(b'D%d' % newrevid, b'phabricator.drev')
+    nodedesc = ui.label(bytes(ctx), b'phabricator.node')
+    desc = ui.label(ctx.description().split(b'\n')[0], b'phabricator.desc')
+    ui.write(_(b'%s - %s - %s: %s\n') % (drevdesc, actiondesc, nodedesc, desc))
+
+
+def _amend_diff_properties(unfi, drevid, newnodes, diff):
+    """update the local commit list for the ``diff`` associated with ``drevid``
+
+    This is a utility function for the amend phase of ``phabsend``, which
+    converts failures to warning messages.
+    """
+    _debug(
+        unfi.ui,
+        b"new commits: %s\n" % stringutil.pprint([short(n) for n in newnodes]),
+    )
+
+    try:
+        writediffproperties([unfi[newnode] for newnode in newnodes], diff)
+    except util.urlerr.urlerror:
+        # If it fails just warn and keep going, otherwise the DREV
+        # associations will be lost
+        unfi.ui.warnnoi18n(b'Failed to update metadata for D%d\n' % drevid)
+
+
 @vcrcommand(
     b'phabsend',
     [
@@ -1071,6 +1254,7 b' def userphids(repo, names):'
             _(b'add a comment to Revisions with new/updated Diffs'),
         ),
         (b'', b'confirm', None, _(b'ask for confirmation before sending')),
+        (b'', b'fold', False, _(b'combine the revisions into one review')),
     ],
     _(b'REV [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
@@ -1099,6 +1283,12 b' def phabsend(ui, repo, *revs, **opts):'
         [phabsend]
         confirm = true
 
+    By default, a separate review will be created for each commit that is
+    selected, and will have the same parent/child relationship in Phabricator.
+    If ``--fold`` is set, multiple commits are rolled up into a single review
+    as if diffed from the parent of the first revision to the last.  The commit
+    messages are concatenated in the summary field on Phabricator.
+
     phabsend will check obsstore and the above association to decide whether to
     update an existing Differential Revision, or create a new one.
     """
@@ -1112,6 +1302,47 b' def phabsend(ui, repo, *revs, **opts):'
     if opts.get(b'amend'):
         cmdutil.checkunfinished(repo)
 
+    ctxs = [repo[rev] for rev in revs]
+
+    if any(c for c in ctxs if c.obsolete()):
+        raise error.Abort(_(b"obsolete commits cannot be posted for review"))
+
+    fold = opts.get(b'fold')
+    if fold:
+        if len(revs) == 1:
+            # TODO: just switch to --no-fold instead?
+            raise error.Abort(_(b"cannot fold a single revision"))
+
+        # There's no clear way to manage multiple commits with a Dxxx tag, so
+        # require the amend option.  (We could append "_nnn", but then it
+        # becomes jumbled if earlier commits are added to an update.)  It should
+        # lock the repo and ensure that the range is editable, but that would
+        # make the code pretty convoluted.  The default behavior of `arc` is to
+        # create a new review anyway.
+        if not opts.get(b"amend"):
+            raise error.Abort(_(b"cannot fold with --no-amend"))
+
+        # Ensure the local commits are an unbroken range
+        revrange = repo.revs(b'(first(%ld)::last(%ld))', revs, revs)
+        if any(r for r in revs if r not in revrange) or any(
+            r for r in revrange if r not in revs
+        ):
+            raise error.Abort(_(b"cannot fold non-linear revisions"))
+
+        # It might be possible to bucketize the revisions by the DREV value, and
+        # iterate over those groups when posting, and then again when amending.
+        # But for simplicity, require all selected revisions to be for the same
+        # DREV (if present).  Adding local revisions to an existing DREV is
+        # acceptable.
+        drevmatchers = [
+            _differentialrevisiondescre.search(ctx.description())
+            for ctx in ctxs
+        ]
+        if len({m.group('url') for m in drevmatchers if m}) > 1:
+            raise error.Abort(
+                _(b"cannot fold revisions with different DREV values")
+            )
+
     # {newnode: (oldnode, olddiff, olddrev}
     oldmap = getoldnodedrevmap(repo, [repo[r].node() for r in revs])
 
@@ -1127,10 +1358,13 b' def phabsend(ui, repo, *revs, **opts):'
     blockers = opts.get(b'blocker', [])
     phids = []
     if reviewers:
-        phids.extend(userphids(repo, reviewers))
+        phids.extend(userphids(repo.ui, reviewers))
     if blockers:
         phids.extend(
-            map(lambda phid: b'blocking(%s)' % phid, userphids(repo, blockers))
+            map(
+                lambda phid: b'blocking(%s)' % phid,
+                userphids(repo.ui, blockers),
+            )
         )
     if phids:
         actions.append({b'type': b'reviewers.add', b'value': phids})
@@ -1141,24 +1375,40 b' def phabsend(ui, repo, *revs, **opts):'
     # Send patches one by one so we know their Differential Revision PHIDs and
     # can provide dependency relationship
     lastrevphid = None
-    for rev in revs:
-        ui.debug(b'sending rev %d\n' % rev)
-        ctx = repo[rev]
+    for ctx in ctxs:
+        if fold:
+            ui.debug(b'sending rev %d::%d\n' % (ctx.rev(), ctxs[-1].rev()))
+        else:
+            ui.debug(b'sending rev %d\n' % ctx.rev())
 
         # Get Differential Revision ID
         oldnode, olddiff, revid = oldmap.get(ctx.node(), (None, None, None))
+        oldbasenode, oldbasediff, oldbaserevid = oldnode, olddiff, revid
+
+        if fold:
+            oldbasenode, oldbasediff, oldbaserevid = oldmap.get(
+                ctxs[-1].node(), (None, None, None)
+            )
+
         if oldnode != ctx.node() or opts.get(b'amend'):
             # Create or update Differential Revision
             revision, diff = createdifferentialrevision(
-                ctx,
+                ctxs if fold else [ctx],
                 revid,
                 lastrevphid,
+                oldbasenode,
                 oldnode,
                 olddiff,
                 actions,
                 opts.get(b'comment'),
             )
-            diffmap[ctx.node()] = diff
+
+            if fold:
+                for ctx in ctxs:
+                    diffmap[ctx.node()] = diff
+            else:
+                diffmap[ctx.node()] = diff
+
             newrevid = int(revision[b'object'][b'id'])
             newrevphid = revision[b'object'][b'phid']
             if revid:
@@ -1168,56 +1418,75 b' def phabsend(ui, repo, *revs, **opts):'
 
             # Create a local tag to note the association, if commit message
             # does not have it already
-            m = _differentialrevisiondescre.search(ctx.description())
-            if not m or int(m.group('id')) != newrevid:
-                tagname = b'D%d' % newrevid
-                tags.tag(
-                    repo,
-                    tagname,
-                    ctx.node(),
-                    message=None,
-                    user=None,
-                    date=None,
-                    local=True,
-                )
+            if not fold:
+                m = _differentialrevisiondescre.search(ctx.description())
+                if not m or int(m.group('id')) != newrevid:
+                    tagname = b'D%d' % newrevid
+                    tags.tag(
+                        repo,
+                        tagname,
+                        ctx.node(),
+                        message=None,
+                        user=None,
+                        date=None,
+                        local=True,
+                    )
         else:
             # Nothing changed. But still set "newrevphid" so the next revision
             # could depend on this one and "newrevid" for the summary line.
-            newrevphid = querydrev(repo, b'%d' % revid)[0][b'phid']
+            newrevphid = querydrev(repo.ui, b'%d' % revid)[0][b'phid']
             newrevid = revid
             action = b'skipped'
 
-        actiondesc = ui.label(
-            {
-                b'created': _(b'created'),
-                b'skipped': _(b'skipped'),
-                b'updated': _(b'updated'),
-            }[action],
-            b'phabricator.action.%s' % action,
-        )
-        drevdesc = ui.label(b'D%d' % newrevid, b'phabricator.drev')
-        nodedesc = ui.label(bytes(ctx), b'phabricator.node')
-        desc = ui.label(ctx.description().split(b'\n')[0], b'phabricator.desc')
-        ui.write(
-            _(b'%s - %s - %s: %s\n') % (drevdesc, actiondesc, nodedesc, desc)
-        )
         drevids.append(newrevid)
         lastrevphid = newrevphid
 
+        if fold:
+            for c in ctxs:
+                if oldmap.get(c.node(), (None, None, None))[2]:
+                    action = b'updated'
+                else:
+                    action = b'created'
+                _print_phabsend_action(ui, c, newrevid, action)
+            break
+
+        _print_phabsend_action(ui, ctx, newrevid, action)
+
     # Update commit messages and remove tags
     if opts.get(b'amend'):
         unfi = repo.unfiltered()
         drevs = callconduit(ui, b'differential.query', {b'ids': drevids})
         with repo.wlock(), repo.lock(), repo.transaction(b'phabsend'):
+            # Eagerly evaluate commits to restabilize before creating new
+            # commits.  The selected revisions are excluded because they are
+            # automatically restacked as part of the submission process.
+            restack = [
+                c
+                for c in repo.set(
+                    b"(%ld::) - (%ld) - unstable() - obsolete() - public()",
+                    revs,
+                    revs,
+                )
+            ]
             wnode = unfi[b'.'].node()
             mapping = {}  # {oldnode: [newnode]}
+            newnodes = []
+
+            drevid = drevids[0]
+
             for i, rev in enumerate(revs):
                 old = unfi[rev]
-                drevid = drevids[i]
+                if not fold:
+                    drevid = drevids[i]
                 drev = [d for d in drevs if int(d[b'id']) == drevid][0]
-                newdesc = getdescfromdrev(drev)
+
+                newdesc = get_amended_desc(drev, old, fold)
                 # Make sure commit message contain "Differential Revision"
-                if old.description() != newdesc:
+                if (
+                    old.description() != newdesc
+                    or old.p1().node() in mapping
+                    or old.p2().node() in mapping
+                ):
                     if old.phase() == phases.public:
                         ui.warn(
                             _(b"warning: not updating public commit %s\n")
@@ -1241,27 +1510,93 b' def phabsend(ui, repo, *revs, **opts):'
                     newnode = new.commit()
 
                     mapping[old.node()] = [newnode]
-                    # Update diff property
-                    # If it fails just warn and keep going, otherwise the DREV
-                    # associations will be lost
-                    try:
-                        writediffproperties(unfi[newnode], diffmap[old.node()])
-                    except util.urlerr.urlerror:
-                        ui.warnnoi18n(
-                            b'Failed to update metadata for D%d\n' % drevid
+
+                    if fold:
+                        # Defer updating the (single) Diff until all nodes are
+                        # collected.  No tags were created, so none need to be
+                        # removed.
+                        newnodes.append(newnode)
+                        continue
+
+                    _amend_diff_properties(
+                        unfi, drevid, [newnode], diffmap[old.node()]
+                    )
+
+                    # Remove local tags since it's no longer necessary
+                    tagname = b'D%d' % drevid
+                    if tagname in repo.tags():
+                        tags.tag(
+                            repo,
+                            tagname,
+                            nullid,
+                            message=None,
+                            user=None,
+                            date=None,
+                            local=True,
                         )
-                # Remove local tags since it's no longer necessary
-                tagname = b'D%d' % drevid
-                if tagname in repo.tags():
-                    tags.tag(
-                        repo,
-                        tagname,
-                        nullid,
-                        message=None,
-                        user=None,
-                        date=None,
-                        local=True,
+                elif fold:
+                    # When folding multiple commits into one review with
+                    # --fold, track even the commits that weren't amended, so
+                    # that their association isn't lost if the properties are
+                    # rewritten below.
+                    newnodes.append(old.node())
+
+            # If the submitted commits are public, no amend takes place so
+            # there are no newnodes and therefore no diff update to do.
+            if fold and newnodes:
+                diff = diffmap[old.node()]
+
+                # The diff object in diffmap doesn't have the local commits
+                # because that could be returned from differential.creatediff,
+                # not differential.querydiffs.  So use the queried diff (if
+                # present), or force the amend (a new revision is being posted.)
+                if not olddiff or set(newnodes) != getlocalcommits(olddiff):
+                    _debug(ui, b"updating local commit list for D%d\n" % drevid)
+                    _amend_diff_properties(unfi, drevid, newnodes, diff)
+                else:
+                    _debug(
+                        ui,
+                        b"local commit list for D%d is already up-to-date\n"
+                        % drevid,
                     )
+            elif fold:
+                _debug(ui, b"no newnodes to update\n")
+
+            # Restack any children of first-time submissions that were orphaned
+            # in the process.  The ctx won't report that it is an orphan until
+            # the cleanup takes place below.
+            for old in restack:
+                parents = [
+                    mapping.get(old.p1().node(), (old.p1(),))[0],
+                    mapping.get(old.p2().node(), (old.p2(),))[0],
+                ]
+                new = context.metadataonlyctx(
+                    repo,
+                    old,
+                    parents=parents,
+                    text=old.description(),
+                    user=old.user(),
+                    date=old.date(),
+                    extra=old.extra(),
+                )
+
+                newnode = new.commit()
+
+                # Don't obsolete unselected descendants of nodes that have not
+                # been changed in this transaction- that results in an error.
+                if newnode != old.node():
+                    mapping[old.node()] = [newnode]
+                    _debug(
+                        ui,
+                        b"restabilizing %s as %s\n"
+                        % (short(old.node()), short(newnode)),
+                    )
+                else:
+                    _debug(
+                        ui,
+                        b"not restabilizing unchanged %s\n" % short(old.node()),
+                    )
+
             scmutil.cleanupnodes(repo, mapping, b'phabsend', fixphase=True)
             if wnode in mapping:
                 unfi.setparents(mapping[wnode][0])
@@ -1398,7 +1733,7 b' def _prefetchdrevs(tree):'
     return drevs, ancestordrevs
 
 
-def querydrev(repo, spec):
+def querydrev(ui, spec):
     """return a list of "Differential Revision" dicts
 
     spec is a string using a simple query language, see docstring in phabread
@@ -1407,46 +1742,49 b' def querydrev(repo, spec):'
     A "Differential Revision dict" looks like:
 
         {
-            "id": "2",
-            "phid": "PHID-DREV-672qvysjcczopag46qty",
-            "title": "example",
-            "uri": "https://phab.example.com/D2",
+            "activeDiffPHID": "PHID-DIFF-xoqnjkobbm6k4dk6hi72",
+            "authorPHID": "PHID-USER-tv3ohwc4v4jeu34otlye",
+            "auxiliary": {
+              "phabricator:depends-on": [
+                "PHID-DREV-gbapp366kutjebt7agcd"
+              ]
+              "phabricator:projects": [],
+            },
+            "branch": "default",
+            "ccs": [],
+            "commits": [],
             "dateCreated": "1499181406",
             "dateModified": "1499182103",
-            "authorPHID": "PHID-USER-tv3ohwc4v4jeu34otlye",
-            "status": "0",
-            "statusName": "Needs Review",
-            "properties": [],
-            "branch": null,
-            "summary": "",
-            "testPlan": "",
-            "lineCount": "2",
-            "activeDiffPHID": "PHID-DIFF-xoqnjkobbm6k4dk6hi72",
             "diffs": [
               "3",
               "4",
             ],
-            "commits": [],
+            "hashes": [],
+            "id": "2",
+            "lineCount": "2",
+            "phid": "PHID-DREV-672qvysjcczopag46qty",
+            "properties": {},
+            "repositoryPHID": "PHID-REPO-hub2hx62ieuqeheznasv",
             "reviewers": [],
-            "ccs": [],
-            "hashes": [],
-            "auxiliary": {
-              "phabricator:projects": [],
-              "phabricator:depends-on": [
-                "PHID-DREV-gbapp366kutjebt7agcd"
-              ]
-            },
-            "repositoryPHID": "PHID-REPO-hub2hx62ieuqeheznasv",
             "sourcePath": null
+            "status": "0",
+            "statusName": "Needs Review",
+            "summary": "",
+            "testPlan": "",
+            "title": "example",
+            "uri": "https://phab.example.com/D2",
         }
     """
+    # TODO: replace differential.query and differential.querydiffs with
+    # differential.diff.search because the former (and their output) are
+    # frozen, and planned to be deprecated and removed.
 
     def fetch(params):
         """params -> single drev or None"""
         key = (params.get(b'ids') or params.get(b'phids') or [None])[0]
         if key in prefetched:
             return prefetched[key]
-        drevs = callconduit(repo.ui, b'differential.query', params)
+        drevs = callconduit(ui, b'differential.query', params)
         # Fill prefetched with the result
         for drev in drevs:
             prefetched[drev[b'phid']] = drev
@@ -1483,7 +1821,7 b' def querydrev(repo, spec):'
     drevs, ancestordrevs = _prefetchdrevs(tree)
 
     # developer config: phabricator.batchsize
-    batchsize = repo.ui.configint(b'phabricator', b'batchsize')
+    batchsize = ui.configint(b'phabricator', b'batchsize')
 
     # Prefetch Differential Revisions in batch
     tofetch = set(drevs)
@@ -1537,6 +1875,48 b' def getdescfromdrev(drev):'
     return b'\n\n'.join(filter(None, [title, summary, testplan, uri]))
 
 
+def get_amended_desc(drev, ctx, folded):
+    """similar to ``getdescfromdrev``, but supports a folded series of commits
+
+    This is used when determining if an individual commit needs to have its
+    message amended after posting it for review.  The determination is made for
+    each individual commit, even when they were folded into one review.
+    """
+    if not folded:
+        return getdescfromdrev(drev)
+
+    uri = b'Differential Revision: %s' % drev[b'uri']
+
+    # Since the commit messages were combined when posting multiple commits
+    # with --fold, the fields can't be read from Phabricator here, or *all*
+    # affected local revisions will end up with the same commit message after
+    # the URI is amended in.  Append in the DREV line, or update it if it
+    # exists.  At worst, this means commit message or test plan updates on
+    # Phabricator aren't propagated back to the repository, but that seems
+    # reasonable for the case where local commits are effectively combined
+    # in Phabricator.
+    m = _differentialrevisiondescre.search(ctx.description())
+    if not m:
+        return b'\n\n'.join([ctx.description(), uri])
+
+    return _differentialrevisiondescre.sub(uri, ctx.description())
+
+
+def getlocalcommits(diff):
+    """get the set of local commits from a diff object
+
+    See ``getdiffmeta()`` for an example diff object.
+    """
+    props = diff.get(b'properties') or {}
+    commits = props.get(b'local:commits') or {}
+    if len(commits) > 1:
+        return {bin(c) for c in commits.keys()}
+
+    # Storing the diff metadata predates storing `local:commits`, so continue
+    # to use that in the --no-fold case.
+    return {bin(getdiffmeta(diff).get(b'node', b'')) or None}
+
+
 def getdiffmeta(diff):
     """get commit metadata (date, node, user, p1) from a diff object
 
@@ -1544,6 +1924,7 b' def getdiffmeta(diff):'
 
         "properties": {
           "hg:meta": {
+            "branch": "default",
             "date": "1499571514 25200",
             "node": "98c08acae292b2faf60a279b4189beb6cff1414d",
             "user": "Foo Bar <[e-mail]>",
@@ -1557,16 +1938,16 b' def getdiffmeta(diff):'
           "local:commits": {
             "98c08acae292b2faf60a279b4189beb6cff1414d": {
               "author": "Foo Bar",
-              "time": 1499546314,
+              "authorEmail": "[e-mail]"
               "branch": "default",
-              "tag": "",
               "commit": "98c08acae292b2faf60a279b4189beb6cff1414d",
+              "local": "1000",
+              "message": "...",
+              "parents": ["6d0abad76b30e4724a37ab8721d630394070fe16"],
               "rev": "98c08acae292b2faf60a279b4189beb6cff1414d",
-              "local": "1000",
-              "parents": ["6d0abad76b30e4724a37ab8721d630394070fe16"],
               "summary": "...",
-              "message": "...",
-              "authorEmail": "[e-mail]"
+              "tag": "",
+              "time": 1499546314,
             }
           }
         }
@@ -1605,24 +1986,47 b' def getdiffmeta(diff):'
     return meta
 
 
-def readpatch(repo, drevs, write):
+def _getdrevs(ui, stack, specs):
+    """convert user supplied DREVSPECs into "Differential Revision" dicts
+
+    See ``hg help phabread`` for how to specify each DREVSPEC.
+    """
+    if len(specs) > 0:
+
+        def _formatspec(s):
+            if stack:
+                s = b':(%s)' % s
+            return b'(%s)' % s
+
+        spec = b'+'.join(pycompat.maplist(_formatspec, specs))
+
+        drevs = querydrev(ui, spec)
+        if drevs:
+            return drevs
+
+    raise error.Abort(_(b"empty DREVSPEC set"))
+
+
+def readpatch(ui, drevs, write):
     """generate plain-text patch readable by 'hg import'
 
-    write is usually ui.write. drevs is what "querydrev" returns, results of
+    write takes a list of (DREV, bytes), where DREV is the differential number
+    (as bytes, without the "D" prefix) and the bytes are the text of a patch
+    to be imported. drevs is what "querydrev" returns, results of
     "differential.query".
     """
     # Prefetch hg:meta property for all diffs
-    diffids = sorted(set(max(int(v) for v in drev[b'diffs']) for drev in drevs))
-    diffs = callconduit(repo.ui, b'differential.querydiffs', {b'ids': diffids})
+    diffids = sorted({max(int(v) for v in drev[b'diffs']) for drev in drevs})
+    diffs = callconduit(ui, b'differential.querydiffs', {b'ids': diffids})
+
+    patches = []
 
     # Generate patch for each drev
     for drev in drevs:
-        repo.ui.note(_(b'reading D%s\n') % drev[b'id'])
+        ui.note(_(b'reading D%s\n') % drev[b'id'])
 
         diffid = max(int(v) for v in drev[b'diffs'])
-        body = callconduit(
-            repo.ui, b'differential.getrawdiff', {b'diffID': diffid}
-        )
+        body = callconduit(ui, b'differential.getrawdiff', {b'diffID': diffid})
         desc = getdescfromdrev(drev)
         header = b'# HG changeset patch\n'
 
@@ -1635,22 +2039,28 b' def readpatch(repo, drevs, write):'
                 header += b'# %s %s\n' % (_metanamemap[k], meta[k])
 
         content = b'%s%s\n%s' % (header, desc, body)
-        write(content)
+        patches.append((drev[b'id'], content))
+
+    # Write patches to the supplied callback
+    write(patches)
 
 
 @vcrcommand(
     b'phabread',
     [(b'', b'stack', False, _(b'read dependencies'))],
-    _(b'DREVSPEC [OPTIONS]'),
+    _(b'DREVSPEC... [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
+    optionalrepo=True,
 )
-def phabread(ui, repo, spec, **opts):
+def phabread(ui, repo, *specs, **opts):
     """print patches from Phabricator suitable for importing
 
     DREVSPEC could be a Differential Revision identity, like ``D123``, or just
     the number ``123``. It could also have common operators like ``+``, ``-``,
     ``&``, ``(``, ``)`` for complex queries. Prefix ``:`` could be used to
-    select a stack.
+    select a stack.  If multiple DREVSPEC values are given, the result is the
+    union of each individually evaluated value.  No attempt is currently made
+    to reorder the values to run from parent to child.
 
     ``abandoned``, ``accepted``, ``closed``, ``needsreview``, ``needsrevision``
     could be used to filter patches by status. For performance reason, they
@@ -1664,10 +2074,74 b' def phabread(ui, repo, spec, **opts):'
     It is equivalent to the ``:`` operator.
     """
     opts = pycompat.byteskwargs(opts)
-    if opts.get(b'stack'):
-        spec = b':(%s)' % spec
-    drevs = querydrev(repo, spec)
-    readpatch(repo, drevs, ui.write)
+    drevs = _getdrevs(ui, opts.get(b'stack'), specs)
+
+    def _write(patches):
+        for drev, content in patches:
+            ui.write(content)
+
+    readpatch(ui, drevs, _write)
+
+
+@vcrcommand(
+    b'phabimport',
+    [(b'', b'stack', False, _(b'import dependencies as well'))],
+    _(b'DREVSPEC... [OPTIONS]'),
+    helpcategory=command.CATEGORY_IMPORT_EXPORT,
+)
+def phabimport(ui, repo, *specs, **opts):
+    """import patches from Phabricator for the specified Differential Revisions
+
+    The patches are read and applied starting at the parent of the working
+    directory.
+
+    See ``hg help phabread`` for how to specify DREVSPEC.
+    """
+    opts = pycompat.byteskwargs(opts)
+
+    # --bypass avoids losing exec and symlink bits when importing on Windows,
+    # and allows importing with a dirty wdir.  It also aborts instead of leaving
+    # rejects.
+    opts[b'bypass'] = True
+
+    # Mandatory default values, synced with commands.import
+    opts[b'strip'] = 1
+    opts[b'prefix'] = b''
+    # Evolve 9.3.0 assumes this key is present in cmdutil.tryimportone()
+    opts[b'obsolete'] = False
+
+    if ui.configbool(b'phabimport', b'secret'):
+        opts[b'secret'] = True
+    if ui.configbool(b'phabimport', b'obsolete'):
+        opts[b'obsolete'] = True  # Handled by evolve wrapping tryimportone()
+
+    def _write(patches):
+        parents = repo[None].parents()
+
+        with repo.wlock(), repo.lock(), repo.transaction(b'phabimport'):
+            for drev, contents in patches:
+                ui.status(_(b'applying patch from D%s\n') % drev)
+
+                with patch.extract(ui, pycompat.bytesio(contents)) as patchdata:
+                    msg, node, rej = cmdutil.tryimportone(
+                        ui,
+                        repo,
+                        patchdata,
+                        parents,
+                        opts,
+                        [],
+                        None,  # Never update wdir to another revision
+                    )
+
+                    if not node:
+                        raise error.Abort(_(b'D%s: no diffs found') % drev)
+
+                    ui.note(msg + b'\n')
+                    parents = [repo[node]]
+
+    drevs = _getdrevs(ui, opts.get(b'stack'), specs)
+
+    readpatch(repo.ui, drevs, _write)
 
 
 @vcrcommand(
@@ -1679,10 +2153,11 b' def phabread(ui, repo, spec, **opts):'
         (b'', b'reclaim', False, _(b'reclaim revisions')),
         (b'm', b'comment', b'', _(b'comment on the last revision')),
     ],
-    _(b'DREVSPEC [OPTIONS]'),
+    _(b'DREVSPEC... [OPTIONS]'),
     helpcategory=command.CATEGORY_IMPORT_EXPORT,
+    optionalrepo=True,
 )
-def phabupdate(ui, repo, spec, **opts):
+def phabupdate(ui, repo, *specs, **opts):
     """update Differential Revision in batch
 
     DREVSPEC selects revisions. See :hg:`help phabread` for its usage.
@@ -1696,7 +2171,7 b' def phabupdate(ui, repo, spec, **opts):'
     for f in flags:
         actions.append({b'type': f, b'value': True})
 
-    drevs = querydrev(repo, spec)
+    drevs = _getdrevs(ui, opts.get(b'stack'), specs)
     for i, drev in enumerate(drevs):
         if i + 1 == len(drevs) and opts.get(b'comment'):
             actions.append({b'type': b'comment', b'value': opts[b'comment']})
@@ -1759,11 +2234,11 b' def phabstatusshowview(ui, repo, display'
     """Phabricator differiential status"""
     revs = repo.revs('sort(_underway(), topo)')
     drevmap = getdrevmap(repo, revs)
-    unknownrevs, drevids, revsbydrevid = [], set([]), {}
+    unknownrevs, drevids, revsbydrevid = [], set(), {}
     for rev, drevid in pycompat.iteritems(drevmap):
         if drevid is not None:
             drevids.add(drevid)
-            revsbydrevid.setdefault(drevid, set([])).add(rev)
+            revsbydrevid.setdefault(drevid, set()).add(rev)
         else:
             unknownrevs.append(rev)
 

hgext/purge.py

@@ -48,6 +48,7 b" testedwith = b'ships-with-hg-core'"
     [
         (b'a', b'abort-on-err', None, _(b'abort if an error occurs')),
         (b'', b'all', None, _(b'purge ignored files too')),
+        (b'i', b'ignored', None, _(b'purge only ignored files')),
         (b'', b'dirs', None, _(b'purge empty directories')),
         (b'', b'files', None, _(b'purge files')),
         (b'p', b'print', None, _(b'print filenames instead of deleting them')),
@@ -80,7 +81,7 b' def purge(ui, repo, *dirs, **opts):'
     But it will leave untouched:
 
     - Modified and unmodified tracked files
-    - Ignored files (unless --all is specified)
+    - Ignored files (unless -i or --all is specified)
     - New files added to the repository (with :hg:`add`)
 
     The --files and --dirs options can be used to direct purge to delete
@@ -96,12 +97,19 b' def purge(ui, repo, *dirs, **opts):'
     option.
     '''
     opts = pycompat.byteskwargs(opts)
+    cmdutil.check_at_most_one_arg(opts, b'all', b'ignored')
 
     act = not opts.get(b'print')
     eol = b'\n'
     if opts.get(b'print0'):
         eol = b'\0'
         act = False  # --print0 implies --print
+    if opts.get(b'all', False):
+        ignored = True
+        unknown = True
+    else:
+        ignored = opts.get(b'ignored', False)
+        unknown = not ignored
 
     removefiles = opts.get(b'files')
     removedirs = opts.get(b'dirs')
@@ -115,7 +123,8 b' def purge(ui, repo, *dirs, **opts):'
     paths = mergemod.purge(
         repo,
         match,
-        ignored=opts.get(b'all', False),
+        unknown=unknown,
+        ignored=ignored,
         removeemptydirs=removedirs,
         removefiles=removefiles,
         abortonerror=opts.get(b'abort_on_err'),

hgext/rebase.py

@@ -37,6 +37,7 b' from mercurial import ('
     hg,
     merge as mergemod,
     mergeutil,
+    node as nodemod,
     obsolete,
     obsutil,
     patch,
@@ -177,6 +178,7 b' class rebaseruntime(object):'
         # --continue or --abort)), the original repo should be used so
         # visibility-dependent revsets are correct.
         self.prepared = False
+        self.resume = False
         self._repo = repo
 
         self.ui = ui
@@ -366,6 +368,7 b' class rebaseruntime(object):'
         _checkobsrebase(self.repo, self.ui, obsoleteset, skippedset)
 
     def _prepareabortorcontinue(self, isabort, backup=True, suppwarns=False):
+        self.resume = True
         try:
             self.restorestatus()
             self.collapsemsg = restorecollapsemsg(self.repo, isabort)
@@ -503,7 +506,7 b' class rebaseruntime(object):'
         p.complete()
         ui.note(_(b'rebase merging completed\n'))
 
-    def _concludenode(self, rev, p1, p2, editor, commitmsg=None):
+    def _concludenode(self, rev, p1, editor, commitmsg=None):
         '''Commit the wd changes with parents p1 and p2.
 
         Reuse commit info from rev but also store useful information in extra.
@@ -527,8 +530,6 b' class rebaseruntime(object):'
             if self.inmemory:
                 newnode = commitmemorynode(
                     repo,
-                    p1,
-                    p2,
                     wctx=self.wctx,
                     extra=extra,
                     commitmsg=commitmsg,
@@ -540,8 +541,6 b' class rebaseruntime(object):'
             else:
                 newnode = commitnode(
                     repo,
-                    p1,
-                    p2,
                     extra=extra,
                     commitmsg=commitmsg,
                     editor=editor,
@@ -549,11 +548,6 b' class rebaseruntime(object):'
                     date=date,
                 )
 
-            if newnode is None:
-                # If it ended up being a no-op commit, then the normal
-                # merge state clean-up path doesn't happen, so do it
-                # here. Fix issue5494
-                mergemod.mergestate.clean(repo)
             return newnode
 
     def _rebasenode(self, tr, rev, allowdivergence, progressfn):
@@ -605,8 +599,9 b' class rebaseruntime(object):'
                 self.skipped,
                 self.obsoletenotrebased,
             )
-            if not self.inmemory and len(repo[None].parents()) == 2:
+            if self.resume and self.wctx.p1().rev() == p1:
                 repo.ui.debug(b'resuming interrupted rebase\n')
+                self.resume = False
             else:
                 overrides = {(b'ui', b'forcemerge'): opts.get(b'tool', b'')}
                 with ui.configoverride(overrides, b'rebase'):
@@ -614,6 +609,7 b' class rebaseruntime(object):'
                         repo,
                         rev,
                         p1,
+                        p2,
                         base,
                         self.collapsef,
                         dest,
@@ -635,13 +631,15 b' class rebaseruntime(object):'
                 editor = cmdutil.getcommiteditor(
                     editform=editform, **pycompat.strkwargs(opts)
                 )
-                newnode = self._concludenode(rev, p1, p2, editor)
+                # We need to set parents again here just in case we're continuing
+                # a rebase started with an old hg version (before 9c9cfecd4600),
+                # because those old versions would have left us with two dirstate
+                # parents, and we don't want to create a merge commit here (unless
+                # we're rebasing a merge commit).
+                self.wctx.setparents(repo[p1].node(), repo[p2].node())
+                newnode = self._concludenode(rev, p1, editor)
             else:
                 # Skip commit if we are collapsing
-                if self.inmemory:
-                    self.wctx.setbase(repo[p1])
-                else:
-                    repo.setparents(repo[p1].node())
                 newnode = None
             # Update the state
             if newnode is not None:
@@ -696,8 +694,9 b' class rebaseruntime(object):'
             editor = cmdutil.getcommiteditor(edit=editopt, editform=editform)
             revtoreuse = max(self.state)
 
+            self.wctx.setparents(repo[p1].node(), repo[self.external].node())
             newnode = self._concludenode(
-                revtoreuse, p1, self.external, editor, commitmsg=commitmsg
+                revtoreuse, p1, editor, commitmsg=commitmsg
             )
 
             if newnode is not None:
@@ -799,9 +798,7 b' class rebaseruntime(object):'
 
                 # Update away from the rebase if necessary
                 if shouldupdate:
-                    mergemod.update(
-                        repo, self.originalwd, branchmerge=False, force=True
-                    )
+                    mergemod.clean_update(repo[self.originalwd])
 
                 # Strip from the first rebased revision
                 if rebased:
@@ -824,14 +821,14 b' class rebaseruntime(object):'
         (
             b's',
             b'source',
-            b'',
-            _(b'rebase the specified changeset and descendants'),
+            [],
+            _(b'rebase the specified changesets and their descendants'),
             _(b'REV'),
         ),
         (
             b'b',
             b'base',
-            b'',
+            [],
             _(b'rebase everything from branching point of specified changeset'),
             _(b'REV'),
         ),
@@ -880,7 +877,7 b' class rebaseruntime(object):'
     + cmdutil.dryrunopts
     + cmdutil.formatteropts
     + cmdutil.confirmopts,
-    _(b'[-s REV | -b REV] [-d REV] [OPTION]'),
+    _(b'[[-s REV]... | [-b REV]... | [-r REV]...] [-d REV] [OPTION]...'),
     helpcategory=command.CATEGORY_CHANGE_MANAGEMENT,
 )
 def rebase(ui, repo, **opts):
@@ -1011,10 +1008,10 b' def rebase(ui, repo, **opts):'
     action = cmdutil.check_at_most_one_arg(opts, b'abort', b'stop', b'continue')
     if action:
         cmdutil.check_incompatible_arguments(
-            opts, action, b'confirm', b'dry_run'
+            opts, action, [b'confirm', b'dry_run']
         )
         cmdutil.check_incompatible_arguments(
-            opts, action, b'rev', b'source', b'base', b'dest'
+            opts, action, [b'rev', b'source', b'base', b'dest']
         )
     cmdutil.check_at_most_one_arg(opts, b'confirm', b'dry_run')
     cmdutil.check_at_most_one_arg(opts, b'rev', b'source', b'base')
@@ -1028,7 +1025,7 b' def rebase(ui, repo, **opts):'
     if opts.get(b'auto_orphans'):
         disallowed_opts = set(opts) - {b'auto_orphans'}
         cmdutil.check_incompatible_arguments(
-            opts, b'auto_orphans', *disallowed_opts
+            opts, b'auto_orphans', disallowed_opts
         )
 
         userrevs = list(repo.revs(opts.get(b'auto_orphans')))
@@ -1195,8 +1192,8 b' def _origrebase('
                 repo,
                 inmemory,
                 opts.get(b'dest', None),
-                opts.get(b'source', None),
-                opts.get(b'base', None),
+                opts.get(b'source', []),
+                opts.get(b'base', []),
                 opts.get(b'rev', []),
                 destspace=destspace,
             )
@@ -1226,16 +1223,7 b' def _origrebase('
                     rbsrt._finishrebase()
 
 
-def _definedestmap(
-    ui,
-    repo,
-    inmemory,
-    destf=None,
-    srcf=None,
-    basef=None,
-    revf=None,
-    destspace=None,
-):
+def _definedestmap(ui, repo, inmemory, destf, srcf, basef, revf, destspace):
     """use revisions argument to define destmap {srcrev: destrev}"""
     if revf is None:
         revf = []
@@ -1261,14 +1249,14 b' def _definedestmap('
             ui.status(_(b'empty "rev" revision set - nothing to rebase\n'))
             return None
     elif srcf:
-        src = scmutil.revrange(repo, [srcf])
+        src = scmutil.revrange(repo, srcf)
         if not src:
             ui.status(_(b'empty "source" revision set - nothing to rebase\n'))
             return None
-        rebaseset = repo.revs(b'(%ld)::', src)
-        assert rebaseset
+        # `+  (%ld)` to work around `wdir()::` being empty
+        rebaseset = repo.revs(b'(%ld):: + (%ld)', src, src)
     else:
-        base = scmutil.revrange(repo, [basef or b'.'])
+        base = scmutil.revrange(repo, basef or [b'.'])
         if not base:
             ui.status(
                 _(b'empty "base" revision set - ' b"can't compute rebase set\n")
@@ -1341,6 +1329,8 b' def _definedestmap('
                 )
             return None
 
+    if nodemod.wdirrev in rebaseset:
+        raise error.Abort(_(b'cannot rebase the working copy'))
     rebasingwcp = repo[b'.'].rev() in rebaseset
     ui.log(
         b"rebase",
@@ -1420,7 +1410,7 b' def externalparent(repo, state, destance'
     )
 
 
-def commitmemorynode(repo, p1, p2, wctx, editor, extra, user, date, commitmsg):
+def commitmemorynode(repo, wctx, editor, extra, user, date, commitmsg):
     '''Commit the memory changes with parents p1 and p2.
     Return node of committed revision.'''
     # Replicates the empty check in ``repo.commit``.
@@ -1433,7 +1423,6 b' def commitmemorynode(repo, p1, p2, wctx,'
     if b'branch' in extra:
         branch = extra[b'branch']
 
-    wctx.setparents(repo[p1].node(), repo[p2].node())
     memctx = wctx.tomemctx(
         commitmsg,
         date=date,
@@ -1447,15 +1436,13 b' def commitmemorynode(repo, p1, p2, wctx,'
     return commitres
 
 
-def commitnode(repo, p1, p2, editor, extra, user, date, commitmsg):
+def commitnode(repo, editor, extra, user, date, commitmsg):
     '''Commit the wd changes with parents p1 and p2.
     Return node of committed revision.'''
     dsguard = util.nullcontextmanager()
     if not repo.ui.configbool(b'rebase', b'singletransaction'):
         dsguard = dirstateguard.dirstateguard(repo, b'rebase')
     with dsguard:
-        repo.setparents(repo[p1].node(), repo[p2].node())
-
         # Commit might fail if unresolved files exist
         newnode = repo.commit(
             text=commitmsg, user=user, date=date, extra=extra, editor=editor
@@ -1465,7 +1452,7 b' def commitnode(repo, p1, p2, editor, ext'
         return newnode
 
 
-def rebasenode(repo, rev, p1, base, collapse, dest, wctx):
+def rebasenode(repo, rev, p1, p2, base, collapse, dest, wctx):
     """Rebase a single revision rev on top of p1 using base as merge ancestor"""
     # Merge phase
     # Update to destination and merge it with local
@@ -1475,7 +1462,7 b' def rebasenode(repo, rev, p1, base, coll'
     else:
         if repo[b'.'].rev() != p1:
             repo.ui.debug(b" update to %d:%s\n" % (p1, p1ctx))
-            mergemod.update(repo, p1, branchmerge=False, force=True)
+            mergemod.clean_update(p1ctx)
         else:
             repo.ui.debug(b" already in destination\n")
         # This is, alas, necessary to invalidate workingctx's manifest cache,
@@ -1499,6 +1486,7 b' def rebasenode(repo, rev, p1, base, coll'
         labels=[b'dest', b'source'],
         wc=wctx,
     )
+    wctx.setparents(p1ctx.node(), repo[p2].node())
     if collapse:
         copies.graftcopies(wctx, ctx, repo[dest])
     else:
@@ -1678,22 +1666,6 b' def defineparents(repo, rev, destmap, st'
             elif p in state and state[p] > 0:
                 np = state[p]
 
-            # "bases" only record "special" merge bases that cannot be
-            # calculated from changelog DAG (i.e. isancestor(p, np) is False).
-            # For example:
-            #
-            #   B'   # rebase -s B -d D, when B was rebased to B'. dest for C
-            #   | C  # is B', but merge base for C is B, instead of
-            #   D |  # changelog.ancestor(C, B') == A. If changelog DAG and
-            #   | B  # "state" edges are merged (so there will be an edge from
-            #   |/   # B to B'), the merge base is still ancestor(C, B') in
-            #   A    # the merged graph.
-            #
-            # Also see https://bz.mercurial-scm.org/show_bug.cgi?id=1950#c8
-            # which uses "virtual null merge" to explain this situation.
-            if isancestor(p, np):
-                bases[i] = nullrev
-
             # If one parent becomes an ancestor of the other, drop the ancestor
             for j, x in enumerate(newps[:i]):
                 if x == nullrev:
@@ -1739,12 +1711,6 b' def defineparents(repo, rev, destmap, st'
     if any(p != nullrev and isancestor(rev, p) for p in newps):
         raise error.Abort(_(b'source is ancestor of destination'))
 
-    # "rebasenode" updates to new p1, use the corresponding merge base.
-    if bases[0] != nullrev:
-        base = bases[0]
-    else:
-        base = None
-
     # Check if the merge will contain unwanted changes. That may happen if
     # there are multiple special (non-changelog ancestor) merge bases, which
     # cannot be handled well by the 3-way merge algorithm. For example:
@@ -1760,15 +1726,16 b' def defineparents(repo, rev, destmap, st'
     # But our merge base candidates (D and E in above case) could still be
     # better than the default (ancestor(F, Z) == null). Therefore still
     # pick one (so choose p1 above).
-    if sum(1 for b in set(bases) if b != nullrev) > 1:
+    if sum(1 for b in set(bases) if b != nullrev and b not in newps) > 1:
         unwanted = [None, None]  # unwanted[i]: unwanted revs if choose bases[i]
         for i, base in enumerate(bases):
-            if base == nullrev:
+            if base == nullrev or base in newps:
                 continue
             # Revisions in the side (not chosen as merge base) branch that
             # might contain "surprising" contents
+            other_bases = set(bases) - {base}
             siderevs = list(
-                repo.revs(b'((%ld-%d) %% (%d+%d))', bases, base, base, dest)
+                repo.revs(b'(%ld %% (%d+%d))', other_bases, base, dest)
             )
 
             # If those revisions are covered by rebaseset, the result is good.
@@ -1786,35 +1753,40 b' def defineparents(repo, rev, destmap, st'
                     )
                 )
 
-        # Choose a merge base that has a minimal number of unwanted revs.
-        l, i = min(
-            (len(revs), i)
-            for i, revs in enumerate(unwanted)
-            if revs is not None
-        )
-        base = bases[i]
-
-        # newps[0] should match merge base if possible. Currently, if newps[i]
-        # is nullrev, the only case is newps[i] and newps[j] (j < i), one is
-        # the other's ancestor. In that case, it's fine to not swap newps here.
-        # (see CASE-1 and CASE-2 above)
-        if i != 0 and newps[i] != nullrev:
-            newps[0], newps[i] = newps[i], newps[0]
+        if any(revs is not None for revs in unwanted):
+            # Choose a merge base that has a minimal number of unwanted revs.
+            l, i = min(
+                (len(revs), i)
+                for i, revs in enumerate(unwanted)
+                if revs is not None
+            )
 
-        # The merge will include unwanted revisions. Abort now. Revisit this if
-        # we have a more advanced merge algorithm that handles multiple bases.
-        if l > 0:
-            unwanteddesc = _(b' or ').join(
-                (
-                    b', '.join(b'%d:%s' % (r, repo[r]) for r in revs)
-                    for revs in unwanted
-                    if revs is not None
+            # The merge will include unwanted revisions. Abort now. Revisit this if
+            # we have a more advanced merge algorithm that handles multiple bases.
+            if l > 0:
+                unwanteddesc = _(b' or ').join(
+                    (
+                        b', '.join(b'%d:%s' % (r, repo[r]) for r in revs)
+                        for revs in unwanted
+                        if revs is not None
+                    )
                 )
-            )
-            raise error.Abort(
-                _(b'rebasing %d:%s will include unwanted changes from %s')
-                % (rev, repo[rev], unwanteddesc)
-            )
+                raise error.Abort(
+                    _(b'rebasing %d:%s will include unwanted changes from %s')
+                    % (rev, repo[rev], unwanteddesc)
+                )
+
+            # newps[0] should match merge base if possible. Currently, if newps[i]
+            # is nullrev, the only case is newps[i] and newps[j] (j < i), one is
+            # the other's ancestor. In that case, it's fine to not swap newps here.
+            # (see CASE-1 and CASE-2 above)
+            if i != 0:
+                if newps[i] != nullrev:
+                    newps[0], newps[i] = newps[i], newps[0]
+                bases[0], bases[i] = bases[i], bases[0]
+
+    # "rebasenode" updates to new p1, use the corresponding merge base.
+    base = bases[0]
 
     repo.ui.debug(b" future parents are %d and %d\n" % tuple(newps))
 
@@ -1962,7 +1934,7 b' def buildstate(repo, destmap, collapse):'
     # applied patch. But it prevents messing up the working directory when
     # a partially completed rebase is blocked by mq.
     if b'qtip' in repo.tags():
-        mqapplied = set(repo[s.node].rev() for s in repo.mq.applied)
+        mqapplied = {repo[s.node].rev() for s in repo.mq.applied}
         if set(destmap.values()) & mqapplied:
             raise error.Abort(_(b'cannot rebase onto an applied mq patch'))
 
@@ -2147,7 +2119,7 b' def pullrebase(orig, ui, repo, *args, **'
 
 def _filterobsoleterevs(repo, revs):
     """returns a set of the obsolete revisions in revs"""
-    return set(r for r in revs if repo[r].obsolete())
+    return {r for r in revs if repo[r].obsolete()}
 
 
 def _computeobsoletenotrebased(repo, rebaseobsrevs, destmap):

hgext/releasenotes.py

@@ -654,7 +654,7 b' def releasenotes(ui, repo, file_=None, *'
     opts = pycompat.byteskwargs(opts)
     sections = releasenotessections(ui, repo)
 
-    cmdutil.check_incompatible_arguments(opts, b'list', b'rev', b'check')
+    cmdutil.check_incompatible_arguments(opts, b'list', [b'rev', b'check'])
 
     if opts.get(b'list'):
         return _getadmonitionlist(ui, sections)

hgext/remotefilelog/__init__.py

@@ -737,7 +737,7 b' def onetimeclientsetup(ui):'
             # "link" is actually wrong here (it is set to len(changelog))
             # if changelog remains unchanged, skip writing file revisions
             # but still do a sanity check about pending multiple revisions
-            if len(set(x[3] for x in pendingfilecommits)) > 1:
+            if len({x[3] for x in pendingfilecommits}) > 1:
                 raise error.ProgrammingError(
                     b'pending multiple integer revisions are not supported'
                 )

hgext/remotefilelog/basepack.py

@@ -101,7 +101,7 b' class _cachebackedpacks(object):'
             self._lastpack = pack
             yield pack
 
-        cachedpacks = set(pack for pack in self._lrucache)
+        cachedpacks = {pack for pack in self._lrucache}
         # Yield for paths not in the cache.
         for pack in self._packs - cachedpacks:
             self._lastpack = pack
@@ -259,7 +259,7 b' class basepackstore(object):'
         newpacks = []
         if now > self.lastrefresh + REFRESHRATE:
             self.lastrefresh = now
-            previous = set(p.path for p in self.packs)
+            previous = {p.path for p in self.packs}
             for filepath, __, __ in self._getavailablepackfilessorted():
                 if filepath not in previous:
                     newpack = self.getpack(filepath)

hgext/remotefilelog/contentstore.py

@@ -300,7 +300,7 b' class manifestrevlogstore(object):'
 
         rl = self._revlog(name)
         ancestors = {}
-        missing = set((node,))
+        missing = {node}
         for ancrev in rl.ancestors([rl.rev(node)], inclusive=True):
             ancnode = rl.node(ancrev)
             missing.discard(ancnode)

hgext/remotefilelog/datapack.py

@@ -271,9 +271,9 b' class datapack(basepack.basepack):'
     def cleanup(self, ledger):
         entries = ledger.sources.get(self, [])
         allkeys = set(self)
-        repackedkeys = set(
+        repackedkeys = {
             (e.filename, e.node) for e in entries if e.datarepacked or e.gced
-        )
+        }
 
         if len(allkeys - repackedkeys) == 0:
             if self.path not in ledger.created:

hgext/remotefilelog/historypack.py

@@ -132,7 +132,7 b' class historypack(basepack.basepack):'
             known = set()
         section = self._findsection(name)
         filename, offset, size, nodeindexoffset, nodeindexsize = section
-        pending = set((node,))
+        pending = {node}
         o = 0
         while o < size:
             if not pending:
@@ -291,9 +291,9 b' class historypack(basepack.basepack):'
     def cleanup(self, ledger):
         entries = ledger.sources.get(self, [])
         allkeys = set(self)
-        repackedkeys = set(
+        repackedkeys = {
             (e.filename, e.node) for e in entries if e.historyrepacked
-        )
+        }
 
         if len(allkeys - repackedkeys) == 0:
             if self.path not in ledger.created:
@@ -452,7 +452,7 b' class mutablehistorypack(basepack.mutabl'
             sectionstart = self.packfp.tell()
 
             # Write the file section content
-            entrymap = dict((e[0], e) for e in entries)
+            entrymap = {e[0]: e for e in entries}
 
             def parentfunc(node):
                 x, p1, p2, x, x, x = entrymap[node]

hgext/remotefilelog/remotefilelog.py

@@ -259,6 +259,10 b' class remotefilelog(object):'
 
         raise RuntimeError(b"len not supported")
 
+    def heads(self):
+        # Fake heads of the filelog to satisfy hgweb.
+        return []
+
     def empty(self):
         return False
 
@@ -429,7 +433,7 b' class remotefilelog(object):'
             return nullid
 
         revmap, parentfunc = self._buildrevgraph(a, b)
-        nodemap = dict(((v, k) for (k, v) in pycompat.iteritems(revmap)))
+        nodemap = {v: k for (k, v) in pycompat.iteritems(revmap)}
 
         ancs = ancestor.ancestors(parentfunc, revmap[a], revmap[b])
         if ancs:
@@ -444,7 +448,7 b' class remotefilelog(object):'
             return nullid
 
         revmap, parentfunc = self._buildrevgraph(a, b)
-        nodemap = dict(((v, k) for (k, v) in pycompat.iteritems(revmap)))
+        nodemap = {v: k for (k, v) in pycompat.iteritems(revmap)}
 
         ancs = ancestor.commonancestorsheads(parentfunc, revmap[a], revmap[b])
         return map(nodemap.__getitem__, ancs)

hgext/remotefilelog/repack.py

@@ -321,7 +321,7 b' def _computeincrementalhistorypack(ui, f'
 
 def _allpackfileswithsuffix(files, packsuffix, indexsuffix):
     result = []
-    fileset = set(fn for fn, mode, stat in files)
+    fileset = {fn for fn, mode, stat in files}
     for filename, mode, stat in files:
         if not filename.endswith(packsuffix):
             continue

hgext/schemes.py

@@ -97,7 +97,7 b' class ShortRepository(object):'
             parts = parts[:-1]
         else:
             tail = b''
-        context = dict((b'%d' % (i + 1), v) for i, v in enumerate(parts))
+        context = {b'%d' % (i + 1): v for i, v in enumerate(parts)}
         return b''.join(self.templater.process(self.url, context)) + tail
 
 

hgext/sparse.py

@@ -246,7 +246,7 b' def _setupdirstate(ui):'
             if changedfiles is not None:
                 # In _rebuild, these files will be deleted from the dirstate
                 # when they are not found to be in allfiles
-                dirstatefilestoremove = set(f for f in self if not matcher(f))
+                dirstatefilestoremove = {f for f in self if not matcher(f)}
                 changedfiles = dirstatefilestoremove.union(changedfiles)
 
         return orig(self, parent, allfiles, changedfiles)

hgext/strip.py

@@ -228,7 +228,7 b' def stripcmd(ui, repo, *revs, **opts):'
             for p in repo.dirstate.parents()
         )
 
-        rootnodes = set(cl.node(r) for r in roots)
+        rootnodes = {cl.node(r) for r in roots}
 
         q = getattr(repo, 'mq', None)
         if q is not None and q.applied:

hgext/transplant.py

@@ -761,12 +761,12 b' def _dotransplant(ui, repo, *revs, **opt'
     def checkopts(opts, revs):
         if opts.get(b'continue'):
             cmdutil.check_incompatible_arguments(
-                opts, b'continue', b'branch', b'all', b'merge'
+                opts, b'continue', [b'branch', b'all', b'merge']
             )
             return
         if opts.get(b'stop'):
             cmdutil.check_incompatible_arguments(
-                opts, b'stop', b'branch', b'all', b'merge'
+                opts, b'stop', [b'branch', b'all', b'merge']
             )
             return
         if not (
@@ -840,10 +840,10 b' def _dotransplant(ui, repo, *revs, **opt'
 
         tf = tp.transplantfilter(repo, source, p1)
         if opts.get(b'prune'):
-            prune = set(
+            prune = {
                 source[r].node()
                 for r in scmutil.revrange(source, opts.get(b'prune'))
-            )
+            }
             matchfn = lambda x: tf(x) and x not in prune
         else:
             matchfn = tf

hgext/uncommit.py

@@ -65,7 +65,7 b' def _commitfiltered('
     base = ctx.p1()
     # ctx
     initialfiles = set(ctx.files())
-    exclude = set(f for f in initialfiles if match(f))
+    exclude = {f for f in initialfiles if match(f)}
 
     # No files matched commit, so nothing excluded
     if not exclude:
@@ -78,9 +78,9 b' def _commitfiltered('
     files = initialfiles - exclude
     # Filter copies
     copied = copiesmod.pathcopies(base, ctx)
-    copied = dict(
-        (dst, src) for dst, src in pycompat.iteritems(copied) if dst in files
-    )
+    copied = {
+        dst: src for dst, src in pycompat.iteritems(copied) if dst in files
+    }
 
     def filectxfn(repo, memctx, path, contentctx=ctx, redirect=()):
         if path not in contentctx:

i18n/polib.py

@@ -722,8 +722,8 b' class POFile(_BaseFile):'
             object POFile, the reference catalog.
         """
         # Store entries in dict/set for faster access
-        self_entries = dict((entry.msgid, entry) for entry in self)
-        refpot_msgids = set(entry.msgid for entry in refpot)
+        self_entries = {entry.msgid: entry for entry in self}
+        refpot_msgids = {entry.msgid for entry in refpot}
         # Merge entries that are in the refpot
         for entry in refpot:
             e = self_entries.get(entry.msgid)
@@ -1808,9 +1808,9 b' class _MOFileParser(object):'
                 entry = self._build_entry(
                     msgid=msgid_tokens[0],
                     msgid_plural=msgid_tokens[1],
-                    msgstr_plural=dict(
-                        (k, v) for k, v in enumerate(msgstr.split(b('\0')))
-                    ),
+                    msgstr_plural={
+                        k: v for k, v in enumerate(msgstr.split(b('\0')))
+                    },
                 )
             else:
                 entry = self._build_entry(msgid=msgid, msgstr=msgstr)

mercurial/ancestor.py

@@ -138,7 +138,7 b' def ancestors(pfunc, *orignodes):'
         k = 0
         for i in interesting:
             k |= i
-        return set(n for (i, n) in mapping if k & i)
+        return {n for (i, n) in mapping if k & i}
 
     gca = commonancestorsheads(pfunc, *orignodes)
 
@@ -393,39 +393,3 b' class lazyancestors(object):'
             # free up memory.
             self._containsiter = None
             return False
-
-
-class rustlazyancestors(object):
-    def __init__(self, index, revs, stoprev=0, inclusive=False):
-        self._index = index
-        self._stoprev = stoprev
-        self._inclusive = inclusive
-        # no need to prefilter out init revs that are smaller than stoprev,
-        # it's done by rustlazyancestors constructor.
-        # we need to convert to a list, because our ruslazyancestors
-        # constructor (from C code) doesn't understand anything else yet
-        self._initrevs = initrevs = list(revs)
-
-        self._containsiter = parsers.rustlazyancestors(
-            index, initrevs, stoprev, inclusive
-        )
-
-    def __nonzero__(self):
-        """False if the set is empty, True otherwise.
-
-        It's better to duplicate this essentially trivial method than
-        to subclass lazyancestors
-        """
-        try:
-            next(iter(self))
-            return True
-        except StopIteration:
-            return False
-
-    def __iter__(self):
-        return parsers.rustlazyancestors(
-            self._index, self._initrevs, self._stoprev, self._inclusive
-        )
-
-    def __contains__(self, target):
-        return target in self._containsiter

mercurial/archival.py

@@ -359,7 +359,7 b' def archive('
         if match(name):
             write(name, 0o644, False, lambda: buildmetadata(ctx))
 
-    files = [f for f in ctx.manifest().matches(match)]
+    files = list(ctx.manifest().walk(match))
     total = len(files)
     if total:
         files.sort()

mercurial/bookmarks.py

@@ -173,6 +173,8 b' class bmstore(object):'
             nrefs.sort()
 
     def _del(self, mark):
+        if mark not in self._refmap:
+            return
         self._clean = False
         node = self._refmap.pop(mark)
         nrefs = self._nodemap[node]
@@ -461,6 +463,10 b' def update(repo, parents, node):'
     return bool(bmchanges)
 
 
+def isdivergent(b):
+    return b'@' in b and not b.endswith(b'@')
+
+
 def listbinbookmarks(repo):
     # We may try to list bookmarks on a repo type that does not
     # support it (e.g., statichttprepository).
@@ -469,7 +475,7 b' def listbinbookmarks(repo):'
     hasnode = repo.changelog.hasnode
     for k, v in pycompat.iteritems(marks):
         # don't expose local divergent bookmarks
-        if hasnode(v) and (b'@' not in k or k.endswith(b'@')):
+        if hasnode(v) and not isdivergent(k):
             yield k, v
 
 
@@ -481,6 +487,8 b' def listbookmarks(repo):'
 
 
 def pushbookmark(repo, key, old, new):
+    if isdivergent(key):
+        return False
     if bookmarksinstore(repo):
         wlock = util.nullcontextmanager()
     else:

mercurial/branchmap.py

@@ -291,8 +291,8 b' class branchcache(object):'
                     % (
                         _branchcachedesc(repo),
                         pycompat.bytestr(
-                            inst  # pytype: disable=wrong-arg-types
-                        ),
+                            inst
+                        ),  # pytype: disable=wrong-arg-types
                     )
                 )
             bcache = None
@@ -446,7 +446,7 b' class branchcache(object):'
         # 1 (branch a) -> 2 (branch b) -> 3 (branch a)
         for branch, newheadrevs in pycompat.iteritems(newbranches):
             bheads = self._entries.setdefault(branch, [])
-            bheadset = set(cl.rev(node) for node in bheads)
+            bheadset = {cl.rev(node) for node in bheads}
 
             # This have been tested True on all internal usage of this function.
             # run it again in case of doubt
@@ -582,7 +582,7 b' class revbranchcache(object):'
 
     @util.propertycache
     def _namesreverse(self):
-        return dict((b, r) for r, b in enumerate(self._names))
+        return {b: r for r, b in enumerate(self._names)}
 
     def branchinfo(self, rev):
         """Return branch name and close flag for rev, using and updating

mercurial/bundle2.py

@@ -2368,6 +2368,11 b' def handlebookmark(op, inpart):'
                     b'prepushkey', throw=True, **pycompat.strkwargs(hookargs)
                 )
 
+        for book, node in changes:
+            if bookmarks.isdivergent(book):
+                msg = _(b'cannot accept divergent bookmark %s!') % book
+                raise error.Abort(msg)
+
         bookstore.applychanges(op.repo, op.gettransaction(), changes)
 
         if pushkeycompat:

mercurial/cext/manifest.c

@@ -53,21 +53,35 b' static PyObject *nodeof(line *l)'
 {
 	char *s = l->start;
 	Py_ssize_t llen = pathlen(l);
+	Py_ssize_t hlen = l->len - llen - 2;
+	Py_ssize_t hlen_raw = 20;
 	PyObject *hash;
 	if (llen + 1 + 40 + 1 > l->len) { /* path '\0' hash '\n' */
 		PyErr_SetString(PyExc_ValueError, "manifest line too short");
 		return NULL;
 	}
-	hash = unhexlify(s + llen + 1, 40);
+	switch (hlen) {
+	case 40: /* sha1 */
+	case 41: /* sha1 with cruft for a merge */
+		break;
+	case 64: /* new hash */
+	case 65: /* new hash with cruft for a merge */
+		hlen_raw = 32;
+		break;
+	default:
+		PyErr_SetString(PyExc_ValueError, "invalid node length in manifest");
+		return NULL;
+	}
+	hash = unhexlify(s + llen + 1, hlen_raw * 2);
 	if (!hash) {
 		return NULL;
 	}
 	if (l->hash_suffix != '\0') {
-		char newhash[21];
-		memcpy(newhash, PyBytes_AsString(hash), 20);
+		char newhash[33];
+		memcpy(newhash, PyBytes_AsString(hash), hlen_raw);
 		Py_DECREF(hash);
-		newhash[20] = l->hash_suffix;
-		hash = PyBytes_FromStringAndSize(newhash, 21);
+		newhash[hlen_raw] = l->hash_suffix;
+		hash = PyBytes_FromStringAndSize(newhash, hlen_raw+1);
 	}
 	return hash;
 }
@@ -78,15 +92,20 b' static PyObject *hashflags(line *l)'
 	char *s = l->start;
 	Py_ssize_t plen = pathlen(l);
 	PyObject *hash = nodeof(l);
-
-	/* 40 for hash, 1 for null byte, 1 for newline */
-	Py_ssize_t hplen = plen + 42;
-	Py_ssize_t flen = l->len - hplen;
+	ssize_t hlen;
+	Py_ssize_t hplen, flen;
 	PyObject *flags;
 	PyObject *tup;
 
 	if (!hash)
 		return NULL;
+	/* hash is either 20 or 21 bytes for an old hash, so we use a
+	   ternary here to get the "real" hexlified sha length. */
+	hlen = PyBytes_GET_SIZE(hash) < 22 ? 40 : 64;
+	/* 1 for null byte, 1 for newline */
+	hplen = plen + hlen + 2;
+	flen = l->len - hplen;
+
 	flags = PyBytes_FromStringAndSize(s + hplen - 1, flen);
 	if (!flags) {
 		Py_DECREF(hash);

mercurial/cext/osutil.c

@@ -810,9 +810,10 b' static PyObject *setprocname(PyObject *s'
 			/* Check the memory we can use. Typically, argv[i] and
 			 * argv[i + 1] are continuous. */
 			for (i = 0; i < argc; ++i) {
+				size_t len;
 				if (argv[i] > argvend || argv[i] < argvstart)
 					break; /* not continuous */
-				size_t len = strlen(argv[i]);
+				len = strlen(argv[i]);
 				argvend = argv[i] + len + 1 /* '\0' */;
 			}
 			if (argvend > argvstart) /* sanity check */
@@ -1169,10 +1170,10 b' static PyObject *getfsmountpoint(PyObjec'
 static PyObject *unblocksignal(PyObject *self, PyObject *args)
 {
 	int sig = 0;
+	sigset_t set;
 	int r;
 	if (!PyArg_ParseTuple(args, "i", &sig))
 		return NULL;
-	sigset_t set;
 	r = sigemptyset(&set);
 	if (r != 0)
 		return PyErr_SetFromErrno(PyExc_OSError);

mercurial/cext/revlog.c

@@ -39,6 +39,8 b' typedef struct {'
 
 typedef struct {
 	int abi_version;
+	Py_ssize_t (*index_length)(const indexObject *);
+	const char *(*index_node)(indexObject *, Py_ssize_t);
 	int (*index_parents)(PyObject *, int, int *);
 } Revlog_CAPI;
 
@@ -212,7 +214,7 b' static inline int index_get_parents(inde'
  *
  * Returns 0 on success or -1 on failure.
  */
-int HgRevlogIndex_GetParents(PyObject *op, int rev, int *ps)
+static int HgRevlogIndex_GetParents(PyObject *op, int rev, int *ps)
 {
 	int tiprev;
 	if (!op || !HgRevlogIndex_Check(op) || !ps) {
@@ -2878,173 +2880,12 b' bail:'
 	return NULL;
 }
 
-#ifdef WITH_RUST
-
-/* rustlazyancestors: iteration over ancestors implemented in Rust
- *
- * This class holds a reference to an index and to the Rust iterator.
- */
-typedef struct rustlazyancestorsObjectStruct rustlazyancestorsObject;
-
-struct rustlazyancestorsObjectStruct {
-	PyObject_HEAD
-	    /* Type-specific fields go here. */
-	    indexObject *index; /* Ref kept to avoid GC'ing the index */
-	void *iter;             /* Rust iterator */
-};
-
-/* FFI exposed from Rust code */
-rustlazyancestorsObject *rustlazyancestors_init(indexObject *index,
-                                                /* intrevs vector */
-                                                Py_ssize_t initrevslen,
-                                                long *initrevs, long stoprev,
-                                                int inclusive);
-void rustlazyancestors_drop(rustlazyancestorsObject *self);
-int rustlazyancestors_next(rustlazyancestorsObject *self);
-int rustlazyancestors_contains(rustlazyancestorsObject *self, long rev);
-
-/* CPython instance methods */
-static int rustla_init(rustlazyancestorsObject *self, PyObject *args)
-{
-	PyObject *initrevsarg = NULL;
-	PyObject *inclusivearg = NULL;
-	long stoprev = 0;
-	long *initrevs = NULL;
-	int inclusive = 0;
-	Py_ssize_t i;
-
-	indexObject *index;
-	if (!PyArg_ParseTuple(args, "O!O!lO!", &HgRevlogIndex_Type, &index,
-	                      &PyList_Type, &initrevsarg, &stoprev,
-	                      &PyBool_Type, &inclusivearg))
-		return -1;
-
-	Py_INCREF(index);
-	self->index = index;
-
-	if (inclusivearg == Py_True)
-		inclusive = 1;
-
-	Py_ssize_t linit = PyList_GET_SIZE(initrevsarg);
-
-	initrevs = (long *)calloc(linit, sizeof(long));
-
-	if (initrevs == NULL) {
-		PyErr_NoMemory();
-		goto bail;
-	}
-
-	for (i = 0; i < linit; i++) {
-		initrevs[i] = PyInt_AsLong(PyList_GET_ITEM(initrevsarg, i));
-	}
-	if (PyErr_Occurred())
-		goto bail;
-
-	self->iter =
-	    rustlazyancestors_init(index, linit, initrevs, stoprev, inclusive);
-	if (self->iter == NULL) {
-		/* if this is because of GraphError::ParentOutOfRange
-		 * HgRevlogIndex_GetParents() has already set the proper
-		 * exception */
-		goto bail;
-	}
-
-	free(initrevs);
-	return 0;
-
-bail:
-	free(initrevs);
-	return -1;
-};
-
-static void rustla_dealloc(rustlazyancestorsObject *self)
-{
-	Py_XDECREF(self->index);
-	if (self->iter != NULL) { /* can happen if rustla_init failed */
-		rustlazyancestors_drop(self->iter);
-	}
-	PyObject_Del(self);
-}
-
-static PyObject *rustla_next(rustlazyancestorsObject *self)
-{
-	int res = rustlazyancestors_next(self->iter);
-	if (res == -1) {
-		/* Setting an explicit exception seems unnecessary
-		 * as examples from Python source code (Objects/rangeobjets.c
-		 * and Modules/_io/stringio.c) seem to demonstrate.
-		 */
-		return NULL;
-	}
-	return PyInt_FromLong(res);
-}
-
-static int rustla_contains(rustlazyancestorsObject *self, PyObject *rev)
-{
-	long lrev;
-	if (!pylong_to_long(rev, &lrev)) {
-		PyErr_Clear();
-		return 0;
-	}
-	return rustlazyancestors_contains(self->iter, lrev);
-}
-
-static PySequenceMethods rustla_sequence_methods = {
-    0,                           /* sq_length */
-    0,                           /* sq_concat */
-    0,                           /* sq_repeat */
-    0,                           /* sq_item */
-    0,                           /* sq_slice */
-    0,                           /* sq_ass_item */
-    0,                           /* sq_ass_slice */
-    (objobjproc)rustla_contains, /* sq_contains */
-};
-
-static PyTypeObject rustlazyancestorsType = {
-    PyVarObject_HEAD_INIT(NULL, 0)                  /* header */
-    "parsers.rustlazyancestors",                    /* tp_name */
-    sizeof(rustlazyancestorsObject),                /* tp_basicsize */
-    0,                                              /* tp_itemsize */
-    (destructor)rustla_dealloc,                     /* tp_dealloc */
-    0,                                              /* tp_print */
-    0,                                              /* tp_getattr */
-    0,                                              /* tp_setattr */
-    0,                                              /* tp_compare */
-    0,                                              /* tp_repr */
-    0,                                              /* tp_as_number */
-    &rustla_sequence_methods,                       /* tp_as_sequence */
-    0,                                              /* tp_as_mapping */
-    0,                                              /* tp_hash */
-    0,                                              /* tp_call */
-    0,                                              /* tp_str */
-    0,                                              /* tp_getattro */
-    0,                                              /* tp_setattro */
-    0,                                              /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                             /* tp_flags */
-    "Iterator over ancestors, implemented in Rust", /* tp_doc */
-    0,                                              /* tp_traverse */
-    0,                                              /* tp_clear */
-    0,                                              /* tp_richcompare */
-    0,                                              /* tp_weaklistoffset */
-    0,                                              /* tp_iter */
-    (iternextfunc)rustla_next,                      /* tp_iternext */
-    0,                                              /* tp_methods */
-    0,                                              /* tp_members */
-    0,                                              /* tp_getset */
-    0,                                              /* tp_base */
-    0,                                              /* tp_dict */
-    0,                                              /* tp_descr_get */
-    0,                                              /* tp_descr_set */
-    0,                                              /* tp_dictoffset */
-    (initproc)rustla_init,                          /* tp_init */
-    0,                                              /* tp_alloc */
-};
-#endif /* WITH_RUST */
-
 static Revlog_CAPI CAPI = {
     /* increment the abi_version field upon each change in the Revlog_CAPI
        struct or in the ABI of the listed functions */
-    1,
+    2,
+    index_length,
+    index_node,
     HgRevlogIndex_GetParents,
 };
 
@@ -3074,13 +2915,4 b' void revlog_module_init(PyObject *mod)'
 	caps = PyCapsule_New(&CAPI, "mercurial.cext.parsers.revlog_CAPI", NULL);
 	if (caps != NULL)
 		PyModule_AddObject(mod, "revlog_CAPI", caps);
-
-#ifdef WITH_RUST
-	rustlazyancestorsType.tp_new = PyType_GenericNew;
-	if (PyType_Ready(&rustlazyancestorsType) < 0)
-		return;
-	Py_INCREF(&rustlazyancestorsType);
-	PyModule_AddObject(mod, "rustlazyancestors",
-	                   (PyObject *)&rustlazyancestorsType);
-#endif
 }

mercurial/cext/revlog.h

@@ -14,6 +14,4 b' extern PyTypeObject HgRevlogIndex_Type;'
 
 #define HgRevlogIndex_Check(op) PyObject_TypeCheck(op, &HgRevlogIndex_Type)
 
-int HgRevlogIndex_GetParents(PyObject *op, int rev, int *ps);
-
 #endif /* _HG_REVLOG_H_ */

mercurial/changegroup.py

@@ -993,7 +993,7 b' class cgpacker(object):'
             ]
 
         manifests.clear()
-        clrevs = set(cl.rev(x) for x in clnodes)
+        clrevs = {cl.rev(x) for x in clnodes}
 
         it = self.generatefiles(
             changedfiles,
@@ -1149,8 +1149,8 b' class cgpacker(object):'
             if fastpathlinkrev:
                 assert not tree
                 return (
-                    manifests.__getitem__  # pytype: disable=unsupported-operands
-                )
+                    manifests.__getitem__
+                )  # pytype: disable=unsupported-operands
 
             def lookupmflinknode(x):
                 """Callback for looking up the linknode for manifests.
@@ -1282,9 +1282,7 b' class cgpacker(object):'
                 flinkrev = store.linkrev
                 fnode = store.node
                 revs = ((r, flinkrev(r)) for r in store)
-                return dict(
-                    (fnode(r), cln(lr)) for r, lr in revs if lr in clrevs
-                )
+                return {fnode(r): cln(lr) for r, lr in revs if lr in clrevs}
 
         clrevtolocalrev = {}
 

mercurial/changelog.py

@@ -161,15 +161,18 b' class appender(object):'
         return self.fp.__exit__(*args)
 
 
-def _divertopener(opener, target):
-    """build an opener that writes in 'target.a' instead of 'target'"""
+class _divertopener(object):
+    def __init__(self, opener, target):
+        self._opener = opener
+        self._target = target
 
-    def _divert(name, mode=b'r', checkambig=False, **kwargs):
-        if name != target:
-            return opener(name, mode, **kwargs)
-        return opener(name + b".a", mode, **kwargs)
+    def __call__(self, name, mode=b'r', checkambig=False, **kwargs):
+        if name != self._target:
+            return self._opener(name, mode, **kwargs)
+        return self._opener(name + b".a", mode, **kwargs)
 
-    return _divert
+    def __getattr__(self, attr):
+        return getattr(self._opener, attr)
 
 
 def _delayopener(opener, target, buf):
@@ -382,6 +385,9 b' class changelog(revlog.revlog):'
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):

mercurial/chgserver.py

@@ -80,9 +80,12 b' def _hashlist(items):'
 # sensitive config sections affecting confighash
 _configsections = [
     b'alias',  # affects global state commands.table
+    b'diff-tools',  # affects whether gui or not in extdiff's uisetup
     b'eol',  # uses setconfig('eol', ...)
     b'extdiff',  # uisetup will register new commands
     b'extensions',
+    b'fastannotate',  # affects annotate command and adds fastannonate cmd
+    b'merge-tools',  # affects whether gui or not in extdiff's uisetup
     b'schemes',  # extsetup will update global hg.schemes
 ]
 
@@ -525,7 +528,7 b' class chgcmdserver(commandserver.server)'
     def _setumask(self, data):
         mask = struct.unpack(b'>I', data)[0]
         self.ui.log(b'chgserver', b'setumask %r\n', mask)
-        os.umask(mask)
+        util.setumask(mask)
 
     def runcommand(self):
         # pager may be attached within the runcommand session, which should
@@ -551,40 +554,6 b' class chgcmdserver(commandserver.server)'
             raise ValueError(b'unexpected value in setenv request')
         self.ui.log(b'chgserver', b'setenv: %r\n', sorted(newenv.keys()))
 
-        # Python3 has some logic to "coerce" the C locale to a UTF-8 capable
-        # one, and it sets LC_CTYPE in the environment to C.UTF-8 if none of
-        # 'LC_CTYPE', 'LC_ALL' or 'LANG' are set (to any value). This can be
-        # disabled with PYTHONCOERCECLOCALE=0 in the environment.
-        #
-        # When fromui is called via _inithashstate, python has already set
-        # this, so that's in the environment right when we start up the hg
-        # process. Then chg will call us and tell us to set the environment to
-        # the one it has; this might NOT have LC_CTYPE, so we'll need to
-        # carry-forward the LC_CTYPE that was coerced in these situations.
-        #
-        # If this is not handled, we will fail config+env validation and fail
-        # to start chg. If this is just ignored instead of carried forward, we
-        # may have different behavior between chg and non-chg.
-        if pycompat.ispy3:
-            # Rename for wordwrapping purposes
-            oldenv = encoding.environ
-            if not any(
-                e.get(b'PYTHONCOERCECLOCALE') == b'0' for e in [oldenv, newenv]
-            ):
-                keys = [b'LC_CTYPE', b'LC_ALL', b'LANG']
-                old_keys = [k for k, v in oldenv.items() if k in keys and v]
-                new_keys = [k for k, v in newenv.items() if k in keys and v]
-                # If the user's environment (from chg) doesn't have ANY of the
-                # keys that python looks for, and the environment (from
-                # initialization) has ONLY LC_CTYPE and it's set to C.UTF-8,
-                # carry it forward.
-                if (
-                    not new_keys
-                    and old_keys == [b'LC_CTYPE']
-                    and oldenv[b'LC_CTYPE'] == b'C.UTF-8'
-                ):
-                    newenv[b'LC_CTYPE'] = oldenv[b'LC_CTYPE']
-
         encoding.environ.clear()
         encoding.environ.update(newenv)
 
@@ -731,6 +700,16 b' def chgunixservice(ui, repo, opts):'
     # environ cleaner.
     if b'CHGINTERNALMARK' in encoding.environ:
         del encoding.environ[b'CHGINTERNALMARK']
+    # Python3.7+ "coerces" the LC_CTYPE environment variable to a UTF-8 one if
+    # it thinks the current value is "C". This breaks the hash computation and
+    # causes chg to restart loop.
+    if b'CHGORIG_LC_CTYPE' in encoding.environ:
+        encoding.environ[b'LC_CTYPE'] = encoding.environ[b'CHGORIG_LC_CTYPE']
+        del encoding.environ[b'CHGORIG_LC_CTYPE']
+    elif b'CHG_CLEAR_LC_CTYPE' in encoding.environ:
+        if b'LC_CTYPE' in encoding.environ:
+            del encoding.environ[b'LC_CTYPE']
+        del encoding.environ[b'CHG_CLEAR_LC_CTYPE']
 
     if repo:
         # one chgserver can serve multiple repos. drop repo information

mercurial/cmdutil.py

@@ -170,7 +170,12 b' logopts = ['
 
 diffopts = [
     (b'a', b'text', None, _(b'treat all files as text')),
-    (b'g', b'git', None, _(b'use git extended diff format')),
+    (
+        b'g',
+        b'git',
+        None,
+        _(b'use git extended diff format (DEFAULT: diff.git)'),
+    ),
     (b'', b'binary', None, _(b'generate binary diffs in git mode (default)')),
     (b'', b'nodates', None, _(b'omit dates from diff headers')),
 ]
@@ -209,7 +214,9 b' diffopts2 = ('
             b'p',
             b'show-function',
             None,
-            _(b'show which function each change is in'),
+            _(
+                b'show which function each change is in (DEFAULT: diff.showfunc)'
+            ),
         ),
         (b'', b'reverse', None, _(b'produce a diff that undoes the changes')),
     ]
@@ -281,11 +288,11 b' def check_at_most_one_arg(opts, *args):'
     return previous
 
 
-def check_incompatible_arguments(opts, first, *others):
+def check_incompatible_arguments(opts, first, others):
     """abort if the first argument is given along with any of the others
 
     Unlike check_at_most_one_arg(), `others` are not mutually exclusive
-    among themselves.
+    among themselves, and they're passed as a single collection.
     """
     for other in others:
         check_at_most_one_arg(opts, first, other)
@@ -584,15 +591,8 b' def dorecord('
             [os.unlink(repo.wjoin(c)) for c in newlyaddedandmodifiedfiles]
             # 3a. apply filtered patch to clean repo  (clean)
             if backups:
-                # Equivalent to hg.revert
                 m = scmutil.matchfiles(repo, set(backups.keys()) | alsorestore)
-                mergemod.update(
-                    repo,
-                    repo.dirstate.p1(),
-                    branchmerge=False,
-                    force=True,
-                    matcher=m,
-                )
+                mergemod.revert_to(repo[b'.'], matcher=m)
 
             # 3b. (apply)
             if dopatch:
@@ -1414,46 +1414,165 b' def openrevlog(repo, cmd, file_, opts):'
 
 
 def copy(ui, repo, pats, opts, rename=False):
+    check_incompatible_arguments(opts, b'forget', [b'dry_run'])
+
     # called with the repo lock held
     #
     # hgsep => pathname that uses "/" to separate directories
     # ossep => pathname that uses os.sep to separate directories
     cwd = repo.getcwd()
     targets = {}
+    forget = opts.get(b"forget")
     after = opts.get(b"after")
     dryrun = opts.get(b"dry_run")
-    wctx = repo[None]
+    rev = opts.get(b'at_rev')
+    if rev:
+        if not forget and not after:
+            # TODO: Remove this restriction and make it also create the copy
+            #       targets (and remove the rename source if rename==True).
+            raise error.Abort(_(b'--at-rev requires --after'))
+        ctx = scmutil.revsingle(repo, rev)
+        if len(ctx.parents()) > 1:
+            raise error.Abort(_(b'cannot mark/unmark copy in merge commit'))
+    else:
+        ctx = repo[None]
+
+    pctx = ctx.p1()
 
     uipathfn = scmutil.getuipathfn(repo, legacyrelativevalue=True)
 
+    if forget:
+        if ctx.rev() is None:
+            new_ctx = ctx
+        else:
+            if len(ctx.parents()) > 1:
+                raise error.Abort(_(b'cannot unmark copy in merge commit'))
+            # avoid cycle context -> subrepo -> cmdutil
+            from . import context
+
+            rewriteutil.precheck(repo, [ctx.rev()], b'uncopy')
+            new_ctx = context.overlayworkingctx(repo)
+            new_ctx.setbase(ctx.p1())
+            mergemod.graft(repo, ctx, wctx=new_ctx)
+
+        match = scmutil.match(ctx, pats, opts)
+
+        current_copies = ctx.p1copies()
+        current_copies.update(ctx.p2copies())
+
+        uipathfn = scmutil.getuipathfn(repo)
+        for f in ctx.walk(match):
+            if f in current_copies:
+                new_ctx[f].markcopied(None)
+            elif match.exact(f):
+                ui.warn(
+                    _(
+                        b'%s: not unmarking as copy - file is not marked as copied\n'
+                    )
+                    % uipathfn(f)
+                )
+
+        if ctx.rev() is not None:
+            with repo.lock():
+                mem_ctx = new_ctx.tomemctx_for_amend(ctx)
+                new_node = mem_ctx.commit()
+
+                if repo.dirstate.p1() == ctx.node():
+                    with repo.dirstate.parentchange():
+                        scmutil.movedirstate(repo, repo[new_node])
+                replacements = {ctx.node(): [new_node]}
+                scmutil.cleanupnodes(
+                    repo, replacements, b'uncopy', fixphase=True
+                )
+
+        return
+
+    pats = scmutil.expandpats(pats)
+    if not pats:
+        raise error.Abort(_(b'no source or destination specified'))
+    if len(pats) == 1:
+        raise error.Abort(_(b'no destination specified'))
+    dest = pats.pop()
+
     def walkpat(pat):
         srcs = []
-        if after:
-            badstates = b'?'
-        else:
-            badstates = b'?r'
-        m = scmutil.match(wctx, [pat], opts, globbed=True)
-        for abs in wctx.walk(m):
-            state = repo.dirstate[abs]
+        m = scmutil.match(ctx, [pat], opts, globbed=True)
+        for abs in ctx.walk(m):
             rel = uipathfn(abs)
             exact = m.exact(abs)
-            if state in badstates:
-                if exact and state == b'?':
-                    ui.warn(_(b'%s: not copying - file is not managed\n') % rel)
-                if exact and state == b'r':
-                    ui.warn(
-                        _(
-                            b'%s: not copying - file has been marked for'
-                            b' remove\n'
+            if abs not in ctx:
+                if abs in pctx:
+                    if not after:
+                        if exact:
+                            ui.warn(
+                                _(
+                                    b'%s: not copying - file has been marked '
+                                    b'for remove\n'
+                                )
+                                % rel
+                            )
+                        continue
+                else:
+                    if exact:
+                        ui.warn(
+                            _(b'%s: not copying - file is not managed\n') % rel
                         )
-                        % rel
-                    )
-                continue
+                    continue
+
             # abs: hgsep
             # rel: ossep
             srcs.append((abs, rel, exact))
         return srcs
 
+    if ctx.rev() is not None:
+        rewriteutil.precheck(repo, [ctx.rev()], b'uncopy')
+        absdest = pathutil.canonpath(repo.root, cwd, dest)
+        if ctx.hasdir(absdest):
+            raise error.Abort(
+                _(b'%s: --at-rev does not support a directory as destination')
+                % uipathfn(absdest)
+            )
+        if absdest not in ctx:
+            raise error.Abort(
+                _(b'%s: copy destination does not exist in %s')
+                % (uipathfn(absdest), ctx)
+            )
+
+        # avoid cycle context -> subrepo -> cmdutil
+        from . import context
+
+        copylist = []
+        for pat in pats:
+            srcs = walkpat(pat)
+            if not srcs:
+                continue
+            for abs, rel, exact in srcs:
+                copylist.append(abs)
+
+        # TODO: Add support for `hg cp --at-rev . foo bar dir` and
+        # `hg cp --at-rev . dir1 dir2`, preferably unifying the code with the
+        # existing functions below.
+        if len(copylist) != 1:
+            raise error.Abort(_(b'--at-rev requires a single source'))
+
+        new_ctx = context.overlayworkingctx(repo)
+        new_ctx.setbase(ctx.p1())
+        mergemod.graft(repo, ctx, wctx=new_ctx)
+
+        new_ctx.markcopied(absdest, copylist[0])
+
+        with repo.lock():
+            mem_ctx = new_ctx.tomemctx_for_amend(ctx)
+            new_node = mem_ctx.commit()
+
+            if repo.dirstate.p1() == ctx.node():
+                with repo.dirstate.parentchange():
+                    scmutil.movedirstate(repo, repo[new_node])
+            replacements = {ctx.node(): [new_node]}
+            scmutil.cleanupnodes(repo, replacements, b'copy', fixphase=True)
+
+        return
+
     # abssrc: hgsep
     # relsrc: ossep
     # otarget: ossep
@@ -1583,13 +1702,13 b' def copy(ui, repo, pats, opts, rename=Fa'
 
         # fix up dirstate
         scmutil.dirstatecopy(
-            ui, repo, wctx, abssrc, abstarget, dryrun=dryrun, cwd=cwd
+            ui, repo, ctx, abssrc, abstarget, dryrun=dryrun, cwd=cwd
         )
         if rename and not dryrun:
             if not after and srcexists and not samefile:
                 rmdir = repo.ui.configbool(b'experimental', b'removeemptydirs')
                 repo.wvfs.unlinkpath(abssrc, rmdir=rmdir)
-            wctx.forget([abssrc])
+            ctx.forget([abssrc])
 
     # pat: ossep
     # dest ossep
@@ -1659,12 +1778,6 b' def copy(ui, repo, pats, opts, rename=Fa'
                     res = lambda p: dest
         return res
 
-    pats = scmutil.expandpats(pats)
-    if not pats:
-        raise error.Abort(_(b'no source or destination specified'))
-    if len(pats) == 1:
-        raise error.Abort(_(b'no destination specified'))
-    dest = pats.pop()
     destdirexists = os.path.isdir(dest) and not os.path.islink(dest)
     if not destdirexists:
         if len(pats) > 1 or matchmod.patkind(pats[0]):
@@ -3012,7 +3125,7 b' def amend(ui, repo, old, extra, pats, op'
         ms = mergemod.mergestate.read(repo)
         mergeutil.checkunresolved(ms)
 
-        filestoamend = set(f for f in wctx.files() if matcher(f))
+        filestoamend = {f for f in wctx.files() if matcher(f)}
 
         changes = len(filestoamend) > 0
         if changes:
@@ -3804,7 +3917,7 b' def _performrevert('
         # Apply changes
         fp = stringio()
         # chunks are serialized per file, but files aren't sorted
-        for f in sorted(set(c.header.filename() for c in chunks if ishunk(c))):
+        for f in sorted({c.header.filename() for c in chunks if ishunk(c)}):
             prntstatusmsg(b'revert', f)
         files = set()
         for c in chunks:

mercurial/color.py

@@ -44,7 +44,7 b' try:'
         b'cyan': (False, curses.COLOR_CYAN, b''),
         b'white': (False, curses.COLOR_WHITE, b''),
     }
-except ImportError:
+except (ImportError, AttributeError):
     curses = None
     _baseterminfoparams = {}
 

mercurial/commands.py

@@ -876,7 +876,7 b' def _dobackout(ui, repo, node=None, rev='
         )
         overrides = {(b'ui', b'forcemerge'): opts.get(b'tool', b'')}
         with ui.configoverride(overrides, b'backout'):
-            return hg.merge(repo, hex(repo.changelog.tip()))
+            return hg.merge(repo[b'tip'])
     return 0
 
 
@@ -1228,7 +1228,7 b' def bookmark(ui, repo, *names, **opts):'
 
     action = cmdutil.check_at_most_one_arg(opts, b'delete', b'rename', b'list')
     if action:
-        cmdutil.check_incompatible_arguments(opts, action, b'rev')
+        cmdutil.check_incompatible_arguments(opts, action, [b'rev'])
     elif names or rev:
         action = b'add'
     elif inactive:
@@ -1236,7 +1236,9 b' def bookmark(ui, repo, *names, **opts):'
     else:
         action = b'list'
 
-    cmdutil.check_incompatible_arguments(opts, b'inactive', b'delete', b'list')
+    cmdutil.check_incompatible_arguments(
+        opts, b'inactive', [b'delete', b'list']
+    )
     if not names and action in {b'add', b'delete'}:
         raise error.Abort(_(b"bookmark name required"))
 
@@ -2307,8 +2309,16 b' def continuecmd(ui, repo, **opts):'
 @command(
     b'copy|cp',
     [
+        (b'', b'forget', None, _(b'unmark a file as copied')),
         (b'A', b'after', None, _(b'record a copy that has already occurred')),
         (
+            b'',
+            b'at-rev',
+            b'',
+            _(b'(un)mark copies in the given revision (EXPERIMENTAL)'),
+            _(b'REV'),
+        ),
+        (
             b'f',
             b'force',
             None,
@@ -2331,8 +2341,11 b' def copy(ui, repo, *pats, **opts):'
     exist in the working directory. If invoked with -A/--after, the
     operation is recorded, but no copying is performed.
 
-    This command takes effect with the next commit. To undo a copy
-    before that, see :hg:`revert`.
+    To undo marking a file as copied, use --forget. With that option,
+    all given (positional) arguments are unmarked as copies. The destination
+    file(s) will be left in place (still tracked).
+
+    This command takes effect with the next commit by default.
 
     Returns 0 on success, 1 if errors are encountered.
     """
@@ -2938,7 +2951,7 b' def graft(ui, repo, *revs, **opts):'
 
     See :hg:`help revisions` for more about specifying revisions.
 
-    Returns 0 on successful completion.
+    Returns 0 on successful completion, 1 if there are unresolved files.
     '''
     with repo.wlock():
         return _dograft(ui, repo, *revs, **opts)
@@ -3199,10 +3212,9 b' def _dograft(ui, repo, *revs, **opts):'
                 statedata[b'nodes'] = nodes
                 stateversion = 1
                 graftstate.save(stateversion, statedata)
-                hint = _(b"use 'hg resolve' and 'hg graft --continue'")
-                raise error.Abort(
-                    _(b"unresolved conflicts, can't continue"), hint=hint
-                )
+                ui.error(_(b"abort: unresolved conflicts, can't continue\n"))
+                ui.error(_(b"(use 'hg resolve' and 'hg graft --continue')\n"))
+                return 1
         else:
             cont = False
 
@@ -3708,9 +3720,9 b' def heads(ui, repo, *branchrevs, **opts)'
         heads = [repo[h] for h in heads]
 
     if branchrevs:
-        branches = set(
+        branches = {
             repo[r].branch() for r in scmutil.revrange(repo, branchrevs)
-        )
+        }
         heads = [h for h in heads if h.branch() in branches]
 
     if opts.get(b'active') and branchrevs:
@@ -3718,7 +3730,7 b' def heads(ui, repo, *branchrevs, **opts)'
         heads = [h for h in heads if h.node() in dagheads]
 
     if branchrevs:
-        haveheads = set(h.branch() for h in heads)
+        haveheads = {h.branch() for h in heads}
         if branches - haveheads:
             headless = b', '.join(b for b in branches - haveheads)
             msg = _(b'no open branch heads found on branches %s')
@@ -4847,6 +4859,7 b' def merge(ui, repo, node=None, **opts):'
     abort = opts.get(b'abort')
     if abort and repo.dirstate.p2() == nullid:
         cmdutil.wrongtooltocontinue(repo, _(b'merge'))
+    cmdutil.check_incompatible_arguments(opts, b'abort', [b'rev', b'preview'])
     if abort:
         state = cmdutil.getunfinishedstate(repo)
         if state and state._opname != b'merge':
@@ -4856,19 +4869,16 b' def merge(ui, repo, node=None, **opts):'
             )
         if node:
             raise error.Abort(_(b"cannot specify a node with --abort"))
-        if opts.get(b'rev'):
-            raise error.Abort(_(b"cannot specify both --rev and --abort"))
-        if opts.get(b'preview'):
-            raise error.Abort(_(b"cannot specify --preview with --abort"))
+        return hg.abortmerge(repo.ui, repo)
+
     if opts.get(b'rev') and node:
         raise error.Abort(_(b"please specify just one revision"))
     if not node:
         node = opts.get(b'rev')
 
     if node:
-        node = scmutil.revsingle(repo, node).node()
-
-    if not node and not abort:
+        ctx = scmutil.revsingle(repo, node)
+    else:
         if ui.configbool(b'commands', b'merge.require-rev'):
             raise error.Abort(
                 _(
@@ -4876,12 +4886,15 b' def merge(ui, repo, node=None, **opts):'
                     b'with'
                 )
             )
-        node = repo[destutil.destmerge(repo)].node()
+        ctx = repo[destutil.destmerge(repo)]
+
+    if ctx.node() is None:
+        raise error.Abort(_(b'merging with the working copy has no effect'))
 
     if opts.get(b'preview'):
         # find nodes that are ancestors of p2 but not of p1
-        p1 = repo.lookup(b'.')
-        p2 = node
+        p1 = repo[b'.'].node()
+        p2 = ctx.node()
         nodes = repo.changelog.findmissing(common=[p1], heads=[p2])
 
         displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
@@ -4895,14 +4908,7 b' def merge(ui, repo, node=None, **opts):'
     with ui.configoverride(overrides, b'merge'):
         force = opts.get(b'force')
         labels = [b'working copy', b'merge rev']
-        return hg.merge(
-            repo,
-            node,
-            force=force,
-            mergeforce=force,
-            labels=labels,
-            abort=abort,
-        )
+        return hg.merge(ctx, force=force, labels=labels)
 
 
 statemod.addunfinished(
@@ -5337,6 +5343,7 b' def postincoming(ui, repo, modheads, opt'
             None,
             _(b'run even when remote repository is unrelated'),
         ),
+        (b'', b'confirm', None, _(b'confirm pull before applying changes'),),
         (
             b'r',
             b'rev',
@@ -5453,6 +5460,7 b' def pull(ui, repo, source=b"default", **'
                 force=opts.get(b'force'),
                 bookmarks=opts.get(b'bookmark', ()),
                 opargs=pullopargs,
+                confirm=opts.get(b'confirm'),
             ).cgresult
 
             # brev is a name, which might be a bookmark to be activated at
@@ -5671,7 +5679,7 b' def push(ui, repo, dest=None, **opts):'
 
 @command(
     b'recover',
-    [(b'', b'verify', True, b"run `hg verify` after successful recover"),],
+    [(b'', b'verify', False, b"run `hg verify` after successful recover"),],
     helpcategory=command.CATEGORY_MAINTENANCE,
 )
 def recover(ui, repo, **opts):
@@ -5946,6 +5954,8 b' def resolve(ui, repo, *pats, **opts):'
             if not m(f):
                 continue
 
+            if ms[f] == mergemod.MERGE_RECORD_MERGED_OTHER:
+                continue
             label, key = mergestateinfo[ms[f]]
             fm.startitem()
             fm.context(ctx=wctx)
@@ -5993,6 +6003,9 b' def resolve(ui, repo, *pats, **opts):'
 
             didwork = True
 
+            if ms[f] == mergemod.MERGE_RECORD_MERGED_OTHER:
+                continue
+
             # don't let driver-resolved files be marked, and run the conclude
             # step if asked to resolve
             if ms[f] == mergemod.MERGE_RECORD_DRIVER_RESOLVED:
@@ -6648,7 +6661,12 b' def shelve(ui, repo, *pats, **opts):'
         (b'i', b'ignored', None, _(b'show only ignored files')),
         (b'n', b'no-status', None, _(b'hide status prefix')),
         (b't', b'terse', _NOTTERSE, _(b'show the terse output (EXPERIMENTAL)')),
-        (b'C', b'copies', None, _(b'show source of copied files')),
+        (
+            b'C',
+            b'copies',
+            None,
+            _(b'show source of copied files (DEFAULT: ui.statuscopies)'),
+        ),
         (
             b'0',
             b'print0',
@@ -7571,7 +7589,7 b' def unshelve(ui, repo, *shelved, **opts)'
        unshelved.
     """
     with repo.wlock():
-        return shelvemod.dounshelve(ui, repo, *shelved, **opts)
+        return shelvemod.unshelvecmd(ui, repo, *shelved, **opts)
 
 
 statemod.addunfinished(
@@ -7653,6 +7671,7 b' def update(ui, repo, node=None, **opts):'
 
     Returns 0 on success, 1 if there are unresolved files.
     """
+    cmdutil.check_at_most_one_arg(opts, 'clean', 'check', 'merge')
     rev = opts.get('rev')
     date = opts.get('date')
     clean = opts.get('clean')
@@ -7674,14 +7693,6 b' def update(ui, repo, node=None, **opts):'
     if date and rev is not None:
         raise error.Abort(_(b"you can't specify a revision and a date"))
 
-    if len([x for x in (clean, check, merge) if x]) > 1:
-        raise error.Abort(
-            _(
-                b"can only specify one of -C/--clean, -c/--check, "
-                b"or -m/--merge"
-            )
-        )
-
     updatecheck = None
     if check:
         updatecheck = b'abort'

mercurial/commandserver.py

@@ -545,6 +545,10 b' class unixforkingservice(object):'
         if maxlen < 0:
             raise error.Abort(_(b'negative max-repo-cache size not allowed'))
         self._repoloader = repocache.repoloader(ui, maxlen)
+        # attempt to avoid crash in CoreFoundation when using chg after fix in
+        # a89381e04c58
+        if pycompat.isdarwin:
+            procutil.gui()
 
     def init(self):
         self._sock = socket.socket(socket.AF_UNIX)

mercurial/configitems.py

@@ -405,6 +405,21 b' coreconfigitem('
 coreconfigitem(
     b'devel', b'legacy.exchange', default=list,
 )
+# TODO before getting `persistent-nodemap` out of experimental
+#
+# * decide for a "status" of the persistent nodemap and associated location
+#   - part of the store next the revlog itself (new requirements)
+#   - part of the cache directory
+#   - part of an `index` directory
+#     (https://www.mercurial-scm.org/wiki/ComputedIndexPlan)
+# * do we want to use this for more than just changelog? if so we need:
+#   - simpler "pending" logic for them
+#   - double check the memory story (we dont want to keep all revlog in memory)
+#   - think about the naming scheme if we are in "cache"
+# * increment the version format to "1" and freeze it.
+coreconfigitem(
+    b'devel', b'persistent-nodemap', default=False,
+)
 coreconfigitem(
     b'devel', b'servercafile', default=b'',
 )
@@ -660,6 +675,12 b' coreconfigitem('
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
@@ -750,7 +771,7 b' coreconfigitem('
 coreconfigitem(
     b'format',
     b'revlog-compression',
-    default=b'zlib',
+    default=lambda: [b'zlib'],
     alias=[(b'experimental', b'format.compression')],
 )
 coreconfigitem(
@@ -1044,6 +1065,9 b' coreconfigitem('
     b'progress', b'width', default=dynamicdefault,
 )
 coreconfigitem(
+    b'pull', b'confirm', default=False,
+)
+coreconfigitem(
     b'push', b'pushvars.server', default=False,
 )
 coreconfigitem(
@@ -1107,7 +1131,7 b' coreconfigitem('
     b'server', b'compressionengines', default=list,
 )
 coreconfigitem(
-    b'server', b'concurrent-push-mode', default=b'strict',
+    b'server', b'concurrent-push-mode', default=b'check-related',
 )
 coreconfigitem(
     b'server', b'disablefullbundle', default=False,

mercurial/context.py

@@ -267,7 +267,7 b' class basectx(object):'
     def _fileinfo(self, path):
         if '_manifest' in self.__dict__:
             try:
-                return self._manifest[path], self._manifest.flags(path)
+                return self._manifest.find(path)
             except KeyError:
                 raise error.ManifestLookupError(
                     self._node, path, _(b'not found in manifest')
@@ -2357,8 +2357,7 b' class overlayworkingctx(committablectx):'
         # Test the other direction -- that this path from p2 isn't a directory
         # in p1 (test that p1 doesn't have any paths matching `path/*`).
         match = self.match([path], default=b'path')
-        matches = self.p1().manifest().matches(match)
-        mfiles = matches.keys()
+        mfiles = list(self.p1().manifest().walk(match))
         if len(mfiles) > 0:
             if len(mfiles) == 1 and mfiles[0] == path:
                 return
@@ -2488,6 +2487,17 b' class overlayworkingctx(committablectx):'
             editor=editor,
         )
 
+    def tomemctx_for_amend(self, precursor):
+        extra = precursor.extra().copy()
+        extra[b'amend_source'] = precursor.hex()
+        return self.tomemctx(
+            text=precursor.description(),
+            branch=precursor.branch(),
+            extra=extra,
+            date=precursor.date(),
+            user=precursor.user(),
+        )
+
     def isdirty(self, path):
         return path in self._cache
 

mercurial/copies.py

@@ -403,13 +403,15 b' def pathcopies(x, y, match=None):'
         )
     if x == y or not x or not y:
         return {}
+    if y.rev() is None and x == y.p1():
+        if debug:
+            repo.ui.debug(b'debug.copies: search mode: dirstate\n')
+        # short-circuit to avoid issues with merge states
+        return _dirstatecopies(repo, match)
     a = y.ancestor(x)
     if a == x:
         if debug:
             repo.ui.debug(b'debug.copies: search mode: forward\n')
-        if y.rev() is None and x == y.p1():
-            # short-circuit to avoid issues with merge states
-            return _dirstatecopies(repo, match)
         copies = _forwardcopies(x, y, match=match)
     elif a == y:
         if debug:
@@ -452,44 +454,34 b' def mergecopies(repo, c1, c2, base):'
 
     ```other changed <file> which local deleted```
 
-    Returns five dicts: "copy", "movewithdir", "diverge", "renamedelete" and
-    "dirmove".
+    Returns a tuple where:
 
-    "copy" is a mapping from destination name -> source name,
-    where source is in c1 and destination is in c2 or vice-versa.
-
-    "movewithdir" is a mapping from source name -> destination name,
-    where the file at source present in one context but not the other
-    needs to be moved to destination by the merge process, because the
-    other context moved the directory it is in.
+    "branch_copies" an instance of branch_copies.
 
     "diverge" is a mapping of source name -> list of destination names
     for divergent renames.
 
-    "renamedelete" is a mapping of source name -> list of destination
-    names for files deleted in c1 that were renamed in c2 or vice-versa.
-
-    "dirmove" is a mapping of detected source dir -> destination dir renames.
-    This is needed for handling changes to new files previously grafted into
-    renamed directories.
-
     This function calls different copytracing algorithms based on config.
     """
     # avoid silly behavior for update from empty dir
     if not c1 or not c2 or c1 == c2:
-        return {}, {}, {}, {}, {}
+        return branch_copies(), branch_copies(), {}
 
     narrowmatch = c1.repo().narrowmatch()
 
     # avoid silly behavior for parent -> working dir
     if c2.node() is None and c1.node() == repo.dirstate.p1():
-        return _dirstatecopies(repo, narrowmatch), {}, {}, {}, {}
+        return (
+            branch_copies(_dirstatecopies(repo, narrowmatch)),
+            branch_copies(),
+            {},
+        )
 
     copytracing = repo.ui.config(b'experimental', b'copytrace')
     if stringutil.parsebool(copytracing) is False:
         # stringutil.parsebool() returns None when it is unable to parse the
         # value, so we should rely on making sure copytracing is on such cases
-        return {}, {}, {}, {}, {}
+        return branch_copies(), branch_copies(), {}
 
     if usechangesetcentricalgo(repo):
         # The heuristics don't make sense when we need changeset-centric algos
@@ -537,15 +529,45 b' def _checksinglesidecopies('
         if src not in m1:
             # renamed on side 1, deleted on side 2
             renamedelete[src] = dsts1
+    elif src not in mb:
+        # Work around the "short-circuit to avoid issues with merge states"
+        # thing in pathcopies(): pathcopies(x, y) can return a copy where the
+        # destination doesn't exist in y.
+        pass
     elif m2[src] != mb[src]:
         if not _related(c2[src], base[src]):
             return
         # modified on side 2
         for dst in dsts1:
-            if dst not in m2:
-                # dst not added on side 2 (handle as regular
-                # "both created" case in manifestmerge otherwise)
-                copy[dst] = src
+            copy[dst] = src
+
+
+class branch_copies(object):
+    """Information about copies made on one side of a merge/graft.
+
+    "copy" is a mapping from destination name -> source name,
+    where source is in c1 and destination is in c2 or vice-versa.
+
+    "movewithdir" is a mapping from source name -> destination name,
+    where the file at source present in one context but not the other
+    needs to be moved to destination by the merge process, because the
+    other context moved the directory it is in.
+
+    "renamedelete" is a mapping of source name -> list of destination
+    names for files deleted in c1 that were renamed in c2 or vice-versa.
+
+    "dirmove" is a mapping of detected source dir -> destination dir renames.
+    This is needed for handling changes to new files previously grafted into
+    renamed directories.
+    """
+
+    def __init__(
+        self, copy=None, renamedelete=None, dirmove=None, movewithdir=None
+    ):
+        self.copy = {} if copy is None else copy
+        self.renamedelete = {} if renamedelete is None else renamedelete
+        self.dirmove = {} if dirmove is None else dirmove
+        self.movewithdir = {} if movewithdir is None else movewithdir
 
 
 def _fullcopytracing(repo, c1, c2, base):
@@ -563,6 +585,9 b' def _fullcopytracing(repo, c1, c2, base)'
     copies1 = pathcopies(base, c1)
     copies2 = pathcopies(base, c2)
 
+    if not (copies1 or copies2):
+        return branch_copies(), branch_copies(), {}
+
     inversecopies1 = {}
     inversecopies2 = {}
     for dst, src in copies1.items():
@@ -570,9 +595,11 b' def _fullcopytracing(repo, c1, c2, base)'
     for dst, src in copies2.items():
         inversecopies2.setdefault(src, []).append(dst)
 
-    copy = {}
+    copy1 = {}
+    copy2 = {}
     diverge = {}
-    renamedelete = {}
+    renamedelete1 = {}
+    renamedelete2 = {}
     allsources = set(inversecopies1) | set(inversecopies2)
     for src in allsources:
         dsts1 = inversecopies1.get(src)
@@ -589,7 +616,8 b' def _fullcopytracing(repo, c1, c2, base)'
                 # and 'd' and deletes 'a'.
                 if dsts1 & dsts2:
                     for dst in dsts1 & dsts2:
-                        copy[dst] = src
+                        copy1[dst] = src
+                        copy2[dst] = src
                 else:
                     diverge[src] = sorted(dsts1 | dsts2)
             elif src in m1 and src in m2:
@@ -597,27 +625,21 b' def _fullcopytracing(repo, c1, c2, base)'
                 dsts1 = set(dsts1)
                 dsts2 = set(dsts2)
                 for dst in dsts1 & dsts2:
-                    copy[dst] = src
+                    copy1[dst] = src
+                    copy2[dst] = src
             # TODO: Handle cases where it was renamed on one side and copied
             # on the other side
         elif dsts1:
             # copied/renamed only on side 1
             _checksinglesidecopies(
-                src, dsts1, m1, m2, mb, c2, base, copy, renamedelete
+                src, dsts1, m1, m2, mb, c2, base, copy1, renamedelete1
             )
         elif dsts2:
             # copied/renamed only on side 2
             _checksinglesidecopies(
-                src, dsts2, m2, m1, mb, c1, base, copy, renamedelete
+                src, dsts2, m2, m1, mb, c1, base, copy2, renamedelete2
             )
 
-    renamedeleteset = set()
-    divergeset = set()
-    for dsts in diverge.values():
-        divergeset.update(dsts)
-    for dsts in renamedelete.values():
-        renamedeleteset.update(dsts)
-
     # find interesting file sets from manifests
     addedinm1 = m1.filesnotin(mb, repo.narrowmatch())
     addedinm2 = m2.filesnotin(mb, repo.narrowmatch())
@@ -630,33 +652,60 b' def _fullcopytracing(repo, c1, c2, base)'
     if u2:
         repo.ui.debug(b"%s:\n   %s\n" % (header % b'other', b"\n   ".join(u2)))
 
-    fullcopy = copies1.copy()
-    fullcopy.update(copies2)
-    if not fullcopy:
-        return copy, {}, diverge, renamedelete, {}
+    if repo.ui.debugflag:
+        renamedeleteset = set()
+        divergeset = set()
+        for dsts in diverge.values():
+            divergeset.update(dsts)
+        for dsts in renamedelete1.values():
+            renamedeleteset.update(dsts)
+        for dsts in renamedelete2.values():
+            renamedeleteset.update(dsts)
 
-    if repo.ui.debugflag:
         repo.ui.debug(
             b"  all copies found (* = to merge, ! = divergent, "
             b"% = renamed and deleted):\n"
         )
-        for f in sorted(fullcopy):
-            note = b""
-            if f in copy:
-                note += b"*"
-            if f in divergeset:
-                note += b"!"
-            if f in renamedeleteset:
-                note += b"%"
-            repo.ui.debug(
-                b"   src: '%s' -> dst: '%s' %s\n" % (fullcopy[f], f, note)
-            )
-    del divergeset
+        for side, copies in ((b"local", copies1), (b"remote", copies2)):
+            if not copies:
+                continue
+            repo.ui.debug(b"   on %s side:\n" % side)
+            for f in sorted(copies):
+                note = b""
+                if f in copy1 or f in copy2:
+                    note += b"*"
+                if f in divergeset:
+                    note += b"!"
+                if f in renamedeleteset:
+                    note += b"%"
+                repo.ui.debug(
+                    b"    src: '%s' -> dst: '%s' %s\n" % (copies[f], f, note)
+                )
+        del renamedeleteset
+        del divergeset
 
     repo.ui.debug(b"  checking for directory renames\n")
 
+    dirmove1, movewithdir2 = _dir_renames(repo, c1, copy1, copies1, u2)
+    dirmove2, movewithdir1 = _dir_renames(repo, c2, copy2, copies2, u1)
+
+    branch_copies1 = branch_copies(copy1, renamedelete1, dirmove1, movewithdir1)
+    branch_copies2 = branch_copies(copy2, renamedelete2, dirmove2, movewithdir2)
+
+    return branch_copies1, branch_copies2, diverge
+
+
+def _dir_renames(repo, ctx, copy, fullcopy, addedfiles):
+    """Finds moved directories and files that should move with them.
+
+    ctx: the context for one of the sides
+    copy: files copied on the same side (as ctx)
+    fullcopy: files copied on the same side (as ctx), including those that
+              merge.manifestmerge() won't care about
+    addedfiles: added files on the other side (compared to ctx)
+    """
     # generate a directory move map
-    d1, d2 = c1.dirs(), c2.dirs()
+    d = ctx.dirs()
     invalid = set()
     dirmove = {}
 
@@ -667,12 +716,9 b' def _fullcopytracing(repo, c1, c2, base)'
         if dsrc in invalid:
             # already seen to be uninteresting
             continue
-        elif dsrc in d1 and ddst in d1:
+        elif dsrc in d and ddst in d:
             # directory wasn't entirely moved locally
             invalid.add(dsrc)
-        elif dsrc in d2 and ddst in d2:
-            # directory wasn't entirely moved remotely
-            invalid.add(dsrc)
         elif dsrc in dirmove and dirmove[dsrc] != ddst:
             # files from the same directory moved to two different places
             invalid.add(dsrc)
@@ -683,10 +729,10 b' def _fullcopytracing(repo, c1, c2, base)'
     for i in invalid:
         if i in dirmove:
             del dirmove[i]
-    del d1, d2, invalid
+    del d, invalid
 
     if not dirmove:
-        return copy, {}, diverge, renamedelete, {}
+        return {}, {}
 
     dirmove = {k + b"/": v + b"/" for k, v in pycompat.iteritems(dirmove)}
 
@@ -697,7 +743,7 b' def _fullcopytracing(repo, c1, c2, base)'
 
     movewithdir = {}
     # check unaccounted nonoverlapping files against directory moves
-    for f in u1 + u2:
+    for f in addedfiles:
         if f not in fullcopy:
             for d in dirmove:
                 if f.startswith(d):
@@ -711,7 +757,7 b' def _fullcopytracing(repo, c1, c2, base)'
                         )
                     break
 
-    return copy, movewithdir, diverge, renamedelete, dirmove
+    return dirmove, movewithdir
 
 
 def _heuristicscopytracing(repo, c1, c2, base):
@@ -744,8 +790,6 b' def _heuristicscopytracing(repo, c1, c2,'
     if c2.rev() is None:
         c2 = c2.p1()
 
-    copies = {}
-
     changedfiles = set()
     m1 = c1.manifest()
     if not repo.revs(b'%d::%d', base.rev(), c2.rev()):
@@ -765,10 +809,11 b' def _heuristicscopytracing(repo, c1, c2,'
         changedfiles.update(ctx.files())
         ctx = ctx.p1()
 
+    copies2 = {}
     cp = _forwardcopies(base, c2)
     for dst, src in pycompat.iteritems(cp):
         if src in m1:
-            copies[dst] = src
+            copies2[dst] = src
 
     # file is missing if it isn't present in the destination, but is present in
     # the base and present in the source.
@@ -777,6 +822,7 b' def _heuristicscopytracing(repo, c1, c2,'
     filt = lambda f: f not in m1 and f in base and f in c2
     missingfiles = [f for f in changedfiles if filt(f)]
 
+    copies1 = {}
     if missingfiles:
         basenametofilename = collections.defaultdict(list)
         dirnametofilename = collections.defaultdict(list)
@@ -818,9 +864,9 b' def _heuristicscopytracing(repo, c1, c2,'
                     # if there are a few related copies then we'll merge
                     # changes into all of them. This matches the behaviour
                     # of upstream copytracing
-                    copies[candidate] = f
+                    copies1[candidate] = f
 
-    return copies, {}, {}, {}, {}
+    return branch_copies(copies1), branch_copies(copies2), {}
 
 
 def _related(f1, f2):

mercurial/crecord.py

@@ -63,13 +63,13 b' try:'
     import curses.ascii
 
     curses.error
-except ImportError:
+except (ImportError, AttributeError):
     # I have no idea if wcurses works with crecord...
     try:
         import wcurses as curses
 
         curses.error
-    except ImportError:
+    except (ImportError, AttributeError):
         # wcurses is not shipped on Windows by default, or python is not
         # compiled with curses
         curses = False

mercurial/dagop.py

@@ -274,6 +274,247 b' def descendantrevs(revs, revsfn, parentr'
                 break
 
 
+class subsetparentswalker(object):
+    r"""Scan adjacent ancestors in the graph given by the subset
+
+    This computes parent-child relations in the sub graph filtered by
+    a revset. Primary use case is to draw a revisions graph.
+
+    In the following example, we consider that the node 'f' has edges to all
+    ancestor nodes, but redundant paths are eliminated. The edge 'f'->'b'
+    is eliminated because there is a path 'f'->'c'->'b' for example.
+
+          - d - e -
+         /         \
+        a - b - c - f
+
+    If the node 'c' is filtered out, the edge 'f'->'b' is activated.
+
+          - d - e -
+         /         \
+        a - b -(c)- f
+
+    Likewise, if 'd' and 'e' are filtered out, this edge is fully eliminated
+    since there is a path 'f'->'c'->'b'->'a' for 'f'->'a'.
+
+           (d) (e)
+
+        a - b - c - f
+
+    Implementation-wise, 'f' is passed down to 'a' as unresolved through the
+    'f'->'e'->'d'->'a' path, whereas we do also remember that 'f' has already
+    been resolved while walking down the 'f'->'c'->'b'->'a' path. When
+    processing the node 'a', the unresolved 'f'->'a' path is eliminated as
+    the 'f' end is marked as resolved.
+
+    Ancestors are searched from the tipmost revision in the subset so the
+    results can be cached. You should specify startrev to narrow the search
+    space to ':startrev'.
+    """
+
+    def __init__(self, repo, subset, startrev=None):
+        if startrev is not None:
+            subset = repo.revs(b'%d:null', startrev) & subset
+
+        # equivalent to 'subset = subset.sorted(reverse=True)', but there's
+        # no such function.
+        fastdesc = subset.fastdesc
+        if fastdesc:
+            desciter = fastdesc()
+        else:
+            if not subset.isdescending() and not subset.istopo():
+                subset = smartset.baseset(subset)
+                subset.sort(reverse=True)
+            desciter = iter(subset)
+
+        self._repo = repo
+        self._changelog = repo.changelog
+        self._subset = subset
+
+        # scanning state (see _scanparents):
+        self._tovisit = []
+        self._pendingcnt = {}
+        self._pointers = {}
+        self._parents = {}
+        self._inputhead = nullrev  # reassigned by self._advanceinput()
+        self._inputtail = desciter
+        self._bottomrev = nullrev
+        self._advanceinput()
+
+    def parentsset(self, rev):
+        """Look up parents of the given revision in the subset, and returns
+        as a smartset"""
+        return smartset.baseset(self.parents(rev))
+
+    def parents(self, rev):
+        """Look up parents of the given revision in the subset
+
+        The returned revisions are sorted by parent index (p1/p2).
+        """
+        self._scanparents(rev)
+        return [r for _c, r in sorted(self._parents.get(rev, []))]
+
+    def _parentrevs(self, rev):
+        try:
+            revs = self._changelog.parentrevs(rev)
+            if revs[-1] == nullrev:
+                return revs[:-1]
+            return revs
+        except error.WdirUnsupported:
+            return tuple(pctx.rev() for pctx in self._repo[None].parents())
+
+    def _advanceinput(self):
+        """Advance the input iterator and set the next revision to _inputhead"""
+        if self._inputhead < nullrev:
+            return
+        try:
+            self._inputhead = next(self._inputtail)
+        except StopIteration:
+            self._bottomrev = self._inputhead
+            self._inputhead = nullrev - 1
+
+    def _scanparents(self, stoprev):
+        """Scan ancestors until the parents of the specified stoprev are
+        resolved"""
+
+        # 'tovisit' is the queue of the input revisions and their ancestors.
+        # It will be populated incrementally to minimize the initial cost
+        # of computing the given subset.
+        #
+        # For to-visit revisions, we keep track of
+        # - the number of the unresolved paths: pendingcnt[rev],
+        # - dict of the unresolved descendants and chains: pointers[rev][0],
+        # - set of the already resolved descendants: pointers[rev][1].
+        #
+        # When a revision is visited, 'pointers[rev]' should be popped and
+        # propagated to its parents accordingly.
+        #
+        # Once all pending paths have been resolved, 'pendingcnt[rev]' becomes
+        # 0 and 'parents[rev]' contains the unsorted list of parent revisions
+        # and p1/p2 chains (excluding linear paths.) The p1/p2 chains will be
+        # used as a sort key preferring p1. 'len(chain)' should be the number
+        # of merges between two revisions.
+
+        subset = self._subset
+        tovisit = self._tovisit  # heap queue of [-rev]
+        pendingcnt = self._pendingcnt  # {rev: count} for visited revisions
+        pointers = self._pointers  # {rev: [{unresolved_rev: chain}, resolved]}
+        parents = self._parents  # {rev: [(chain, rev)]}
+
+        while tovisit or self._inputhead >= nullrev:
+            if pendingcnt.get(stoprev) == 0:
+                return
+
+            # feed greater revisions from input set to queue
+            if not tovisit:
+                heapq.heappush(tovisit, -self._inputhead)
+                self._advanceinput()
+            while self._inputhead >= -tovisit[0]:
+                heapq.heappush(tovisit, -self._inputhead)
+                self._advanceinput()
+
+            rev = -heapq.heappop(tovisit)
+            if rev < self._bottomrev:
+                return
+            if rev in pendingcnt and rev not in pointers:
+                continue  # already visited
+
+            curactive = rev in subset
+            pendingcnt.setdefault(rev, 0)  # mark as visited
+            if curactive:
+                assert rev not in parents
+                parents[rev] = []
+            unresolved, resolved = pointers.pop(rev, ({}, set()))
+
+            if curactive:
+                # reached to active rev, resolve pending descendants' parents
+                for r, c in unresolved.items():
+                    pendingcnt[r] -= 1
+                    assert pendingcnt[r] >= 0
+                    if r in resolved:
+                        continue  # eliminate redundant path
+                    parents[r].append((c, rev))
+                    # mark the descendant 'r' as resolved through this path if
+                    # there are still pending pointers. the 'resolved' set may
+                    # be concatenated later at a fork revision.
+                    if pendingcnt[r] > 0:
+                        resolved.add(r)
+                unresolved.clear()
+                # occasionally clean resolved markers. otherwise the set
+                # would grow indefinitely.
+                resolved = {r for r in resolved if pendingcnt[r] > 0}
+
+            parentrevs = self._parentrevs(rev)
+            bothparentsactive = all(p in subset for p in parentrevs)
+
+            # set up or propagate tracking pointers if
+            # - one of the parents is not active,
+            # - or descendants' parents are unresolved.
+            if not bothparentsactive or unresolved or resolved:
+                if len(parentrevs) <= 1:
+                    # can avoid copying the tracking pointer
+                    parentpointers = [(unresolved, resolved)]
+                else:
+                    parentpointers = [
+                        (unresolved, resolved),
+                        (unresolved.copy(), resolved.copy()),
+                    ]
+                    # 'rev' is a merge revision. increment the pending count
+                    # as the 'unresolved' dict will be duplicated, and append
+                    # p1/p2 code to the existing chains.
+                    for r in unresolved:
+                        pendingcnt[r] += 1
+                        parentpointers[0][0][r] += b'1'
+                        parentpointers[1][0][r] += b'2'
+                for i, p in enumerate(parentrevs):
+                    assert p < rev
+                    heapq.heappush(tovisit, -p)
+                    if p in pointers:
+                        # 'p' is a fork revision. concatenate tracking pointers
+                        # and decrement the pending count accordingly.
+                        knownunresolved, knownresolved = pointers[p]
+                        unresolved, resolved = parentpointers[i]
+                        for r, c in unresolved.items():
+                            if r in knownunresolved:
+                                # unresolved at both paths
+                                pendingcnt[r] -= 1
+                                assert pendingcnt[r] > 0
+                                # take shorter chain
+                                knownunresolved[r] = min(c, knownunresolved[r])
+                            else:
+                                knownunresolved[r] = c
+                        # simply propagate the 'resolved' set as deduplicating
+                        # 'unresolved' here would be slightly complicated.
+                        knownresolved.update(resolved)
+                    else:
+                        pointers[p] = parentpointers[i]
+
+            # then, populate the active parents directly and add the current
+            # 'rev' to the tracking pointers of the inactive parents.
+            # 'pointers[p]' may be optimized out if both parents are active.
+            chaincodes = [b''] if len(parentrevs) <= 1 else [b'1', b'2']
+            if curactive and bothparentsactive:
+                for i, p in enumerate(parentrevs):
+                    c = chaincodes[i]
+                    parents[rev].append((c, p))
+                    # no need to mark 'rev' as resolved since the 'rev' should
+                    # be fully resolved (i.e. pendingcnt[rev] == 0)
+                assert pendingcnt[rev] == 0
+            elif curactive:
+                for i, p in enumerate(parentrevs):
+                    unresolved, resolved = pointers[p]
+                    assert rev not in unresolved
+                    c = chaincodes[i]
+                    if p in subset:
+                        parents[rev].append((c, p))
+                        # mark 'rev' as resolved through this path
+                        resolved.add(rev)
+                    else:
+                        pendingcnt[rev] += 1
+                        unresolved[rev] = c
+                assert 0 < pendingcnt[rev] <= 2
+
+
 def _reachablerootspure(pfunc, minroot, roots, heads, includepath):
     """See revlog.reachableroots"""
     if not roots:

mercurial/debugcommands.py

@@ -11,8 +11,10 b' import codecs'
 import collections
 import difflib
 import errno
+import glob
 import operator
 import os
+import platform
 import random
 import re
 import socket
@@ -27,7 +29,6 b' from .i18n import _'
 from .node import (
     bin,
     hex,
-    nullhex,
     nullid,
     nullrev,
     short,
@@ -38,6 +39,7 b' from .pycompat import ('
 )
 from . import (
     bundle2,
+    bundlerepo,
     changegroup,
     cmdutil,
     color,
@@ -75,6 +77,7 b' from . import ('
     sshpeer,
     sslutil,
     streamclone,
+    tags as tagsmod,
     templater,
     treediscovery,
     upgrade,
@@ -93,7 +96,10 b' from .utils import ('
     stringutil,
 )
 
-from .revlogutils import deltas as deltautil
+from .revlogutils import (
+    deltas as deltautil,
+    nodemap,
+)
 
 release = lockmod.release
 
@@ -578,7 +584,7 b' def debugdag(ui, repo, file_=None, *revs'
     dots = opts.get('dots')
     if file_:
         rlog = revlog.revlog(vfsmod.vfs(encoding.getcwd(), audit=False), file_)
-        revs = set((int(r) for r in revs))
+        revs = {int(r) for r in revs}
 
         def events():
             for r in rlog:
@@ -1128,7 +1134,7 b' def debugfileset(ui, repo, expr, **opts)'
         (b'analyzed', filesetlang.analyze),
         (b'optimized', filesetlang.optimize),
     ]
-    stagenames = set(n for n, f in stages)
+    stagenames = {n for n, f in stages}
 
     showalways = set()
     if ui.verbose and not opts[b'show_stage']:
@@ -1487,6 +1493,11 b' def debuginstall(ui, **opts):'
         pycompat.sysexecutable or _(b"unknown"),
     )
     fm.write(
+        b'pythonimplementation',
+        _(b"checking Python implementation (%s)\n"),
+        pycompat.sysbytes(platform.python_implementation()),
+    )
+    fm.write(
         b'pythonver',
         _(b"checking Python version (%s)\n"),
         (b"%d.%d.%d" % sys.version_info[:3]),
@@ -1497,6 +1508,13 b' def debuginstall(ui, **opts):'
         pythonlib or _(b"unknown"),
     )
 
+    try:
+        from . import rustext
+
+        rustext.__doc__  # trigger lazy import
+    except ImportError:
+        rustext = None
+
     security = set(sslutil.supportedprotocols)
     if sslutil.hassni:
         security.add(b'sni')
@@ -1524,6 +1542,13 b' def debuginstall(ui, **opts):'
             )
         )
 
+    fm.plain(
+        _(
+            b"checking Rust extensions (%s)\n"
+            % (b'missing' if rustext is None else b'installed')
+        ),
+    )
+
     # TODO print CA cert info
 
     # hg version
@@ -1625,6 +1650,13 b' def debuginstall(ui, **opts):'
     fm.plain(_(b'checking "re2" regexp engine (%s)\n') % re2)
     fm.data(re2=bool(util._re2))
 
+    rust_debug_mod = policy.importrust("debug")
+    if rust_debug_mod is not None:
+        re2_rust = b'installed' if rust_debug_mod.re2_installed else b'missing'
+
+        msg = b'checking "re2" regexp engine Rust bindings (%s)\n'
+        fm.plain(_(msg % re2_rust))
+
     # templates
     p = templater.templatepaths()
     fm.write(b'templatedirs', b'checking templates (%s)...\n', b' '.join(p))
@@ -1934,120 +1966,100 b' def debugmanifestfulltextcache(ui, repo,'
         )
 
 
-@command(b'debugmergestate', [], b'')
-def debugmergestate(ui, repo, *args):
+@command(b'debugmergestate', [] + cmdutil.templateopts, b'')
+def debugmergestate(ui, repo, *args, **opts):
     """print merge state
 
     Use --verbose to print out information about whether v1 or v2 merge state
     was chosen."""
 
-    def _hashornull(h):
-        if h == nullhex:
-            return b'null'
-        else:
-            return h
-
-    def printrecords(version):
-        ui.writenoi18n(b'* version %d records\n' % version)
-        if version == 1:
-            records = v1records
+    if ui.verbose:
+        ms = mergemod.mergestate(repo)
+
+        # sort so that reasonable information is on top
+        v1records = ms._readrecordsv1()
+        v2records = ms._readrecordsv2()
+
+        if not v1records and not v2records:
+            pass
+        elif not v2records:
+            ui.writenoi18n(b'no version 2 merge state\n')
+        elif ms._v1v2match(v1records, v2records):
+            ui.writenoi18n(b'v1 and v2 states match: using v2\n')
         else:
-            records = v2records
-
-        for rtype, record in records:
-            # pretty print some record types
-            if rtype == b'L':
-                ui.writenoi18n(b'local: %s\n' % record)
-            elif rtype == b'O':
-                ui.writenoi18n(b'other: %s\n' % record)
-            elif rtype == b'm':
-                driver, mdstate = record.split(b'\0', 1)
-                ui.writenoi18n(
-                    b'merge driver: %s (state "%s")\n' % (driver, mdstate)
-                )
-            elif rtype in b'FDC':
-                r = record.split(b'\0')
-                f, state, hash, lfile, afile, anode, ofile = r[0:7]
-                if version == 1:
-                    onode = b'not stored in v1 format'
-                    flags = r[7]
-                else:
-                    onode, flags = r[7:9]
-                ui.writenoi18n(
-                    b'file: %s (record type "%s", state "%s", hash %s)\n'
-                    % (f, rtype, state, _hashornull(hash))
-                )
-                ui.writenoi18n(
-                    b'  local path: %s (flags "%s")\n' % (lfile, flags)
-                )
-                ui.writenoi18n(
-                    b'  ancestor path: %s (node %s)\n'
-                    % (afile, _hashornull(anode))
-                )
-                ui.writenoi18n(
-                    b'  other path: %s (node %s)\n'
-                    % (ofile, _hashornull(onode))
-                )
-            elif rtype == b'f':
-                filename, rawextras = record.split(b'\0', 1)
-                extras = rawextras.split(b'\0')
-                i = 0
-                extrastrings = []
-                while i < len(extras):
-                    extrastrings.append(b'%s = %s' % (extras[i], extras[i + 1]))
-                    i += 2
-
-                ui.writenoi18n(
-                    b'file extras: %s (%s)\n'
-                    % (filename, b', '.join(extrastrings))
-                )
-            elif rtype == b'l':
-                labels = record.split(b'\0', 2)
-                labels = [l for l in labels if len(l) > 0]
-                ui.writenoi18n(b'labels:\n')
-                ui.write((b'  local: %s\n' % labels[0]))
-                ui.write((b'  other: %s\n' % labels[1]))
-                if len(labels) > 2:
-                    ui.write((b'  base:  %s\n' % labels[2]))
-            else:
-                ui.writenoi18n(
-                    b'unrecognized entry: %s\t%s\n'
-                    % (rtype, record.replace(b'\0', b'\t'))
-                )
-
-    # Avoid mergestate.read() since it may raise an exception for unsupported
-    # merge state records. We shouldn't be doing this, but this is OK since this
-    # command is pretty low-level.
-    ms = mergemod.mergestate(repo)
-
-    # sort so that reasonable information is on top
-    v1records = ms._readrecordsv1()
-    v2records = ms._readrecordsv2()
-    order = b'LOml'
-
-    def key(r):
-        idx = order.find(r[0])
-        if idx == -1:
-            return (1, r[1])
-        else:
-            return (0, idx)
-
-    v1records.sort(key=key)
-    v2records.sort(key=key)
-
-    if not v1records and not v2records:
-        ui.writenoi18n(b'no merge state found\n')
-    elif not v2records:
-        ui.notenoi18n(b'no version 2 merge state\n')
-        printrecords(1)
-    elif ms._v1v2match(v1records, v2records):
-        ui.notenoi18n(b'v1 and v2 states match: using v2\n')
-        printrecords(2)
-    else:
-        ui.notenoi18n(b'v1 and v2 states mismatch: using v1\n')
-        printrecords(1)
-        if ui.verbose:
-            printrecords(2)
+            ui.writenoi18n(b'v1 and v2 states mismatch: using v1\n')
+
+    opts = pycompat.byteskwargs(opts)
+    if not opts[b'template']:
+        opts[b'template'] = (
+            b'{if(commits, "", "no merge state found\n")}'
+            b'{commits % "{name}{if(label, " ({label})")}: {node}\n"}'
+            b'{files % "file: {path} (state \\"{state}\\")\n'
+            b'{if(local_path, "'
+            b'  local path: {local_path} (hash {local_key}, flags \\"{local_flags}\\")\n'
+            b'  ancestor path: {ancestor_path} (node {ancestor_node})\n'
+            b'  other path: {other_path} (node {other_node})\n'
+            b'")}'
+            b'{if(rename_side, "'
+            b'  rename side: {rename_side}\n'
+            b'  renamed path: {renamed_path}\n'
+            b'")}'
+            b'{extras % "  extra: {key} = {value}\n"}'
+            b'"}'
+        )
+
+    ms = mergemod.mergestate.read(repo)
+
+    fm = ui.formatter(b'debugmergestate', opts)
+    fm.startitem()
+
+    fm_commits = fm.nested(b'commits')
+    if ms.active():
+        for name, node, label_index in (
+            (b'local', ms.local, 0),
+            (b'other', ms.other, 1),
+        ):
+            fm_commits.startitem()
+            fm_commits.data(name=name)
+            fm_commits.data(node=hex(node))
+            if ms._labels and len(ms._labels) > label_index:
+                fm_commits.data(label=ms._labels[label_index])
+    fm_commits.end()
+
+    fm_files = fm.nested(b'files')
+    if ms.active():
+        for f in ms:
+            fm_files.startitem()
+            fm_files.data(path=f)
+            state = ms._state[f]
+            fm_files.data(state=state[0])
+            if state[0] in (
+                mergemod.MERGE_RECORD_UNRESOLVED,
+                mergemod.MERGE_RECORD_RESOLVED,
+            ):
+                fm_files.data(local_key=state[1])
+                fm_files.data(local_path=state[2])
+                fm_files.data(ancestor_path=state[3])
+                fm_files.data(ancestor_node=state[4])
+                fm_files.data(other_path=state[5])
+                fm_files.data(other_node=state[6])
+                fm_files.data(local_flags=state[7])
+            elif state[0] in (
+                mergemod.MERGE_RECORD_UNRESOLVED_PATH,
+                mergemod.MERGE_RECORD_RESOLVED_PATH,
+            ):
+                fm_files.data(renamed_path=state[1])
+                fm_files.data(rename_side=state[2])
+            fm_extras = fm_files.nested(b'extras')
+            for k, v in ms.extras(f).items():
+                fm_extras.startitem()
+                fm_extras.data(key=k)
+                fm_extras.data(value=v)
+            fm_extras.end()
+
+    fm_files.end()
+
+    fm.end()
 
 
 @command(b'debugnamecomplete', [], _(b'NAME...'))
@@ -2075,6 +2087,70 b' def debugnamecomplete(ui, repo, *args):'
 
 
 @command(
+    b'debugnodemap',
+    [
+        (
+            b'',
+            b'dump-new',
+            False,
+            _(b'write a (new) persistent binary nodemap on stdin'),
+        ),
+        (b'', b'dump-disk', False, _(b'dump on-disk data on stdin')),
+        (
+            b'',
+            b'check',
+            False,
+            _(b'check that the data on disk data are correct.'),
+        ),
+        (
+            b'',
+            b'metadata',
+            False,
+            _(b'display the on disk meta data for the nodemap'),
+        ),
+    ],
+)
+def debugnodemap(ui, repo, **opts):
+    """write and inspect on disk nodemap
+    """
+    if opts['dump_new']:
+        unfi = repo.unfiltered()
+        cl = unfi.changelog
+        if util.safehasattr(cl.index, "nodemap_data_all"):
+            data = cl.index.nodemap_data_all()
+        else:
+            data = nodemap.persistent_data(cl.index)
+        ui.write(data)
+    elif opts['dump_disk']:
+        unfi = repo.unfiltered()
+        cl = unfi.changelog
+        nm_data = nodemap.persisted_data(cl)
+        if nm_data is not None:
+            docket, data = nm_data
+            ui.write(data[:])
+    elif opts['check']:
+        unfi = repo.unfiltered()
+        cl = unfi.changelog
+        nm_data = nodemap.persisted_data(cl)
+        if nm_data is not None:
+            docket, data = nm_data
+            return nodemap.check_data(ui, cl.index, data)
+    elif opts['metadata']:
+        unfi = repo.unfiltered()
+        cl = unfi.changelog
+        nm_data = nodemap.persisted_data(cl)
+        if nm_data is not None:
+            docket, data = nm_data
+            ui.write((b"uid: %s\n") % docket.uid)
+            ui.write((b"tip-rev: %d\n") % docket.tip_rev)
+            ui.write((b"tip-node: %s\n") % hex(docket.tip_node))
+            ui.write((b"data-length: %d\n") % docket.data_length)
+            ui.write((b"data-unused: %d\n") % docket.data_unused)
+            unused_perc = docket.data_unused * 100.0 / docket.data_length
+            ui.write((b"data-unused: %2.3f%%\n") % unused_perc)
+
+
+@command(
     b'debugobsolete',
     [
         (b'', b'flags', 0, _(b'markers flag')),
@@ -2549,7 +2625,7 b' def debugrebuilddirstate(ui, repo, rev, '
             dirstatefiles = set(dirstate)
             manifestonly = manifestfiles - dirstatefiles
             dsonly = dirstatefiles - manifestfiles
-            dsnotadded = set(f for f in dsonly if dirstate[f] != b'a')
+            dsnotadded = {f for f in dsonly if dirstate[f] != b'a'}
             changedfiles = manifestonly | dsnotadded
 
         dirstate.rebuild(ctx.node(), ctx.manifest(), changedfiles)
@@ -3116,7 +3192,7 b' def debugrevspec(ui, repo, expr, **opts)'
         raise error.Abort(
             _(b'cannot use --verify-optimized with --no-optimized')
         )
-    stagenames = set(n for n, f in stages)
+    stagenames = {n for n, f in stages}
 
     showalways = set()
     showchanged = set()
@@ -3355,6 +3431,143 b' def debugssl(ui, repo, source=None, **op'
 
 
 @command(
+    b"debugbackupbundle",
+    [
+        (
+            b"",
+            b"recover",
+            b"",
+            b"brings the specified changeset back into the repository",
+        )
+    ]
+    + cmdutil.logopts,
+    _(b"hg debugbackupbundle [--recover HASH]"),
+)
+def debugbackupbundle(ui, repo, *pats, **opts):
+    """lists the changesets available in backup bundles
+
+    Without any arguments, this command prints a list of the changesets in each
+    backup bundle.
+
+    --recover takes a changeset hash and unbundles the first bundle that
+    contains that hash, which puts that changeset back in your repository.
+
+    --verbose will print the entire commit message and the bundle path for that
+    backup.
+    """
+    backups = list(
+        filter(
+            os.path.isfile, glob.glob(repo.vfs.join(b"strip-backup") + b"/*.hg")
+        )
+    )
+    backups.sort(key=lambda x: os.path.getmtime(x), reverse=True)
+
+    opts = pycompat.byteskwargs(opts)
+    opts[b"bundle"] = b""
+    opts[b"force"] = None
+    limit = logcmdutil.getlimit(opts)
+
+    def display(other, chlist, displayer):
+        if opts.get(b"newest_first"):
+            chlist.reverse()
+        count = 0
+        for n in chlist:
+            if limit is not None and count >= limit:
+                break
+            parents = [True for p in other.changelog.parents(n) if p != nullid]
+            if opts.get(b"no_merges") and len(parents) == 2:
+                continue
+            count += 1
+            displayer.show(other[n])
+
+    recovernode = opts.get(b"recover")
+    if recovernode:
+        if scmutil.isrevsymbol(repo, recovernode):
+            ui.warn(_(b"%s already exists in the repo\n") % recovernode)
+            return
+    elif backups:
+        msg = _(
+            b"Recover changesets using: hg debugbackupbundle --recover "
+            b"<changeset hash>\n\nAvailable backup changesets:"
+        )
+        ui.status(msg, label=b"status.removed")
+    else:
+        ui.status(_(b"no backup changesets found\n"))
+        return
+
+    for backup in backups:
+        # Much of this is copied from the hg incoming logic
+        source = ui.expandpath(os.path.relpath(backup, encoding.getcwd()))
+        source, branches = hg.parseurl(source, opts.get(b"branch"))
+        try:
+            other = hg.peer(repo, opts, source)
+        except error.LookupError as ex:
+            msg = _(b"\nwarning: unable to open bundle %s") % source
+            hint = _(b"\n(missing parent rev %s)\n") % short(ex.name)
+            ui.warn(msg, hint=hint)
+            continue
+        revs, checkout = hg.addbranchrevs(
+            repo, other, branches, opts.get(b"rev")
+        )
+
+        if revs:
+            revs = [other.lookup(rev) for rev in revs]
+
+        quiet = ui.quiet
+        try:
+            ui.quiet = True
+            other, chlist, cleanupfn = bundlerepo.getremotechanges(
+                ui, repo, other, revs, opts[b"bundle"], opts[b"force"]
+            )
+        except error.LookupError:
+            continue
+        finally:
+            ui.quiet = quiet
+
+        try:
+            if not chlist:
+                continue
+            if recovernode:
+                with repo.lock(), repo.transaction(b"unbundle") as tr:
+                    if scmutil.isrevsymbol(other, recovernode):
+                        ui.status(_(b"Unbundling %s\n") % (recovernode))
+                        f = hg.openpath(ui, source)
+                        gen = exchange.readbundle(ui, f, source)
+                        if isinstance(gen, bundle2.unbundle20):
+                            bundle2.applybundle(
+                                repo,
+                                gen,
+                                tr,
+                                source=b"unbundle",
+                                url=b"bundle:" + source,
+                            )
+                        else:
+                            gen.apply(repo, b"unbundle", b"bundle:" + source)
+                        break
+            else:
+                backupdate = encoding.strtolocal(
+                    time.strftime(
+                        "%a %H:%M, %Y-%m-%d",
+                        time.localtime(os.path.getmtime(source)),
+                    )
+                )
+                ui.status(b"\n%s\n" % (backupdate.ljust(50)))
+                if ui.verbose:
+                    ui.status(b"%s%s\n" % (b"bundle:".ljust(13), source))
+                else:
+                    opts[
+                        b"template"
+                    ] = b"{label('status.modified', node|short)} {desc|firstline}\n"
+                displayer = logcmdutil.changesetdisplayer(
+                    ui, other, opts, False
+                )
+                display(other, chlist, displayer)
+                displayer.close()
+        finally:
+            cleanupfn()
+
+
+@command(
     b'debugsub',
     [(b'r', b'rev', b'', _(b'revision to check'), _(b'REV'))],
     _(b'[-r REV] [REV]'),
@@ -3423,6 +3636,17 b' def debugsuccessorssets(ui, repo, *revs,'
             ui.write(b'\n')
 
 
+@command(b'debugtagscache', [])
+def debugtagscache(ui, repo):
+    """display the contents of .hg/cache/hgtagsfnodes1"""
+    cache = tagsmod.hgtagsfnodescache(repo.unfiltered())
+    for r in repo:
+        node = repo[r].node()
+        tagsnode = cache.getfnode(node, computemissing=False)
+        tagsnodedisplay = hex(tagsnode) if tagsnode else b'missing/invalid'
+        ui.write(b'%d %s %s\n' % (r, hex(node), tagsnodedisplay))
+
+
 @command(
     b'debugtemplate',
     [
@@ -3497,7 +3721,7 b' def debugtemplate(ui, repo, tmpl, **opts'
 def debuguigetpass(ui, prompt=b''):
     """show prompt to type password"""
     r = ui.getpass(prompt)
-    ui.writenoi18n(b'respose: %s\n' % r)
+    ui.writenoi18n(b'response: %s\n' % r)
 
 
 @command(

mercurial/dirstate.py

@@ -27,6 +27,7 b' from . import ('
     policy,
     pycompat,
     scmutil,
+    sparse,
     txnutil,
     util,
 )
@@ -1083,7 +1084,7 b' class dirstate(object):'
                     results[next(iv)] = st
         return results
 
-    def _rust_status(self, matcher, list_clean):
+    def _rust_status(self, matcher, list_clean, list_ignored, list_unknown):
         # Force Rayon (Rust parallelism library) to respect the number of
         # workers. This is a temporary workaround until Rust code knows
         # how to read the config file.
@@ -1101,16 +1102,45 b' class dirstate(object):'
             added,
             removed,
             deleted,
+            clean,
+            ignored,
             unknown,
-            clean,
+            warnings,
+            bad,
         ) = rustmod.status(
             self._map._rustmap,
             matcher,
             self._rootdir,
-            bool(list_clean),
+            self._ignorefiles(),
+            self._checkexec,
             self._lastnormaltime,
-            self._checkexec,
+            bool(list_clean),
+            bool(list_ignored),
+            bool(list_unknown),
         )
+        if self._ui.warn:
+            for item in warnings:
+                if isinstance(item, tuple):
+                    file_path, syntax = item
+                    msg = _(b"%s: ignoring invalid syntax '%s'\n") % (
+                        file_path,
+                        syntax,
+                    )
+                    self._ui.warn(msg)
+                else:
+                    msg = _(b"skipping unreadable pattern file '%s': %s\n")
+                    self._ui.warn(
+                        msg
+                        % (
+                            pathutil.canonpath(
+                                self._rootdir, self._rootdir, item
+                            ),
+                            b"No such file or directory",
+                        )
+                    )
+
+        for (fn, message) in bad:
+            matcher.bad(fn, encoding.strtolocal(message))
 
         status = scmutil.status(
             modified=modified,
@@ -1118,7 +1148,7 b' class dirstate(object):'
             removed=removed,
             deleted=deleted,
             unknown=unknown,
-            ignored=[],
+            ignored=ignored,
             clean=clean,
         )
         return (lookup, status)
@@ -1148,26 +1178,34 b' class dirstate(object):'
 
         use_rust = True
 
-        allowed_matchers = (matchmod.alwaysmatcher, matchmod.exactmatcher)
+        allowed_matchers = (
+            matchmod.alwaysmatcher,
+            matchmod.exactmatcher,
+            matchmod.includematcher,
+        )
 
         if rustmod is None:
             use_rust = False
+        elif self._checkcase:
+            # Case-insensitive filesystems are not handled yet
+            use_rust = False
         elif subrepos:
             use_rust = False
-        elif bool(listunknown):
-            # Pathauditor does not exist yet in Rust, unknown files
-            # can't be trusted.
+        elif sparse.enabled:
             use_rust = False
-        elif self._ignorefiles() and listignored:
-            # Rust has no ignore mechanism yet, so don't use Rust for
-            # commands that need ignore.
+        elif match.traversedir is not None:
             use_rust = False
         elif not isinstance(match, allowed_matchers):
             # Matchers have yet to be implemented
             use_rust = False
 
         if use_rust:
-            return self._rust_status(match, listclean)
+            try:
+                return self._rust_status(
+                    match, listclean, listignored, listunknown
+                )
+            except rustmod.FallbackError:
+                pass
 
         def noop(f):
             pass
@@ -1249,19 +1287,19 b' class dirstate(object):'
                 aadd(fn)
             elif state == b'r':
                 radd(fn)
-
-        return (
-            lookup,
-            scmutil.status(
-                modified, added, removed, deleted, unknown, ignored, clean
-            ),
+        status = scmutil.status(
+            modified, added, removed, deleted, unknown, ignored, clean
         )
+        return (lookup, status)
 
     def matches(self, match):
         '''
         return files in the dirstate (in whatever state) filtered by match
         '''
         dmap = self._map
+        if rustmod is not None:
+            dmap = self._map._rustmap
+
         if match.always():
             return dmap.keys()
         files = match.files()

mercurial/discovery.py

@@ -192,7 +192,7 b' def findcommonoutgoing('
         # ancestors of missing
         og._computecommonmissing()
         cl = repo.changelog
-        missingrevs = set(cl.rev(n) for n in og._missing)
+        missingrevs = {cl.rev(n) for n in og._missing}
         og._common = set(cl.ancestors(missingrevs)) - missingrevs
         commonheads = set(og.commonheads)
         og.missingheads = [h for h in og.missingheads if h not in commonheads]
@@ -268,8 +268,8 b' def _headssummary(pushop):'
     # If there are no obsstore, no post processing are needed.
     if repo.obsstore:
         torev = repo.changelog.rev
-        futureheads = set(torev(h) for h in outgoing.missingheads)
-        futureheads |= set(torev(h) for h in outgoing.commonheads)
+        futureheads = {torev(h) for h in outgoing.missingheads}
+        futureheads |= {torev(h) for h in outgoing.commonheads}
         allfuturecommon = repo.changelog.ancestors(futureheads, inclusive=True)
         for branch, heads in sorted(pycompat.iteritems(headssum)):
             remoteheads, newheads, unsyncedheads, placeholder = heads
@@ -452,7 +452,7 b' def checkheads(pushop):'
                 if branch not in (b'default', None):
                     errormsg = _(
                         b"push creates new remote head %s on branch '%s'!"
-                    ) % (short(dhs[0]), branch)
+                    ) % (short(dhs[0]), branch,)
                 elif repo[dhs[0]].bookmarks():
                     errormsg = _(
                         b"push creates new remote head %s "

mercurial/dispatch.py

@@ -10,6 +10,7 b' from __future__ import absolute_import, '
 import difflib
 import errno
 import getopt
+import io
 import os
 import pdb
 import re
@@ -144,7 +145,50 b' def run():'
 if pycompat.ispy3:
 
     def initstdio():
-        pass
+        # stdio streams on Python 3 are io.TextIOWrapper instances proxying another
+        # buffer. These streams will normalize \n to \r\n by default. Mercurial's
+        # preferred mechanism for writing output (ui.write()) uses io.BufferedWriter
+        # instances, which write to the underlying stdio file descriptor in binary
+        # mode. ui.write() uses \n for line endings and no line ending normalization
+        # is attempted through this interface. This "just works," even if the system
+        # preferred line ending is not \n.
+        #
+        # But some parts of Mercurial (e.g. hooks) can still send data to sys.stdout
+        # and sys.stderr. They will inherit the line ending normalization settings,
+        # potentially causing e.g. \r\n to be emitted. Since emitting \n should
+        # "just work," here we change the sys.* streams to disable line ending
+        # normalization, ensuring compatibility with our ui type.
+
+        # write_through is new in Python 3.7.
+        kwargs = {
+            "newline": "\n",
+            "line_buffering": sys.stdout.line_buffering,
+        }
+        if util.safehasattr(sys.stdout, "write_through"):
+            kwargs["write_through"] = sys.stdout.write_through
+        sys.stdout = io.TextIOWrapper(
+            sys.stdout.buffer, sys.stdout.encoding, sys.stdout.errors, **kwargs
+        )
+
+        kwargs = {
+            "newline": "\n",
+            "line_buffering": sys.stderr.line_buffering,
+        }
+        if util.safehasattr(sys.stderr, "write_through"):
+            kwargs["write_through"] = sys.stderr.write_through
+        sys.stderr = io.TextIOWrapper(
+            sys.stderr.buffer, sys.stderr.encoding, sys.stderr.errors, **kwargs
+        )
+
+        # No write_through on read-only stream.
+        sys.stdin = io.TextIOWrapper(
+            sys.stdin.buffer,
+            sys.stdin.encoding,
+            sys.stdin.errors,
+            # None is universal newlines mode.
+            newline=None,
+            line_buffering=sys.stdin.line_buffering,
+        )
 
     def _silencestdio():
         for fp in (sys.stdout, sys.stderr):
@@ -514,7 +558,7 b' def aliasinterpolate(name, args, cmd):'
     '''
     # util.interpolate can't deal with "$@" (with quotes) because it's only
     # built to match prefix + patterns.
-    replacemap = dict((b'$%d' % (i + 1), arg) for i, arg in enumerate(args))
+    replacemap = {b'$%d' % (i + 1): arg for i, arg in enumerate(args)}
     replacemap[b'$0'] = name
     replacemap[b'$$'] = b'$'
     replacemap[b'$@'] = b' '.join(args)
@@ -624,7 +668,7 b' class cmdalias(object):'
         except error.AmbiguousCommand:
             self.badalias = _(
                 b"alias '%s' resolves to ambiguous command '%s'"
-            ) % (self.name, cmd)
+            ) % (self.name, cmd,)
 
     def _populatehelp(self, ui, name, cmd, fn, defaulthelp=None):
         # confine strings to be passed to i18n.gettext()

mercurial/encoding.py

@@ -86,10 +86,10 b' elif _nativeenviron:'
 else:
     # preferred encoding isn't known yet; use utf-8 to avoid unicode error
     # and recreate it once encoding is settled
-    environ = dict(
-        (k.encode('utf-8'), v.encode('utf-8'))
+    environ = {
+        k.encode('utf-8'): v.encode('utf-8')
         for k, v in os.environ.items()  # re-exports
-    )
+    }
 
 _encodingrewrites = {
     b'646': b'ascii',
@@ -285,10 +285,10 b' else:'
 if not _nativeenviron:
     # now encoding and helper functions are available, recreate the environ
     # dict to be exported to other modules
-    environ = dict(
-        (tolocal(k.encode('utf-8')), tolocal(v.encode('utf-8')))
+    environ = {
+        tolocal(k.encode('utf-8')): tolocal(v.encode('utf-8'))
         for k, v in os.environ.items()  # re-exports
-    )
+    }
 
 if pycompat.ispy3:
     # os.getcwd() on Python 3 returns string, but it has os.getcwdb() which

mercurial/exchange.py

@@ -8,6 +8,7 b''
 from __future__ import absolute_import
 
 import collections
+import weakref
 
 from .i18n import _
 from .node import (
@@ -856,7 +857,11 b' def _processcompared(pushop, pushed, exp'
     for b, scid, dcid in addsrc:
         if b in explicit:
             explicit.remove(b)
-            pushop.outbookmarks.append((b, b'', scid))
+            if bookmod.isdivergent(b):
+                pushop.ui.warn(_(b'cannot push divergent bookmark %s!\n') % b)
+                pushop.bkresult = 2
+            else:
+                pushop.outbookmarks.append((b, b'', scid))
     # search for overwritten bookmark
     for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
         if b in explicit:
@@ -1675,12 +1680,12 b' def _fullpullbundle2(repo, pullop):'
     def headsofdiff(h1, h2):
         """Returns heads(h1 % h2)"""
         res = unfi.set(b'heads(%ln %% %ln)', h1, h2)
-        return set(ctx.node() for ctx in res)
+        return {ctx.node() for ctx in res}
 
     def headsofunion(h1, h2):
         """Returns heads((h1 + h2) - null)"""
         res = unfi.set(b'heads((%ln + %ln - null))', h1, h2)
-        return set(ctx.node() for ctx in res)
+        return {ctx.node() for ctx in res}
 
     while True:
         old_heads = unficl.heads()
@@ -1701,6 +1706,25 b' def _fullpullbundle2(repo, pullop):'
         pullop.rheads = set(pullop.rheads) - pullop.common
 
 
+def add_confirm_callback(repo, pullop):
+    """ adds a finalize callback to transaction which can be used to show stats
+    to user and confirm the pull before committing transaction """
+
+    tr = pullop.trmanager.transaction()
+    scmutil.registersummarycallback(
+        repo, tr, txnname=b'pull', as_validator=True
+    )
+    reporef = weakref.ref(repo.unfiltered())
+
+    def prompt(tr):
+        repo = reporef()
+        cm = _(b'accept incoming changes (yn)?$$ &Yes $$ &No')
+        if repo.ui.promptchoice(cm):
+            raise error.Abort("user aborted")
+
+    tr.addvalidator(b'900-pull-prompt', prompt)
+
+
 def pull(
     repo,
     remote,
@@ -1712,6 +1736,7 b' def pull('
     includepats=None,
     excludepats=None,
     depth=None,
+    confirm=None,
 ):
     """Fetch repository data from a remote.
 
@@ -1736,6 +1761,8 b' def pull('
     ``depth`` is an integer indicating the DAG depth of history we're
     interested in. If defined, for each revision specified in ``heads``, we
     will fetch up to this many of its ancestors and data associated with them.
+    ``confirm`` is a boolean indicating whether the pull should be confirmed
+    before committing the transaction. This overrides HGPLAIN.
 
     Returns the ``pulloperation`` created for this pull.
     """
@@ -1782,6 +1809,11 b' def pull('
     if not bookmod.bookmarksinstore(repo):
         wlock = repo.wlock()
     with wlock, repo.lock(), pullop.trmanager:
+        if confirm or (
+            repo.ui.configbool(b"pull", b"confirm") and not repo.ui.plain()
+        ):
+            add_confirm_callback(repo, pullop)
+
         # Use the modern wire protocol, if available.
         if remote.capable(b'command-changesetdata'):
             exchangev2.pull(pullop)
@@ -3068,7 +3100,15 b' def sortclonebundleentries(ui, entries):'
     if not prefers:
         return list(entries)
 
-    prefers = [p.split(b'=', 1) for p in prefers]
+    def _split(p):
+        if b'=' not in p:
+            hint = _(b"each comma separated item should be key=value pairs")
+            raise error.Abort(
+                _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint
+            )
+        return p.split(b'=', 1)
+
+    prefers = [_split(p) for p in prefers]
 
     items = sorted(clonebundleentry(v, prefers) for v in entries)
     return [i.value for i in items]

mercurial/extensions.py

@@ -787,11 +787,11 b' def disabled():'
     try:
         from hgext import __index__  # pytype: disable=import-error
 
-        return dict(
-            (name, gettext(desc))
+        return {
+            name: gettext(desc)
             for name, desc in pycompat.iteritems(__index__.docs)
             if name not in _order
-        )
+        }
     except (ImportError, AttributeError):
         pass
 
@@ -808,18 +808,8 b' def disabled():'
     return exts
 
 
-def disabledext(name):
-    '''find a specific disabled extension from hgext. returns desc'''
-    try:
-        from hgext import __index__  # pytype: disable=import-error
-
-        if name in _order:  # enabled
-            return
-        else:
-            return gettext(__index__.docs.get(name))
-    except (ImportError, AttributeError):
-        pass
-
+def disabled_help(name):
+    """Obtain the full help text for a disabled extension, or None."""
     paths = _disabledpaths()
     if name in paths:
         return _disabledhelp(paths[name])

mercurial/fancyopts.py

@@ -314,7 +314,7 b' def fancyopts(args, options, state, gnu='
     argmap = {}
     defmap = {}
     negations = {}
-    alllong = set(o[1] for o in options)
+    alllong = {o[1] for o in options}
 
     for option in options:
         if len(option) == 5:

mercurial/graphmod.py

@@ -58,7 +58,7 b' def dagwalker(repo, revs):'
         # partition into parents in the rev set and missing parents, then
         # augment the lists with markers, to inform graph drawing code about
         # what kind of edge to draw between nodes.
-        pset = set(p.rev() for p in ctx.parents() if p.rev() in revs)
+        pset = {p.rev() for p in ctx.parents() if p.rev() in revs}
         mpars = [
             p.rev()
             for p in ctx.parents()
@@ -95,9 +95,9 b' def nodes(repo, nodes):'
     include = set(nodes)
     for node in nodes:
         ctx = repo[node]
-        parents = set(
+        parents = {
             (PARENT, p.rev()) for p in ctx.parents() if p.node() in include
-        )
+        }
         yield (ctx.rev(), CHANGESET, ctx, sorted(parents))
 
 

mercurial/hbisect.py

@@ -137,7 +137,7 b' def extendrange(repo, state, nodes, good'
             side = state[b'bad']
         else:
             side = state[b'good']
-        num = len(set(i.node() for i in parents) & set(side))
+        num = len({i.node() for i in parents} & set(side))
         if num == 1:
             return parents[0].ancestor(parents[1])
     return None

mercurial/help.py

@@ -153,7 +153,18 b' def extshelp(ui):'
     return doc
 
 
-def optrst(header, options, verbose):
+def parsedefaultmarker(text):
+    """given a text 'abc (DEFAULT: def.ghi)',
+    returns (b'abc', (b'def', b'ghi')). Otherwise return None"""
+    if text[-1:] == b')':
+        marker = b' (DEFAULT: '
+        pos = text.find(marker)
+        if pos >= 0:
+            item = text[pos + len(marker) : -1]
+            return text[:pos], item.split(b'.', 2)
+
+
+def optrst(header, options, verbose, ui):
     data = []
     multioccur = False
     for option in options:
@@ -165,7 +176,14 b' def optrst(header, options, verbose):'
 
         if not verbose and any(w in desc for w in _exclkeywords):
             continue
-
+        defaultstrsuffix = b''
+        if default is None:
+            parseresult = parsedefaultmarker(desc)
+            if parseresult is not None:
+                (desc, (section, name)) = parseresult
+                if ui.configbool(section, name):
+                    default = True
+                    defaultstrsuffix = _(b' from config')
         so = b''
         if shortopt:
             so = b'-' + shortopt
@@ -183,7 +201,7 b' def optrst(header, options, verbose):'
             defaultstr = pycompat.bytestr(default)
             if default is True:
                 defaultstr = _(b"on")
-            desc += _(b" (default: %s)") % defaultstr
+            desc += _(b" (default: %s)") % (defaultstr + defaultstrsuffix)
 
         if isinstance(default, list):
             lo += b" %s [+]" % optlabel
@@ -714,11 +732,13 b' def help_('
 
         # options
         if not ui.quiet and entry[1]:
-            rst.append(optrst(_(b"options"), entry[1], ui.verbose))
+            rst.append(optrst(_(b"options"), entry[1], ui.verbose, ui))
 
         if ui.verbose:
             rst.append(
-                optrst(_(b"global options"), commands.globalopts, ui.verbose)
+                optrst(
+                    _(b"global options"), commands.globalopts, ui.verbose, ui
+                )
             )
 
         if not ui.verbose:
@@ -858,7 +878,9 b' def help_('
         elif ui.verbose:
             rst.append(
                 b'\n%s\n'
-                % optrst(_(b"global options"), commands.globalopts, ui.verbose)
+                % optrst(
+                    _(b"global options"), commands.globalopts, ui.verbose, ui
+                )
             )
             if name == b'shortlist':
                 rst.append(
@@ -944,7 +966,7 b' def help_('
             doc = gettext(pycompat.getdoc(mod)) or _(b'no help text available')
         except KeyError:
             mod = None
-            doc = extensions.disabledext(name)
+            doc = extensions.disabled_help(name)
             if not doc:
                 raise error.UnknownCommand(name)
 

mercurial/helptext/config.txt

@@ -888,7 +888,8 b' https://www.mercurial-scm.org/wiki/Missi'
     Compression algorithm used by revlog. Supported values are `zlib` and
     `zstd`. The `zlib` engine is the historical default of Mercurial. `zstd` is
     a newer format that is usually a net win over `zlib`, operating faster at
-    better compression rates. Use `zstd` to reduce CPU usage.
+    better compression rates. Use `zstd` to reduce CPU usage. Multiple values
+    can be specified, the first available one will be used.
 
     On some systems, the Mercurial installation may lack `zstd` support.
 
@@ -2005,12 +2006,12 b' Controls generic server settings.'
     Level of allowed race condition between two pushing clients.
 
     - 'strict': push is abort if another client touched the repository
-      while the push was preparing. (default)
+      while the push was preparing.
     - 'check-related': push is only aborted if it affects head that got also
-      affected while the push was preparing.
-
-    This requires compatible client (version 4.3 and later). Old client will
-    use 'strict'.
+      affected while the push was preparing. (default since 5.4)
+
+    'check-related' only takes effect for compatible clients (version
+    4.3 and later). Older clients will use 'strict'.
 
 ``validate``
     Whether to validate the completeness of pushed changesets by

mercurial/hg.py

@@ -60,12 +60,19 b' def _local(path):'
     path = util.expandpath(util.urllocalpath(path))
 
     try:
-        isfile = os.path.isfile(path)
+        # we use os.stat() directly here instead of os.path.isfile()
+        # because the latter started returning `False` on invalid path
+        # exceptions starting in 3.8 and we care about handling
+        # invalid paths specially here.
+        st = os.stat(path)
+        isfile = stat.S_ISREG(st.st_mode)
     # Python 2 raises TypeError, Python 3 ValueError.
     except (TypeError, ValueError) as e:
         raise error.Abort(
             _(b'invalid path %s: %s') % (path, pycompat.bytestr(e))
         )
+    except OSError:
+        isfile = False
 
     return isfile and bundlerepo or localrepo
 
@@ -688,7 +695,7 b' def clone('
         # data.
         createopts[b'lfs'] = True
 
-        if extensions.disabledext(b'lfs'):
+        if extensions.disabled_help(b'lfs'):
             ui.status(
                 _(
                     b'(remote is using large file support (lfs), but it is '
@@ -1040,10 +1047,9 b' def update(repo, node, quietempty=False,'
 def clean(repo, node, show_stats=True, quietempty=False):
     """forcibly switch the working directory to node, clobbering changes"""
     stats = updaterepo(repo, node, True)
-    repo.vfs.unlinkpath(b'graftstate', ignoremissing=True)
+    assert stats.unresolvedcount == 0
     if show_stats:
         _showstats(repo, stats, quietempty)
-    return stats.unresolvedcount > 0
 
 
 # naming conflict in updatetotally()
@@ -1138,27 +1144,12 b' def updatetotally(ui, repo, checkout, br'
 
 
 def merge(
-    repo,
-    node,
-    force=None,
-    remind=True,
-    mergeforce=False,
-    labels=None,
-    abort=False,
+    ctx, force=False, remind=True, labels=None,
 ):
     """Branch merge with node, resolving changes. Return true if any
     unresolved conflicts."""
-    if abort:
-        return abortmerge(repo.ui, repo)
-
-    stats = mergemod.update(
-        repo,
-        node,
-        branchmerge=True,
-        force=force,
-        mergeforce=mergeforce,
-        labels=labels,
-    )
+    repo = ctx.repo()
+    stats = mergemod.merge(ctx, force=force, labels=labels)
     _showstats(repo, stats)
     if stats.unresolvedcount:
         repo.ui.status(
@@ -1182,9 +1173,9 b' def abortmerge(ui, repo):'
         node = repo[b'.'].hex()
 
     repo.ui.status(_(b"aborting the merge, updating back to %s\n") % node[:12])
-    stats = mergemod.update(repo, node, branchmerge=False, force=True)
+    stats = mergemod.clean_update(repo[node])
+    assert stats.unresolvedcount == 0
     _showstats(repo, stats)
-    return stats.unresolvedcount > 0
 
 
 def _incoming(

mercurial/hgweb/webutil.py

@@ -936,5 +936,5 b' def getwebsubs(repo):'
 
 def getgraphnode(repo, ctx):
     return templatekw.getgraphnodecurrent(
-        repo, ctx
+        repo, ctx, {}
     ) + templatekw.getgraphnodesymbol(ctx)

mercurial/hook.py

@@ -7,6 +7,7 b''
 
 from __future__ import absolute_import
 
+import contextlib
 import os
 import sys
 
@@ -259,26 +260,45 b' def hook(ui, repo, htype, throw=False, *'
     return r
 
 
[e-mail]
+def redirect_stdio():
+    """Redirects stdout to stderr, if possible."""
+
+    oldstdout = -1
+    try:
+        if _redirect:
+            try:
+                stdoutno = procutil.stdout.fileno()
+                stderrno = procutil.stderr.fileno()
+                # temporarily redirect stdout to stderr, if possible
+                if stdoutno >= 0 and stderrno >= 0:
+                    procutil.stdout.flush()
+                    oldstdout = os.dup(stdoutno)
+                    os.dup2(stderrno, stdoutno)
+            except (OSError, AttributeError):
+                # files seem to be bogus, give up on redirecting (WSGI, etc)
+                pass
+
+        yield
+
+    finally:
+        # The stderr is fully buffered on Windows when connected to a pipe.
+        # A forcible flush is required to make small stderr data in the
+        # remote side available to the client immediately.
+        procutil.stderr.flush()
+
+        if _redirect and oldstdout >= 0:
+            procutil.stdout.flush()  # write hook output to stderr fd
+            os.dup2(oldstdout, stdoutno)
+            os.close(oldstdout)
+
+
 def runhooks(ui, repo, htype, hooks, throw=False, **args):
     args = pycompat.byteskwargs(args)
     res = {}
-    oldstdout = -1
 
-    try:
+    with redirect_stdio():
         for hname, cmd in hooks:
-            if oldstdout == -1 and _redirect:
-                try:
-                    stdoutno = procutil.stdout.fileno()
-                    stderrno = procutil.stderr.fileno()
-                    # temporarily redirect stdout to stderr, if possible
-                    if stdoutno >= 0 and stderrno >= 0:
-                        procutil.stdout.flush()
-                        oldstdout = os.dup(stdoutno)
-                        os.dup2(stderrno, stdoutno)
-                except (OSError, AttributeError):
-                    # files seem to be bogus, give up on redirecting (WSGI, etc)
-                    pass
-
             if cmd is _fromuntrusted:
                 if throw:
                     raise error.HookAbort(
@@ -312,15 +332,5 b' def runhooks(ui, repo, htype, hooks, thr'
                 raised = False
 
             res[hname] = r, raised
-    finally:
-        # The stderr is fully buffered on Windows when connected to a pipe.
-        # A forcible flush is required to make small stderr data in the
-        # remote side available to the client immediately.
-        procutil.stderr.flush()
-
-        if _redirect and oldstdout >= 0:
-            procutil.stdout.flush()  # write hook output to stderr fd
-            os.dup2(oldstdout, stdoutno)
-            os.close(oldstdout)
 
     return res

mercurial/httpconnection.py

@@ -39,12 +39,15 b' class httpsendfile(object):'
         self.write = self._data.write
         self.length = os.fstat(self._data.fileno()).st_size
         self._pos = 0
+        self._progress = self._makeprogress()
+
+    def _makeprogress(self):
         # We pass double the max for total because we currently have
         # to send the bundle twice in the case of a server that
         # requires authentication. Since we can't know until we try
         # once whether authentication will be required, just lie to
         # the user and maybe the push succeeds suddenly at 50%.
-        self._progress = ui.makeprogress(
+        return self.ui.makeprogress(
             _(b'sending'), unit=_(b'kb'), total=(self.length // 1024 * 2)
         )
 

mercurial/interfaces/repository.py

@@ -985,18 +985,9 b' class imanifestdict(interfaceutil.Interf'
     def hasdir(dir):
         """Returns a bool indicating if a directory is in this manifest."""
 
-    def matches(match):
-        """Generate a new manifest filtered through a matcher.
-
-        Returns an object conforming to the ``imanifestdict`` interface.
-        """
-
     def walk(match):
         """Generator of paths in manifest satisfying a matcher.
 
-        This is equivalent to ``self.matches(match).iterkeys()`` except a new
-        manifest object is not created.
-
         If the matcher has explicit files listed and they don't exist in
         the manifest, ``match.bad()`` is called for each missing file.
         """
@@ -1027,8 +1018,8 b' class imanifestdict(interfaceutil.Interf'
     def get(path, default=None):
         """Obtain the node value for a path or a default value if missing."""
 
-    def flags(path, default=b''):
-        """Return the flags value for a path or a default value if missing."""
+    def flags(path):
+        """Return the flags value for a path (default: empty bytestring)."""
 
     def copy():
         """Return a copy of this manifest."""
@@ -1061,6 +1052,9 b' class imanifestdict(interfaceutil.Interf'
 
         Returns a 2-tuple containing ``bytearray(self.text())`` and the
         delta between ``base`` and this manifest.
+
+        If this manifest implementation can't support ``fastdelta()``,
+        raise ``mercurial.manifest.FastdeltaUnavailable``.
         """
 
 
@@ -1071,14 +1065,6 b' class imanifestrevisionbase(interfaceuti'
     as part of a larger interface.
     """
 
-    def new():
-        """Obtain a new manifest instance.
-
-        Returns an object conforming to the ``imanifestrevisionwritable``
-        interface. The instance will be associated with the same
-        ``imanifestlog`` collection as this instance.
-        """
-
     def copy():
         """Obtain a copy of this manifest instance.
 

mercurial/localrepo.py

@@ -699,6 +699,7 b' def afterhgrcload(ui, wdirvfs, hgvfs, re'
     # Map of requirements to list of extensions to load automatically when
     # requirement is present.
     autoextensions = {
+        b'git': [b'git'],
         b'largefiles': [b'largefiles'],
         b'lfs': [b'lfs'],
     }
@@ -932,6 +933,12 b' def resolverevlogstorevfsoptions(ui, req'
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
+    if ui.configbool(b'devel', b'persistent-nodemap'):
+        options[b'devel-force-nodemap'] = True
 
     return options
 
@@ -1803,7 +1810,7 b' class localrepository(object):'
         # map tag name to (node, hist)
         alltags = tagsmod.findglobaltags(self.ui, self)
         # map tag name to tag type
-        tagtypes = dict((tag, b'global') for tag in alltags)
+        tagtypes = {tag: b'global' for tag in alltags}
 
         tagsmod.readlocaltags(self.ui, self, alltags, tagtypes)
 
@@ -1816,12 +1823,10 b' class localrepository(object):'
             if node != nullid:
                 tags[encoding.tolocal(name)] = node
         tags[b'tip'] = self.changelog.tip()
-        tagtypes = dict(
-            [
-                (encoding.tolocal(name), value)
-                for (name, value) in pycompat.iteritems(tagtypes)
-            ]
-        )
+        tagtypes = {
+            encoding.tolocal(name): value
+            for (name, value) in pycompat.iteritems(tagtypes)
+        }
         return (tags, tagtypes)
 
     def tagtype(self, tagname):
@@ -2173,15 +2178,16 b' class localrepository(object):'
                     )
             if hook.hashook(repo.ui, b'pretxnclose-phase'):
                 cl = repo.unfiltered().changelog
-                for rev, (old, new) in tr.changes[b'phases'].items():
-                    args = tr.hookargs.copy()
-                    node = hex(cl.node(rev))
-                    args.update(phases.preparehookargs(node, old, new))
-                    repo.hook(
-                        b'pretxnclose-phase',
-                        throw=True,
-                        **pycompat.strkwargs(args)
-                    )
+                for revs, (old, new) in tr.changes[b'phases']:
+                    for rev in revs:
+                        args = tr.hookargs.copy()
+                        node = hex(cl.node(rev))
+                        args.update(phases.preparehookargs(node, old, new))
+                        repo.hook(
+                            b'pretxnclose-phase',
+                            throw=True,
+                            **pycompat.strkwargs(args)
+                        )
 
             repo.hook(
                 b'pretxnclose', throw=True, **pycompat.strkwargs(tr.hookargs)
@@ -2226,7 +2232,7 b' class localrepository(object):'
         )
         tr.changes[b'origrepolen'] = len(self)
         tr.changes[b'obsmarkers'] = set()
-        tr.changes[b'phases'] = {}
+        tr.changes[b'phases'] = []
         tr.changes[b'bookmarks'] = {}
 
         tr.hookargs[b'txnid'] = txnid
@@ -2260,16 +2266,19 b' class localrepository(object):'
 
                 if hook.hashook(repo.ui, b'txnclose-phase'):
                     cl = repo.unfiltered().changelog
-                    phasemv = sorted(tr.changes[b'phases'].items())
-                    for rev, (old, new) in phasemv:
-                        args = tr.hookargs.copy()
-                        node = hex(cl.node(rev))
-                        args.update(phases.preparehookargs(node, old, new))
-                        repo.hook(
-                            b'txnclose-phase',
-                            throw=False,
-                            **pycompat.strkwargs(args)
-                        )
+                    phasemv = sorted(
+                        tr.changes[b'phases'], key=lambda r: r[0][0]
+                    )
+                    for revs, (old, new) in phasemv:
+                        for rev in revs:
+                            args = tr.hookargs.copy()
+                            node = hex(cl.node(rev))
+                            args.update(phases.preparehookargs(node, old, new))
+                            repo.hook(
+                                b'txnclose-phase',
+                                throw=False,
+                                **pycompat.strkwargs(args)
+                            )
 
                 repo.hook(
                     b'txnclose', throw=False, **pycompat.strkwargs(hookargs)
@@ -2498,6 +2507,9 b' class localrepository(object):'
 
         if full:
             unfi = self.unfiltered()
+
+            self.changelog.update_caches(transaction=tr)
+
             rbc = unfi.revbranchcache()
             for r in unfi.changelog:
                 rbc.branchinfo(r)
@@ -2843,6 +2855,14 b' class localrepository(object):'
                 fparent1, fparent2 = fparent2, nullid
             elif fparent2 in fparentancestors:
                 fparent2 = nullid
+            elif not fparentancestors:
+                # TODO: this whole if-else might be simplified much more
+                ms = mergemod.mergestate.read(self)
+                if (
+                    fname in ms
+                    and ms[fname] == mergemod.MERGE_RECORD_MERGED_OTHER
+                ):
+                    fparent1, fparent2 = fparent2, nullid
 
         # is the file changed?
         text = fctx.data()
@@ -2938,6 +2958,9 b' class localrepository(object):'
                 self, status, text, user, date, extra
             )
 
+            ms = mergemod.mergestate.read(self)
+            mergeutil.checkunresolved(ms)
+
             # internal config: ui.allowemptycommit
             allowemptycommit = (
                 wctx.branch() != wctx.p1().branch()
@@ -2947,14 +2970,13 b' class localrepository(object):'
                 or self.ui.configbool(b'ui', b'allowemptycommit')
             )
             if not allowemptycommit:
+                self.ui.debug(b'nothing to commit, clearing merge state\n')
+                ms.reset()
                 return None
 
             if merge and cctx.deleted():
                 raise error.Abort(_(b"cannot commit merge with missing files"))
 
-            ms = mergemod.mergestate.read(self)
-            mergeutil.checkunresolved(ms)
-
             if editor:
                 cctx._text = editor(self, cctx, subs)
             edited = text != cctx._text
@@ -3572,14 +3594,17 b' def newreporequirements(ui, createopts):'
             if ui.configbool(b'format', b'dotencode'):
                 requirements.add(b'dotencode')
 
-    compengine = ui.config(b'format', b'revlog-compression')
-    if compengine not in util.compengines:
+    compengines = ui.configlist(b'format', b'revlog-compression')
+    for compengine in compengines:
+        if compengine in util.compengines:
+            break
+    else:
         raise error.Abort(
             _(
-                b'compression engine %s defined by '
+                b'compression engines %s defined by '
                 b'format.revlog-compression not available'
             )
-            % compengine,
+            % b', '.join(b'"%s"' % e for e in compengines),
             hint=_(
                 b'run "hg debuginstall" to list available '
                 b'compression engines'
@@ -3587,7 +3612,7 b' def newreporequirements(ui, createopts):'
         )
 
     # zlib is the historical default and doesn't need an explicit requirement.
-    elif compengine == b'zstd':
+    if compengine == b'zstd':
         requirements.add(b'revlog-compression-zstd')
     elif compengine != b'zlib':
         requirements.add(b'exp-compression-%s' % compengine)

mercurial/logcmdutil.py

@@ -1004,7 +1004,7 b' def _graphnodeformatter(ui, displayer):'
         ui, spec, defaults=templatekw.keywords, resources=tres
     )
 
-    def formatnode(repo, ctx):
+    def formatnode(repo, ctx, cache):
         props = {b'ctx': ctx, b'repo': repo}
         return templ.renderdefault(props)
 
@@ -1038,8 +1038,9 b' def displaygraph(ui, repo, dag, displaye'
         # experimental config: experimental.graphshorten
         state.graphshorten = ui.configbool(b'experimental', b'graphshorten')
 
+    formatnode_cache = {}
     for rev, type, ctx, parents in dag:
-        char = formatnode(repo, ctx)
+        char = formatnode(repo, ctx, formatnode_cache)
         copies = getcopies(ctx) if getcopies else None
         edges = edgefn(type, char, state, rev, parents)
         firstedge = next(edges)

mercurial/manifest.py

@@ -23,6 +23,7 b' from .pycompat import getattr'
 from . import (
     encoding,
     error,
+    match as matchmod,
     mdiff,
     pathutil,
     policy,
@@ -56,7 +57,12 b' def _parse(data):'
             raise ValueError(b'Manifest lines not in sorted order.')
         prev = l
         f, n = l.split(b'\0')
-        if len(n) > 40:
+        nl = len(n)
+        if 64 < nl:
+            # modern hash, full width
+            yield f, bin(n[:64]), n[64:]
+        elif 40 < nl < 45:
+            # legacy hash, always sha1
             yield f, bin(n[:40]), n[40:]
         else:
             yield f, bin(n), b''
@@ -264,9 +270,15 b' class _lazymanifest(object):'
         if pos == -1:
             return (data[1], data[2])
         zeropos = data.find(b'\x00', pos)
+        nlpos = data.find(b'\n', zeropos)
         assert 0 <= needle <= len(self.positions)
         assert len(self.extrainfo) == len(self.positions)
-        hashval = unhexlify(data, self.extrainfo[needle], zeropos + 1, 40)
+        hlen = nlpos - zeropos - 1
+        # Hashes sometimes have an extra byte tucked on the end, so
+        # detect that.
+        if hlen % 2:
+            hlen -= 1
+        hashval = unhexlify(data, self.extrainfo[needle], zeropos + 1, hlen)
         flags = self._getflags(data, needle, zeropos)
         return (hashval, flags)
 
@@ -291,8 +303,13 b' class _lazymanifest(object):'
                 b"Manifest values must be a tuple of (node, flags)."
             )
         hashval = value[0]
-        if not isinstance(hashval, bytes) or not 20 <= len(hashval) <= 22:
-            raise TypeError(b"node must be a 20-byte byte string")
+        # hashes are either 20 or 32 bytes (sha1 or its replacement),
+        # and allow one extra byte taht won't be persisted to disk but
+        # is sometimes used in memory.
+        if not isinstance(hashval, bytes) or not (
+            20 <= len(hashval) <= 22 or 32 <= len(hashval) <= 34
+        ):
+            raise TypeError(b"node must be a 20-byte or 32-byte byte string")
         flags = value[1]
         if len(hashval) == 22:
             hashval = hashval[:-1]
@@ -376,8 +393,13 b' class _lazymanifest(object):'
                     t = self.extradata[-cur - 1]
                     l.append(self._pack(t))
                     self.positions[i] = offset
-                    if len(t[1]) > 20:
-                        self.extrainfo[i] = ord(t[1][21])
+                    # Hashes are either 20 bytes (old sha1s) or 32
+                    # bytes (new non-sha1).
+                    hlen = 20
+                    if len(t[1]) > 25:
+                        hlen = 32
+                    if len(t[1]) > hlen:
+                        self.extrainfo[i] = ord(t[1][hlen + 1])
                     offset += len(l[-1])
                     i += 1
         self.data = b''.join(l)
@@ -385,7 +407,11 b' class _lazymanifest(object):'
         self.extradata = []
 
     def _pack(self, d):
-        return d[0] + b'\x00' + hex(d[1][:20]) + d[2] + b'\n'
+        n = d[1]
+        if len(n) == 21 or len(n) == 33:
+            n = n[:-1]
+        assert len(n) == 20 or len(n) == 32
+        return d[0] + b'\x00' + hex(n) + d[2] + b'\n'
 
     def text(self):
         self._compact()
@@ -461,7 +487,7 b' class manifestdict(object):'
     __bool__ = __nonzero__
 
     def __setitem__(self, key, node):
-        self._lm[key] = node, self.flags(key, b'')
+        self._lm[key] = node, self.flags(key)
 
     def __contains__(self, key):
         if key is None:
@@ -482,17 +508,11 b' class manifestdict(object):'
 
     def filesnotin(self, m2, match=None):
         '''Set of files in this manifest that are not in the other'''
-        if match:
-            m1 = self.matches(match)
-            m2 = m2.matches(match)
-            return m1.filesnotin(m2)
-        diff = self.diff(m2)
-        files = set(
-            filepath
-            for filepath, hashflags in pycompat.iteritems(diff)
-            if hashflags[1][0] is None
-        )
-        return files
+        if match is not None:
+            match = matchmod.badmatch(match, lambda path, msg: None)
+            sm2 = set(m2.walk(match))
+            return {f for f in self.walk(match) if f not in sm2}
+        return {f for f in self if f not in m2}
 
     @propertycache
     def _dirs(self):
@@ -531,7 +551,8 b' class manifestdict(object):'
         # avoid the entire walk if we're only looking for specific files
         if self._filesfastpath(match):
             for fn in sorted(fset):
-                yield fn
+                if fn in self:
+                    yield fn
             return
 
         for fn in self:
@@ -549,7 +570,7 b' class manifestdict(object):'
             if not self.hasdir(fn):
                 match.bad(fn, None)
 
-    def matches(self, match):
+    def _matches(self, match):
         '''generate a new manifest filtered by the match argument'''
         if match.always():
             return self.copy()
@@ -582,8 +603,8 b' class manifestdict(object):'
         string.
         '''
         if match:
-            m1 = self.matches(match)
-            m2 = m2.matches(match)
+            m1 = self._matches(match)
+            m2 = m2._matches(match)
             return m1.diff(m2, clean=clean)
         return self._lm.diff(m2._lm, clean)
 
@@ -596,11 +617,11 b' class manifestdict(object):'
         except KeyError:
             return default
 
-    def flags(self, key, default=b''):
+    def flags(self, key):
         try:
             return self._lm[key][1]
         except KeyError:
-            return default
+            return b''
 
     def copy(self):
         c = manifestdict()
@@ -764,6 +785,7 b' def _splittopdir(f):'
 _noop = lambda s: None
 
 
[e-mail](repository.imanifestdict)
 class treemanifest(object):
     def __init__(self, dir=b'', text=b''):
         self._dir = dir
@@ -1026,7 +1048,12 b' class treemanifest(object):'
                 self._dirs[dir] = treemanifest(self._subpath(dir))
             self._dirs[dir].__setitem__(subpath, n)
         else:
-            self._files[f] = n[:21]  # to match manifestdict's behavior
+            # manifest nodes are either 20 bytes or 32 bytes,
+            # depending on the hash in use. An extra byte is
+            # occasionally used by hg, but won't ever be
+            # persisted. Trim to 21 or 33 bytes as appropriate.
+            trim = 21 if len(n) < 25 else 33
+            self._files[f] = n[:trim]  # to match manifestdict's behavior
         self._dirty = True
 
     def _load(self):
@@ -1079,8 +1106,8 b' class treemanifest(object):'
     def filesnotin(self, m2, match=None):
         '''Set of files in this manifest that are not in the other'''
         if match and not match.always():
-            m1 = self.matches(match)
-            m2 = m2.matches(match)
+            m1 = self._matches(match)
+            m2 = m2._matches(match)
             return m1.filesnotin(m2)
 
         files = set()
@@ -1126,9 +1153,6 b' class treemanifest(object):'
     def walk(self, match):
         '''Generates matching file names.
 
-        Equivalent to manifest.matches(match).iterkeys(), but without creating
-        an entirely new manifest.
-
         It also reports nonexistent files by marking them bad with match.bad().
         '''
         if match.always():
@@ -1171,16 +1195,16 b' class treemanifest(object):'
                     for f in self._dirs[p]._walk(match):
                         yield f
 
-    def matches(self, match):
-        '''generate a new manifest filtered by the match argument'''
-        if match.always():
-            return self.copy()
-
-        return self._matches(match)
-
     def _matches(self, match):
         '''recursively generate a new manifest filtered by the match argument.
         '''
+        if match.always():
+            return self.copy()
+        return self._matches_inner(match)
+
+    def _matches_inner(self, match):
+        if match.always():
+            return self.copy()
 
         visit = match.visitchildrenset(self._dir[:-1])
         if visit == b'all':
@@ -1211,7 +1235,7 b' class treemanifest(object):'
         for dir, subm in pycompat.iteritems(self._dirs):
             if visit and dir[:-1] not in visit:
                 continue
-            m = subm._matches(match)
+            m = subm._matches_inner(match)
             if not m._isempty():
                 ret._dirs[dir] = m
 
@@ -1219,6 +1243,9 b' class treemanifest(object):'
             ret._dirty = True
         return ret
 
+    def fastdelta(self, base, changes):
+        raise FastdeltaUnavailable()
+
     def diff(self, m2, match=None, clean=False):
         '''Finds changes between the current manifest and m2.
 
@@ -1235,8 +1262,8 b' class treemanifest(object):'
         string.
         '''
         if match and not match.always():
-            m1 = self.matches(match)
-            m2 = m2.matches(match)
+            m1 = self._matches(match)
+            m2 = m2._matches(match)
             return m1.diff(m2, clean=clean)
         result = {}
         emptytree = treemanifest()
@@ -1405,6 +1432,7 b' class manifestfulltextcache(util.lrucach'
                 set = super(manifestfulltextcache, self).__setitem__
                 # ignore trailing data, this is a cache, corruption is skipped
                 while True:
+                    # TODO do we need to do work here for sha1 portability?
                     node = fp.read(20)
                     if len(node) < 20:
                         break
@@ -1495,6 +1523,10 b' class manifestfulltextcache(util.lrucach'
 MAXCOMPRESSION = 3
 
 
+class FastdeltaUnavailable(Exception):
+    """Exception raised when fastdelta isn't usable on a manifest."""
+
+
 @interfaceutil.implementer(repository.imanifeststorage)
 class manifestrevlog(object):
     '''A revlog that stores manifest texts. This is responsible for caching the
@@ -1621,7 +1653,9 b' class manifestrevlog(object):'
         readtree=None,
         match=None,
     ):
-        if p1 in self.fulltextcache and util.safehasattr(m, b'fastdelta'):
+        try:
+            if p1 not in self.fulltextcache:
+                raise FastdeltaUnavailable()
             # If our first parent is in the manifest cache, we can
             # compute a delta here using properties we know about the
             # manifest up-front, which may save time later for the
@@ -1640,11 +1674,12 b' class manifestrevlog(object):'
             n = self._revlog.addrevision(
                 text, transaction, link, p1, p2, cachedelta
             )
-        else:
-            # The first parent manifest isn't already loaded, so we'll
-            # just encode a fulltext of the manifest and pass that
-            # through to the revlog layer, and let it handle the delta
-            # process.
+        except FastdeltaUnavailable:
+            # The first parent manifest isn't already loaded or the
+            # manifest implementation doesn't support fastdelta, so
+            # we'll just encode a fulltext of the manifest and pass
+            # that through to the revlog layer, and let it handle the
+            # delta process.
             if self._treeondisk:
                 assert readtree, b"readtree must be set for treemanifest writes"
                 assert match, b"match must be specified for treemanifest writes"
@@ -1923,9 +1958,6 b' class memmanifestctx(object):'
     def _storage(self):
         return self._manifestlog.getstorage(b'')
 
-    def new(self):
-        return memmanifestctx(self._manifestlog)
-
     def copy(self):
         memmf = memmanifestctx(self._manifestlog)
         memmf._manifestdict = self.read().copy()
@@ -1972,9 +2004,6 b' class manifestctx(object):'
     def node(self):
         return self._node
 
-    def new(self):
-        return memmanifestctx(self._manifestlog)
-
     def copy(self):
         memmf = memmanifestctx(self._manifestlog)
         memmf._manifestdict = self.read().copy()
@@ -2039,9 +2068,6 b' class memtreemanifestctx(object):'
     def _storage(self):
         return self._manifestlog.getstorage(b'')
 
-    def new(self, dir=b''):
-        return memtreemanifestctx(self._manifestlog, dir=dir)
-
     def copy(self):
         memmf = memtreemanifestctx(self._manifestlog, dir=self._dir)
         memmf._treemanifest = self._treemanifest.copy()
@@ -2124,9 +2150,6 b' class treemanifestctx(object):'
     def node(self):
         return self._node
 
-    def new(self, dir=b''):
-        return memtreemanifestctx(self._manifestlog, dir=dir)
-
     def copy(self):
         memmf = memtreemanifestctx(self._manifestlog, dir=self._dir)
         memmf._treemanifest = self.read().copy()

mercurial/match.py

@@ -24,7 +24,7 b' from . import ('
 )
 from .utils import stringutil
 
-rustmod = policy.importrust('filepatterns')
+rustmod = policy.importrust('dirstate')
 
 allpatternkinds = (
     b're',
@@ -666,7 +666,10 b' class _dirchildren(object):'
 class includematcher(basematcher):
     def __init__(self, root, kindpats, badfn=None):
         super(includematcher, self).__init__(badfn)
-
+        if rustmod is not None:
+            # We need to pass the patterns to Rust because they can contain
+            # patterns from the user interface
+            self._kindpats = kindpats
         self._pats, self.matchfn = _buildmatch(kindpats, b'(?:/|$)', root)
         self._prefix = _prefix(kindpats)
         roots, dirs, parents = _rootsdirsandparents(kindpats)
@@ -772,7 +775,7 b' class exactmatcher(basematcher):'
         candidates = self._fileset | self._dirs - {b''}
         if dir != b'':
             d = dir + b'/'
-            candidates = set(c[len(d) :] for c in candidates if c.startswith(d))
+            candidates = {c[len(d) :] for c in candidates if c.startswith(d)}
         # self._dirs includes all of the directories, recursively, so if
         # we're attempting to match foo/bar/baz.txt, it'll have '', 'foo',
         # 'foo/bar' in it. Thus we can safely ignore a candidate that has a
@@ -1273,15 +1276,6 b' def _regex(kind, pat, globsuffix):'
     '''Convert a (normalized) pattern of any kind into a
     regular expression.
     globsuffix is appended to the regexp of globs.'''
-
-    if rustmod is not None:
-        try:
-            return rustmod.build_single_regex(kind, pat, globsuffix)
-        except rustmod.PatternError:
-            raise error.ProgrammingError(
-                b'not a regex pattern: %s:%s' % (kind, pat)
-            )
-
     if not pat and kind in (b'glob', b'relpath'):
         return b''
     if kind == b're':
@@ -1554,18 +1548,6 b' def readpatternfile(filepath, warn, sour'
     This is useful to debug ignore patterns.
     '''
 
-    if rustmod is not None:
-        result, warnings = rustmod.read_pattern_file(
-            filepath, bool(warn), sourceinfo,
-        )
-
-        for warning_params in warnings:
-            # Can't be easily emitted from Rust, because it would require
-            # a mechanism for both gettext and calling the `warn` function.
-            warn(_(b"%s: ignoring invalid syntax '%s'\n") % warning_params)
-
-        return result
-
     syntaxes = {
         b're': b'relre:',
         b'regexp': b'relre:',

mercurial/mdiff.py

@@ -91,7 +91,7 b' class diffopts(object):'
             )
 
     def copy(self, **kwargs):
-        opts = dict((k, getattr(self, k)) for k in self.defaults)
+        opts = {k: getattr(self, k) for k in self.defaults}
         opts = pycompat.strkwargs(opts)
         opts.update(kwargs)
         return diffopts(**opts)

mercurial/merge.py

@@ -64,6 +64,7 b" RECORD_LABELS = b'l'"
 RECORD_OVERRIDE = b't'
 RECORD_UNSUPPORTED_MANDATORY = b'X'
 RECORD_UNSUPPORTED_ADVISORY = b'x'
+RECORD_RESOLVED_OTHER = b'R'
 
 MERGE_DRIVER_STATE_UNMARKED = b'u'
 MERGE_DRIVER_STATE_MARKED = b'm'
@@ -74,6 +75,9 b" MERGE_RECORD_RESOLVED = b'r'"
 MERGE_RECORD_UNRESOLVED_PATH = b'pu'
 MERGE_RECORD_RESOLVED_PATH = b'pr'
 MERGE_RECORD_DRIVER_RESOLVED = b'd'
+# represents that the file was automatically merged in favor
+# of other version. This info is used on commit.
+MERGE_RECORD_MERGED_OTHER = b'o'
 
 ACTION_FORGET = b'f'
 ACTION_REMOVE = b'r'
@@ -91,6 +95,8 b" ACTION_DIR_RENAME_MOVE_LOCAL = b'dm'"
 ACTION_KEEP = b'k'
 ACTION_EXEC = b'e'
 ACTION_CREATED_MERGE = b'cm'
+# GET the other/remote side and store this info in mergestate
+ACTION_GET_OTHER_AND_STORE = b'gs'
 
 
 class mergestate(object):
@@ -227,6 +233,7 b' class mergestate(object):'
                 RECORD_CHANGEDELETE_CONFLICT,
                 RECORD_PATH_CONFLICT,
                 RECORD_MERGE_DRIVER_MERGE,
+                RECORD_RESOLVED_OTHER,
             ):
                 bits = record.split(b'\0')
                 self._state[bits[0]] = bits[1:]
@@ -386,18 +393,26 b' class mergestate(object):'
         return configmergedriver
 
     @util.propertycache
-    def localctx(self):
+    def local(self):
         if self._local is None:
-            msg = b"localctx accessed but self._local isn't set"
+            msg = b"local accessed but self._local isn't set"
             raise error.ProgrammingError(msg)
-        return self._repo[self._local]
+        return self._local
+
+    @util.propertycache
+    def localctx(self):
+        return self._repo[self.local]
+
+    @util.propertycache
+    def other(self):
+        if self._other is None:
+            msg = b"other accessed but self._other isn't set"
+            raise error.ProgrammingError(msg)
+        return self._other
 
     @util.propertycache
     def otherctx(self):
-        if self._other is None:
-            msg = b"otherctx accessed but self._other isn't set"
-            raise error.ProgrammingError(msg)
-        return self._repo[self._other]
+        return self._repo[self.other]
 
     def active(self):
         """Whether mergestate is active.
@@ -405,14 +420,7 b' class mergestate(object):'
         Returns True if there appears to be mergestate. This is a rough proxy
         for "is a merge in progress."
         """
-        # Check local variables before looking at filesystem for performance
-        # reasons.
-        return (
-            bool(self._local)
-            or bool(self._state)
-            or self._repo.vfs.exists(self.statepathv1)
-            or self._repo.vfs.exists(self.statepathv2)
-        )
+        return bool(self._local) or bool(self._state)
 
     def commit(self):
         """Write current state on disk (if necessary)"""
@@ -452,6 +460,10 b' class mergestate(object):'
                 records.append(
                     (RECORD_PATH_CONFLICT, b'\0'.join([filename] + v))
                 )
+            elif v[0] == MERGE_RECORD_MERGED_OTHER:
+                records.append(
+                    (RECORD_RESOLVED_OTHER, b'\0'.join([filename] + v))
+                )
             elif v[1] == nullhex or v[6] == nullhex:
                 # Change/Delete or Delete/Change conflicts. These are stored in
                 # 'C' records. v[1] is the local file, and is nullhex when the
@@ -550,6 +562,10 b' class mergestate(object):'
         self._state[path] = [MERGE_RECORD_UNRESOLVED_PATH, frename, forigin]
         self._dirty = True
 
+    def addmergedother(self, path):
+        self._state[path] = [MERGE_RECORD_MERGED_OTHER, nullhex, nullhex]
+        self._dirty = True
+
     def __contains__(self, dfile):
         return dfile in self._state
 
@@ -593,6 +609,8 b' class mergestate(object):'
         """rerun merge process for file path `dfile`"""
         if self[dfile] in (MERGE_RECORD_RESOLVED, MERGE_RECORD_DRIVER_RESOLVED):
             return True, 0
+        if self._state[dfile][0] == MERGE_RECORD_MERGED_OTHER:
+            return True, 0
         stateentry = self._state[dfile]
         state, localkey, lfile, afile, anode, ofile, onode, flags = stateentry
         octx = self._repo[self._other]
@@ -989,11 +1007,10 b' def _checkcollision(repo, wmf, actions):'
     """
     Check for case-folding collisions.
     """
-
     # If the repo is narrowed, filter out files outside the narrowspec.
     narrowmatch = repo.narrowmatch()
     if not narrowmatch.always():
-        wmf = wmf.matches(narrowmatch)
+        pmmf = set(wmf.walk(narrowmatch))
         if actions:
             narrowactions = {}
             for m, actionsfortype in pycompat.iteritems(actions):
@@ -1002,9 +1019,9 b' def _checkcollision(repo, wmf, actions):'
                     if narrowmatch(f):
                         narrowactions[m].append((f, args, msg))
             actions = narrowactions
-
-    # build provisional merged manifest up
-    pmmf = set(wmf)
+    else:
+        # build provisional merged manifest up
+        pmmf = set(wmf)
 
     if actions:
         # KEEP and EXEC are no-op
@@ -1209,7 +1226,7 b' def _filternarrowactions(narrowmatch, br'
     narrowed.
     """
     nooptypes = {b'k'}  # TODO: handle with nonconflicttypes
-    nonconflicttypes = set(b'a am c cm f g r e'.split())
+    nonconflicttypes = set(b'a am c cm f g gs r e'.split())
     # We mutate the items in the dict during iteration, so iterate
     # over a copy.
     for f, action in list(actions.items()):
@@ -1256,17 +1273,19 b' def manifestmerge('
     if matcher is not None and matcher.always():
         matcher = None
 
-    copy, movewithdir, diverge, renamedelete, dirmove = {}, {}, {}, {}, {}
-
     # manifests fetched in order are going to be faster, so prime the caches
     [
         x.manifest()
         for x in sorted(wctx.parents() + [p2, pa], key=scmutil.intrev)
     ]
 
+    branch_copies1 = copies.branch_copies()
+    branch_copies2 = copies.branch_copies()
+    diverge = {}
     if followcopies:
-        ret = copies.mergecopies(repo, wctx, p2, pa)
-        copy, movewithdir, diverge, renamedelete, dirmove = ret
+        branch_copies1, branch_copies2, diverge = copies.mergecopies(
+            repo, wctx, p2, pa
+        )
 
     boolbm = pycompat.bytestr(bool(branchmerge))
     boolf = pycompat.bytestr(bool(force))
@@ -1278,8 +1297,10 b' def manifestmerge('
     repo.ui.debug(b" ancestor: %s, local: %s, remote: %s\n" % (pa, wctx, p2))
 
     m1, m2, ma = wctx.manifest(), p2.manifest(), pa.manifest()
-    copied = set(copy.values())
-    copied.update(movewithdir.values())
+    copied1 = set(branch_copies1.copy.values())
+    copied1.update(branch_copies1.movewithdir.values())
+    copied2 = set(branch_copies2.copy.values())
+    copied2.update(branch_copies2.movewithdir.values())
 
     if b'.hgsubstate' in m1 and wctx.rev() is None:
         # Check whether sub state is modified, and overwrite the manifest
@@ -1299,10 +1320,10 b' def manifestmerge('
         relevantfiles = set(ma.diff(m2).keys())
 
         # For copied and moved files, we need to add the source file too.
-        for copykey, copyvalue in pycompat.iteritems(copy):
+        for copykey, copyvalue in pycompat.iteritems(branch_copies1.copy):
             if copyvalue in relevantfiles:
                 relevantfiles.add(copykey)
-        for movedirkey in movewithdir:
+        for movedirkey in branch_copies1.movewithdir:
             relevantfiles.add(movedirkey)
         filesmatcher = scmutil.matchfiles(repo, relevantfiles)
         matcher = matchmod.intersectmatchers(matcher, filesmatcher)
@@ -1313,7 +1334,10 b' def manifestmerge('
     for f, ((n1, fl1), (n2, fl2)) in pycompat.iteritems(diff):
         if n1 and n2:  # file exists on both local and remote side
             if f not in ma:
-                fa = copy.get(f, None)
+                # TODO: what if they're renamed from different sources?
+                fa = branch_copies1.copy.get(
+                    f, None
+                ) or branch_copies2.copy.get(f, None)
                 if fa is not None:
                     actions[f] = (
                         ACTION_MERGE,
@@ -1341,14 +1365,22 b' def manifestmerge('
                         )
                     else:
                         actions[f] = (
-                            ACTION_GET,
+                            ACTION_GET_OTHER_AND_STORE
+                            if branchmerge
+                            else ACTION_GET,
                             (fl2, False),
                             b'remote is newer',
                         )
                 elif nol and n2 == a:  # remote only changed 'x'
                     actions[f] = (ACTION_EXEC, (fl2,), b'update permissions')
                 elif nol and n1 == a:  # local only changed 'x'
-                    actions[f] = (ACTION_GET, (fl1, False), b'remote is newer')
+                    actions[f] = (
+                        ACTION_GET_OTHER_AND_STORE
+                        if branchmerge
+                        else ACTION_GET,
+                        (fl1, False),
+                        b'remote is newer',
+                    )
                 else:  # both changed something
                     actions[f] = (
                         ACTION_MERGE,
@@ -1356,10 +1388,12 b' def manifestmerge('
                         b'versions differ',
                     )
         elif n1:  # file exists only on local side
-            if f in copied:
+            if f in copied2:
                 pass  # we'll deal with it on m2 side
-            elif f in movewithdir:  # directory rename, move local
-                f2 = movewithdir[f]
+            elif (
+                f in branch_copies1.movewithdir
+            ):  # directory rename, move local
+                f2 = branch_copies1.movewithdir[f]
                 if f2 in m2:
                     actions[f2] = (
                         ACTION_MERGE,
@@ -1372,8 +1406,8 b' def manifestmerge('
                         (f, fl1),
                         b'remote directory rename - move from %s' % f,
                     )
-            elif f in copy:
-                f2 = copy[f]
+            elif f in branch_copies1.copy:
+                f2 = branch_copies1.copy[f]
                 actions[f] = (
                     ACTION_MERGE,
                     (f, f2, f2, False, pa.node()),
@@ -1397,10 +1431,10 b' def manifestmerge('
                 else:
                     actions[f] = (ACTION_REMOVE, None, b'other deleted')
         elif n2:  # file exists only on remote side
-            if f in copied:
+            if f in copied1:
                 pass  # we'll deal with it on m1 side
-            elif f in movewithdir:
-                f2 = movewithdir[f]
+            elif f in branch_copies2.movewithdir:
+                f2 = branch_copies2.movewithdir[f]
                 if f2 in m1:
                     actions[f2] = (
                         ACTION_MERGE,
@@ -1413,8 +1447,8 b' def manifestmerge('
                         (f, fl2),
                         b'local directory rename - get from %s' % f,
                     )
-            elif f in copy:
-                f2 = copy[f]
+            elif f in branch_copies2.copy:
+                f2 = branch_copies2.copy[f]
                 if f2 in m2:
                     actions[f] = (
                         ACTION_MERGE,
@@ -1451,10 +1485,10 b' def manifestmerge('
                     )
             elif n2 != ma[f]:
                 df = None
-                for d in dirmove:
+                for d in branch_copies1.dirmove:
                     if f.startswith(d):
                         # new file added in a directory that was moved
-                        df = dirmove[d] + f[len(d) :]
+                        df = branch_copies1.dirmove[d] + f[len(d) :]
                         break
                 if df is not None and df in m1:
                     actions[df] = (
@@ -1481,6 +1515,9 b' def manifestmerge('
         # Updates "actions" in place
         _filternarrowactions(narrowmatch, branchmerge, actions)
 
+    renamedelete = branch_copies1.renamedelete
+    renamedelete.update(branch_copies2.renamedelete)
+
     return actions, diverge, renamedelete
 
 
@@ -1576,6 +1613,8 b' def calculateupdates('
 
             for f, a in sorted(pycompat.iteritems(actions)):
                 m, args, msg = a
+                if m == ACTION_GET_OTHER_AND_STORE:
+                    m = ACTION_GET
                 repo.ui.debug(b' %s: %s -> %s\n' % (f, msg, m))
                 if f in fbids:
                     d = fbids[f]
@@ -1784,8 +1823,8 b' class updateresult(object):'
 
 def emptyactions():
     """create an actions dict, to be populated and passed to applyupdates()"""
-    return dict(
-        (m, [])
+    return {
+        m: []
         for m in (
             ACTION_ADD,
             ACTION_ADD_MODIFIED,
@@ -1801,8 +1840,9 b' def emptyactions():'
             ACTION_KEEP,
             ACTION_PATH_CONFLICT,
             ACTION_PATH_CONFLICT_RESOLVE,
+            ACTION_GET_OTHER_AND_STORE,
         )
-    )
+    }
 
 
 def applyupdates(
@@ -1823,6 +1863,11 b' def applyupdates('
 
     updated, merged, removed = 0, 0, 0
     ms = mergestate.clean(repo, wctx.p1().node(), mctx.node(), labels)
+
+    # add ACTION_GET_OTHER_AND_STORE to mergestate
+    for e in actions[ACTION_GET_OTHER_AND_STORE]:
+        ms.addmergedother(e[0])
+
     moves = []
     for m, l in actions.items():
         l.sort()
@@ -2058,7 +2103,7 b' def applyupdates('
 
     extraactions = ms.actions()
     if extraactions:
-        mfiles = set(a[0] for a in actions[ACTION_MERGE])
+        mfiles = {a[0] for a in actions[ACTION_MERGE]}
         for k, acts in pycompat.iteritems(extraactions):
             actions[k].extend(acts)
             if k == ACTION_GET and wantfiledata:
@@ -2205,6 +2250,7 b' def update('
     labels=None,
     matcher=None,
     mergeforce=False,
+    updatedirstate=True,
     updatecheck=None,
     wc=None,
 ):
@@ -2288,13 +2334,6 b' def update('
                     ),
                 )
             )
-    # If we're doing a partial update, we need to skip updating
-    # the dirstate, so make a note of any partial-ness to the
-    # update here.
-    if matcher is None or matcher.always():
-        partial = False
-    else:
-        partial = True
     with repo.wlock():
         if wc is None:
             wc = repo[None]
@@ -2409,6 +2448,7 b' def update('
                     ACTION_EXEC,
                     ACTION_REMOVE,
                     ACTION_PATH_CONFLICT_RESOLVE,
+                    ACTION_GET_OTHER_AND_STORE,
                 ):
                     msg = _(b"conflicting changes")
                     hint = _(b"commit or update --clean to discard changes")
@@ -2471,6 +2511,10 b' def update('
                 actions[m] = []
             actions[m].append((f, args, msg))
 
+        # ACTION_GET_OTHER_AND_STORE is a ACTION_GET + store in mergestate
+        for e in actions[ACTION_GET_OTHER_AND_STORE]:
+            actions[ACTION_GET].append(e)
+
         if not util.fscasesensitive(repo.path):
             # check collision between files only in p2 for clean update
             if not branchmerge and (
@@ -2507,7 +2551,11 b' def update('
         ### apply phase
         if not branchmerge:  # just jump to the new rev
             fp1, fp2, xp1, xp2 = fp2, nullid, xp2, b''
-        if not partial and not wc.isinmemory():
+        # If we're doing a partial update, we need to skip updating
+        # the dirstate.
+        always = matcher is None or matcher.always()
+        updatedirstate = updatedirstate and always and not wc.isinmemory()
+        if updatedirstate:
             repo.hook(b'preupdate', throw=True, parent1=xp1, parent2=xp2)
             # note that we're in the middle of an update
             repo.vfs.write(b'updatestate', p2.hex())
@@ -2553,7 +2601,6 b' def update('
                 )
             )
 
-        updatedirstate = not partial and not wc.isinmemory()
         wantfiledata = updatedirstate and not branchmerge
         stats, getfiledata = applyupdates(
             repo, actions, wc, p2, overwrite, wantfiledata, labels=labels
@@ -2574,15 +2621,65 b' def update('
     if not branchmerge:
         sparse.prunetemporaryincludes(repo)
 
-    if not partial:
+    if updatedirstate:
         repo.hook(
             b'update', parent1=xp1, parent2=xp2, error=stats.unresolvedcount
         )
     return stats
 
 
+def merge(ctx, labels=None, force=False, wc=None):
+    """Merge another topological branch into the working copy.
+
+    force = whether the merge was run with 'merge --force' (deprecated)
+    """
+
+    return update(
+        ctx.repo(),
+        ctx.rev(),
+        labels=labels,
+        branchmerge=True,
+        force=force,
+        mergeforce=force,
+        wc=wc,
+    )
+
+
+def clean_update(ctx, wc=None):
+    """Do a clean update to the given commit.
+
+    This involves updating to the commit and discarding any changes in the
+    working copy.
+    """
+    return update(ctx.repo(), ctx.rev(), branchmerge=False, force=True, wc=wc)
+
+
+def revert_to(ctx, matcher=None, wc=None):
+    """Revert the working copy to the given commit.
+
+    The working copy will keep its current parent(s) but its content will
+    be the same as in the given commit.
+    """
+
+    return update(
+        ctx.repo(),
+        ctx.rev(),
+        branchmerge=False,
+        force=True,
+        updatedirstate=False,
+        matcher=matcher,
+        wc=wc,
+    )
+
+
 def graft(
-    repo, ctx, base, labels=None, keepparent=False, keepconflictparent=False
+    repo,
+    ctx,
+    base=None,
+    labels=None,
+    keepparent=False,
+    keepconflictparent=False,
+    wctx=None,
 ):
     """Do a graft-like merge.
 
@@ -2593,7 +2690,7 b' def graft('
     renames/copies appropriately.
 
     ctx - changeset to rebase
-    base - merge base, usually ctx.p1()
+    base - merge base, or ctx.p1() if not specified
     labels - merge labels eg ['local', 'graft']
     keepparent - keep second parent if any
     keepconflictparent - if unresolved, keep parent used for the merge
@@ -2605,9 +2702,15 b' def graft('
     # to copy commits), and 2) informs update that the incoming changes are
     # newer than the destination so it doesn't prompt about "remote changed foo
     # which local deleted".
-    wctx = repo[None]
+    # We also pass mergeancestor=True when base is the same revision as p1. 2)
+    # doesn't matter as there can't possibly be conflicts, but 1) is necessary.
+    wctx = wctx or repo[None]
     pctx = wctx.p1()
-    mergeancestor = repo.changelog.isancestor(pctx.node(), ctx.node())
+    base = base or ctx.p1()
+    mergeancestor = (
+        repo.changelog.isancestor(pctx.node(), ctx.node())
+        or pctx.rev() == base.rev()
+    )
 
     stats = update(
         repo,
@@ -2617,6 +2720,7 b' def graft('
         base.node(),
         mergeancestor=mergeancestor,
         labels=labels,
+        wc=wctx,
     )
 
     if keepconflictparent and stats.unresolvedcount:
@@ -2631,17 +2735,23 b' def graft('
     if pother == pctx.node():
         pother = nullid
 
-    with repo.dirstate.parentchange():
-        repo.setparents(pctx.node(), pother)
-        repo.dirstate.write(repo.currenttransaction())
+    if wctx.isinmemory():
+        wctx.setparents(pctx.node(), pother)
         # fix up dirstate for copies and renames
         copies.graftcopies(wctx, ctx, base)
+    else:
+        with repo.dirstate.parentchange():
+            repo.setparents(pctx.node(), pother)
+            repo.dirstate.write(repo.currenttransaction())
+            # fix up dirstate for copies and renames
+            copies.graftcopies(wctx, ctx, base)
     return stats
 
 
 def purge(
     repo,
     matcher,
+    unknown=True,
     ignored=False,
     removeemptydirs=True,
     removefiles=True,
@@ -2653,7 +2763,9 b' def purge('
     ``matcher`` is a matcher configured to scan the working directory -
     potentially a subset.
 
-    ``ignored`` controls whether ignored files should also be purged.
+    ``unknown`` controls whether unknown files should be purged.
+
+    ``ignored`` controls whether ignored files should be purged.
 
     ``removeemptydirs`` controls whether empty directories should be removed.
 
@@ -2690,7 +2802,7 b' def purge('
             directories = []
             matcher.traversedir = directories.append
 
-        status = repo.status(match=matcher, ignored=ignored, unknown=True)
+        status = repo.status(match=matcher, ignored=ignored, unknown=unknown)
 
         if removefiles:
             for f in sorted(status.unknown + status.ignored):

mercurial/namespaces.py

@@ -83,6 +83,9 b' class namespaces(object):'
     def __iter__(self):
         return self._names.__iter__()
 
+    def get(self, namespace, default=None):
+        return self._names.get(namespace, default)
+
     def items(self):
         return pycompat.iteritems(self._names)
 

mercurial/narrowspec.py

@@ -233,21 +233,6 b' def restrictpatterns(req_includes, req_e'
     :param repo_includes: repo includes
     :param repo_excludes: repo excludes
     :return: include patterns, exclude patterns, and invalid include patterns.
-
-    >>> restrictpatterns({'f1','f2'}, {}, ['f1'], [])
-    (set(['f1']), {}, [])
-    >>> restrictpatterns({'f1'}, {}, ['f1','f2'], [])
-    (set(['f1']), {}, [])
-    >>> restrictpatterns({'f1/fc1', 'f3/fc3'}, {}, ['f1','f2'], [])
-    (set(['f1/fc1']), {}, [])
-    >>> restrictpatterns({'f1_fc1'}, {}, ['f1','f2'], [])
-    ([], set(['path:.']), [])
-    >>> restrictpatterns({'f1/../f2/fc2'}, {}, ['f1','f2'], [])
-    (set(['f2/fc2']), {}, [])
-    >>> restrictpatterns({'f1/../f3/fc3'}, {}, ['f1','f2'], [])
-    ([], set(['path:.']), [])
-    >>> restrictpatterns({'f1/$non_exitent_var'}, {}, ['f1','f2'], [])
-    (set(['f1/$non_exitent_var']), {}, [])
     """
     res_excludes = set(req_excludes)
     res_excludes.update(repo_excludes)

mercurial/obsolete.py

@@ -939,7 +939,7 b' def _computeobsoleteset(repo):'
     getnode = repo.changelog.node
     notpublic = _mutablerevs(repo)
     isobs = repo.obsstore.successors.__contains__
-    obs = set(r for r in notpublic if isobs(getnode(r)))
+    obs = {r for r in notpublic if isobs(getnode(r))}
     return obs
 
 
@@ -965,7 +965,7 b' def _computeorphanset(repo):'
 def _computesuspendedset(repo):
     """the set of obsolete parents with non obsolete descendants"""
     suspended = repo.changelog.ancestors(getrevs(repo, b'orphan'))
-    return set(r for r in getrevs(repo, b'obsolete') if r in suspended)
+    return {r for r in getrevs(repo, b'obsolete') if r in suspended}
 
 
 @cachefor(b'extinct')

mercurial/obsutil.py

@@ -194,7 +194,7 b' def allsuccessors(obsstore, nodes, ignor'
 
 def _filterprunes(markers):
     """return a set with no prune markers"""
-    return set(m for m in markers if m[1])
+    return {m for m in markers if m[1]}
 
 
 def exclusivemarkers(repo, nodes):
@@ -338,12 +338,12 b' def foreground(repo, nodes):'
         # compute the whole set of successors or descendants
         while len(foreground) != plen:
             plen = len(foreground)
-            succs = set(c.node() for c in foreground)
+            succs = {c.node() for c in foreground}
             mutable = [c.node() for c in foreground if c.mutable()]
             succs.update(allsuccessors(repo.obsstore, mutable))
             known = (n for n in succs if has_node(n))
             foreground = set(repo.set(b'%ln::', known))
-    return set(c.node() for c in foreground)
+    return {c.node() for c in foreground}
 
 
 # effectflag field
@@ -855,11 +855,11 b' def markersusers(markers):'
     """ Returns a sorted list of markers users without duplicates
     """
     markersmeta = [dict(m[3]) for m in markers]
-    users = set(
+    users = {
         encoding.tolocal(meta[b'user'])
         for meta in markersmeta
         if meta.get(b'user')
-    )
+    }
 
     return sorted(users)
 
@@ -868,9 +868,9 b' def markersoperations(markers):'
     """ Returns a sorted list of markers operations without duplicates
     """
     markersmeta = [dict(m[3]) for m in markers]
-    operations = set(
+    operations = {
         meta.get(b'operation') for meta in markersmeta if meta.get(b'operation')
-    )
+    }
 
     return sorted(operations)
 

mercurial/patch.py

@@ -2888,7 +2888,7 b' def _filepairs(modified, added, removed,'
     or 'rename' (the latter two only if opts.git is set).'''
     gone = set()
 
-    copyto = dict([(v, k) for k, v in copy.items()])
+    copyto = {v: k for k, v in copy.items()}
 
     addedset, removedset = set(added), set(removed)
 

mercurial/pathutil.py

@@ -84,7 +84,7 b' class pathauditor(object):'
                         _(b"path contains illegal component: %s") % path
                     )
         if b'.hg' in _lowerclean(path):
-            lparts = [_lowerclean(p.lower()) for p in parts]
+            lparts = [_lowerclean(p) for p in parts]
             for p in b'.hg', b'.hg.':
                 if p in lparts[1:]:
                     pos = lparts.index(p)
@@ -99,10 +99,11 b' class pathauditor(object):'
 
         parts.pop()
         normparts.pop()
-        prefixes = []
         # It's important that we check the path parts starting from the root.
-        # This means we won't accidentally traverse a symlink into some other
-        # filesystem (which is potentially expensive to access).
+        # We don't want to add "foo/bar/baz" to auditeddir before checking if
+        # there's a "foo/.hg" directory. This also means we won't accidentally
+        # traverse a symlink into some other filesystem (which is potentially
+        # expensive to access).
         for i in range(len(parts)):
             prefix = pycompat.ossep.join(parts[: i + 1])
             normprefix = pycompat.ossep.join(normparts[: i + 1])
@@ -110,13 +111,11 b' class pathauditor(object):'
                 continue
             if self._realfs:
                 self._checkfs(prefix, path)
-            prefixes.append(normprefix)
+            if self._cached:
+                self.auditeddir.add(normprefix)
 
         if self._cached:
             self.audited.add(normpath)
-            # only add prefixes to the cache after checking everything: we don't
-            # want to add "foo/bar/baz" before checking if there's a "foo/.hg"
-            self.auditeddir.update(prefixes)
 
     def _checkfs(self, prefix, path):
         """raise exception if a file system backed check fails"""
@@ -287,6 +286,9 b' class dirs(object):'
     '''a multiset of directory names from a set of file paths'''
 
     def __init__(self, map, skip=None):
+        '''
+        a dict map indicates a dirstate while a list indicates a manifest
+        '''
         self._dirs = {}
         addpath = self.addpath
         if isinstance(map, dict) and skip is not None:

mercurial/phases.py

@@ -216,17 +216,101 b' def binarydecode(stream):'
     return headsbyphase
 
 
+def _sortedrange_insert(data, idx, rev, t):
+    merge_before = False
+    if idx:
+        r1, t1 = data[idx - 1]
+        merge_before = r1[-1] + 1 == rev and t1 == t
+    merge_after = False
+    if idx < len(data):
+        r2, t2 = data[idx]
+        merge_after = r2[0] == rev + 1 and t2 == t
+
+    if merge_before and merge_after:
+        data[idx - 1] = (pycompat.xrange(r1[0], r2[-1] + 1), t)
+        data.pop(idx)
+    elif merge_before:
+        data[idx - 1] = (pycompat.xrange(r1[0], rev + 1), t)
+    elif merge_after:
+        data[idx] = (pycompat.xrange(rev, r2[-1] + 1), t)
+    else:
+        data.insert(idx, (pycompat.xrange(rev, rev + 1), t))
+
+
+def _sortedrange_split(data, idx, rev, t):
+    r1, t1 = data[idx]
+    if t == t1:
+        return
+    t = (t1[0], t[1])
+    if len(r1) == 1:
+        data.pop(idx)
+        _sortedrange_insert(data, idx, rev, t)
+    elif r1[0] == rev:
+        data[idx] = (pycompat.xrange(rev + 1, r1[-1] + 1), t1)
+        _sortedrange_insert(data, idx, rev, t)
+    elif r1[-1] == rev:
+        data[idx] = (pycompat.xrange(r1[0], rev), t1)
+        _sortedrange_insert(data, idx + 1, rev, t)
+    else:
+        data[idx : idx + 1] = [
+            (pycompat.xrange(r1[0], rev), t1),
+            (pycompat.xrange(rev, rev + 1), t),
+            (pycompat.xrange(rev + 1, r1[-1] + 1), t1),
+        ]
+
+
 def _trackphasechange(data, rev, old, new):
-    """add a phase move the <data> dictionnary
+    """add a phase move to the <data> list of ranges
 
     If data is None, nothing happens.
     """
     if data is None:
         return
-    existing = data.get(rev)
-    if existing is not None:
-        old = existing[0]
-    data[rev] = (old, new)
+
+    # If data is empty, create a one-revision range and done
+    if not data:
+        data.insert(0, (pycompat.xrange(rev, rev + 1), (old, new)))
+        return
+
+    low = 0
+    high = len(data)
+    t = (old, new)
+    while low < high:
+        mid = (low + high) // 2
+        revs = data[mid][0]
+
+        if rev in revs:
+            _sortedrange_split(data, mid, rev, t)
+            return
+
+        if revs[0] == rev + 1:
+            if mid and data[mid - 1][0][-1] == rev:
+                _sortedrange_split(data, mid - 1, rev, t)
+            else:
+                _sortedrange_insert(data, mid, rev, t)
+            return
+
+        if revs[-1] == rev - 1:
+            if mid + 1 < len(data) and data[mid + 1][0][0] == rev:
+                _sortedrange_split(data, mid + 1, rev, t)
+            else:
+                _sortedrange_insert(data, mid + 1, rev, t)
+            return
+
+        if revs[0] > rev:
+            high = mid
+        else:
+            low = mid + 1
+
+    if low == len(data):
+        data.append((pycompat.xrange(rev, rev + 1), t))
+        return
+
+    r1, t1 = data[low]
+    if r1[0] > rev:
+        data.insert(low, (pycompat.xrange(rev, rev + 1), t))
+    else:
+        data.insert(low + 1, (pycompat.xrange(rev, rev + 1), t))
 
 
 class phasecache(object):
@@ -400,8 +484,9 b' class phasecache(object):'
             phasetracking = tr.changes[b'phases']
             torev = repo.changelog.rev
             phase = self.phase
-            for n in nodes:
-                rev = torev(n)
+            revs = [torev(node) for node in nodes]
+            revs.sort()
+            for rev in revs:
                 revphase = phase(repo, rev)
                 _trackphasechange(phasetracking, rev, None, revphase)
         repo.invalidatevolatilesets()
@@ -445,10 +530,10 b' class phasecache(object):'
                     phasetracking, r, self.phase(repo, r), targetphase
                 )
 
-            roots = set(
+            roots = {
                 ctx.node()
                 for ctx in repo.set(b'roots((%ln::) - %ld)', olds, affected)
-            )
+            }
             if olds != roots:
                 self._updateroots(phase, roots, tr)
                 # some roots may need to be declared for lower phases
@@ -485,7 +570,7 b' class phasecache(object):'
                     affected -= revs
                 else:  # public phase
                     revs = affected
-                for r in revs:
+                for r in sorted(revs):
                     _trackphasechange(phasetracking, r, phase, targetphase)
         repo.invalidatevolatilesets()
 
@@ -518,9 +603,7 b' class phasecache(object):'
             ]
             updatedroots = repo.set(b'roots(%ln::)', aboveroots)
 
-            finalroots = set(
-                n for n in currentroots if repo[n].rev() < minnewroot
-            )
+            finalroots = {n for n in currentroots if repo[n].rev() < minnewroot}
             finalroots.update(ctx.node() for ctx in updatedroots)
         if finalroots != oldroots:
             self._updateroots(targetphase, finalroots, tr)
@@ -760,7 +843,7 b' def newheads(repo, heads, roots):'
     if not heads or heads == [nullid]:
         return []
     # The logic operated on revisions, convert arguments early for convenience
-    new_heads = set(rev(n) for n in heads if n != nullid)
+    new_heads = {rev(n) for n in heads if n != nullid}
     roots = [rev(n) for n in roots]
     # compute the area we need to remove
     affected_zone = repo.revs(b"(%ld::%ld)", roots, new_heads)

mercurial/posix.py

@@ -324,9 +324,8 b' def checklink(path):'
                     open(fullpath, b'w').close()
                 except IOError as inst:
                     if (
-                        inst[0]  # pytype: disable=unsupported-operands
-                        == errno.EACCES
-                    ):
+                        inst[0] == errno.EACCES
+                    ):  # pytype: disable=unsupported-operands
                         # If we can't write to cachedir, just pretend
                         # that the fs is readonly and by association
                         # that the fs won't support symlinks. This

mercurial/profiling.py

@@ -186,6 +186,7 b' class profile(object):'
         self._output = None
         self._fp = None
         self._fpdoclose = True
+        self._flushfp = None
         self._profiler = None
         self._enabled = enabled
         self._entered = False
@@ -246,6 +247,8 b' class profile(object):'
             else:
                 self._fpdoclose = False
                 self._fp = self._ui.ferr
+                # Ensure we've flushed fout before writing to ferr.
+                self._flushfp = self._ui.fout
 
             if proffn is not None:
                 pass
@@ -265,6 +268,7 b' class profile(object):'
     def __exit__(self, exception_type, exception_value, traceback):
         propagate = None
         if self._profiler is not None:
+            self._uiflush()
             propagate = self._profiler.__exit__(
                 exception_type, exception_value, traceback
             )
@@ -280,3 +284,7 b' class profile(object):'
     def _closefp(self):
         if self._fpdoclose and self._fp is not None:
             self._fp.close()
+
+    def _uiflush(self):
+        if self._flushfp:
+            self._flushfp.flush()

mercurial/pure/parsers.py

@@ -141,6 +141,50 b' class IndexObject(BaseIndexObject):'
             self._extra = self._extra[: i - self._lgt]
 
 
+class PersistentNodeMapIndexObject(IndexObject):
+    """a Debug oriented class to test persistent nodemap
+
+    We need a simple python object to test API and higher level behavior. See
+    the Rust implementation for  more serious usage. This should be used only
+    through the dedicated `devel.persistent-nodemap` config.
+    """
+
+    def nodemap_data_all(self):
+        """Return bytes containing a full serialization of a nodemap
+
+        The nodemap should be valid for the full set of revisions in the
+        index."""
+        return nodemaputil.persistent_data(self)
+
+    def nodemap_data_incremental(self):
+        """Return bytes containing a incremental update to persistent nodemap
+
+        This containst the data for an append-only update of the data provided
+        in the last call to `update_nodemap_data`.
+        """
+        if self._nm_root is None:
+            return None
+        docket = self._nm_docket
+        changed, data = nodemaputil.update_persistent_data(
+            self, self._nm_root, self._nm_max_idx, self._nm_docket.tip_rev
+        )
+
+        self._nm_root = self._nm_max_idx = self._nm_docket = None
+        return docket, changed, data
+
+    def update_nodemap_data(self, docket, nm_data):
+        """provide full block of persisted binary data for a nodemap
+
+        The data are expected to come from disk. See `nodemap_data_all` for a
+        produceur of such data."""
+        if nm_data is not None:
+            self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data)
+            if self._nm_root:
+                self._nm_docket = docket
+            else:
+                self._nm_root = self._nm_max_idx = self._nm_docket = None
+
+
 class InlinedIndexObject(BaseIndexObject):
     def __init__(self, data, inline=0):
         self._data = data
@@ -188,6 +232,12 b' def parse_index2(data, inline):'
     return InlinedIndexObject(data, inline), (0, data)
 
 
+def parse_index_devel_nodemap(data, inline):
+    """like parse_index2, but alway return a PersistentNodeMapIndexObject
+    """
+    return PersistentNodeMapIndexObject(data), None
+
+
 def parse_dirstate(dmap, copymap, st):
     parents = [st[:20], st[20:40]]
     # dereference fields so they will be local in loop

mercurial/pvec.py

@@ -48,7 +48,7 b' Uses:'
   different branches
 '''
 
-from __future__ import absolute_import, division
+from __future__ import absolute_import
 
 from .node import nullrev
 from . import (

mercurial/pycompat.py

@@ -98,6 +98,7 b' if ispy3:'
     import codecs
     import functools
     import io
+    import locale
     import struct
 
     if os.name == r'nt' and sys.version_info >= (3, 6):
@@ -148,15 +149,36 b' if ispy3:'
     stdout = sys.stdout.buffer
     stderr = sys.stderr.buffer
 
-    # Since Python 3 converts argv to wchar_t type by Py_DecodeLocale() on Unix,
-    # we can use os.fsencode() to get back bytes argv.
-    #
-    # https://hg.python.org/cpython/file/v3.5.1/Programs/python.c#l55
-    #
-    # On Windows, the native argv is unicode and is converted to MBCS bytes
-    # since we do enable the legacy filesystem encoding.
     if getattr(sys, 'argv', None) is not None:
-        sysargv = list(map(os.fsencode, sys.argv))
+        # On POSIX, the char** argv array is converted to Python str using
+        # Py_DecodeLocale(). The inverse of this is Py_EncodeLocale(), which isn't
+        # directly callable from Python code. So, we need to emulate it.
+        # Py_DecodeLocale() calls mbstowcs() and falls back to mbrtowc() with
+        # surrogateescape error handling on failure. These functions take the
+        # current system locale into account. So, the inverse operation is to
+        # .encode() using the system locale's encoding and using the
+        # surrogateescape error handler. The only tricky part here is getting
+        # the system encoding correct, since `locale.getlocale()` can return
+        # None. We fall back to the filesystem encoding if lookups via `locale`
+        # fail, as this seems like a reasonable thing to do.
+        #
+        # On Windows, the wchar_t **argv is passed into the interpreter as-is.
+        # Like POSIX, we need to emulate what Py_EncodeLocale() would do. But
+        # there's an additional wrinkle. What we really want to access is the
+        # ANSI codepage representation of the arguments, as this is what
+        # `int main()` would receive if Python 3 didn't define `int wmain()`
+        # (this is how Python 2 worked). To get that, we encode with the mbcs
+        # encoding, which will pass CP_ACP to the underlying Windows API to
+        # produce bytes.
+        if os.name == r'nt':
+            sysargv = [a.encode("mbcs", "ignore") for a in sys.argv]
+        else:
+            encoding = (
+                locale.getlocale()[1]
+                or locale.getdefaultlocale()[1]
+                or sys.getfilesystemencoding()
+            )
+            sysargv = [a.encode(encoding, "surrogateescape") for a in sys.argv]
 
     bytechr = struct.Struct('>B').pack
     byterepr = b'%r'.__mod__
@@ -334,7 +356,7 b' if ispy3:'
         they can be passed as keyword arguments as dictonaries with bytes keys
         can't be passed as keyword arguments to functions on Python 3.
         """
-        dic = dict((k.decode('latin-1'), v) for k, v in dic.items())
+        dic = {k.decode('latin-1'): v for k, v in dic.items()}
         return dic
 
     def byteskwargs(dic):
@@ -342,7 +364,7 b' if ispy3:'
         Converts keys of python dictonaries to bytes as they were converted to
         str to pass that dictonary as a keyword argument on Python 3.
         """
-        dic = dict((k.encode('latin-1'), v) for k, v in dic.items())
+        dic = {k.encode('latin-1'): v for k, v in dic.items()}
         return dic
 
     # TODO: handle shlex.shlex().

mercurial/repair.py

@@ -351,7 +351,7 b' def _createstripbackup(repo, stripbases,'
 def safestriproots(ui, repo, nodes):
     """return list of roots of nodes where descendants are covered by nodes"""
     torev = repo.unfiltered().changelog.rev
-    revs = set(torev(n) for n in nodes)
+    revs = {torev(n) for n in nodes}
     # tostrip = wanted - unsafe = wanted - ancestors(orphaned)
     # orphaned = affected - wanted
     # affected = descendants(roots(wanted))

mercurial/revlog.py

@@ -352,6 +352,21 b' class revlogio(object):'
         return p
 
 
+NodemapRevlogIO = None
+
+if util.safehasattr(parsers, 'parse_index_devel_nodemap'):
+
+    class NodemapRevlogIO(revlogio):
+        """A debug oriented IO class that return a PersistentNodeMapIndexObject
+
+        The PersistentNodeMapIndexObject object is meant to test the persistent nodemap feature.
+        """
+
+        def parseindex(self, data, inline):
+            index, cache = parsers.parse_index_devel_nodemap(data, inline)
+            return index, cache
+
+
 class rustrevlogio(revlogio):
     def parseindex(self, data, inline):
         index, cache = super(rustrevlogio, self).parseindex(data, inline)
@@ -407,6 +422,7 b' class revlog(object):'
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +434,17 b' class revlog(object):'
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            if indexfile.endswith(b'.a'):
+                pending_path = indexfile[:-4] + b".n.a"
+                if opener.exists(pending_path):
+                    self.nodemap_file = pending_path
+                else:
+                    self.nodemap_file = indexfile[:-4] + b".n"
+            else:
+                self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -435,6 +462,7 b' class revlog(object):'
         self._maxchainlen = None
         self._deltabothparents = True
         self.index = None
+        self._nodemap_docket = None
         # Mapping of partial identifiers to full nodes.
         self._pcache = {}
         # Mapping of revision integer to full node.
@@ -591,13 +619,42 b' class revlog(object):'
 
         self._storedeltachains = True
 
+        devel_nodemap = (
+            self.nodemap_file
+            and opts.get(b'devel-force-nodemap', False)
+            and NodemapRevlogIO is not None
+        )
+
+        use_rust_index = False
+        if rustrevlog is not None:
+            if self.nodemap_file is not None:
+                use_rust_index = True
+            else:
+                use_rust_index = self.opener.options.get(b'rust.index')
+
         self._io = revlogio()
         if self.version == REVLOGV0:
             self._io = revlogoldio()
-        elif rustrevlog is not None and self.opener.options.get(b'rust.index'):
+        elif devel_nodemap:
+            self._io = NodemapRevlogIO()
+        elif use_rust_index:
             self._io = rustrevlogio()
         try:
             d = self._io.parseindex(indexdata, self._inline)
+            index, _chunkcache = d
+            use_nodemap = (
+                not self._inline
+                and self.nodemap_file is not None
+                and util.safehasattr(index, 'update_nodemap_data')
+            )
+            if use_nodemap:
+                nodemap_data = nodemaputil.persisted_data(self)
+                if nodemap_data is not None:
+                    docket = nodemap_data[0]
+                    if d[0][docket.tip_rev][7] == docket.tip_node:
+                        # no changelog tampering
+                        self._nodemap_docket = docket
+                        index.update_nodemap_data(*nodemap_data)
         except (ValueError, IndexError):
             raise error.RevlogError(
                 _(b"index %s is corrupted") % self.indexfile
@@ -708,12 +765,32 b' class revlog(object):'
             return False
         return True
 
+    def update_caches(self, transaction):
+        if self.nodemap_file is not None:
+            if transaction is None:
+                nodemaputil.update_persistent_nodemap(self)
+            else:
+                nodemaputil.setup_persistent_nodemap(transaction, self)
+
     def clearcaches(self):
         self._revisioncache = None
         self._chainbasecache.clear()
         self._chunkcache = (0, b'')
         self._pcache = {}
+        self._nodemap_docket = None
         self.index.clearcaches()
+        # The python code is the one responsible for validating the docket, we
+        # end up having to refresh it here.
+        use_nodemap = (
+            not self._inline
+            and self.nodemap_file is not None
+            and util.safehasattr(self.index, 'update_nodemap_data')
+        )
+        if use_nodemap:
+            nodemap_data = nodemaputil.persisted_data(self)
+            if nodemap_data is not None:
+                self._nodemap_docket = nodemap_data[0]
+                self.index.update_nodemap_data(*nodemap_data)
 
     def rev(self, node):
         try:
@@ -898,9 +975,6 b' class revlog(object):'
         if rustancestor is not None:
             lazyancestors = rustancestor.LazyAncestors
             arg = self.index
-        elif util.safehasattr(parsers, b'rustlazyancestors'):
-            lazyancestors = ancestor.rustlazyancestors
-            arg = self.index
         else:
             lazyancestors = ancestor.lazyancestors
             arg = self._uncheckedparentrevs
@@ -1239,7 +1313,7 b' class revlog(object):'
         else:
             start = self.rev(start)
 
-        stoprevs = set(self.rev(n) for n in stop or [])
+        stoprevs = {self.rev(n) for n in stop or []}
 
         revs = dagop.headrevssubset(
             self.revs, self.parentrevs, startrev=start, stoprevs=stoprevs
@@ -1960,6 +2034,7 b' class revlog(object):'
             # manager
 
         tr.replace(self.indexfile, trindex * self._io.size)
+        nodemaputil.setup_persistent_nodemap(tr, self)
         self._chunkclear()
 
     def _nodeduplicatecallback(self, transaction, node):
@@ -2286,6 +2361,7 b' class revlog(object):'
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """

mercurial/revlogutils/nodemap.py

@@ -7,9 +7,622 b''
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
-from .. import error
+
+import errno
+import os
+import re
+import struct
+
+from .. import (
+    error,
+    node as nodemod,
+    util,
+)
 
 
 class NodeMap(dict):
     def __missing__(self, x):
         raise error.RevlogError(b'unknown node: %s' % x)
+
+
+def persisted_data(revlog):
+    """read the nodemap for a revlog from disk"""
+    if revlog.nodemap_file is None:
+        return None
+    pdata = revlog.opener.tryread(revlog.nodemap_file)
+    if not pdata:
+        return None
+    offset = 0
+    (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size])
+    if version != ONDISK_VERSION:
+        return None
+    offset += S_VERSION.size
+    headers = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
+    uid_size, tip_rev, data_length, data_unused, tip_node_size = headers
+    offset += S_HEADER.size
+    docket = NodeMapDocket(pdata[offset : offset + uid_size])
+    offset += uid_size
+    docket.tip_rev = tip_rev
+    docket.tip_node = pdata[offset : offset + tip_node_size]
+    docket.data_length = data_length
+    docket.data_unused = data_unused
+
+    filename = _rawdata_filepath(revlog, docket)
+    use_mmap = revlog.opener.options.get(b"exp-persistent-nodemap.mmap")
+    try:
+        with revlog.opener(filename) as fd:
+            if use_mmap:
+                data = util.buffer(util.mmapread(fd, data_length))
+            else:
+                data = fd.read(data_length)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    if len(data) < data_length:
+        return None
+    return docket, data
+
+
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog._inline:
+        return  # inlined revlog are too small for this to be relevant
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+
+    # we need to happen after the changelog finalization, in that use "cl-"
+    callback_id = b"nm-revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addpending(
+        callback_id, lambda tr: _persist_nodemap(tr, revlog, pending=True)
+    )
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+class _NoTransaction(object):
+    """transaction like object to update the nodemap outside a transaction
+    """
+
+    def __init__(self):
+        self._postclose = {}
+
+    def addpostclose(self, callback_id, callback_func):
+        self._postclose[callback_id] = callback_func
+
+    def registertmp(self, *args, **kwargs):
+        pass
+
+    def addbackup(self, *args, **kwargs):
+        pass
+
+    def add(self, *args, **kwargs):
+        pass
+
+    def addabort(self, *args, **kwargs):
+        pass
+
+
+def update_persistent_nodemap(revlog):
+    """update the persistent nodemap right now
+
+    To be used for updating the nodemap on disk outside of a normal transaction
+    setup (eg, `debugupdatecache`).
+    """
+    notr = _NoTransaction()
+    _persist_nodemap(notr, revlog)
+    for k in sorted(notr._postclose):
+        notr._postclose[k](None)
+
+
+def _persist_nodemap(tr, revlog, pending=False):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+
+    can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
+    ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get(b"exp-persistent-nodemap.mmap")
+
+    data = None
+    # first attemp an incremental update of the data
+    if can_incremental and ondisk_docket is not None:
+        target_docket = revlog._nodemap_docket.copy()
+        (
+            src_docket,
+            data_changed_count,
+            data,
+        ) = revlog.index.nodemap_data_incremental()
+        new_length = target_docket.data_length + len(data)
+        new_unused = target_docket.data_unused + data_changed_count
+        if src_docket != target_docket:
+            data = None
+        elif new_length <= (new_unused * 10):  # under 10% of unused data
+            data = None
+        else:
+            datafile = _rawdata_filepath(revlog, target_docket)
+            # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+            # store vfs
+            tr.add(datafile, target_docket.data_length)
+            with revlog.opener(datafile, b'r+') as fd:
+                fd.seek(target_docket.data_length)
+                fd.write(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
+            target_docket.data_unused = new_unused
+
+    if data is None:
+        # otherwise fallback to a full new export
+        target_docket = NodeMapDocket()
+        datafile = _rawdata_filepath(revlog, target_docket)
+        if util.safehasattr(revlog.index, "nodemap_data_all"):
+            data = revlog.index.nodemap_data_all()
+        else:
+            data = persistent_data(revlog.index)
+        # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+        # store vfs
+
+        tryunlink = revlog.opener.tryunlink
+
+        def abortck(tr):
+            tryunlink(datafile)
+
+        callback_id = b"delete-%s" % datafile
+
+        # some flavor of the transaction abort does not cleanup new file, it
+        # simply empty them.
+        tr.addabort(callback_id, abortck)
+        with revlog.opener(datafile, b'w+') as fd:
+            fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
+        target_docket.data_length = len(data)
+    target_docket.tip_rev = revlog.tiprev()
+    target_docket.tip_node = revlog.node(target_docket.tip_rev)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    file_path = revlog.nodemap_file
+    if pending:
+        file_path += b'.a'
+        tr.registertmp(file_path)
+    else:
+        tr.addbackup(file_path)
+
+    with revlog.opener(file_path, b'w', atomictemp=True) as fp:
+        fp.write(target_docket.serialize())
+    revlog._nodemap_docket = target_docket
+    if feed_data:
+        revlog.index.update_nodemap_data(target_docket, new_data)
+
+    # search for old index file in all cases, some older process might have
+    # left one behind.
+    olds = _other_rawdata_filepath(revlog, target_docket)
+    if olds:
+        realvfs = getattr(revlog, '_realopener', revlog.opener)
+
+        def cleanup(tr):
+            for oldfile in olds:
+                realvfs.tryunlink(oldfile)
+
+        callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file
+        tr.addpostclose(callback_id, cleanup)
+
+
+### Nodemap docket file
+#
+# The nodemap data are stored on disk using 2 files:
+#
+# * a raw data files containing a persistent nodemap
+#   (see `Nodemap Trie` section)
+#
+# * a small "docket" file containing medatadata
+#
+# While the nodemap data can be multiple tens of megabytes, the "docket" is
+# small, it is easy to update it automatically or to duplicated its content
+# during a transaction.
+#
+# Multiple raw data can exist at the same time (The currently valid one and a
+# new one beind used by an in progress transaction). To accomodate this, the
+# filename hosting the raw data has a variable parts. The exact filename is
+# specified inside the "docket" file.
+#
+# The docket file contains information to find, qualify and validate the raw
+# data. Its content is currently very light, but it will expand as the on disk
+# nodemap gains the necessary features to be used in production.
+
+# version 0 is experimental, no BC garantee, do no use outside of tests.
+ONDISK_VERSION = 0
+S_VERSION = struct.Struct(">B")
+S_HEADER = struct.Struct(">BQQQQ")
+
+ID_SIZE = 8
+
+
+def _make_uid():
+    """return a new unique identifier.
+
+    The identifier is random and composed of ascii characters."""
+    return nodemod.hex(os.urandom(ID_SIZE))
+
+
+class NodeMapDocket(object):
+    """metadata associated with persistent nodemap data
+
+    The persistent data may come from disk or be on their way to disk.
+    """
+
+    def __init__(self, uid=None):
+        if uid is None:
+            uid = _make_uid()
+        # a unique identifier for the data file:
+        #   - When new data are appended, it is preserved.
+        #   - When a new data file is created, a new identifier is generated.
+        self.uid = uid
+        # the tipmost revision stored in the data file. This revision and all
+        # revision before it are expected to be encoded in the data file.
+        self.tip_rev = None
+        # the node of that tipmost revision, if it mismatch the current index
+        # data the docket is not valid for the current index and should be
+        # discarded.
+        #
+        # note: this method is not perfect as some destructive operation could
+        # preserve the same tip_rev + tip_node while altering lower revision.
+        # However this multiple other caches have the same vulnerability (eg:
+        # brancmap cache).
+        self.tip_node = None
+        # the size (in bytes) of the persisted data to encode the nodemap valid
+        # for `tip_rev`.
+        #   - data file shorter than this are corrupted,
+        #   - any extra data should be ignored.
+        self.data_length = None
+        # the amount (in bytes) of "dead" data, still in the data file but no
+        # longer used for the nodemap.
+        self.data_unused = 0
+
+    def copy(self):
+        new = NodeMapDocket(uid=self.uid)
+        new.tip_rev = self.tip_rev
+        new.tip_node = self.tip_node
+        new.data_length = self.data_length
+        new.data_unused = self.data_unused
+        return new
+
+    def __cmp__(self, other):
+        if self.uid < other.uid:
+            return -1
+        if self.uid > other.uid:
+            return 1
+        elif self.data_length < other.data_length:
+            return -1
+        elif self.data_length > other.data_length:
+            return 1
+        return 0
+
+    def __eq__(self, other):
+        return self.uid == other.uid and self.data_length == other.data_length
+
+    def serialize(self):
+        """return serialized bytes for a docket using the passed uid"""
+        data = []
+        data.append(S_VERSION.pack(ONDISK_VERSION))
+        headers = (
+            len(self.uid),
+            self.tip_rev,
+            self.data_length,
+            self.data_unused,
+            len(self.tip_node),
+        )
+        data.append(S_HEADER.pack(*headers))
+        data.append(self.uid)
+        data.append(self.tip_node)
+        return b''.join(data)
+
+
+def _rawdata_filepath(revlog, docket):
+    """The (vfs relative) nodemap's rawdata file for a given uid"""
+    if revlog.nodemap_file.endswith(b'.n.a'):
+        prefix = revlog.nodemap_file[:-4]
+    else:
+        prefix = revlog.nodemap_file[:-2]
+    return b"%s-%s.nd" % (prefix, docket.uid)
+
+
+def _other_rawdata_filepath(revlog, docket):
+    prefix = revlog.nodemap_file[:-2]
+    pattern = re.compile(br"(^|/)%s-[0-9a-f]+\.nd$" % prefix)
+    new_file_path = _rawdata_filepath(revlog, docket)
+    new_file_name = revlog.opener.basename(new_file_path)
+    dirpath = revlog.opener.dirname(new_file_path)
+    others = []
+    for f in revlog.opener.listdir(dirpath):
+        if pattern.match(f) and f != new_file_name:
+            others.append(f)
+    return others
+
+
+### Nodemap Trie
+#
+# This is a simple reference implementation to compute and persist a nodemap
+# trie. This reference implementation is write only. The python version of this
+# is not expected to be actually used, since it wont provide performance
+# improvement over existing non-persistent C implementation.
+#
+# The nodemap is persisted as Trie using 4bits-address/16-entries block. each
+# revision can be adressed using its node shortest prefix.
+#
+# The trie is stored as a sequence of block. Each block contains 16 entries
+# (signed 64bit integer, big endian). Each entry can be one of the following:
+#
+#  * value >=  0 -> index of sub-block
+#  * value == -1 -> no value
+#  * value <  -1 -> a revision value: rev = -(value+10)
+#
+# The implementation focus on simplicity, not on performance. A Rust
+# implementation should provide a efficient version of the same binary
+# persistence. This reference python implementation is never meant to be
+# extensively use in production.
+
+
+def persistent_data(index):
+    """return the persistent binary form for a nodemap for a given index
+    """
+    trie = _build_trie(index)
+    return _persist_trie(trie)
+
+
+def update_persistent_data(index, root, max_idx, last_rev):
+    """return the incremental update for persistent nodemap from a given index
+    """
+    changed_block, trie = _update_trie(index, root, last_rev)
+    return (
+        changed_block * S_BLOCK.size,
+        _persist_trie(trie, existing_idx=max_idx),
+    )
+
+
+S_BLOCK = struct.Struct(">" + ("l" * 16))
+
+NO_ENTRY = -1
+# rev 0 need to be -2 because 0 is used by block, -1 is a special value.
+REV_OFFSET = 2
+
+
+def _transform_rev(rev):
+    """Return the number used to represent the rev in the tree.
+
+    (or retrieve a rev number from such representation)
+
+    Note that this is an involution, a function equal to its inverse (i.e.
+    which gives the identity when applied to itself).
+    """
+    return -(rev + REV_OFFSET)
+
+
+def _to_int(hex_digit):
+    """turn an hexadecimal digit into a proper integer"""
+    return int(hex_digit, 16)
+
+
+class Block(dict):
+    """represent a block of the Trie
+
+    contains up to 16 entry indexed from 0 to 15"""
+
+    def __init__(self):
+        super(Block, self).__init__()
+        # If this block exist on disk, here is its ID
+        self.ondisk_id = None
+
+    def __iter__(self):
+        return iter(self.get(i) for i in range(16))
+
+
+def _build_trie(index):
+    """build a nodemap trie
+
+    The nodemap stores revision number for each unique prefix.
+
+    Each block is a dictionary with keys in `[0, 15]`. Values are either
+    another block or a revision number.
+    """
+    root = Block()
+    for rev in range(len(index)):
+        hex = nodemod.hex(index[rev][7])
+        _insert_into_block(index, 0, root, rev, hex)
+    return root
+
+
+def _update_trie(index, root, last_rev):
+    """consume"""
+    changed = 0
+    for rev in range(last_rev + 1, len(index)):
+        hex = nodemod.hex(index[rev][7])
+        changed += _insert_into_block(index, 0, root, rev, hex)
+    return changed, root
+
+
+def _insert_into_block(index, level, block, current_rev, current_hex):
+    """insert a new revision in a block
+
+    index: the index we are adding revision for
+    level: the depth of the current block in the trie
+    block: the block currently being considered
+    current_rev: the revision number we are adding
+    current_hex: the hexadecimal representation of the of that revision
+    """
+    changed = 1
+    if block.ondisk_id is not None:
+        block.ondisk_id = None
+    hex_digit = _to_int(current_hex[level : level + 1])
+    entry = block.get(hex_digit)
+    if entry is None:
+        # no entry, simply store the revision number
+        block[hex_digit] = current_rev
+    elif isinstance(entry, dict):
+        # need to recurse to an underlying block
+        changed += _insert_into_block(
+            index, level + 1, entry, current_rev, current_hex
+        )
+    else:
+        # collision with a previously unique prefix, inserting new
+        # vertices to fit both entry.
+        other_hex = nodemod.hex(index[entry][7])
+        other_rev = entry
+        new = Block()
+        block[hex_digit] = new
+        _insert_into_block(index, level + 1, new, other_rev, other_hex)
+        _insert_into_block(index, level + 1, new, current_rev, current_hex)
+    return changed
+
+
+def _persist_trie(root, existing_idx=None):
+    """turn a nodemap trie into persistent binary data
+
+    See `_build_trie` for nodemap trie structure"""
+    block_map = {}
+    if existing_idx is not None:
+        base_idx = existing_idx + 1
+    else:
+        base_idx = 0
+    chunks = []
+    for tn in _walk_trie(root):
+        if tn.ondisk_id is not None:
+            block_map[id(tn)] = tn.ondisk_id
+        else:
+            block_map[id(tn)] = len(chunks) + base_idx
+            chunks.append(_persist_block(tn, block_map))
+    return b''.join(chunks)
+
+
+def _walk_trie(block):
+    """yield all the block in a trie
+
+    Children blocks are always yield before their parent block.
+    """
+    for (__, item) in sorted(block.items()):
+        if isinstance(item, dict):
+            for sub_block in _walk_trie(item):
+                yield sub_block
+    yield block
+
+
+def _persist_block(block_node, block_map):
+    """produce persistent binary data for a single block
+
+    Children block are assumed to be already persisted and present in
+    block_map.
+    """
+    data = tuple(_to_value(v, block_map) for v in block_node)
+    return S_BLOCK.pack(*data)
+
+
+def _to_value(item, block_map):
+    """persist any value as an integer"""
+    if item is None:
+        return NO_ENTRY
+    elif isinstance(item, dict):
+        return block_map[id(item)]
+    else:
+        return _transform_rev(item)
+
+
+def parse_data(data):
+    """parse parse nodemap data into a nodemap Trie"""
+    if (len(data) % S_BLOCK.size) != 0:
+        msg = "nodemap data size is not a multiple of block size (%d): %d"
+        raise error.Abort(msg % (S_BLOCK.size, len(data)))
+    if not data:
+        return Block(), None
+    block_map = {}
+    new_blocks = []
+    for i in range(0, len(data), S_BLOCK.size):
+        block = Block()
+        block.ondisk_id = len(block_map)
+        block_map[block.ondisk_id] = block
+        block_data = data[i : i + S_BLOCK.size]
+        values = S_BLOCK.unpack(block_data)
+        new_blocks.append((block, values))
+    for b, values in new_blocks:
+        for idx, v in enumerate(values):
+            if v == NO_ENTRY:
+                continue
+            elif v >= 0:
+                b[idx] = block_map[v]
+            else:
+                b[idx] = _transform_rev(v)
+    return block, i // S_BLOCK.size
+
+
+# debug utility
+
+
+def check_data(ui, index, data):
+    """verify that the provided nodemap data are valid for the given idex"""
+    ret = 0
+    ui.status((b"revision in index:   %d\n") % len(index))
+    root, __ = parse_data(data)
+    all_revs = set(_all_revisions(root))
+    ui.status((b"revision in nodemap: %d\n") % len(all_revs))
+    for r in range(len(index)):
+        if r not in all_revs:
+            msg = b"  revision missing from nodemap: %d\n" % r
+            ui.write_err(msg)
+            ret = 1
+        else:
+            all_revs.remove(r)
+        nm_rev = _find_node(root, nodemod.hex(index[r][7]))
+        if nm_rev is None:
+            msg = b"  revision node does not match any entries: %d\n" % r
+            ui.write_err(msg)
+            ret = 1
+        elif nm_rev != r:
+            msg = (
+                b"  revision node does not match the expected revision: "
+                b"%d != %d\n" % (r, nm_rev)
+            )
+            ui.write_err(msg)
+            ret = 1
+
+    if all_revs:
+        for r in sorted(all_revs):
+            msg = b"  extra revision in  nodemap: %d\n" % r
+            ui.write_err(msg)
+        ret = 1
+    return ret
+
+
+def _all_revisions(root):
+    """return all revisions stored in a Trie"""
+    for block in _walk_trie(root):
+        for v in block:
+            if v is None or isinstance(v, Block):
+                continue
+            yield v
+
+
+def _find_node(block, node):
+    """find the revision associated with a given node"""
+    entry = block.get(_to_int(node[0:1]))
+    if isinstance(entry, dict):
+        return _find_node(entry, node[1:])
+    return entry

mercurial/revset.py

@@ -247,7 +247,15 b' def notset(repo, subset, x, order):'
 
 
 def relationset(repo, subset, x, y, order):
-    raise error.ParseError(_(b"can't use a relation in this context"))
+    # this is pretty basic implementation of 'x#y' operator, still
+    # experimental so undocumented. see the wiki for further ideas.
+    # https://www.mercurial-scm.org/wiki/RevsetOperatorPlan
+    rel = getsymbol(y)
+    if rel in relations:
+        return relations[rel](repo, subset, x, rel, order)
+
+    relnames = [r for r in relations.keys() if len(r) > 1]
+    raise error.UnknownIdentifier(rel, relnames)
 
 
 def _splitrange(a, b):
@@ -281,7 +289,12 b' def _splitrange(a, b):'
     return ancdepths, descdepths
 
 
-def generationsrel(repo, subset, x, rel, z, order):
+def generationsrel(repo, subset, x, rel, order):
+    z = (b'rangeall', None)
+    return generationssubrel(repo, subset, x, rel, z, order)
+
+
+def generationssubrel(repo, subset, x, rel, z, order):
     # TODO: rewrite tests, and drop startdepth argument from ancestors() and
     # descendants() predicates
     a, b = getintrange(
@@ -769,6 +782,38 b' def commonancestors(repo, subset, x):'
     return subset
 
 
+@predicate(b'conflictlocal()', safe=True)
+def conflictlocal(repo, subset, x):
+    """The local side of the merge, if currently in an unresolved merge.
+
+    "merge" here includes merge conflicts from e.g. 'hg rebase' or 'hg graft'.
+    """
+    getargs(x, 0, 0, _(b"conflictlocal takes no arguments"))
+    from . import merge
+
+    mergestate = merge.mergestate.read(repo)
+    if mergestate.active() and repo.changelog.hasnode(mergestate.local):
+        return subset & {repo.changelog.rev(mergestate.local)}
+
+    return baseset()
+
+
+@predicate(b'conflictother()', safe=True)
+def conflictother(repo, subset, x):
+    """The other side of the merge, if currently in an unresolved merge.
+
+    "merge" here includes merge conflicts from e.g. 'hg rebase' or 'hg graft'.
+    """
+    getargs(x, 0, 0, _(b"conflictother takes no arguments"))
+    from . import merge
+
+    mergestate = merge.mergestate.read(repo)
+    if mergestate.active() and repo.changelog.hasnode(mergestate.other):
+        return subset & {repo.changelog.rev(mergestate.other)}
+
+    return baseset()
+
+
 @predicate(b'contains(pattern)', weight=100)
 def contains(repo, subset, x):
     """The revision's manifest contains a file matching pattern (but might not
@@ -1022,7 +1067,7 b' def extdata(repo, subset, x):'
 
 @predicate(b'extinct()', safe=True)
 def extinct(repo, subset, x):
-    """Obsolete changesets with obsolete descendants only.
+    """Obsolete changesets with obsolete descendants only. (EXPERIMENTAL)
     """
     # i18n: "extinct" is a keyword
     getargs(x, 0, 0, _(b"extinct takes no arguments"))
@@ -1670,7 +1715,7 b' def none(repo, subset, x):'
 
 @predicate(b'obsolete()', safe=True)
 def obsolete(repo, subset, x):
-    """Mutable changeset with a newer version."""
+    """Mutable changeset with a newer version. (EXPERIMENTAL)"""
     # i18n: "obsolete" is a keyword
     getargs(x, 0, 0, _(b"obsolete takes no arguments"))
     obsoletes = obsmod.getrevs(repo, b'obsolete')
@@ -1843,7 +1888,7 b' def parents(repo, subset, x):'
     The set of all parents for all changesets in set, or the working directory.
     """
     if x is None:
-        ps = set(p.rev() for p in repo[x].parents())
+        ps = {p.rev() for p in repo[x].parents()}
     else:
         ps = set()
         cl = repo.changelog
@@ -2050,19 +2095,11 b' def removes(repo, subset, x):'
 
 @predicate(b'rev(number)', safe=True)
 def rev(repo, subset, x):
-    """Revision with the given numeric identifier.
-    """
-    # i18n: "rev" is a keyword
-    l = getargs(x, 1, 1, _(b"rev requires one argument"))
+    """Revision with the given numeric identifier."""
     try:
-        # i18n: "rev" is a keyword
-        l = int(getstring(l[0], _(b"rev requires a number")))
-    except (TypeError, ValueError):
-        # i18n: "rev" is a keyword
-        raise error.ParseError(_(b"rev expects a number"))
-    if l not in repo.changelog and l not in _virtualrevs:
+        return _rev(repo, subset, x)
+    except error.RepoLookupError:
         return baseset()
-    return subset & baseset([l])
 
 
 @predicate(b'_rev(number)', safe=True)
@@ -2076,7 +2113,11 b' def _rev(repo, subset, x):'
     except (TypeError, ValueError):
         # i18n: "rev" is a keyword
         raise error.ParseError(_(b"rev expects a number"))
-    repo.changelog.node(l)  # check that the rev exists
+    if l not in _virtualrevs:
+        try:
+            repo.changelog.node(l)  # check that the rev exists
+        except IndexError:
+            raise error.RepoLookupError(_(b"unknown revision '%d'") % l)
     return subset & baseset([l])
 
 
@@ -2405,14 +2446,15 b' def _mapbynodefunc(repo, s, f):'
     cl = repo.unfiltered().changelog
     torev = cl.index.get_rev
     tonode = cl.node
-    result = set(torev(n) for n in f(tonode(r) for r in s))
+    result = {torev(n) for n in f(tonode(r) for r in s)}
     result.discard(None)
     return smartset.baseset(result - repo.changelog.filteredrevs)
 
 
 @predicate(b'successors(set)', safe=True)
 def successors(repo, subset, x):
-    """All successors for set, including the given set themselves"""
+    """All successors for set, including the given set themselves.
+    (EXPERIMENTAL)"""
     s = getset(repo, fullreposet(repo), x)
     f = lambda nodes: obsutil.allsuccessors(repo.obsstore, nodes)
     d = _mapbynodefunc(repo, s, f)
@@ -2479,6 +2521,19 b' def orphan(repo, subset, x):'
     return subset & orphan
 
 
+@predicate(b'unstable()', safe=True)
+def unstable(repo, subset, x):
+    """Changesets with instabilities. (EXPERIMENTAL)
+    """
+    # i18n: "unstable" is a keyword
+    getargs(x, 0, 0, b'unstable takes no arguments')
+    _unstable = set()
+    _unstable.update(obsmod.getrevs(repo, b'orphan'))
+    _unstable.update(obsmod.getrevs(repo, b'phasedivergent'))
+    _unstable.update(obsmod.getrevs(repo, b'contentdivergent'))
+    return subset & baseset(_unstable)
+
+
 @predicate(b'user(string)', safe=True, weight=10)
 def user(repo, subset, x):
     """User name contains string. The match is case-insensitive.
@@ -2605,11 +2660,16 b' methods = {'
     b"smartset": rawsmartset,
 }
 
-subscriptrelations = {
+relations = {
     b"g": generationsrel,
     b"generations": generationsrel,
 }
 
+subscriptrelations = {
+    b"g": generationssubrel,
+    b"generations": generationssubrel,
+}
+
 
 def lookupfn(repo):
     return lambda symbol: scmutil.isrevsymbol(repo, symbol)

mercurial/scmutil.py

@@ -1457,10 +1457,10 b' def movedirstate(repo, newctx, match=Non'
     # Merge old parent and old working dir copies
     oldcopies = copiesmod.pathcopies(newctx, oldctx, match)
     oldcopies.update(copies)
-    copies = dict(
-        (dst, oldcopies.get(src, src))
+    copies = {
+        dst: oldcopies.get(src, src)
         for dst, src in pycompat.iteritems(oldcopies)
-    )
+    }
     # Adjust the dirstate copies
     for dst, src in pycompat.iteritems(copies):
         if src not in newctx or dst in newctx or ds[dst] != b'a':
@@ -1900,8 +1900,11 b' fileprefetchhooks = util.hooks()'
 _reportstroubledchangesets = True
 
 
-def registersummarycallback(repo, otr, txnname=b''):
+def registersummarycallback(repo, otr, txnname=b'', as_validator=False):
     """register a callback to issue a summary after the transaction is closed
+
+    If as_validator is true, then the callbacks are registered as transaction
+    validators instead
     """
 
     def txmatch(sources):
@@ -1927,7 +1930,10 b' def registersummarycallback(repo, otr, t'
             func(repo, tr)
 
         newcat = b'%02i-txnreport' % len(categories)
-        otr.addpostclose(newcat, wrapped)
+        if as_validator:
+            otr.addvalidator(newcat, wrapped)
+        else:
+            otr.addpostclose(newcat, wrapped)
         categories.append(newcat)
         return wrapped
 
@@ -1942,6 +1948,8 b' def registersummarycallback(repo, otr, t'
             if cgheads:
                 htext = _(b" (%+d heads)") % cgheads
             msg = _(b"added %d changesets with %d changes to %d files%s\n")
+            if as_validator:
+                msg = _(b"adding %d changesets with %d changes to %d files%s\n")
             assert repo is not None  # help pytype
             repo.ui.status(msg % (cgchangesets, cgrevisions, cgfiles, htext))
 
@@ -1954,7 +1962,10 b' def registersummarycallback(repo, otr, t'
             if newmarkers:
                 repo.ui.status(_(b'%i new obsolescence markers\n') % newmarkers)
             if obsoleted:
-                repo.ui.status(_(b'obsoleted %i changesets\n') % len(obsoleted))
+                msg = _(b'obsoleted %i changesets\n')
+                if as_validator:
+                    msg = _(b'obsoleting %i changesets\n')
+                repo.ui.status(msg % len(obsoleted))
 
     if obsolete.isenabled(
         repo, obsolete.createmarkersopt
@@ -2047,19 +2058,17 b' def registersummarycallback(repo, otr, t'
             pull/unbundle.
             """
             origrepolen = tr.changes.get(b'origrepolen', len(repo))
-            phasetracking = tr.changes.get(b'phases', {})
-            if not phasetracking:
-                return
-            published = [
-                rev
-                for rev, (old, new) in pycompat.iteritems(phasetracking)
-                if new == phases.public and rev < origrepolen
-            ]
+            published = []
+            for revs, (old, new) in tr.changes.get(b'phases', []):
+                if new != phases.public:
+                    continue
+                published.extend(rev for rev in revs if rev < origrepolen)
             if not published:
                 return
-            repo.ui.status(
-                _(b'%d local changesets published\n') % len(published)
-            )
+            msg = _(b'%d local changesets published\n')
+            if as_validator:
+                msg = _(b'%d local changesets will be published\n')
+            repo.ui.status(msg % len(published))
 
 
 def getinstabilitymessage(delta, instability):

mercurial/shelve.py

@@ -745,7 +745,7 b' def unshelveabort(ui, repo, state):'
         try:
             checkparents(repo, state)
 
-            merge.update(repo, state.pendingctx, branchmerge=False, force=True)
+            merge.clean_update(state.pendingctx)
             if state.activebookmark and state.activebookmark in repo._bookmarks:
                 bookmarks.activate(repo, state.activebookmark)
             mergefiles(ui, repo, state.wctx, state.pendingctx)
@@ -827,10 +827,6 b' def unshelvecontinue(ui, repo, state, op'
                 )
 
         if newnode is None:
-            # If it ended up being a no-op commit, then the normal
-            # merge state clean-up path doesn't happen, so do it
-            # here. Fix issue5494
-            merge.mergestate.clean(repo)
             shelvectx = state.pendingctx
             msg = _(
                 b'note: unshelved changes already existed '
@@ -996,7 +992,6 b' def _rebaserestoredcommit('
         stats = merge.graft(
             repo,
             shelvectx,
-            shelvectx.p1(),
             labels=[b'working-copy', b'shelve'],
             keepconflictparent=True,
         )
@@ -1032,10 +1027,6 b' def _rebaserestoredcommit('
             )
 
         if newnode is None:
-            # If it ended up being a no-op commit, then the normal
-            # merge state clean-up path doesn't happen, so do it
-            # here. Fix issue5494
-            merge.mergestate.clean(repo)
             shelvectx = tmpwctx
             msg = _(
                 b'note: unshelved changes already existed '
@@ -1083,7 +1074,7 b' def _checkunshelveuntrackedproblems(ui, '
         raise error.Abort(m, hint=hint)
 
 
-def dounshelve(ui, repo, *shelved, **opts):
+def unshelvecmd(ui, repo, *shelved, **opts):
     opts = pycompat.byteskwargs(opts)
     abortf = opts.get(b'abort')
     continuef = opts.get(b'continue')
@@ -1130,6 +1121,10 b' def dounshelve(ui, repo, *shelved, **opt'
     if not shelvedfile(repo, basename, patchextension).exists():
         raise error.Abort(_(b"shelved change '%s' not found") % basename)
 
+    return _dounshelve(ui, repo, basename, opts)
+
+
+def _dounshelve(ui, repo, basename, opts):
     repo = repo.unfiltered()
     lock = tr = None
     try:

mercurial/store.py

@@ -137,7 +137,7 b' def _buildencodefun():'
     asciistr = list(map(xchr, range(127)))
     capitals = list(range(ord(b"A"), ord(b"Z") + 1))
 
-    cmap = dict((x, x) for x in asciistr)
+    cmap = {x: x for x in asciistr}
     for x in _reserved():
         cmap[xchr(x)] = b"~%02x" % x
     for x in capitals + [ord(e)]:
@@ -200,7 +200,7 b' def _buildlowerencodefun():'
     'the~07quick~adshot'
     '''
     xchr = pycompat.bytechr
-    cmap = dict([(xchr(x), xchr(x)) for x in pycompat.xrange(127)])
+    cmap = {xchr(x): xchr(x) for x in pycompat.xrange(127)}
     for x in _reserved():
         cmap[xchr(x)] = b"~%02x" % x
     for x in range(ord(b"A"), ord(b"Z") + 1):

mercurial/subrepo.py

@@ -806,7 +806,7 b' class hgsubrepo(abstractsubrepo):'
                 self.ui.debug(
                     b'merging subrepository "%s"\n' % subrelpath(self)
                 )
-                hg.merge(self._repo, state[1], remind=False)
+                hg.merge(dst, remind=False)
 
         wctx = self._repo[None]
         if self.dirty():

mercurial/tags.py

@@ -720,15 +720,20 b' class hgtagsfnodescache(object):'
 
         self._dirtyoffset = None
 
-        if rawlen < wantedlen:
-            self._dirtyoffset = rawlen
-            self._raw.extend(b'\xff' * (wantedlen - rawlen))
-        elif rawlen > wantedlen:
+        rawlentokeep = min(
+            wantedlen, (rawlen // _fnodesrecsize) * _fnodesrecsize
+        )
+        if rawlen > rawlentokeep:
             # There's no easy way to truncate array instances. This seems
             # slightly less evil than copying a potentially large array slice.
-            for i in range(rawlen - wantedlen):
+            for i in range(rawlen - rawlentokeep):
                 self._raw.pop()
-            self._dirtyoffset = len(self._raw)
+            rawlen = len(self._raw)
+            self._dirtyoffset = rawlen
+        if rawlen < wantedlen:
+            if self._dirtyoffset is None:
+                self._dirtyoffset = rawlen
+            self._raw.extend(b'\xff' * (wantedlen - rawlen))
 
     def getfnode(self, node, computemissing=True):
         """Obtain the filenode of the .hgtags file at a specified revision.