merge default into stable for 5.1 release

contrib/automation/hgautomation/linux.py

@@ -0,0 +1,545 b''
+# linux.py - Linux specific automation functionality
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import os
+import pathlib
+import shlex
+import subprocess
+import tempfile
+
+from .ssh import (
+    exec_command,
+)
+
+
+# Linux distributions that are supported.
+DISTROS = {
+    'debian9',
+    'ubuntu18.04',
+    'ubuntu18.10',
+    'ubuntu19.04',
+}
+
+INSTALL_PYTHONS = r'''
+PYENV2_VERSIONS="2.7.16 pypy2.7-7.1.1"
+PYENV3_VERSIONS="3.5.7 3.6.8 3.7.3 3.8-dev pypy3.5-7.0.0 pypy3.6-7.1.1"
+
+git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
+pushd /hgdev/pyenv
+git checkout 3faeda67bb33e07750d1a104271369a7384ca45c
+popd
+
+export PYENV_ROOT="/hgdev/pyenv"
+export PATH="$PYENV_ROOT/bin:$PATH"
+
+# pip 19.0.3.
+PIP_SHA256=efe99298f3fbb1f56201ce6b81d2658067d2f7d7dfc2d412e0d3cacc9a397c61
+wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/fee32c376da1ff6496a798986d7939cd51e1644f/get-pip.py
+echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
+
+VIRTUALENV_SHA256=984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39
+VIRTUALENV_TARBALL=virtualenv-16.4.3.tar.gz
+wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/${VIRTUALENV_TARBALL}
+echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
+
+for v in ${PYENV2_VERSIONS}; do
+    pyenv install -v ${v}
+    ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
+    ${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
+    ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
+done
+
+for v in ${PYENV3_VERSIONS}; do
+    pyenv install -v ${v}
+    ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
+    ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
+done
+
+pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
+'''.lstrip().replace('\r\n', '\n')
+
+
+BOOTSTRAP_VIRTUALENV = r'''
+/usr/bin/virtualenv /hgdev/venv-bootstrap
+
+HG_SHA256=1bdd21bb87d1e05fb5cd395d488d0e0cc2f2f90ce0fd248e31a03595da5ccb47
+HG_TARBALL=mercurial-4.9.1.tar.gz
+
+wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
+echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
+
+/hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
+'''.lstrip().replace('\r\n', '\n')
+
+
+BOOTSTRAP_DEBIAN = r'''
+#!/bin/bash
+
+set -ex
+
+DISTRO=`grep DISTRIB_ID /etc/lsb-release  | awk -F= '{{print $2}}'`
+DEBIAN_VERSION=`cat /etc/debian_version`
+LSB_RELEASE=`lsb_release -cs`
+
+sudo /usr/sbin/groupadd hg
+sudo /usr/sbin/groupadd docker
+sudo /usr/sbin/useradd -g hg -G sudo,docker -d /home/hg -m -s /bin/bash hg
+sudo mkdir /home/hg/.ssh
+sudo cp ~/.ssh/authorized_keys /home/hg/.ssh/authorized_keys
+sudo chown -R hg:hg /home/hg/.ssh
+sudo chmod 700 /home/hg/.ssh
+sudo chmod 600 /home/hg/.ssh/authorized_keys
+
+cat << EOF | sudo tee /etc/sudoers.d/90-hg
+hg ALL=(ALL) NOPASSWD:ALL
+EOF
+
+sudo apt-get update
+sudo DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade
+
+# Install packages necessary to set up Docker Apt repo.
+sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends \
+    apt-transport-https \
+    gnupg
+
+cat > docker-apt-key << EOF
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth
+lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh
+38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq
+L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7
+UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N
+cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht
+ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo
+vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD
+G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ
+XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj
+q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB
+tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
+BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO
+v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd
+tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk
+jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m
+6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P
+XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc
+FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8
+g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm
+ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh
+9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5
+G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW
+FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB
+EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF
+M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx
+Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu
+w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk
+z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8
+eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb
+VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa
+1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X
+zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ
+pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7
+ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ
+BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY
+1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp
+YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI
+mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES
+KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7
+JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ
+cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0
+6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5
+U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z
+VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f
+irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk
+SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz
+QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W
+9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw
+24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe
+dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y
+Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR
+H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh
+/nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ
+M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S
+xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O
+jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG
+YT90qFF93M3v01BbxP+EIY2/9tiIPbrd
+=0YYh
+-----END PGP PUBLIC KEY BLOCK-----
+EOF
+
+sudo apt-key add docker-apt-key
+
+if [ "$DEBIAN_VERSION" = "9.8" ]; then
+cat << EOF | sudo tee -a /etc/apt/sources.list
+# Need backports for clang-format-6.0
+deb http://deb.debian.org/debian stretch-backports main
+
+# Sources are useful if we want to compile things locally.
+deb-src http://deb.debian.org/debian stretch main
+deb-src http://security.debian.org/debian-security stretch/updates main
+deb-src http://deb.debian.org/debian stretch-updates main
+deb-src http://deb.debian.org/debian stretch-backports main
+
+deb [arch=amd64] https://download.docker.com/linux/debian stretch stable
+EOF
+
+elif [ "$DISTRO" = "Ubuntu" ]; then
+cat << EOF | sudo tee -a /etc/apt/sources.list
+deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
+EOF
+
+fi
+
+sudo apt-get update
+
+PACKAGES="\
+    btrfs-progs \
+    build-essential \
+    bzr \
+    clang-format-6.0 \
+    cvs \
+    darcs \
+    debhelper \
+    devscripts \
+    dpkg-dev \
+    dstat \
+    emacs \
+    gettext \
+    git \
+    htop \
+    iotop \
+    jfsutils \
+    libbz2-dev \
+    libexpat1-dev \
+    libffi-dev \
+    libgdbm-dev \
+    liblzma-dev \
+    libncurses5-dev \
+    libnss3-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    netbase \
+    ntfs-3g \
+    nvme-cli \
+    pyflakes \
+    pyflakes3 \
+    pylint \
+    pylint3 \
+    python-all-dev \
+    python-dev \
+    python-docutils \
+    python-fuzzywuzzy \
+    python-pygments \
+    python-subversion \
+    python-vcr \
+    python3-dev \
+    python3-docutils \
+    python3-fuzzywuzzy \
+    python3-pygments \
+    python3-vcr \
+    rsync \
+    sqlite3 \
+    subversion \
+    tcl-dev \
+    tk-dev \
+    tla \
+    unzip \
+    uuid-dev \
+    vim \
+    virtualenv \
+    wget \
+    xfsprogs \
+    zip \
+    zlib1g-dev"
+
+if [ "$DEBIAN_VERSION" = "9.8" ]; then
+    PACKAGES="$PACKAGES linux-perf"
+elif [ "$DISTRO" = "Ubuntu" ]; then
+    PACKAGES="$PACKAGES linux-tools-common"
+fi
+
+# Ubuntu 19.04 removes monotone.
+if [ "$LSB_RELEASE" != "disco" ]; then
+    PACKAGES="$PACKAGES monotone"
+fi
+
+# As of April 27, 2019, Docker hasn't published packages for
+# Ubuntu 19.04 yet.
+if [ "$LSB_RELEASE" != "disco" ]; then
+    PACKAGES="$PACKAGES docker-ce"
+fi
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
+
+# Create clang-format symlink so test harness finds it.
+sudo update-alternatives --install /usr/bin/clang-format clang-format \
+    /usr/bin/clang-format-6.0 1000
+
+sudo mkdir /hgdev
+# Will be normalized to hg:hg later.
+sudo chown `whoami` /hgdev
+
+cp requirements-py2.txt /hgdev/requirements-py2.txt
+cp requirements-py3.txt /hgdev/requirements-py3.txt
+
+# Disable the pip version check because it uses the network and can
+# be annoying.
+cat << EOF | sudo tee -a /etc/pip.conf
+[global]
+disable-pip-version-check = True
+EOF
+
+{install_pythons}
+{bootstrap_virtualenv}
+
+/hgdev/venv-bootstrap/bin/hg clone https://www.mercurial-scm.org/repo/hg /hgdev/src
+
+# Mark the repo as non-publishing.
+cat >> /hgdev/src/.hg/hgrc << EOF
+[phases]
+publish = false
+EOF
+
+sudo chown -R hg:hg /hgdev
+'''.lstrip().format(
+    install_pythons=INSTALL_PYTHONS,
+    bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV
+).replace('\r\n', '\n')
+
+
+# Prepares /hgdev for operations.
+PREPARE_HGDEV = '''
+#!/bin/bash
+
+set -e
+
+FS=$1
+
+ensure_device() {
+    if [ -z "${DEVICE}" ]; then
+        echo "could not find block device to format"
+        exit 1
+    fi
+}
+
+# Determine device to partition for extra filesystem.
+# If only 1 volume is present, it will be the root volume and
+# should be /dev/nvme0. If multiple volumes are present, the
+# root volume could be nvme0 or nvme1. Use whichever one doesn't have
+# a partition.
+if [ -e /dev/nvme1n1 ]; then
+    if [ -e /dev/nvme0n1p1 ]; then
+        DEVICE=/dev/nvme1n1
+    else
+        DEVICE=/dev/nvme0n1
+    fi
+else
+    DEVICE=
+fi
+
+sudo mkdir /hgwork
+
+if [ "${FS}" != "default" -a "${FS}" != "tmpfs" ]; then
+    ensure_device
+    echo "creating ${FS} filesystem on ${DEVICE}"
+fi
+
+if [ "${FS}" = "default" ]; then
+    :
+
+elif [ "${FS}" = "btrfs" ]; then
+    sudo mkfs.btrfs ${DEVICE}
+    sudo mount ${DEVICE} /hgwork
+
+elif [ "${FS}" = "ext3" ]; then
+    # lazy_journal_init speeds up filesystem creation at the expense of
+    # integrity if things crash. We are an ephemeral instance, so we don't
+    # care about integrity.
+    sudo mkfs.ext3 -E lazy_journal_init=1 ${DEVICE}
+    sudo mount ${DEVICE} /hgwork
+
+elif [ "${FS}" = "ext4" ]; then
+    sudo mkfs.ext4 -E lazy_journal_init=1 ${DEVICE}
+    sudo mount ${DEVICE} /hgwork
+
+elif [ "${FS}" = "jfs" ]; then
+    sudo mkfs.jfs ${DEVICE}
+    sudo mount ${DEVICE} /hgwork
+
+elif [ "${FS}" = "tmpfs" ]; then
+    echo "creating tmpfs volume in /hgwork"
+    sudo mount -t tmpfs -o size=1024M tmpfs /hgwork
+
+elif [ "${FS}" = "xfs" ]; then
+    sudo mkfs.xfs ${DEVICE}
+    sudo mount ${DEVICE} /hgwork
+
+else
+    echo "unsupported filesystem: ${FS}"
+    exit 1
+fi
+
+echo "/hgwork ready"
+
+sudo chown hg:hg /hgwork
+mkdir /hgwork/tmp
+chown hg:hg /hgwork/tmp
+
+rsync -a /hgdev/src /hgwork/
+'''.lstrip().replace('\r\n', '\n')
+
+
+HG_UPDATE_CLEAN = '''
+set -ex
+
+HG=/hgdev/venv-bootstrap/bin/hg
+
+cd /hgwork/src
+${HG} --config extensions.purge= purge --all
+${HG} update -C $1
+${HG} log -r .
+'''.lstrip().replace('\r\n', '\n')
+
+
+def prepare_exec_environment(ssh_client, filesystem='default'):
+    """Prepare an EC2 instance to execute things.
+
+    The AMI has an ``/hgdev`` bootstrapped with various Python installs
+    and a clone of the Mercurial repo.
+
+    In EC2, EBS volumes launched from snapshots have wonky performance behavior.
+    Notably, blocks have to be copied on first access, which makes volume
+    I/O extremely slow on fresh volumes.
+
+    Furthermore, we may want to run operations, tests, etc on alternative
+    filesystems so we examine behavior on different filesystems.
+
+    This function is used to facilitate executing operations on alternate
+    volumes.
+    """
+    sftp = ssh_client.open_sftp()
+
+    with sftp.open('/hgdev/prepare-hgdev', 'wb') as fh:
+        fh.write(PREPARE_HGDEV)
+        fh.chmod(0o0777)
+
+    command = 'sudo /hgdev/prepare-hgdev %s' % filesystem
+    chan, stdin, stdout = exec_command(ssh_client, command)
+    stdin.close()
+
+    for line in stdout:
+        print(line, end='')
+
+    res = chan.recv_exit_status()
+
+    if res:
+        raise Exception('non-0 exit code updating working directory; %d'
+                        % res)
+
+
+def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
+    """Synchronize a local Mercurial source path to remote EC2 instance."""
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_dir = pathlib.Path(temp_dir)
+
+        ssh_dir = temp_dir / '.ssh'
+        ssh_dir.mkdir()
+        ssh_dir.chmod(0o0700)
+
+        public_ip = ec2_instance.public_ip_address
+
+        ssh_config = ssh_dir / 'config'
+
+        with ssh_config.open('w', encoding='utf-8') as fh:
+            fh.write('Host %s\n' % public_ip)
+            fh.write('  User hg\n')
+            fh.write('  StrictHostKeyChecking no\n')
+            fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
+            fh.write('  IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
+
+        if not (source_path / '.hg').is_dir():
+            raise Exception('%s is not a Mercurial repository; synchronization '
+                            'not yet supported' % source_path)
+
+        env = dict(os.environ)
+        env['HGPLAIN'] = '1'
+        env['HGENCODING'] = 'utf-8'
+
+        hg_bin = source_path / 'hg'
+
+        res = subprocess.run(
+            ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
+            cwd=str(source_path), env=env, check=True, capture_output=True)
+
+        full_revision = res.stdout.decode('ascii')
+
+        args = [
+            'python2.7', str(hg_bin),
+            '--config', 'ui.ssh=ssh -F %s' % ssh_config,
+            '--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
+            'push', '-f', '-r', full_revision,
+            'ssh://%s//hgwork/src' % public_ip,
+        ]
+
+        subprocess.run(args, cwd=str(source_path), env=env, check=True)
+
+        # TODO support synchronizing dirty working directory.
+
+        sftp = ec2_instance.ssh_client.open_sftp()
+
+        with sftp.open('/hgdev/hgup', 'wb') as fh:
+            fh.write(HG_UPDATE_CLEAN)
+            fh.chmod(0o0700)
+
+        chan, stdin, stdout = exec_command(
+            ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision)
+        stdin.close()
+
+        for line in stdout:
+            print(line, end='')
+
+        res = chan.recv_exit_status()
+
+        if res:
+            raise Exception('non-0 exit code updating working directory; %d'
+                            % res)
+
+
+def run_tests(ssh_client, python_version, test_flags=None):
+    """Run tests on a remote Linux machine via an SSH client."""
+    test_flags = test_flags or []
+
+    print('running tests')
+
+    if python_version == 'system2':
+        python = '/usr/bin/python2'
+    elif python_version == 'system3':
+        python = '/usr/bin/python3'
+    elif python_version.startswith('pypy'):
+        python = '/hgdev/pyenv/shims/%s' % python_version
+    else:
+        python = '/hgdev/pyenv/shims/python%s' % python_version
+
+    test_flags = ' '.join(shlex.quote(a) for a in test_flags)
+
+    command = (
+        '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
+        'cd /hgwork/src/tests && %s run-tests.py %s"' % (
+            python, test_flags))
+
+    chan, stdin, stdout = exec_command(ssh_client, command)
+
+    stdin.close()
+
+    for line in stdout:
+        print(line, end='')
+
+    return chan.recv_exit_status()

contrib/automation/hgautomation/ssh.py

@@ -0,0 +1,67 b''
+# ssh.py - Interact with remote SSH servers
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import socket
+import time
+import warnings
+
+from cryptography.utils import (
+    CryptographyDeprecationWarning,
+)
+import paramiko
+
+
+def wait_for_ssh(hostname, port, timeout=60, username=None, key_filename=None):
+    """Wait for an SSH server to start on the specified host and port."""
+    class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy):
+        def missing_host_key(self, client, hostname, key):
+            return
+
+    end_time = time.time() + timeout
+
+    # paramiko triggers a CryptographyDeprecationWarning in the cryptography
+    # package. Let's suppress
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore',
+                                category=CryptographyDeprecationWarning)
+
+        while True:
+            client = paramiko.SSHClient()
+            client.set_missing_host_key_policy(IgnoreHostKeyPolicy())
+            try:
+                client.connect(hostname, port=port, username=username,
+                               key_filename=key_filename,
+                               timeout=5.0, allow_agent=False,
+                               look_for_keys=False)
+
+                return client
+            except socket.error:
+                pass
+            except paramiko.AuthenticationException:
+                raise
+            except paramiko.SSHException:
+                pass
+
+            if time.time() >= end_time:
+                raise Exception('Timeout reached waiting for SSH')
+
+            time.sleep(1.0)
+
+
+def exec_command(client, command):
+    """exec_command wrapper that combines stderr/stdout and returns channel"""
+    chan = client.get_transport().open_session()
+
+    chan.exec_command(command)
+    chan.set_combine_stderr(True)
+
+    stdin = chan.makefile('wb', -1)
+    stdout = chan.makefile('r', -1)
+
+    return chan, stdin, stdout

contrib/automation/linux-requirements-py2.txt

@@ -0,0 +1,130 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -U --generate-hashes --output-file contrib/automation/linux-requirements-py2.txt contrib/automation/linux-requirements.txt.in
+#
+astroid==1.6.6 \
+    --hash=sha256:87de48a92e29cedf7210ffa853d11441e7ad94cb47bacd91b023499b51cbc756 \
+    --hash=sha256:d25869fc7f44f1d9fb7d24fd7ea0639656f5355fc3089cd1f3d18c6ec6b124c7 \
+    # via pylint
+backports.functools-lru-cache==1.5 \
+    --hash=sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a \
+    --hash=sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd \
+    # via astroid, isort, pylint
+bzr==2.7.0 ; python_version <= "2.7" and platform_python_implementation == "CPython" \
+    --hash=sha256:c9f6bbe0a50201dadc5fddadd94ba50174193c6cf6e39e16f6dd0ad98a1df338
+configparser==3.7.4 \
+    --hash=sha256:8be81d89d6e7b4c0d4e44bcc525845f6da25821de80cb5e06e7e0238a2899e32 \
+    --hash=sha256:da60d0014fd8c55eb48c1c5354352e363e2d30bbf7057e5e171a468390184c75 \
+    # via pylint
+contextlib2==0.5.5 \
+    --hash=sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48 \
+    --hash=sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00 \
+    # via vcrpy
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
+enum34==1.1.6 \
+    --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
+    --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
+    --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
+    --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
+    # via astroid
+funcsigs==1.0.2 \
+    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
+    # via mock
+futures==3.2.0 \
+    --hash=sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265 \
+    --hash=sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1 \
+    # via isort
+fuzzywuzzy==0.17.0 \
+    --hash=sha256:5ac7c0b3f4658d2743aa17da53a55598144edbc5bee3c6863840636e6926f254 \
+    --hash=sha256:6f49de47db00e1c71d40ad16da42284ac357936fa9b66bea1df63fed07122d62
+isort==4.3.17 \
+    --hash=sha256:01cb7e1ca5e6c5b3f235f0385057f70558b70d2f00320208825fa62887292f43 \
+    --hash=sha256:268067462aed7eb2a1e237fcb287852f22077de3fb07964e87e00f829eea2d1a \
+    # via pylint
+lazy-object-proxy==1.3.1 \
+    --hash=sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33 \
+    --hash=sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39 \
+    --hash=sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019 \
+    --hash=sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088 \
+    --hash=sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b \
+    --hash=sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e \
+    --hash=sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6 \
+    --hash=sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b \
+    --hash=sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5 \
+    --hash=sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff \
+    --hash=sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd \
+    --hash=sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7 \
+    --hash=sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff \
+    --hash=sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d \
+    --hash=sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2 \
+    --hash=sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35 \
+    --hash=sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4 \
+    --hash=sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514 \
+    --hash=sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252 \
+    --hash=sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109 \
+    --hash=sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f \
+    --hash=sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c \
+    --hash=sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92 \
+    --hash=sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577 \
+    --hash=sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d \
+    --hash=sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d \
+    --hash=sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f \
+    --hash=sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a \
+    --hash=sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b \
+    # via astroid
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
+    # via pylint
+mock==2.0.0 \
+    --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+    --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba \
+    # via vcrpy
+pbr==5.1.3 \
+    --hash=sha256:8257baf496c8522437e8a6cfe0f15e00aedc6c0e0e7c9d55eeeeab31e0853843 \
+    --hash=sha256:8c361cc353d988e4f5b998555c88098b9d5964c2e11acf7b0d21925a66bb5824 \
+    # via mock
+pyflakes==2.1.1 \
+    --hash=sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0 \
+    --hash=sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2
+pygments==2.3.1 \
+    --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \
+    --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d
+pylint==1.9.4 \
+    --hash=sha256:02c2b6d268695a8b64ad61847f92e611e6afcff33fd26c3a2125370c4662905d \
+    --hash=sha256:ee1e85575587c5b58ddafa25e1c1b01691ef172e139fc25585e5d3f02451da93
+python-levenshtein==0.12.0 \
+    --hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1
+pyyaml==5.1 \
+    --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \
+    --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \
+    --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \
+    --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \
+    --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \
+    --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \
+    --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \
+    --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \
+    --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \
+    --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \
+    --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 \
+    # via vcrpy
+singledispatch==3.4.0.3 \
+    --hash=sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c \
+    --hash=sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8 \
+    # via astroid, pylint
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
+    # via astroid, mock, pylint, singledispatch, vcrpy
+vcrpy==2.0.1 \
+    --hash=sha256:127e79cf7b569d071d1bd761b83f7b62b2ce2a2eb63ceca7aa67cba8f2602ea3 \
+    --hash=sha256:57be64aa8e9883a4117d0b15de28af62275c001abcdb00b6dc2d4406073d9a4f
+wrapt==1.11.1 \
+    --hash=sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533 \
+    # via astroid, vcrpy

contrib/automation/linux-requirements-py3.txt

@@ -0,0 +1,159 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -U --generate-hashes --output-file contrib/automation/linux-requirements-py3.txt contrib/automation/linux-requirements.txt.in
+#
+astroid==2.2.5 \
+    --hash=sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4 \
+    --hash=sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4 \
+    # via pylint
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
+fuzzywuzzy==0.17.0 \
+    --hash=sha256:5ac7c0b3f4658d2743aa17da53a55598144edbc5bee3c6863840636e6926f254 \
+    --hash=sha256:6f49de47db00e1c71d40ad16da42284ac357936fa9b66bea1df63fed07122d62
+idna==2.8 \
+    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
+    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+    # via yarl
+isort==4.3.17 \
+    --hash=sha256:01cb7e1ca5e6c5b3f235f0385057f70558b70d2f00320208825fa62887292f43 \
+    --hash=sha256:268067462aed7eb2a1e237fcb287852f22077de3fb07964e87e00f829eea2d1a \
+    # via pylint
+lazy-object-proxy==1.3.1 \
+    --hash=sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33 \
+    --hash=sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39 \
+    --hash=sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019 \
+    --hash=sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088 \
+    --hash=sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b \
+    --hash=sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e \
+    --hash=sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6 \
+    --hash=sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b \
+    --hash=sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5 \
+    --hash=sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff \
+    --hash=sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd \
+    --hash=sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7 \
+    --hash=sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff \
+    --hash=sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d \
+    --hash=sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2 \
+    --hash=sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35 \
+    --hash=sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4 \
+    --hash=sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514 \
+    --hash=sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252 \
+    --hash=sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109 \
+    --hash=sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f \
+    --hash=sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c \
+    --hash=sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92 \
+    --hash=sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577 \
+    --hash=sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d \
+    --hash=sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d \
+    --hash=sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f \
+    --hash=sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a \
+    --hash=sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b \
+    # via astroid
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
+    # via pylint
+multidict==4.5.2 \
+    --hash=sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f \
+    --hash=sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3 \
+    --hash=sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef \
+    --hash=sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b \
+    --hash=sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73 \
+    --hash=sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc \
+    --hash=sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3 \
+    --hash=sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd \
+    --hash=sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351 \
+    --hash=sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941 \
+    --hash=sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d \
+    --hash=sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1 \
+    --hash=sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b \
+    --hash=sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a \
+    --hash=sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3 \
+    --hash=sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7 \
+    --hash=sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0 \
+    --hash=sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0 \
+    --hash=sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014 \
+    --hash=sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5 \
+    --hash=sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036 \
+    --hash=sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d \
+    --hash=sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a \
+    --hash=sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce \
+    --hash=sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1 \
+    --hash=sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a \
+    --hash=sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9 \
+    --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \
+    --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b \
+    # via yarl
+pyflakes==2.1.1 \
+    --hash=sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0 \
+    --hash=sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2
+pygments==2.3.1 \
+    --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \
+    --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d
+pylint==2.3.1 \
+    --hash=sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09 \
+    --hash=sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1
+python-levenshtein==0.12.0 \
+    --hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1
+pyyaml==5.1 \
+    --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \
+    --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \
+    --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \
+    --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \
+    --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \
+    --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \
+    --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \
+    --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \
+    --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \
+    --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \
+    --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 \
+    # via vcrpy
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
+    # via astroid, vcrpy
+typed-ast==1.3.4 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \
+    --hash=sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200 \
+    --hash=sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0 \
+    --hash=sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c \
+    --hash=sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99 \
+    --hash=sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7 \
+    --hash=sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1 \
+    --hash=sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d \
+    --hash=sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8 \
+    --hash=sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de \
+    --hash=sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682 \
+    --hash=sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db \
+    --hash=sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8 \
+    --hash=sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7 \
+    --hash=sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f \
+    --hash=sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15 \
+    --hash=sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae \
+    --hash=sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3 \
+    --hash=sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e \
+    --hash=sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a \
+    --hash=sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7
+vcrpy==2.0.1 \
+    --hash=sha256:127e79cf7b569d071d1bd761b83f7b62b2ce2a2eb63ceca7aa67cba8f2602ea3 \
+    --hash=sha256:57be64aa8e9883a4117d0b15de28af62275c001abcdb00b6dc2d4406073d9a4f
+wrapt==1.11.1 \
+    --hash=sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533 \
+    # via astroid, vcrpy
+yarl==1.3.0 \
+    --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \
+    --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \
+    --hash=sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb \
+    --hash=sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320 \
+    --hash=sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842 \
+    --hash=sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0 \
+    --hash=sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829 \
+    --hash=sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310 \
+    --hash=sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4 \
+    --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \
+    --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1 \
+    # via vcrpy

contrib/automation/linux-requirements.txt.in

@@ -0,0 +1,12 b''
+# Bazaar doesn't work with Python 3 nor PyPy.
+bzr ; python_version <= '2.7' and platform_python_implementation == 'CPython'
+docutils
+fuzzywuzzy
+pyflakes
+pygments
+pylint
+# Needed to avoid warnings from fuzzywuzzy.
+python-Levenshtein
+# typed-ast dependency doesn't install on PyPy.
+typed-ast ; python_version >= '3.0' and platform_python_implementation != 'PyPy'
+vcrpy

hgext/lfs/TODO.rst

@@ -0,0 +1,195 b''
+Prior to removing (EXPERIMENTAL)
+--------------------------------
+
+These things affect UI and/or behavior, and should probably be implemented (or
+ruled out) prior to taking off the experimental shrinkwrap.
+
+#. Finish the `hg convert` story
+
+   * Add an argument to accept a rules file to apply during conversion?
+     Currently `lfs.track` is the only way to affect the conversion.
+   * drop `lfs.track` config settings
+   * splice in `.hglfs` file for normal repo -> lfs conversions?
+
+#. Stop uploading blobs when pushing between local repos
+
+   * Could probably hardlink directly to the other local repo's store
+   * Support inferring `lfs.url` for local push/pull (currently only supports
+     http)
+
+#. Stop uploading blobs on strip/amend/histedit/etc.
+
+   * This seems to be a side effect of doing it for `hg bundle`, which probably
+     makes sense.
+
+#. Handle a server with the extension loaded and a client without the extension
+   more gracefully.
+
+   * `changegroup3` is still experimental, and not enabled by default.
+   * Figure out how to `introduce LFS to the server repo
+     <https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-September/122281.html>`_.
+     See the TODO in test-lfs-serve.t.
+
+#. Remove `lfs.retry` hack in client?  This came from FB, but it's not clear why
+   it is/was needed.
+
+#. `hg export` currently writes out the LFS blob.  Should it write the pointer
+   instead?
+
+   * `hg diff` is similar, and probably shouldn't see the pointer file
+
+#. `Fix https multiplexing, and re-enable workers
+   <https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-January/109916.html>`_.
+
+#. Show to-be-applied rules with `hg files -r 'wdir()' 'set:lfs()'`
+
+   * `debugignore` can show file + line number, so a dedicated command could be
+     useful too.
+
+#. Filesets, revsets and templates
+
+   * A dedicated revset should be faster than `'file(set:lfs())'`
+   * Attach `{lfsoid}` and `{lfspointer}` to `general keywords
+     <https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-January/110251.html>`_,
+     IFF the file is a blob
+   * Drop existing items that would be redundant with general support
+
+#. Can `grep` avoid downloading most things?
+
+   * Add a command option to skip LFS blobs?
+
+#. Add a flag that's visible in `hg files -v` to indicate external storage?
+
+#. Server side issues
+
+   * Check for local disk space before allowing upload.  (I've got a patch for
+     this.)
+   * Make sure the http codes used are appropriate.
+   * `Why is copying the Authorization header into the JSON payload necessary
+     <https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-April/116230.html>`_?
+   * `LFS-Authenticate` header support in client and server(?)
+
+#. Add locks on cache and blob store
+
+   * This is complicated with a global store, and multiple potentially unrelated
+     local repositories that reference the same blob.
+   * Alternately, maybe just handle collisions when trying to create the same
+     blob in the store somehow.
+
+#. Are proper file sizes reported in `debugupgraderepo`?
+
+#. Finish prefetching files
+
+   * `-T {data}`  (other than cat?)
+   * `verify`
+   * `grep`
+
+#. Output cleanup
+
+   * Can we print the url when connecting to the blobstore?  (A sudden
+     connection refused after pulling commits looks confusing.)  Problem is,
+     'pushing to main url' is printed, and then lfs wants to upload before going
+     back to the main repo transfer, so then *that* could be confusing with
+     extra output. (This is kinda improved with 380f5131ee7b and 9f78d10742af.)
+
+   * Add more progress indicators?  Uploading a large repo looks idle for a long
+     time while it scans for blobs in each outgoing revision.
+
+   * Print filenames instead of hashes in error messages
+
+     * subrepo aware paths, where necessary
+
+   * Is existing output at the right status/note/debug level?
+
+#. Can `verify` be done without downloading everything?
+
+   * If we know that we are talking to an hg server, we can leverage the fact
+     that it validates in the Batch API portion, and skip d/l altogether.  OTOH,
+     maybe we should download the files unconditionally for forensics.  The
+     alternative is to define a custom transfer handler that definitively
+     verifies without transferring, and then cache those results.  When verify
+     comes looking, look in the cache instead of actually opening the file and
+     processing it.
+
+   * Yuya has concerns about when blob fetch takes place vs when revlog is
+     verified.  Since the visible hash matches the blob content, I don't think
+     there's a way to verify the pointer file that's actually stored in the
+     filelog (other than basic JSON checks).  Full verification requires the
+     blob.  See
+     https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-April/116133.html
+
+   * Opening a corrupt pointer file aborts.  It probably shouldn't for verify.
+
+
+Future ideas/features/polishing
+-------------------------------
+
+These aren't in any particular order, and are things that don't have obvious BC
+concerns.
+
+#. Garbage collection `(issue5790) <https://bz.mercurial-scm.org/show_bug.cgi?id=5790>`_
+
+   * This gets complicated because of the global cache, which may or may not
+     consist of hardlinks to the repo, and may be in use by other repos.  (So
+     the gc may be pointless.)
+
+#. `Compress blobs <https://github.com/git-lfs/git-lfs/issues/260>`_
+
+   * 700MB repo becomes 2.5GB with all lfs blobs
+   * What implications are there for filesystem paths that don't indicate
+     compression?  (i.e. how to share with global cache and other local repos?)
+   * Probably needs to be stored under `.hg/store/lfs/zstd`, with a repo
+     requirement.
+   * Allow tuneable compression type and settings?
+   * Support compression over the wire if both sides understand the compression?
+   * `debugupgraderepo` to convert?
+   * Probably not worth supporting compressed and uncompressed concurrently
+
+#. Determine things to upload with `readfast()
+   <https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-August/121315.html>`_
+
+   * Significantly faster when pushing an entire large repo to http.
+   * Causes test changes to fileset and templates; may need both this and
+     current methods of lookup.
+
+#. Is a command to download everything needed?  This would allow copying the
+   whole to a portable drive.  Currently this can be effected by running
+   `hg verify`.
+
+#. Stop reading in entire file into one buffer when passing through filelog
+   interface
+
+   * `Requires major replumbing to core
+     <https://www.mercurial-scm.org/wiki/HandlingLargeFiles>`_
+
+#. Keep corrupt files around in 'store/lfs/incoming' for forensics?
+
+   * Files should be downloaded to 'incoming', and moved to normal location when
+     done.
+
+#. Client side path enhancements
+
+   * Support paths.default:lfs = ... style paths
+   * SSH -> https server inference
+
+     * https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-April/115416.html
+     * https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md#guessing-the-server
+
+#. Server enhancements
+
+   * Add support for transfer quotas?
+   * Download should be able to send the file in chunks, without reading the
+     whole thing into memory
+     (https://www.mercurial-scm.org/pipermail/mercurial-devel/2018-March/114584.html)
+   * Support for resuming transfers
+
+#. Handle 3rd party server storage.
+
+   * Teach client to handle lfs `verify` action.  This is needed after the
+     server instructs the client to upload the file to another server, in order
+     to tell the server that the upload completed.
+   * Teach the server to send redirects if configured, and process `verify`
+     requests.
+
+#. `Is any hg-git work needed
+   <https://groups.google.com/d/msg/hg-git/XYNQuudteeM/ivt8gXoZAAAJ>`_?

mercurial/help/internals/mergestate.txt

@@ -0,0 +1,80 b''
+The active mergestate is stored in ``.hg/merge`` when a merge is triggered
+by commands like ``hg merge``, ``hg rebase``, etc. until the merge is
+completed or aborted to track the 3-way merge state of individual files.
+
+The contents of the directory are:
+
+Conflicting files
+-----------------
+
+The local version of the conflicting files are stored with their
+filenames as the hash of their paths.
+
+state
+-----
+
+This mergestate file record is used by hg version prior to 2.9.1
+and contains less data than ``state2``. If there is no contradiction
+with ``state2``, we can assume that both are written at the same time.
+In this case, data from ``state2`` is used. Otherwise, we use ``state``.
+We read/write both ``state`` and ``state2`` records to ensure backward
+compatibility.
+
+state2
+------
+
+This record stores a superset of data in ``state``, including new kinds
+of records in the future.
+
+Each record can contain arbitrary content and has an associated type. This
+`type` should be a letter. If `type` is uppercase, the record is mandatory:
+versions of Mercurial that don't support it should abort. If `type` is
+lowercase, the record can be safely ignored.
+
+Currently known records:
+
+| * L: the node of the "local" part of the merge (hexified version)
+| * O: the node of the "other" part of the merge (hexified version)
+| * F: a file to be merged entry
+| * C: a change/delete or delete/change conflict
+| * D: a file that the external merge driver will merge internally
+|      (experimental)
+| * P: a path conflict (file vs directory)
+| * m: the external merge driver defined for this merge plus its run state
+|      (experimental)
+| * f: a (filename, dictionary) tuple of optional values for a given file
+| * X: unsupported mandatory record type (used in tests)
+| * x: unsupported advisory record type (used in tests)
+| * l: the labels for the parts of the merge.
+
+Merge driver run states (experimental):
+
+| * u: driver-resolved files unmarked -- needs to be run next time we're
+|      about to resolve or commit
+| * m: driver-resolved files marked -- only needs to be run before commit
+| * s: success/skipped -- does not need to be run any more
+
+Merge record states (indexed by filename):
+
+| * u: unresolved conflict
+| * r: resolved conflict
+| * pu: unresolved path conflict (file conflicts with directory)
+| * pr: resolved path conflict
+| * d: driver-resolved conflict
+
+The resolve command transitions between 'u' and 'r' for conflicts and
+'pu' and 'pr' for path conflicts.
+
+This format is a list of arbitrary records of the form:
+
+[type][length][content]
+
+`type` is a single character, `length` is a 4 byte integer, and
+`content` is an arbitrary byte sequence of length `length`.
+
+Mercurial versions prior to 3.7 have a bug where if there are
+unsupported mandatory merge records, attempting to clear out the merge
+state with hg update --clean or similar aborts. The 't' record type
+works around that by writing out what those versions treat as an
+advisory record, but later versions interpret as special: the first
+character is the 'real' record type and everything onwards is the data.

contrib/all-revsets.txt

@@ -154,3 +154,6 b' roots(-10000:-1)'
 roots(matching(tip, "author"))
 roots(matching(tip, "author")) and -10000:-1
 (-10000:-1) and roots(matching(tip, "author"))
+only(max(head()))
+only(max(head()), min(head()))
+only(max(head()), limit(head(), 1, 1))

contrib/automation/README.rst

@@ -101,9 +101,14 b' Under normal operation, recurring costs '
 * Storage costs for AMI / EBS snapshots. This should be just a few pennies
   per month.
 
-When running EC2 instances, you'll be billed accordingly. By default, we
-use *small* instances, like ``t3.medium``. This instance type costs ~$0.07 per
-hour.
+When running EC2 instances, you'll be billed accordingly. Default instance
+types vary by operation. We try to be respectful of your money when choosing
+defaults. e.g. for Windows instances which are billed per hour, we use e.g.
+``t3.medium`` instances, which cost ~$0.07 per hour. For operations that
+scale well to many CPUs like running Linux tests, we may use a more powerful
+instance like ``c5.9xlarge``. However, since Linux instances are billed
+per second and the cost of running an e.g. ``c5.9xlarge`` for half the time
+of a ``c5.4xlarge`` is roughly the same, the choice is justified.
 
 .. note::
 
@@ -125,3 +130,54 b' To terminate all EC2 instances that we m'
 To purge all EC2 resources that we manage::
 
    $ automation.py purge-ec2-resources
+
+Remote Machine Interfaces
+=========================
+
+The code that connects to a remote machine and executes things is
+theoretically machine agnostic as long as the remote machine conforms to
+an *interface*. In other words, to perform actions like running tests
+remotely or triggering packaging, it shouldn't matter if the remote machine
+is an EC2 instance, a virtual machine, etc. This section attempts to document
+the interface that remote machines need to provide in order to be valid
+*targets* for remote execution. These interfaces are often not ideal nor
+the most flexible. Instead, they have often evolved as the requirements of
+our automation code have evolved.
+
+Linux
+-----
+
+Remote Linux machines expose an SSH server on port 22. The SSH server
+must allow the ``hg`` user to authenticate using the SSH key generated by
+the automation code. The ``hg`` user should be part of the ``hg`` group
+and it should have ``sudo`` access without password prompting.
+
+The SSH channel must support SFTP to facilitate transferring files from
+client to server.
+
+``/bin/bash`` must be executable and point to a bash shell executable.
+
+The ``/hgdev`` directory must exist and all its content owned by ``hg::hg``.
+
+The ``/hgdev/pyenv`` directory should contain an installation of
+``pyenv``. Various Python distributions should be installed. The exact
+versions shouldn't matter. ``pyenv global`` should have been run so
+``/hgdev/pyenv/shims/`` is populated with redirector scripts that point
+to the appropriate Python executable.
+
+The ``/hgdev/venv-bootstrap`` directory must contain a virtualenv
+with Mercurial installed. The ``/hgdev/venv-bootstrap/bin/hg`` executable
+is referenced by various scripts and the client.
+
+The ``/hgdev/src`` directory MUST contain a clone of the Mercurial
+source code. The state of the working directory is not important.
+
+In order to run tests, the ``/hgwork`` directory will be created.
+This may require running various ``mkfs.*`` executables and ``mount``
+to provision a new filesystem. This will require elevated privileges
+via ``sudo``.
+
+Various dependencies to run the Mercurial test harness are also required.
+Documenting them is beyond the scope of this document. Various tests
+also require other optional dependencies and missing dependencies will
+be printed by the test runner when a test is skipped.

contrib/automation/hgautomation/__init__.py

@@ -53,7 +53,7 b' class HGAutomation:'
 
         return password
 
-    def aws_connection(self, region: str):
+    def aws_connection(self, region: str, ensure_ec2_state: bool=True):
         """Obtain an AWSConnection instance bound to a specific region."""
 
-        return AWSConnection(self, region)
+        return AWSConnection(self, region, ensure_ec2_state=ensure_ec2_state)

contrib/automation/hgautomation/aws.py

@@ -19,6 +19,13 b' import time'
 import boto3
 import botocore.exceptions
 
+from .linux import (
+    BOOTSTRAP_DEBIAN,
+)
+from .ssh import (
+    exec_command as ssh_exec_command,
+    wait_for_ssh,
+)
 from .winrm import (
     run_powershell,
     wait_for_winrm,
@@ -31,12 +38,46 b' INSTALL_WINDOWS_DEPENDENCIES = (SOURCE_R'
                                 'install-windows-dependencies.ps1')
 
 
+INSTANCE_TYPES_WITH_STORAGE = {
+    'c5d',
+    'd2',
+    'h1',
+    'i3',
+    'm5ad',
+    'm5d',
+    'r5d',
+    'r5ad',
+    'x1',
+    'z1d',
+}
+
+
+DEBIAN_ACCOUNT_ID = '379101102735'
+UBUNTU_ACCOUNT_ID = '099720109477'
+
+
 KEY_PAIRS = {
     'automation',
 }
 
 
 SECURITY_GROUPS = {
+    'linux-dev-1': {
+        'description': 'Mercurial Linux instances that perform build/test automation',
+        'ingress': [
+            {
+                'FromPort': 22,
+                'ToPort': 22,
+                'IpProtocol': 'tcp',
+                'IpRanges': [
+                    {
+                        'CidrIp': '0.0.0.0/0',
+                        'Description': 'SSH from entire Internet',
+                    },
+                ],
+            },
+        ],
+    },
     'windows-dev-1': {
         'description': 'Mercurial Windows instances that perform build automation',
         'ingress': [
@@ -180,7 +221,7 b' Install-WindowsFeature -Name Net-Framewo'
 class AWSConnection:
     """Manages the state of a connection with AWS."""
 
-    def __init__(self, automation, region: str):
+    def __init__(self, automation, region: str, ensure_ec2_state: bool=True):
         self.automation = automation
         self.local_state_path = automation.state_path
 
@@ -191,11 +232,12 b' class AWSConnection:'
         self.ec2resource = self.session.resource('ec2')
         self.iamclient = self.session.client('iam')
         self.iamresource = self.session.resource('iam')
-
-        ensure_key_pairs(automation.state_path, self.ec2resource)
+        self.security_groups = {}
 
-        self.security_groups = ensure_security_groups(self.ec2resource)
-        ensure_iam_state(self.iamresource)
+        if ensure_ec2_state:
+            ensure_key_pairs(automation.state_path, self.ec2resource)
+            self.security_groups = ensure_security_groups(self.ec2resource)
+            ensure_iam_state(self.iamclient, self.iamresource)
 
     def key_pair_path_private(self, name):
         """Path to a key pair private key file."""
@@ -324,7 +366,7 b' def delete_instance_profile(profile):'
     profile.delete()
 
 
-def ensure_iam_state(iamresource, prefix='hg-'):
+def ensure_iam_state(iamclient, iamresource, prefix='hg-'):
     """Ensure IAM state is in sync with our canonical definition."""
 
     remote_profiles = {}
@@ -360,6 +402,10 b' def ensure_iam_state(iamresource, prefix'
             InstanceProfileName=actual)
         remote_profiles[name] = profile
 
+        waiter = iamclient.get_waiter('instance_profile_exists')
+        waiter.wait(InstanceProfileName=actual)
+        print('IAM instance profile %s is available' % actual)
+
     for name in sorted(set(IAM_ROLES) - set(remote_roles)):
         entry = IAM_ROLES[name]
 
@@ -372,6 +418,10 b' def ensure_iam_state(iamresource, prefix'
             AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
         )
 
+        waiter = iamclient.get_waiter('role_exists')
+        waiter.wait(RoleName=actual)
+        print('IAM role %s is available' % actual)
+
         remote_roles[name] = role
 
         for arn in entry['policy_arns']:
@@ -393,14 +443,14 b' def ensure_iam_state(iamresource, prefix'
             profile.add_role(RoleName=role)
 
 
-def find_windows_server_2019_image(ec2resource):
-    """Find the Amazon published Windows Server 2019 base image."""
+def find_image(ec2resource, owner_id, name):
+    """Find an AMI by its owner ID and name."""
 
     images = ec2resource.images.filter(
         Filters=[
             {
-                'Name': 'owner-alias',
-                'Values': ['amazon'],
+                'Name': 'owner-id',
+                'Values': [owner_id],
             },
             {
                 'Name': 'state',
@@ -412,14 +462,14 b' def find_windows_server_2019_image(ec2re'
             },
             {
                 'Name': 'name',
-                'Values': ['Windows_Server-2019-English-Full-Base-2019.02.13'],
+                'Values': [name],
             },
         ])
 
     for image in images:
         return image
 
-    raise Exception('unable to find Windows Server 2019 image')
+    raise Exception('unable to find image for %s' % name)
 
 
 def ensure_security_groups(ec2resource, prefix='hg-'):
@@ -490,7 +540,7 b" def remove_resources(c, prefix='hg-'):"
 
     terminate_ec2_instances(ec2resource, prefix=prefix)
 
-    for image in ec2resource.images.all():
+    for image in ec2resource.images.filter(Owners=['self']):
         if image.name.startswith(prefix):
             remove_ami(ec2resource, image)
 
@@ -505,6 +555,10 b" def remove_resources(c, prefix='hg-'):"
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
+            for p in role.attached_policies.all():
+                print('detaching policy %s from %s' % (p.arn, role.name))
+                role.detach_policy(PolicyArn=p.arn)
+
             print('removing role %s' % role.name)
             role.delete()
 
@@ -671,6 +725,309 b' def create_temp_windows_ec2_instances(c:'
         yield instances
 
 
+def resolve_fingerprint(fingerprint):
+    fingerprint = json.dumps(fingerprint, sort_keys=True)
+    return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
+
+
+def find_and_reconcile_image(ec2resource, name, fingerprint):
+    """Attempt to find an existing EC2 AMI with a name and fingerprint.
+
+    If an image with the specified fingerprint is found, it is returned.
+    Otherwise None is returned.
+
+    Existing images for the specified name that don't have the specified
+    fingerprint or are missing required metadata or deleted.
+    """
+    # Find existing AMIs with this name and delete the ones that are invalid.
+    # Store a reference to a good image so it can be returned one the
+    # image state is reconciled.
+    images = ec2resource.images.filter(
+        Filters=[{'Name': 'name', 'Values': [name]}])
+
+    existing_image = None
+
+    for image in images:
+        if image.tags is None:
+            print('image %s for %s lacks required tags; removing' % (
+                image.id, image.name))
+            remove_ami(ec2resource, image)
+        else:
+            tags = {t['Key']: t['Value'] for t in image.tags}
+
+            if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
+                existing_image = image
+            else:
+                print('image %s for %s has wrong fingerprint; removing' % (
+                      image.id, image.name))
+                remove_ami(ec2resource, image)
+
+    return existing_image
+
+
+def create_ami_from_instance(ec2client, instance, name, description,
+                             fingerprint):
+    """Create an AMI from a running instance.
+
+    Returns the ``ec2resource.Image`` representing the created AMI.
+    """
+    instance.stop()
+
+    ec2client.get_waiter('instance_stopped').wait(
+        InstanceIds=[instance.id],
+        WaiterConfig={
+            'Delay': 5,
+        })
+    print('%s is stopped' % instance.id)
+
+    image = instance.create_image(
+        Name=name,
+        Description=description,
+    )
+
+    image.create_tags(Tags=[
+        {
+            'Key': 'HGIMAGEFINGERPRINT',
+            'Value': fingerprint,
+        },
+    ])
+
+    print('waiting for image %s' % image.id)
+
+    ec2client.get_waiter('image_available').wait(
+        ImageIds=[image.id],
+    )
+
+    print('image %s available as %s' % (image.id, image.name))
+
+    return image
+
+
+def ensure_linux_dev_ami(c: AWSConnection, distro='debian9', prefix='hg-'):
+    """Ensures a Linux development AMI is available and up-to-date.
+
+    Returns an ``ec2.Image`` of either an existing AMI or a newly-built one.
+    """
+    ec2client = c.ec2client
+    ec2resource = c.ec2resource
+
+    name = '%s%s-%s' % (prefix, 'linux-dev', distro)
+
+    if distro == 'debian9':
+        image = find_image(
+            ec2resource,
+            DEBIAN_ACCOUNT_ID,
+            'debian-stretch-hvm-x86_64-gp2-2019-02-19-26620',
+        )
+        ssh_username = 'admin'
+    elif distro == 'ubuntu18.04':
+        image = find_image(
+            ec2resource,
+            UBUNTU_ACCOUNT_ID,
+            'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190403',
+        )
+        ssh_username = 'ubuntu'
+    elif distro == 'ubuntu18.10':
+        image = find_image(
+            ec2resource,
+            UBUNTU_ACCOUNT_ID,
+            'ubuntu/images/hvm-ssd/ubuntu-cosmic-18.10-amd64-server-20190402',
+        )
+        ssh_username = 'ubuntu'
+    elif distro == 'ubuntu19.04':
+        image = find_image(
+            ec2resource,
+            UBUNTU_ACCOUNT_ID,
+            'ubuntu/images/hvm-ssd/ubuntu-disco-19.04-amd64-server-20190417',
+        )
+        ssh_username = 'ubuntu'
+    else:
+        raise ValueError('unsupported Linux distro: %s' % distro)
+
+    config = {
+        'BlockDeviceMappings': [
+            {
+                'DeviceName': image.block_device_mappings[0]['DeviceName'],
+                'Ebs': {
+                    'DeleteOnTermination': True,
+                    'VolumeSize': 8,
+                    'VolumeType': 'gp2',
+                },
+            },
+        ],
+        'EbsOptimized': True,
+        'ImageId': image.id,
+        'InstanceInitiatedShutdownBehavior': 'stop',
+        # 8 VCPUs for compiling Python.
+        'InstanceType': 't3.2xlarge',
+        'KeyName': '%sautomation' % prefix,
+        'MaxCount': 1,
+        'MinCount': 1,
+        'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
+    }
+
+    requirements2_path = (pathlib.Path(__file__).parent.parent /
+                          'linux-requirements-py2.txt')
+    requirements3_path = (pathlib.Path(__file__).parent.parent /
+                          'linux-requirements-py3.txt')
+    with requirements2_path.open('r', encoding='utf-8') as fh:
+        requirements2 = fh.read()
+    with requirements3_path.open('r', encoding='utf-8') as fh:
+        requirements3 = fh.read()
+
+    # Compute a deterministic fingerprint to determine whether image needs to
+    # be regenerated.
+    fingerprint = resolve_fingerprint({
+        'instance_config': config,
+        'bootstrap_script': BOOTSTRAP_DEBIAN,
+        'requirements_py2': requirements2,
+        'requirements_py3': requirements3,
+    })
+
+    existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
+
+    if existing_image:
+        return existing_image
+
+    print('no suitable %s image found; creating one...' % name)
+
+    with temporary_ec2_instances(ec2resource, config) as instances:
+        wait_for_ip_addresses(instances)
+
+        instance = instances[0]
+
+        client = wait_for_ssh(
+            instance.public_ip_address, 22,
+            username=ssh_username,
+            key_filename=str(c.key_pair_path_private('automation')))
+
+        home = '/home/%s' % ssh_username
+
+        with client:
+            print('connecting to SSH server')
+            sftp = client.open_sftp()
+
+            print('uploading bootstrap files')
+            with sftp.open('%s/bootstrap' % home, 'wb') as fh:
+                fh.write(BOOTSTRAP_DEBIAN)
+                fh.chmod(0o0700)
+
+            with sftp.open('%s/requirements-py2.txt' % home, 'wb') as fh:
+                fh.write(requirements2)
+                fh.chmod(0o0700)
+
+            with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
+                fh.write(requirements3)
+                fh.chmod(0o0700)
+
+            print('executing bootstrap')
+            chan, stdin, stdout = ssh_exec_command(client,
+                                                   '%s/bootstrap' % home)
+            stdin.close()
+
+            for line in stdout:
+                print(line, end='')
+
+            res = chan.recv_exit_status()
+            if res:
+                raise Exception('non-0 exit from bootstrap: %d' % res)
+
+            print('bootstrap completed; stopping %s to create %s' % (
+                  instance.id, name))
+
+        return create_ami_from_instance(ec2client, instance, name,
+                                        'Mercurial Linux development environment',
+                                        fingerprint)
+
+
[e-mail]
+def temporary_linux_dev_instances(c: AWSConnection, image, instance_type,
+                                  prefix='hg-', ensure_extra_volume=False):
+    """Create temporary Linux development EC2 instances.
+
+    Context manager resolves to a list of ``ec2.Instance`` that were created
+    and are running.
+
+    ``ensure_extra_volume`` can be set to ``True`` to require that instances
+    have a 2nd storage volume available other than the primary AMI volume.
+    For instance types with instance storage, this does nothing special.
+    But for instance types without instance storage, an additional EBS volume
+    will be added to the instance.
+
+    Instances have an ``ssh_client`` attribute containing a paramiko SSHClient
+    instance bound to the instance.
+
+    Instances have an ``ssh_private_key_path`` attributing containing the
+    str path to the SSH private key to connect to the instance.
+    """
+
+    block_device_mappings = [
+        {
+            'DeviceName': image.block_device_mappings[0]['DeviceName'],
+            'Ebs': {
+                'DeleteOnTermination': True,
+                'VolumeSize': 8,
+                'VolumeType': 'gp2',
+            },
+        }
+    ]
+
+    # This is not an exhaustive list of instance types having instance storage.
+    # But
+    if (ensure_extra_volume
+        and not instance_type.startswith(tuple(INSTANCE_TYPES_WITH_STORAGE))):
+        main_device = block_device_mappings[0]['DeviceName']
+
+        if main_device == 'xvda':
+            second_device = 'xvdb'
+        elif main_device == '/dev/sda1':
+            second_device = '/dev/sdb'
+        else:
+            raise ValueError('unhandled primary EBS device name: %s' %
+                             main_device)
+
+        block_device_mappings.append({
+            'DeviceName': second_device,
+            'Ebs': {
+                'DeleteOnTermination': True,
+                'VolumeSize': 8,
+                'VolumeType': 'gp2',
+            }
+        })
+
+    config = {
+        'BlockDeviceMappings': block_device_mappings,
+        'EbsOptimized': True,
+        'ImageId': image.id,
+        'InstanceInitiatedShutdownBehavior': 'terminate',
+        'InstanceType': instance_type,
+        'KeyName': '%sautomation' % prefix,
+        'MaxCount': 1,
+        'MinCount': 1,
+        'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
+    }
+
+    with temporary_ec2_instances(c.ec2resource, config) as instances:
+        wait_for_ip_addresses(instances)
+
+        ssh_private_key_path = str(c.key_pair_path_private('automation'))
+
+        for instance in instances:
+            client = wait_for_ssh(
+                instance.public_ip_address, 22,
+                username='hg',
+                key_filename=ssh_private_key_path)
+
+            instance.ssh_client = client
+            instance.ssh_private_key_path = ssh_private_key_path
+
+        try:
+            yield instances
+        finally:
+            for instance in instances:
+                instance.ssh_client.close()
+
+
 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'):
     """Ensure Windows Development AMI is available and up-to-date.
 
@@ -689,6 +1046,10 b' def ensure_windows_dev_ami(c: AWSConnect'
 
     name = '%s%s' % (prefix, 'windows-dev')
 
+    image = find_image(ec2resource,
+                       '801119661308',
+                       'Windows_Server-2019-English-Full-Base-2019.02.13')
+
     config = {
         'BlockDeviceMappings': [
             {
@@ -700,7 +1061,7 b' def ensure_windows_dev_ami(c: AWSConnect'
                 },
             }
         ],
-        'ImageId': find_windows_server_2019_image(ec2resource).id,
+        'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         'InstanceType': 't3.medium',
         'KeyName': '%sautomation' % prefix,
@@ -735,38 +1096,14 b' def ensure_windows_dev_ami(c: AWSConnect'
 
     # Compute a deterministic fingerprint to determine whether image needs
     # to be regenerated.
-    fingerprint = {
+    fingerprint = resolve_fingerprint({
         'instance_config': config,
         'user_data': WINDOWS_USER_DATA,
         'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
         'bootstrap_commands': commands,
-    }
-
-    fingerprint = json.dumps(fingerprint, sort_keys=True)
-    fingerprint = hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
-
-    # Find existing AMIs with this name and delete the ones that are invalid.
-    # Store a reference to a good image so it can be returned one the
-    # image state is reconciled.
-    images = ec2resource.images.filter(
-        Filters=[{'Name': 'name', 'Values': [name]}])
-
-    existing_image = None
+    })
 
-    for image in images:
-        if image.tags is None:
-            print('image %s for %s lacks required tags; removing' % (
-                image.id, image.name))
-            remove_ami(ec2resource, image)
-        else:
-            tags = {t['Key']: t['Value'] for t in image.tags}
-
-            if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
-                existing_image = image
-            else:
-                print('image %s for %s has wrong fingerprint; removing' % (
-                      image.id, image.name))
-                remove_ami(ec2resource, image)
+    existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
@@ -795,10 +1132,26 b' def ensure_windows_dev_ami(c: AWSConnect'
         )
 
         # Reboot so all updates are fully applied.
+        #
+        # We don't use instance.reboot() here because it is asynchronous and
+        # we don't know when exactly the instance has rebooted. It could take
+        # a while to stop and we may start trying to interact with the instance
+        # before it has rebooted.
         print('rebooting instance %s' % instance.id)
-        ec2client.reboot_instances(InstanceIds=[instance.id])
+        instance.stop()
+        ec2client.get_waiter('instance_stopped').wait(
+            InstanceIds=[instance.id],
+            WaiterConfig={
+                'Delay': 5,
+            })
 
-        time.sleep(15)
+        instance.start()
+        wait_for_ip_addresses([instance])
+
+        # There is a race condition here between the User Data PS script running
+        # and us connecting to WinRM. This can manifest as
+        # "AuthorizationManager check failed" failures during run_powershell().
+        # TODO figure out a workaround.
 
         print('waiting for Windows Remote Management to come back...')
         client = wait_for_winrm(instance.public_ip_address, 'Administrator',
@@ -810,36 +1163,9 b' def ensure_windows_dev_ami(c: AWSConnect'
         run_powershell(instance.winrm_client, '\n'.join(commands))
 
         print('bootstrap completed; stopping %s to create image' % instance.id)
-        instance.stop()
-
-        ec2client.get_waiter('instance_stopped').wait(
-            InstanceIds=[instance.id],
-            WaiterConfig={
-                'Delay': 5,
-            })
-        print('%s is stopped' % instance.id)
-
-        image = instance.create_image(
-            Name=name,
-            Description='Mercurial Windows development environment',
-        )
-
-        image.create_tags(Tags=[
-            {
-                'Key': 'HGIMAGEFINGERPRINT',
-                'Value': fingerprint,
-            },
-        ])
-
-        print('waiting for image %s' % image.id)
-
-        ec2client.get_waiter('image_available').wait(
-            ImageIds=[image.id],
-        )
-
-        print('image %s available as %s' % (image.id, image.name))
-
-        return image
+        return create_ami_from_instance(ec2client, instance, name,
+                                        'Mercurial Windows development environment',
+                                        fingerprint)
 
 
 @contextlib.contextmanager

contrib/automation/hgautomation/cli.py

@@ -8,12 +8,15 b''
 # no-check-code because Python 3 native.
 
 import argparse
+import concurrent.futures as futures
 import os
 import pathlib
+import time
 
 from . import (
     aws,
     HGAutomation,
+    linux,
     windows,
 )
 
@@ -22,6 +25,33 b' SOURCE_ROOT = pathlib.Path(os.path.abspa'
 DIST_PATH = SOURCE_ROOT / 'dist'
 
 
+def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None,
+                        parallel=False):
+    c = hga.aws_connection(aws_region)
+
+    if distros:
+        distros = distros.split(',')
+    else:
+        distros = sorted(linux.DISTROS)
+
+    # TODO There is a wonky interaction involving KeyboardInterrupt whereby
+    # the context manager that is supposed to terminate the temporary EC2
+    # instance doesn't run. Until we fix this, make parallel building opt-in
+    # so we don't orphan instances.
+    if parallel:
+        fs = []
+
+        with futures.ThreadPoolExecutor(len(distros)) as e:
+            for distro in distros:
+                fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
+
+            for f in fs:
+                f.result()
+    else:
+        for distro in distros:
+            aws.ensure_linux_dev_ami(c, distro=distro)
+
+
 def bootstrap_windows_dev(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c)
@@ -73,7 +103,8 b' def build_windows_wheel(hga: HGAutomatio'
             windows.build_wheel(instance.winrm_client, a, DIST_PATH)
 
 
-def build_all_windows_packages(hga: HGAutomation, aws_region, revision):
+def build_all_windows_packages(hga: HGAutomation, aws_region, revision,
+                               version):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c)
     DIST_PATH.mkdir(exist_ok=True)
@@ -89,19 +120,52 b' def build_all_windows_packages(hga: HGAu'
             windows.purge_hg(winrm_client)
             windows.build_wheel(winrm_client, arch, DIST_PATH)
             windows.purge_hg(winrm_client)
-            windows.build_inno_installer(winrm_client, arch, DIST_PATH)
+            windows.build_inno_installer(winrm_client, arch, DIST_PATH,
+                                         version=version)
             windows.purge_hg(winrm_client)
-            windows.build_wix_installer(winrm_client, arch, DIST_PATH)
+            windows.build_wix_installer(winrm_client, arch, DIST_PATH,
+                                        version=version)
 
 
 def terminate_ec2_instances(hga: HGAutomation, aws_region):
-    c = hga.aws_connection(aws_region)
+    c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.terminate_ec2_instances(c.ec2resource)
 
 
 def purge_ec2_resources(hga: HGAutomation, aws_region):
+    c = hga.aws_connection(aws_region, ensure_ec2_state=False)
+    aws.remove_resources(c)
+
+
+def run_tests_linux(hga: HGAutomation, aws_region, instance_type,
+                    python_version, test_flags, distro, filesystem):
     c = hga.aws_connection(aws_region)
-    aws.remove_resources(c)
+    image = aws.ensure_linux_dev_ami(c, distro=distro)
+
+    t_start = time.time()
+
+    ensure_extra_volume = filesystem not in ('default', 'tmpfs')
+
+    with aws.temporary_linux_dev_instances(
+        c, image, instance_type,
+        ensure_extra_volume=ensure_extra_volume) as insts:
+
+        instance = insts[0]
+
+        linux.prepare_exec_environment(instance.ssh_client,
+                                       filesystem=filesystem)
+        linux.synchronize_hg(SOURCE_ROOT, instance, '.')
+        t_prepared = time.time()
+        linux.run_tests(instance.ssh_client, python_version,
+                        test_flags)
+        t_done = time.time()
+
+    t_setup = t_prepared - t_start
+    t_all = t_done - t_start
+
+    print(
+        'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
+        % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0))
 
 
 def run_tests_windows(hga: HGAutomation, aws_region, instance_type,
@@ -135,6 +199,21 b' def get_parser():'
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser(
+        'bootstrap-linux-dev',
+        help='Bootstrap Linux development environments',
+    )
+    sp.add_argument(
+        '--distros',
+        help='Comma delimited list of distros to bootstrap',
+    )
+    sp.add_argument(
+        '--parallel',
+        action='store_true',
+        help='Generate AMIs in parallel (not CTRL-c safe)'
+    )
+    sp.set_defaults(func=bootstrap_linux_dev)
+
+    sp = subparsers.add_parser(
         'bootstrap-windows-dev',
         help='Bootstrap the Windows development environment',
     )
@@ -149,6 +228,10 b' def get_parser():'
         help='Mercurial revision to build',
         default='.',
     )
+    sp.add_argument(
+        '--version',
+        help='Mercurial version string to use',
+    )
     sp.set_defaults(func=build_all_windows_packages)
 
     sp = subparsers.add_parser(
@@ -226,6 +309,41 b' def get_parser():'
     sp.set_defaults(func=purge_ec2_resources)
 
     sp = subparsers.add_parser(
+        'run-tests-linux',
+        help='Run tests on Linux',
+    )
+    sp.add_argument(
+        '--distro',
+        help='Linux distribution to run tests on',
+        choices=linux.DISTROS,
+        default='debian9',
+    )
+    sp.add_argument(
+        '--filesystem',
+        help='Filesystem type to use',
+        choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
+        default='default',
+    )
+    sp.add_argument(
+        '--instance-type',
+        help='EC2 instance type to use',
+        default='c5.9xlarge',
+    )
+    sp.add_argument(
+        '--python-version',
+        help='Python version to use',
+        choices={'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8',
+                 'pypy', 'pypy3.5', 'pypy3.6'},
+        default='system2',
+    )
+    sp.add_argument(
+        'test_flags',
+        help='Extra command line flags to pass to run-tests.py',
+        nargs='*',
+    )
+    sp.set_defaults(func=run_tests_linux)
+
+    sp = subparsers.add_parser(
         'run-tests-windows',
         help='Run tests on Windows',
     )

contrib/automation/hgautomation/windows.py

@@ -39,7 +39,7 b' Write-Output "activating Visual Studio 2'
 $Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE"
 $Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB"
-$Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib:$Env:LIBPATH"
+$Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib;$Env:LIBPATH"
 '''.lstrip()
 
 HG_PURGE = r'''
@@ -156,6 +156,10 b' def synchronize_hg(hg_repo: pathlib.Path'
             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
             fh.write('  IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
 
+        if not (hg_repo / '.hg').is_dir():
+            raise Exception('%s is not a Mercurial repository; '
+                            'synchronization not yet supported' % hg_repo)
+
         env = dict(os.environ)
         env['HGPLAIN'] = '1'
         env['HGENCODING'] = 'utf-8'
@@ -172,7 +176,8 b' def synchronize_hg(hg_repo: pathlib.Path'
             'python2.7', hg_bin,
             '--config', 'ui.ssh=ssh -F %s' % ssh_config,
             '--config', 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
-            'push', '-r', full_revision, 'ssh://%s/c:/hgdev/src' % public_ip,
+            'push', '-f', '-r', full_revision,
+            'ssh://%s/c:/hgdev/src' % public_ip,
         ]
 
         subprocess.run(args, cwd=str(hg_repo), env=env, check=True)

contrib/automation/hgautomation/winrm.py

@@ -25,7 +25,7 b' import requests.exceptions'
 logger = logging.getLogger(__name__)
 
 
-def wait_for_winrm(host, username, password, timeout=120, ssl=False):
+def wait_for_winrm(host, username, password, timeout=180, ssl=False):
     """Wait for the Windows Remoting (WinRM) service to become available.
 
     Returns a ``psrpclient.Client`` instance.

contrib/automation/requirements.txt

@@ -8,47 +8,68 b' asn1crypto==0.24.0 \\'
     --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
     --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \
     # via cryptography
-boto3==1.9.111 \
-    --hash=sha256:06414c75d1f62af7d04fd652b38d1e4fd3cfd6b35bad978466af88e2aaecd00d \
-    --hash=sha256:f3b77dff382374773d02411fa47ee408f4f503aeebd837fd9dc9ed8635bc5e8e
-botocore==1.12.111 \
-    --hash=sha256:6af473c52d5e3e7ff82de5334e9fee96b2d5ec2df5d78bc00cd9937e2573a7a8 \
-    --hash=sha256:9f5123c7be704b17aeacae99b5842ab17bda1f799dd29134de8c70e0a50a45d7 \
+bcrypt==3.1.6 \
+    --hash=sha256:0ba875eb67b011add6d8c5b76afbd92166e98b1f1efab9433d5dc0fafc76e203 \
+    --hash=sha256:21ed446054c93e209434148ef0b362432bb82bbdaf7beef70a32c221f3e33d1c \
+    --hash=sha256:28a0459381a8021f57230954b9e9a65bb5e3d569d2c253c5cac6cb181d71cf23 \
+    --hash=sha256:2aed3091eb6f51c26b7c2fad08d6620d1c35839e7a362f706015b41bd991125e \
+    --hash=sha256:2fa5d1e438958ea90eaedbf8082c2ceb1a684b4f6c75a3800c6ec1e18ebef96f \
+    --hash=sha256:3a73f45484e9874252002793518da060fb11eaa76c30713faa12115db17d1430 \
+    --hash=sha256:3e489787638a36bb466cd66780e15715494b6d6905ffdbaede94440d6d8e7dba \
+    --hash=sha256:44636759d222baa62806bbceb20e96f75a015a6381690d1bc2eda91c01ec02ea \
+    --hash=sha256:678c21b2fecaa72a1eded0cf12351b153615520637efcadc09ecf81b871f1596 \
+    --hash=sha256:75460c2c3786977ea9768d6c9d8957ba31b5fbeb0aae67a5c0e96aab4155f18c \
+    --hash=sha256:8ac06fb3e6aacb0a95b56eba735c0b64df49651c6ceb1ad1cf01ba75070d567f \
+    --hash=sha256:8fdced50a8b646fff8fa0e4b1c5fd940ecc844b43d1da5a980cb07f2d1b1132f \
+    --hash=sha256:9b2c5b640a2da533b0ab5f148d87fb9989bf9bcb2e61eea6a729102a6d36aef9 \
+    --hash=sha256:a9083e7fa9adb1a4de5ac15f9097eb15b04e2c8f97618f1b881af40abce382e1 \
+    --hash=sha256:b7e3948b8b1a81c5a99d41da5fb2dc03ddb93b5f96fcd3fd27e643f91efa33e1 \
+    --hash=sha256:b998b8ca979d906085f6a5d84f7b5459e5e94a13fc27c28a3514437013b6c2f6 \
+    --hash=sha256:dd08c50bc6f7be69cd7ba0769acca28c846ec46b7a8ddc2acf4b9ac6f8a7457e \
+    --hash=sha256:de5badee458544ab8125e63e39afeedfcf3aef6a6e2282ac159c95ae7472d773 \
+    --hash=sha256:ede2a87333d24f55a4a7338a6ccdccf3eaa9bed081d1737e0db4dbd1a4f7e6b6 \
+    # via paramiko
+boto3==1.9.137 \
+    --hash=sha256:882cc4869b47b51dae4b4a900769e72171ff00e0b6bca644b2d7a7ad7378f324 \
+    --hash=sha256:cd503a7e7a04f1c14d2801f9727159dfa88c393b4004e98940fa4aa205d920c8
+botocore==1.12.137 \
+    --hash=sha256:0d95794f6b1239c75e2c5f966221bcd4b68020fddb5676f757531eedbb612ed8 \
+    --hash=sha256:3213cf48cf2ceee10fc3b93221f2cd1c38521cca7584f547d5c086213cc60f35 \
     # via boto3, s3transfer
 certifi==2019.3.9 \
     --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \
     --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \
     # via requests
-cffi==1.12.2 \
-    --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \
-    --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \
-    --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \
-    --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \
-    --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \
-    --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \
-    --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \
-    --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \
-    --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \
-    --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \
-    --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \
-    --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \
-    --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \
-    --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \
-    --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \
-    --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \
-    --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \
-    --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \
-    --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \
-    --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \
-    --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \
-    --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \
-    --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \
-    --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \
-    --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \
-    --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \
-    --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \
-    --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889 \
-    # via cryptography
+cffi==1.12.3 \
+    --hash=sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774 \
+    --hash=sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d \
+    --hash=sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90 \
+    --hash=sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b \
+    --hash=sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63 \
+    --hash=sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45 \
+    --hash=sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25 \
+    --hash=sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3 \
+    --hash=sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b \
+    --hash=sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647 \
+    --hash=sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016 \
+    --hash=sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4 \
+    --hash=sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb \
+    --hash=sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753 \
+    --hash=sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7 \
+    --hash=sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9 \
+    --hash=sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f \
+    --hash=sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8 \
+    --hash=sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f \
+    --hash=sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc \
+    --hash=sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42 \
+    --hash=sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3 \
+    --hash=sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909 \
+    --hash=sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45 \
+    --hash=sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d \
+    --hash=sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512 \
+    --hash=sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff \
+    --hash=sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201 \
+    # via bcrypt, cryptography, pynacl
 chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
@@ -73,7 +94,7 b' cryptography==2.6.1 \\'
     --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \
     --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \
     --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6 \
-    # via pypsrp
+    # via paramiko, pypsrp
 docutils==0.14 \
     --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
     --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
@@ -87,13 +108,41 b' jmespath==0.9.4 \\'
     --hash=sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6 \
     --hash=sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c \
     # via boto3, botocore
-ntlm-auth==1.2.0 \
-    --hash=sha256:7bc02a3fbdfee7275d3dc20fce8028ed8eb6d32364637f28be9e9ae9160c6d5c \
-    --hash=sha256:9b13eaf88f16a831637d75236a93d60c0049536715aafbf8190ba58a590b023e \
+ntlm-auth==1.3.0 \
+    --hash=sha256:bb2fd03c665f0f62c5f65695b62dcdb07fb7a45df6ebc86c770be2054d6902dd \
+    --hash=sha256:ce5b4483ed761f341a538a426a71a52e5a9cf5fd834ebef1d2090f9eef14b3f8 \
     # via pypsrp
+paramiko==2.4.2 \
+    --hash=sha256:3c16b2bfb4c0d810b24c40155dbfd113c0521e7e6ee593d704e84b4c658a1f3b \
+    --hash=sha256:a8975a7df3560c9f1e2b43dc54ebd40fd00a7017392ca5445ce7df409f900fcb
+pyasn1==0.4.5 \
+    --hash=sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7 \
+    --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e \
+    # via paramiko
 pycparser==2.19 \
     --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
     # via cffi
+pynacl==1.3.0 \
+    --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
+    --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
+    --hash=sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e \
+    --hash=sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae \
+    --hash=sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621 \
+    --hash=sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56 \
+    --hash=sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39 \
+    --hash=sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310 \
+    --hash=sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1 \
+    --hash=sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a \
+    --hash=sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786 \
+    --hash=sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b \
+    --hash=sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b \
+    --hash=sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f \
+    --hash=sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20 \
+    --hash=sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415 \
+    --hash=sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715 \
+    --hash=sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1 \
+    --hash=sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0 \
+    # via paramiko
 pypsrp==0.3.1 \
     --hash=sha256:309853380fe086090a03cc6662a778ee69b1cae355ae4a932859034fd76e9d0b \
     --hash=sha256:90f946254f547dc3493cea8493c819ab87e152a755797c93aa2668678ba8ae85
@@ -112,8 +161,8 b' s3transfer==0.2.0 \\'
 six==1.12.0 \
     --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
     --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
-    # via cryptography, pypsrp, python-dateutil
-urllib3==1.24.1 \
-    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
-    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via bcrypt, cryptography, pynacl, pypsrp, python-dateutil
+urllib3==1.24.2 \
+    --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \
+    --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3 \
     # via botocore, requests

contrib/automation/requirements.txt.in

@@ -1,2 +1,3 b''
 boto3
+paramiko
 pypsrp

contrib/byteify-strings.py

@@ -7,7 +7,7 b''
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
 import argparse
 import contextlib
@@ -227,4 +227,7 b' def main():'
                 process(fin, fout, opts)
 
 if __name__ == '__main__':
+    if sys.version_info.major < 3:
+        print('This script must be run under Python 3.')
+        sys.exit(3)
     main()

contrib/catapipe.py

@@ -44,6 +44,7 b' import timeit'
 _TYPEMAP = {
     'START': 'B',
     'END': 'E',
+    'COUNTER': 'C',
 }
 
 _threadmap = {}
@@ -78,6 +79,11 b' def main():'
                 verb, session, label = ev.split(' ', 2)
                 if session not in _threadmap:
                     _threadmap[session] = len(_threadmap)
+                if verb == 'COUNTER':
+                    amount, label = label.split(' ', 1)
+                    payload_args = {'value': int(amount)}
+                else:
+                    payload_args = {}
                 pid = _threadmap[session]
                 ts_micros = (now - start) * 1000000
                 out.write(json.dumps(
@@ -88,7 +94,7 b' def main():'
                         "ts": ts_micros,
                         "pid": pid,
                         "tid": 1,
-                        "args": {}
+                        "args": payload_args,
                     }))
                 out.write(',\n')
     finally:

contrib/import-checker.py

@@ -649,15 +649,15 b' def embedded(f, modname, src):'
     ...         print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
     ...         print(repr(_forcestr(s)))
     >>> lines = [
-    ...   b'comment',
-    ...   b'  >>> from __future__ import print_function',
-    ...   b"  >>> ' multiline",
-    ...   b"  ... string'",
-    ...   b'  ',
-    ...   b'comment',
-    ...   b'  $ cat > foo.py <<EOF',
-    ...   b'  > from __future__ import print_function',
-    ...   b'  > EOF',
+    ...   'comment',
+    ...   '  >>> from __future__ import print_function',
+    ...   "  >>> ' multiline",
+    ...   "  ... string'",
+    ...   '  ',
+    ...   'comment',
+    ...   '  $ cat > foo.py <<EOF',
+    ...   '  > from __future__ import print_function',
+    ...   '  > EOF',
     ... ]
     >>> test(b"example.t", lines)
     example[2] doctest.py 1
@@ -694,7 +694,7 b' def sources(f, modname):'
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
-        with open(f, 'rb') as src:
+        with open(f, 'r') as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname.encode('utf8'), t, line
 

contrib/packaging/inno/readme.rst

@@ -32,7 +32,7 b' From the prompt, change to the Mercurial'
 ``cd c:\src\hg``.
 
 Next, invoke ``build.py`` to produce an Inno installer. You will
-need to supply the path to the Python interpreter to use.:
+need to supply the path to the Python interpreter to use.::
 
    $ python3.exe contrib\packaging\inno\build.py \
        --python c:\python27\python.exe

contrib/packaging/wix/help.wxs

@@ -49,6 +49,7 b''
             <File Id="internals.config.txt"       Name="config.txt" />
             <File Id="internals.extensions.txt"   Name="extensions.txt" />
             <File Id="internals.linelog.txt"      Name="linelog.txt" />
+            <File Id="internals.mergestate.txt"   Name="mergestate.txt" />
             <File Id="internals.requirements.txt" Name="requirements.txt" />
             <File Id="internals.revlogs.txt"      Name="revlogs.txt" />
             <File Id="internals.wireprotocol.txt" Name="wireprotocol.txt" />

contrib/perf.py

@@ -15,6 +15,13 b' Configurations'
 ``presleep``
   number of second to wait before any group of runs (default: 1)
 
+``pre-run``
+  number of run to perform before starting measurement.
+
+``profile-benchmark``
+  Enable profiling for the benchmarked section.
+  (The first iteration is benchmarked)
+
 ``run-limits``
   Control the number of runs each benchmark will perform. The option value
   should be a list of `<time>-<numberofrun>` pairs. After each run the
@@ -106,6 +113,10 b' try:'
 except ImportError:
     pass
 
+try:
+    from mercurial import profiling
+except ImportError:
+    profiling = None
 
 def identity(a):
     return a
@@ -240,6 +251,12 b' try:'
     configitem(b'perf', b'all-timing',
         default=mercurial.configitems.dynamicdefault,
     )
+    configitem(b'perf', b'pre-run',
+        default=mercurial.configitems.dynamicdefault,
+    )
+    configitem(b'perf', b'profile-benchmark',
+        default=mercurial.configitems.dynamicdefault,
+    )
     configitem(b'perf', b'run-limits',
         default=mercurial.configitems.dynamicdefault,
     )
@@ -251,6 +268,15 b' def getlen(ui):'
         return lambda x: 1
     return len
 
+class noop(object):
+    """dummy context manager"""
+    def __enter__(self):
+        pass
+    def __exit__(self, *args):
+        pass
+
+NOOPCTX = noop()
+
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
@@ -341,7 +367,14 b' def gettimer(ui, opts=None):'
     if not limits:
         limits = DEFAULTLIMITS
 
-    t = functools.partial(_timer, fm, displayall=displayall, limits=limits)
+    profiler = None
+    if profiling is not None:
+        if ui.configbool(b"perf", b"profile-benchmark", False):
+            profiler = profiling.profile(ui)
+
+    prerun = getint(ui, b"perf", b"pre-run", 0)
+    t = functools.partial(_timer, fm, displayall=displayall, limits=limits,
+                          prerun=prerun, profiler=profiler)
     return t, fm
 
 def stub_timer(fm, func, setup=None, title=None):
@@ -368,17 +401,25 b' DEFAULTLIMITS = ('
 )
 
 def _timer(fm, func, setup=None, title=None, displayall=False,
-           limits=DEFAULTLIMITS):
+           limits=DEFAULTLIMITS, prerun=0, profiler=None):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
+    if profiler is None:
+        profiler = NOOPCTX
+    for i in range(prerun):
+        if setup is not None:
+            setup()
+        func()
     keepgoing = True
     while keepgoing:
         if setup is not None:
             setup()
-        with timeone() as item:
-            r = func()
+        with profiler:
+            with timeone() as item:
+                r = func()
+        profiler = NOOPCTX
         count += 1
         results.append(item[0])
         cstop = util.timer()
@@ -922,17 +963,39 b' def perfdirstatewrite(ui, repo, **opts):'
     timer(d)
     fm.end()
 
+def _getmergerevs(repo, opts):
+    """parse command argument to return rev involved in merge
+
+    input: options dictionnary with `rev`, `from` and `bse`
+    output: (localctx, otherctx, basectx)
+    """
+    if opts[b'from']:
+        fromrev = scmutil.revsingle(repo, opts[b'from'])
+        wctx = repo[fromrev]
+    else:
+        wctx = repo[None]
+        # we don't want working dir files to be stat'd in the benchmark, so
+        # prime that cache
+        wctx.dirty()
+    rctx = scmutil.revsingle(repo, opts[b'rev'], opts[b'rev'])
+    if opts[b'base']:
+        fromrev = scmutil.revsingle(repo, opts[b'base'])
+        ancestor = repo[fromrev]
+    else:
+        ancestor = wctx.ancestor(rctx)
+    return (wctx, rctx, ancestor)
+
 @command(b'perfmergecalculate',
-         [(b'r', b'rev', b'.', b'rev to merge against')] + formatteropts)
-def perfmergecalculate(ui, repo, rev, **opts):
+         [
+             (b'r', b'rev', b'.', b'rev to merge against'),
+             (b'', b'from', b'', b'rev to merge from'),
+             (b'', b'base', b'', b'the revision to use as base'),
+         ] + formatteropts)
+def perfmergecalculate(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
-    wctx = repo[None]
-    rctx = scmutil.revsingle(repo, rev, rev)
-    ancestor = wctx.ancestor(rctx)
-    # we don't want working dir files to be stat'd in the benchmark, so prime
-    # that cache
-    wctx.dirty()
+
+    wctx, rctx, ancestor = _getmergerevs(repo, opts)
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
@@ -941,6 +1004,24 b' def perfmergecalculate(ui, repo, rev, **'
     timer(d)
     fm.end()
 
+@command(b'perfmergecopies',
+         [
+             (b'r', b'rev', b'.', b'rev to merge against'),
+             (b'', b'from', b'', b'rev to merge from'),
+             (b'', b'base', b'', b'the revision to use as base'),
+         ] + formatteropts)
+def perfmergecopies(ui, repo, **opts):
+    """measure runtime of `copies.mergecopies`"""
+    opts = _byteskwargs(opts)
+    timer, fm = gettimer(ui, opts)
+    wctx, rctx, ancestor = _getmergerevs(repo, opts)
+    def d():
+        # acceptremote is True because we don't want prompts in the middle of
+        # our benchmark
+        copies.mergecopies(repo, wctx, rctx, ancestor)
+    timer(d)
+    fm.end()
+
 @command(b'perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
     """benchmark the copy tracing logic"""
@@ -1390,6 +1471,111 b' def perftemplating(ui, repo, testedtempl'
     timer(format)
     fm.end()
 
+@command(b'perfhelper-mergecopies', formatteropts +
+         [
+          (b'r', b'revs', [], b'restrict search to these revisions'),
+          (b'', b'timing', False, b'provides extra data (costly)'),
+         ])
+def perfhelpermergecopies(ui, repo, revs=[], **opts):
+    """find statistics about potential parameters for `perfmergecopies`
+
+    This command find (base, p1, p2) triplet relevant for copytracing
+    benchmarking in the context of a merge.  It reports values for some of the
+    parameters that impact merge copy tracing time during merge.
+
+    If `--timing` is set, rename detection is run and the associated timing
+    will be reported. The extra details come at the cost of slower command
+    execution.
+
+    Since rename detection is only run once, other factors might easily
+    affect the precision of the timing. However it should give a good
+    approximation of which revision triplets are very costly.
+    """
+    opts = _byteskwargs(opts)
+    fm = ui.formatter(b'perf', opts)
+    dotiming = opts[b'timing']
+
+    output_template = [
+        ("base", "%(base)12s"),
+        ("p1", "%(p1.node)12s"),
+        ("p2", "%(p2.node)12s"),
+        ("p1.nb-revs", "%(p1.nbrevs)12d"),
+        ("p1.nb-files", "%(p1.nbmissingfiles)12d"),
+        ("p1.renames", "%(p1.renamedfiles)12d"),
+        ("p1.time", "%(p1.time)12.3f"),
+        ("p2.nb-revs", "%(p2.nbrevs)12d"),
+        ("p2.nb-files", "%(p2.nbmissingfiles)12d"),
+        ("p2.renames", "%(p2.renamedfiles)12d"),
+        ("p2.time", "%(p2.time)12.3f"),
+        ("renames", "%(nbrenamedfiles)12d"),
+        ("total.time", "%(time)12.3f"),
+        ]
+    if not dotiming:
+        output_template = [i for i in output_template
+                           if not ('time' in i[0] or 'renames' in i[0])]
+    header_names = [h for (h, v) in output_template]
+    output = ' '.join([v for (h, v) in output_template]) + '\n'
+    header = ' '.join(['%12s'] * len(header_names)) + '\n'
+    fm.plain(header % tuple(header_names))
+
+    if not revs:
+        revs = ['all()']
+    revs = scmutil.revrange(repo, revs)
+
+    roi = repo.revs('merge() and %ld', revs)
+    for r in roi:
+        ctx = repo[r]
+        p1 = ctx.p1()
+        p2 = ctx.p2()
+        bases = repo.changelog._commonancestorsheads(p1.rev(), p2.rev())
+        for b in bases:
+            b = repo[b]
+            p1missing = copies._computeforwardmissing(b, p1)
+            p2missing = copies._computeforwardmissing(b, p2)
+            data = {
+                b'base': b.hex(),
+                b'p1.node': p1.hex(),
+                b'p1.nbrevs': len(repo.revs('%d::%d', b.rev(), p1.rev())),
+                b'p1.nbmissingfiles': len(p1missing),
+                b'p2.node': p2.hex(),
+                b'p2.nbrevs': len(repo.revs('%d::%d', b.rev(), p2.rev())),
+                b'p2.nbmissingfiles': len(p2missing),
+            }
+            if dotiming:
+                begin = util.timer()
+                mergedata = copies.mergecopies(repo, p1, p2, b)
+                end = util.timer()
+                # not very stable timing since we did only one run
+                data['time'] = end - begin
+                # mergedata contains five dicts: "copy", "movewithdir",
+                # "diverge", "renamedelete" and "dirmove".
+                # The first 4 are about renamed file so lets count that.
+                renames = len(mergedata[0])
+                renames += len(mergedata[1])
+                renames += len(mergedata[2])
+                renames += len(mergedata[3])
+                data['nbrenamedfiles'] = renames
+                begin = util.timer()
+                p1renames = copies.pathcopies(b, p1)
+                end = util.timer()
+                data['p1.time'] = end - begin
+                begin = util.timer()
+                p2renames = copies.pathcopies(b, p2)
+                data['p2.time'] = end - begin
+                end = util.timer()
+                data['p1.renamedfiles'] = len(p1renames)
+                data['p2.renamedfiles'] = len(p2renames)
+            fm.startitem()
+            fm.data(**data)
+            # make node pretty for the human output
+            out = data.copy()
+            out['base'] = fm.hexfunc(b.node())
+            out['p1.node'] = fm.hexfunc(p1.node())
+            out['p2.node'] = fm.hexfunc(p2.node())
+            fm.plain(output % out)
+
+    fm.end()
+
 @command(b'perfhelper-pathcopies', formatteropts +
          [
           (b'r', b'revs', [], b'restrict search to these revisions'),
@@ -1890,7 +2076,7 b' def perfrevlogrevisions(ui, repo, file_='
 @command(b'perfrevlogwrite', revlogopts + formatteropts +
          [(b's', b'startrev', 1000, b'revision to start writing at'),
           (b'', b'stoprev', -1, b'last revision to write'),
-          (b'', b'count', 3, b'last revision to write'),
+          (b'', b'count', 3, b'number of passes to perform'),
           (b'', b'details', False, b'print timing for every revisions tested'),
           (b'', b'source', b'full', b'the kind of data feed in the revlog'),
           (b'', b'lazydeltabase', True, b'try the provided delta first'),
@@ -1907,6 +2093,16 b' def perfrevlogwrite(ui, repo, file_=None'
                   (use a delta from the first parent otherwise)
     * `parent-smallest`: add from the smallest delta (either p1 or p2)
     * `storage`: add from the existing precomputed deltas
+
+    Note: This performance command measures performance in a custom way. As a
+    result some of the global configuration of the 'perf' command does not
+    apply to it:
+
+    * ``pre-run``: disabled
+
+    * ``profile-benchmark``: disabled
+
+    * ``run-limits``: disabled use --count instead
     """
     opts = _byteskwargs(opts)
 
@@ -2081,6 +2277,10 b' def _temprevlog(ui, orig, truncaterev):'
 
     if orig._inline:
         raise error.Abort('not supporting inline revlog (yet)')
+    revlogkwargs = {}
+    k = 'upperboundcomp'
+    if util.safehasattr(orig, k):
+        revlogkwargs[k] = getattr(orig, k)
 
     origindexpath = orig.opener.join(orig.indexfile)
     origdatapath = orig.opener.join(orig.datafile)
@@ -2112,7 +2312,7 b' def _temprevlog(ui, orig, truncaterev):'
 
         dest = revlog.revlog(vfs,
                              indexfile=indexname,
-                             datafile=dataname)
+                             datafile=dataname, **revlogkwargs)
         if dest._inline:
             raise error.Abort('not supporting inline revlog (yet)')
         # make sure internals are initialized

contrib/python3-whitelist

@@ -4,6 +4,7 b' test-absorb-filefixupstate.py'
 test-absorb-phase.t
 test-absorb-rename.t
 test-absorb-strip.t
+test-absorb-unfinished.t
 test-absorb.t
 test-acl.t
 test-add.t
@@ -35,6 +36,7 b' test-bisect2.t'
 test-bisect3.t
 test-blackbox.t
 test-bookflow.t
+test-bookmarks-corner-case.t
 test-bookmarks-current.t
 test-bookmarks-merge.t
 test-bookmarks-pushpull.t
@@ -104,6 +106,7 b' test-context.py'
 test-contrib-check-code.t
 test-contrib-check-commit.t
 test-contrib-dumprevlog.t
+test-contrib-emacs.t
 test-contrib-perf.t
 test-contrib-relnotes.t
 test-contrib-testparseutil.t
@@ -126,6 +129,8 b' test-convert-splicemap.t'
 test-convert-svn-sink.t
 test-convert-tagsbranch-topology.t
 test-convert.t
+test-copies-in-changeset.t
+test-copies-unrelated.t
 test-copies.t
 test-copy-move-merge.t
 test-copy.t
@@ -139,6 +144,7 b' test-debugindexdot.t'
 test-debugrename.t
 test-default-push.t
 test-demandimport.py
+test-devel-warnings.t
 test-diff-antipatience.t
 test-diff-binary-file.t
 test-diff-change.t
@@ -159,6 +165,7 b' test-directaccess.t'
 test-dirstate-backup.t
 test-dirstate-nonnormalset.t
 test-dirstate-race.t
+test-dirstate-race2.t
 test-dirstate.t
 test-dispatch.py
 test-dispatch.t
@@ -230,6 +237,7 b' test-filecache.py'
 test-filelog.py
 test-fileset-generated.t
 test-fileset.t
+test-fix-metadata.t
 test-fix-topology.t
 test-fix.t
 test-flagprocessor.t
@@ -511,6 +519,7 b' test-pathconflicts-basic.t'
 test-pathconflicts-merge.t
 test-pathconflicts-update.t
 test-pathencode.py
+test-paths.t
 test-pending.t
 test-permissions.t
 test-phabricator.t
@@ -597,6 +606,7 b' test-releasenotes-formatting.t'
 test-releasenotes-merging.t
 test-releasenotes-parsing.t
 test-relink.t
+test-remote-hidden.t
 test-remotefilelog-bad-configs.t
 test-remotefilelog-bgprefetch.t
 test-remotefilelog-blame.t
@@ -658,10 +668,12 b' test-rollback.t'
 test-run-tests.py
 test-run-tests.t
 test-rust-ancestor.py
+test-rust-discovery.py
 test-schemes.t
 test-serve.t
 test-server-view.t
 test-setdiscovery.t
+test-share-bookmarks.t
 test-share.t
 test-shelve.t
 test-shelve2.t

contrib/testparseutil.py

@@ -38,12 +38,6 b' def rapply(f, xs):'
 if ispy3:
     import builtins
 
-    # TODO: .buffer might not exist if std streams were replaced; we'll need
-    # a silly wrapper to make a bytes stream backed by a unicode one.
-    stdin = sys.stdin.buffer
-    stdout = sys.stdout.buffer
-    stderr = sys.stderr.buffer
-
     def bytestr(s):
         # tiny version of pycompat.bytestr
         return s.encode('latin1')
@@ -54,12 +48,8 b' if ispy3:'
         return s.decode(u'latin-1')
 
     def opentext(f):
-        return open(f, 'rb')
+        return open(f, 'r')
 else:
-    stdin = sys.stdin
-    stdout = sys.stdout
-    stderr = sys.stderr
-
     bytestr = str
     sysstr = identity
 
@@ -71,11 +61,11 b' def b2s(x):'
 
 def writeout(data):
     # write "data" in BYTES into stdout
-    stdout.write(data)
+    sys.stdout.write(data)
 
 def writeerr(data):
     # write "data" in BYTES into stderr
-    stderr.write(data)
+    sys.stderr.write(data)
 
 ####################
 
@@ -164,14 +154,14 b' def embedded(basefile, lines, errors, ma'
     ...         self.matchfunc = matchfunc
     ...     def startsat(self, line):
     ...         return self.matchfunc(line)
-    >>> ambig1 = ambigmatcher(b'ambiguous #1',
-    ...                       lambda l: l.startswith(b'  $ cat '))
-    >>> ambig2 = ambigmatcher(b'ambiguous #2',
-    ...                       lambda l: l.endswith(b'<< EOF\\n'))
-    >>> lines = [b'  $ cat > foo.py << EOF\\n']
+    >>> ambig1 = ambigmatcher('ambiguous #1',
+    ...                       lambda l: l.startswith('  $ cat '))
+    >>> ambig2 = ambigmatcher('ambiguous #2',
+    ...                       lambda l: l.endswith('<< EOF\\n'))
+    >>> lines = ['  $ cat > foo.py << EOF\\n']
     >>> errors = []
     >>> matchers = [ambig1, ambig2]
-    >>> list(t for t in embedded(b'<dummy>', lines, errors, matchers))
+    >>> list(t for t in embedded('<dummy>', lines, errors, matchers))
     []
     >>> b2s(errors)
     ['<dummy>:1: ambiguous line for "ambiguous #1", "ambiguous #2"']
@@ -181,21 +171,21 b' def embedded(basefile, lines, errors, ma'
     ctx = filename = code = startline = None # for pyflakes
 
     for lineno, line in enumerate(lines, 1):
-        if not line.endswith(b'\n'):
-            line += b'\n' # to normalize EOF line
+        if not line.endswith('\n'):
+            line += '\n' # to normalize EOF line
         if matcher: # now, inside embedded code
             if matcher.endsat(ctx, line):
                 codeatend = matcher.codeatend(ctx, line)
                 if codeatend is not None:
                     code.append(codeatend)
                 if not matcher.ignores(ctx):
-                    yield (filename, startline, lineno, b''.join(code))
+                    yield (filename, startline, lineno, ''.join(code))
                 matcher = None
                 # DO NOT "continue", because line might start next fragment
             elif not matcher.isinside(ctx, line):
                 # this is an error of basefile
                 # (if matchers are implemented correctly)
-                errors.append(b'%s:%d: unexpected line for "%s"'
+                errors.append('%s:%d: unexpected line for "%s"'
                               % (basefile, lineno, matcher.desc))
                 # stop extracting embedded code by current 'matcher',
                 # because appearance of unexpected line might mean
@@ -218,9 +208,9 b' def embedded(basefile, lines, errors, ma'
         if matched:
             if len(matched) > 1:
                 # this is an error of matchers, maybe
-                errors.append(b'%s:%d: ambiguous line for %s' %
+                errors.append('%s:%d: ambiguous line for %s' %
                               (basefile, lineno,
-                               b', '.join([b'"%s"' % m.desc
+                               ', '.join(['"%s"' % m.desc
                                            for m, c in matched])))
                 # omit extracting embedded code, because choosing
                 # arbitrary matcher from matched ones might fail to
@@ -239,20 +229,20 b' def embedded(basefile, lines, errors, ma'
     if matcher:
         # examine whether EOF ends embedded code, because embedded
         # code isn't yet ended explicitly
-        if matcher.endsat(ctx, b'\n'):
-            codeatend = matcher.codeatend(ctx, b'\n')
+        if matcher.endsat(ctx, '\n'):
+            codeatend = matcher.codeatend(ctx, '\n')
             if codeatend is not None:
                 code.append(codeatend)
             if not matcher.ignores(ctx):
-                yield (filename, startline, lineno + 1, b''.join(code))
+                yield (filename, startline, lineno + 1, ''.join(code))
         else:
             # this is an error of basefile
             # (if matchers are implemented correctly)
-            errors.append(b'%s:%d: unexpected end of file for "%s"'
+            errors.append('%s:%d: unexpected end of file for "%s"'
                           % (basefile, lineno, matcher.desc))
 
 # heredoc limit mark to ignore embedded code at check-code.py or so
-heredocignorelimit = b'NO_CHECK_EOF'
+heredocignorelimit = 'NO_CHECK_EOF'
 
 # the pattern to match against cases below, and to return a limit mark
 # string as 'lname' group
@@ -260,47 +250,47 b" heredocignorelimit = b'NO_CHECK_EOF'"
 # - << LIMITMARK
 # - << "LIMITMARK"
 # - << 'LIMITMARK'
-heredoclimitpat = br'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)'
+heredoclimitpat = r'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)'
 
 class fileheredocmatcher(embeddedmatcher):
     """Detect "cat > FILE << LIMIT" style embedded code
 
-    >>> matcher = fileheredocmatcher(b'heredoc .py file', br'[^<]+\\.py')
-    >>> b2s(matcher.startsat(b'  $ cat > file.py << EOF\\n'))
+    >>> matcher = fileheredocmatcher('heredoc .py file', r'[^<]+\\.py')
+    >>> b2s(matcher.startsat('  $ cat > file.py << EOF\\n'))
     ('file.py', '  > EOF\\n')
-    >>> b2s(matcher.startsat(b'  $ cat   >>file.py   <<EOF\\n'))
+    >>> b2s(matcher.startsat('  $ cat   >>file.py   <<EOF\\n'))
     ('file.py', '  > EOF\\n')
-    >>> b2s(matcher.startsat(b'  $ cat>  \\x27any file.py\\x27<<  "EOF"\\n'))
+    >>> b2s(matcher.startsat('  $ cat>  \\x27any file.py\\x27<<  "EOF"\\n'))
     ('any file.py', '  > EOF\\n')
-    >>> b2s(matcher.startsat(b"  $ cat > file.py << 'ANYLIMIT'\\n"))
+    >>> b2s(matcher.startsat("  $ cat > file.py << 'ANYLIMIT'\\n"))
     ('file.py', '  > ANYLIMIT\\n')
-    >>> b2s(matcher.startsat(b'  $ cat<<ANYLIMIT>"file.py"\\n'))
+    >>> b2s(matcher.startsat('  $ cat<<ANYLIMIT>"file.py"\\n'))
     ('file.py', '  > ANYLIMIT\\n')
-    >>> start = b'  $ cat > file.py << EOF\\n'
+    >>> start = '  $ cat > file.py << EOF\\n'
     >>> ctx = matcher.startsat(start)
     >>> matcher.codeatstart(ctx, start)
     >>> b2s(matcher.filename(ctx))
     'file.py'
     >>> matcher.ignores(ctx)
     False
-    >>> inside = b'  > foo = 1\\n'
+    >>> inside = '  > foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
-    >>> end = b'  > EOF\\n'
+    >>> end = '  > EOF\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
-    >>> matcher.endsat(ctx, b'  > EOFEOF\\n')
+    >>> matcher.endsat(ctx, '  > EOFEOF\\n')
     False
-    >>> ctx = matcher.startsat(b'  $ cat > file.py << NO_CHECK_EOF\\n')
+    >>> ctx = matcher.startsat('  $ cat > file.py << NO_CHECK_EOF\\n')
     >>> matcher.ignores(ctx)
     True
     """
-    _prefix = b'  > '
+    _prefix = '  > '
 
     def __init__(self, desc, namepat):
         super(fileheredocmatcher, self).__init__(desc)
@@ -312,13 +302,13 b' class fileheredocmatcher(embeddedmatcher'
         # - > NAMEPAT
         # - > "NAMEPAT"
         # - > 'NAMEPAT'
-        namepat = (br'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)'
+        namepat = (r'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)'
                    % namepat)
         self._fileres = [
             # "cat > NAME << LIMIT" case
-            re.compile(br'  \$ \s*cat' + namepat + heredoclimitpat),
+            re.compile(r'  \$ \s*cat' + namepat + heredoclimitpat),
             # "cat << LIMIT > NAME" case
-            re.compile(br'  \$ \s*cat' + heredoclimitpat + namepat),
+            re.compile(r'  \$ \s*cat' + heredoclimitpat + namepat),
         ]
 
     def startsat(self, line):
@@ -327,7 +317,7 b' class fileheredocmatcher(embeddedmatcher'
             matched = filere.match(line)
             if matched:
                 return (matched.group('name'),
-                        b'  > %s\n' % matched.group('limit'))
+                        '  > %s\n' % matched.group('limit'))
 
     def endsat(self, ctx, line):
         return ctx[1] == line
@@ -336,7 +326,7 b' class fileheredocmatcher(embeddedmatcher'
         return line.startswith(self._prefix)
 
     def ignores(self, ctx):
-        return b'  > %s\n' % heredocignorelimit == ctx[1]
+        return '  > %s\n' % heredocignorelimit == ctx[1]
 
     def filename(self, ctx):
         return ctx[0]
@@ -357,10 +347,10 b' class pydoctestmatcher(embeddedmatcher):'
     """Detect ">>> code" style embedded python code
 
     >>> matcher = pydoctestmatcher()
-    >>> startline = b'  >>> foo = 1\\n'
+    >>> startline = '  >>> foo = 1\\n'
     >>> matcher.startsat(startline)
     True
-    >>> matcher.startsat(b'  ... foo = 1\\n')
+    >>> matcher.startsat('  ... foo = 1\\n')
     False
     >>> ctx = matcher.startsat(startline)
     >>> matcher.filename(ctx)
@@ -368,45 +358,45 b' class pydoctestmatcher(embeddedmatcher):'
     False
     >>> b2s(matcher.codeatstart(ctx, startline))
     'foo = 1\\n'
-    >>> inside = b'  >>> foo = 1\\n'
+    >>> inside = '  >>> foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
-    >>> inside = b'  ... foo = 1\\n'
+    >>> inside = '  ... foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
-    >>> inside = b'  expected output\\n'
+    >>> inside = '  expected output\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     '\\n'
-    >>> inside = b'  \\n'
+    >>> inside = '  \\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     '\\n'
-    >>> end = b'  $ foo bar\\n'
+    >>> end = '  $ foo bar\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
-    >>> end = b'\\n'
+    >>> end = '\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     """
-    _prefix = b'  >>> '
-    _prefixre = re.compile(br'  (>>>|\.\.\.) ')
+    _prefix = '  >>> '
+    _prefixre = re.compile(r'  (>>>|\.\.\.) ')
 
     # If a line matches against not _prefixre but _outputre, that line
     # is "an expected output line" (= not a part of code fragment).
@@ -416,10 +406,10 b' class pydoctestmatcher(embeddedmatcher):'
     # run-tests.py. But "directive line inside inline python code"
     # should be rejected by Mercurial reviewers. Therefore, this
     # regexp does not matche against such directive lines.
-    _outputre = re.compile(br'  $|  [^$]')
+    _outputre = re.compile(r'  $|  [^$]')
 
     def __init__(self):
-        super(pydoctestmatcher, self).__init__(b"doctest style python code")
+        super(pydoctestmatcher, self).__init__("doctest style python code")
 
     def startsat(self, line):
         # ctx is "True"
@@ -446,57 +436,57 b' class pydoctestmatcher(embeddedmatcher):'
     def codeinside(self, ctx, line):
         if self._prefixre.match(line):
             return line[len(self._prefix):] # strip prefix '  >>> '/'  ... '
-        return b'\n' # an expected output line is treated as an empty line
+        return '\n' # an expected output line is treated as an empty line
 
 class pyheredocmatcher(embeddedmatcher):
     """Detect "python << LIMIT" style embedded python code
 
     >>> matcher = pyheredocmatcher()
-    >>> b2s(matcher.startsat(b'  $ python << EOF\\n'))
+    >>> b2s(matcher.startsat('  $ python << EOF\\n'))
     '  > EOF\\n'
-    >>> b2s(matcher.startsat(b'  $ $PYTHON   <<EOF\\n'))
+    >>> b2s(matcher.startsat('  $ $PYTHON   <<EOF\\n'))
     '  > EOF\\n'
-    >>> b2s(matcher.startsat(b'  $ "$PYTHON"<<  "EOF"\\n'))
+    >>> b2s(matcher.startsat('  $ "$PYTHON"<<  "EOF"\\n'))
     '  > EOF\\n'
-    >>> b2s(matcher.startsat(b"  $ $PYTHON << 'ANYLIMIT'\\n"))
+    >>> b2s(matcher.startsat("  $ $PYTHON << 'ANYLIMIT'\\n"))
     '  > ANYLIMIT\\n'
-    >>> matcher.startsat(b'  $ "$PYTHON" < EOF\\n')
-    >>> start = b'  $ python << EOF\\n'
+    >>> matcher.startsat('  $ "$PYTHON" < EOF\\n')
+    >>> start = '  $ python << EOF\\n'
     >>> ctx = matcher.startsat(start)
     >>> matcher.codeatstart(ctx, start)
     >>> matcher.filename(ctx)
     >>> matcher.ignores(ctx)
     False
-    >>> inside = b'  > foo = 1\\n'
+    >>> inside = '  > foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
-    >>> end = b'  > EOF\\n'
+    >>> end = '  > EOF\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
-    >>> matcher.endsat(ctx, b'  > EOFEOF\\n')
+    >>> matcher.endsat(ctx, '  > EOFEOF\\n')
     False
-    >>> ctx = matcher.startsat(b'  $ python << NO_CHECK_EOF\\n')
+    >>> ctx = matcher.startsat('  $ python << NO_CHECK_EOF\\n')
     >>> matcher.ignores(ctx)
     True
     """
-    _prefix = b'  > '
+    _prefix = '  > '
 
-    _startre = re.compile(br'  \$ (\$PYTHON|"\$PYTHON"|python).*' +
+    _startre = re.compile(r'  \$ (\$PYTHON|"\$PYTHON"|python).*' +
                           heredoclimitpat)
 
     def __init__(self):
-        super(pyheredocmatcher, self).__init__(b"heredoc python invocation")
+        super(pyheredocmatcher, self).__init__("heredoc python invocation")
 
     def startsat(self, line):
         # ctx is END-LINE-OF-EMBEDDED-CODE
         matched = self._startre.match(line)
         if matched:
-            return b'  > %s\n' % matched.group('limit')
+            return '  > %s\n' % matched.group('limit')
 
     def endsat(self, ctx, line):
         return ctx == line
@@ -505,7 +495,7 b' class pyheredocmatcher(embeddedmatcher):'
         return line.startswith(self._prefix)
 
     def ignores(self, ctx):
-        return b'  > %s\n' % heredocignorelimit == ctx
+        return '  > %s\n' % heredocignorelimit == ctx
 
     def filename(self, ctx):
         return None # no filename
@@ -524,7 +514,7 b' class pyheredocmatcher(embeddedmatcher):'
     pyheredocmatcher(),
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
-    fileheredocmatcher(b'heredoc .py file', br'[^<]+\.py'),
+    fileheredocmatcher('heredoc .py file', r'[^<]+\.py'),
 ]
 
 def pyembedded(basefile, lines, errors):
@@ -536,7 +526,7 b' def pyembedded(basefile, lines, errors):'
 _shmatchers = [
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
-    fileheredocmatcher(b'heredoc .sh file', br'[^<]+\.sh'),
+    fileheredocmatcher('heredoc .sh file', r'[^<]+\.sh'),
 ]
 
 def shembedded(basefile, lines, errors):
@@ -548,8 +538,8 b' def shembedded(basefile, lines, errors):'
 _hgrcmatchers = [
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
-    fileheredocmatcher(b'heredoc hgrc file',
-                       br'(([^/<]+/)+hgrc|\$HGRCPATH|\${HGRCPATH})'),
+    fileheredocmatcher('heredoc hgrc file',
+                       r'(([^/<]+/)+hgrc|\$HGRCPATH|\${HGRCPATH})'),
 ]
 
 def hgrcembedded(basefile, lines, errors):
@@ -565,14 +555,14 b' if __name__ == "__main__":'
         errors = []
         for name, starts, ends, code in embeddedfunc(basefile, lines, errors):
             if not name:
-                name = b'<anonymous>'
-            writeout(b"%s:%d: %s starts\n" % (basefile, starts, name))
+                name = '<anonymous>'
+            writeout("%s:%d: %s starts\n" % (basefile, starts, name))
             if opts.verbose and code:
-                writeout(b"  |%s\n" %
-                         b"\n  |".join(l for l in code.splitlines()))
-            writeout(b"%s:%d: %s ends\n" % (basefile, ends, name))
+                writeout("  |%s\n" %
+                         "\n  |".join(l for l in code.splitlines()))
+            writeout("%s:%d: %s ends\n" % (basefile, ends, name))
         for e in errors:
-            writeerr(b"%s\n" % e)
+            writeerr("%s\n" % e)
         return len(errors)
 
     def applyembedded(args, embeddedfunc, opts):
@@ -580,11 +570,11 b' if __name__ == "__main__":'
         if args:
             for f in args:
                 with opentext(f) as fp:
-                    if showembedded(bytestr(f), fp, embeddedfunc, opts):
+                    if showembedded(f, fp, embeddedfunc, opts):
                         ret = 1
         else:
-            lines = [l for l in stdin.readlines()]
-            if showembedded(b'<stdin>', lines, embeddedfunc, opts):
+            lines = [l for l in sys.stdin.readlines()]
+            if showembedded('<stdin>', lines, embeddedfunc, opts):
                 ret = 1
         return ret
 

contrib/win32/mercurial.ini

@@ -64,7 +64,6 b' editor = notepad'
 ;relink =
 ;schemes =
 ;share =
-;shelve =
 ;transplant =
 ;win32mbcs =
 ;zeroconf =

contrib/zsh_completion

@@ -1,4 +1,4 b''
-#compdef hg
+#compdef hg chg
 
 # Zsh completion script for mercurial.  Rename this file to _hg and copy
 # it into your zsh function path (/usr/share/zsh/site-functions for

doc/gendoc.py

@@ -120,7 +120,7 b' def showdoc(ui):'
 
     # print cmds
     ui.write(minirst.section(_(b"Commands")))
-    commandprinter(ui, table, minirst.subsection)
+    commandprinter(ui, table, minirst.subsection, minirst.subsubsection)
 
     # print help topics
     # The config help topic is included in the hgrc.5 man page.
@@ -143,7 +143,8 b' def showdoc(ui):'
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
             ui.write(minirst.subsubsection(_(b'Commands')))
-            commandprinter(ui, cmdtable, minirst.subsubsubsection)
+            commandprinter(ui, cmdtable, minirst.subsubsubsection,
+                    minirst.subsubsubsubsection)
 
 def showtopic(ui, topic):
     extrahelptable = [
@@ -177,7 +178,27 b' def helpprinter(ui, helptable, sectionfu'
         ui.write(doc)
         ui.write(b"\n")
 
-def commandprinter(ui, cmdtable, sectionfunc):
+def commandprinter(ui, cmdtable, sectionfunc, subsectionfunc):
+    """Render restructuredtext describing a list of commands and their
+    documentations, grouped by command category.
+
+    Args:
+      ui: UI object to write the output to
+      cmdtable: a dict that maps a string of the command name plus its aliases
+        (separated with pipes) to a 3-tuple of (the command's function, a list
+        of its option descriptions, and a string summarizing available
+        options). Example, with aliases added for demonstration purposes:
+
+          'phase|alias1|alias2': (
+             <function phase at 0x7f0816b05e60>,
+             [ ('p', 'public', False, 'set changeset phase to public'),
+               ...,
+               ('r', 'rev', [], 'target revision', 'REV')],
+             '[-p|-d|-s] [-f] [-r] [REV...]'
+          )
+      sectionfunc: minirst function to format command category headers
+      subsectionfunc: minirst function to format command headers
+    """
     h = {}
     for c, attr in cmdtable.items():
         f = c.split(b"|")[0]
@@ -185,45 +206,76 b' def commandprinter(ui, cmdtable, section'
         h[f] = c
     cmds = h.keys()
 
-    for f in sorted(cmds):
-        if f.startswith(b"debug"):
+    def helpcategory(cmd):
+        """Given a canonical command name from `cmds` (above), retrieve its
+        help category. If helpcategory is None, default to CATEGORY_NONE.
+        """
+        fullname = h[cmd]
+        details = cmdtable[fullname]
+        helpcategory = details[0].helpcategory
+        return helpcategory or help.registrar.command.CATEGORY_NONE
+
+    cmdsbycategory = {category: [] for category in help.CATEGORY_ORDER}
+    for cmd in cmds:
+        # If a command category wasn't registered, the command won't get
+        # rendered below, so we raise an AssertionError.
+        if helpcategory(cmd) not in cmdsbycategory:
+            raise AssertionError(
+                "The following command did not register its (category) in "
+                "help.CATEGORY_ORDER: %s (%s)" % (cmd, helpcategory(cmd)))
+        cmdsbycategory[helpcategory(cmd)].append(cmd)
+
+    # Print the help for each command. We present the commands grouped by
+    # category, and we use help.CATEGORY_ORDER as a guide for a helpful order
+    # in which to present the categories.
+    for category in help.CATEGORY_ORDER:
+        categorycmds = cmdsbycategory[category]
+        if not categorycmds:
+            # Skip empty categories
             continue
-        d = get_cmd(h[f], cmdtable)
-        ui.write(sectionfunc(d[b'cmd']))
-        # short description
-        ui.write(d[b'desc'][0])
-        # synopsis
-        ui.write(b"::\n\n")
-        synopsislines = d[b'synopsis'].splitlines()
-        for line in synopsislines:
-            # some commands (such as rebase) have a multi-line
+        # Print a section header for the category.
+        # For now, the category header is at the same level as the headers for
+        # the commands in the category; this is fixed in the next commit.
+        ui.write(sectionfunc(help.CATEGORY_NAMES[category]))
+        # Print each command in the category
+        for f in sorted(categorycmds):
+            if f.startswith(b"debug"):
+                continue
+            d = get_cmd(h[f], cmdtable)
+            ui.write(subsectionfunc(d[b'cmd']))
+            # short description
+            ui.write(d[b'desc'][0])
             # synopsis
-            ui.write(b"   %s\n" % line)
-        ui.write(b'\n')
-        # description
-        ui.write(b"%s\n\n" % d[b'desc'][1])
-        # options
-        opt_output = list(d[b'opts'])
-        if opt_output:
-            opts_len = max([len(line[0]) for line in opt_output])
-            ui.write(_(b"Options:\n\n"))
-            multioccur = False
-            for optstr, desc in opt_output:
-                if desc:
-                    s = b"%-*s  %s" % (opts_len, optstr, desc)
-                else:
-                    s = optstr
-                ui.write(b"%s\n" % s)
-                if optstr.endswith(b"[+]>"):
-                    multioccur = True
-            if multioccur:
-                ui.write(_(b"\n[+] marked option can be specified"
-                           b" multiple times\n"))
-            ui.write(b"\n")
-        # aliases
-        if d[b'aliases']:
-            ui.write(_(b"    aliases: %s\n\n") % b" ".join(d[b'aliases']))
-
+            ui.write(b"::\n\n")
+            synopsislines = d[b'synopsis'].splitlines()
+            for line in synopsislines:
+                # some commands (such as rebase) have a multi-line
+                # synopsis
+                ui.write(b"   %s\n" % line)
+            ui.write(b'\n')
+            # description
+            ui.write(b"%s\n\n" % d[b'desc'][1])
+            # options
+            opt_output = list(d[b'opts'])
+            if opt_output:
+                opts_len = max([len(line[0]) for line in opt_output])
+                ui.write(_(b"Options:\n\n"))
+                multioccur = False
+                for optstr, desc in opt_output:
+                    if desc:
+                        s = b"%-*s  %s" % (opts_len, optstr, desc)
+                    else:
+                        s = optstr
+                    ui.write(b"%s\n" % s)
+                    if optstr.endswith(b"[+]>"):
+                        multioccur = True
+                if multioccur:
+                    ui.write(_(b"\n[+] marked option can be specified"
+                               b" multiple times\n"))
+                ui.write(b"\n")
+            # aliases
+            if d[b'aliases']:
+                ui.write(_(b"    aliases: %s\n\n") % b" ".join(d[b'aliases']))
 
 def allextensionnames():
     return set(extensions.enabled().keys()) | set(extensions.disabled().keys())

hgdemandimport/demandimportpy3.py

@@ -32,6 +32,8 b' import importlib.machinery'
 import importlib.util
 import sys
 
+from . import tracing
+
 _deactivated = False
 
 class _lazyloaderex(importlib.util.LazyLoader):
@@ -40,10 +42,11 b' class _lazyloaderex(importlib.util.LazyL'
     """
     def exec_module(self, module):
         """Make the module load lazily."""
-        if _deactivated or module.__name__ in ignores:
-            self.loader.exec_module(module)
-        else:
-            super().exec_module(module)
+        with tracing.log('demandimport %s', module):
+            if _deactivated or module.__name__ in ignores:
+                self.loader.exec_module(module)
+            else:
+                super().exec_module(module)
 
 # This is 3.6+ because with Python 3.5 it isn't possible to lazily load
 # extensions. See the discussion in https://bugs.python.org/issue26186 for more.

hgdemandimport/tracing.py

@@ -13,19 +13,23 b' import os'
 _pipe = None
 _checked = False
 
[e-mail]
-def log(whencefmt, *whenceargs):
+def _isactive():
     global _pipe, _session, _checked
     if _pipe is None:
         if _checked:
-            yield
-            return
+            return False
         _checked = True
         if 'HGCATAPULTSERVERPIPE' not in os.environ:
-            yield
-            return
+            return False
         _pipe = open(os.environ['HGCATAPULTSERVERPIPE'], 'w', 1)
         _session = os.environ.get('HGCATAPULTSESSION', 'none')
+    return True
+
[e-mail]
+def log(whencefmt, *whenceargs):
+    if not _isactive():
+        yield
+        return
     whence = whencefmt % whenceargs
     try:
         # Both writes to the pipe are wrapped in try/except to ignore
@@ -42,3 +46,13 b' def log(whencefmt, *whenceargs):'
             _pipe.write('END %s %s\n' % (_session, whence))
         except IOError:
             pass
+
+def counter(label, amount, *labelargs):
+    if not _isactive():
+        return
+    l = label % labelargs
+    # See above in log() for why this is in a try/except.
+    try:
+        _pipe.write('COUNTER %s %d %s\n' % (_session, amount, l))
+    except IOError:
+        pass

hgext/absorb.py

@@ -871,7 +871,7 b' def _parsechunk(hunk):'
     patchlines = mdiff.splitnewlines(buf.getvalue())
     # hunk.prettystr() will update hunk.removed
     a2 = a1 + hunk.removed
-    blines = [l[1:] for l in patchlines[1:] if l[0] != '-']
+    blines = [l[1:] for l in patchlines[1:] if not l.startswith('-')]
     return path, (a1, a2, blines)
 
 def overlaydiffcontext(ctx, chunks):
@@ -914,7 +914,10 b' def absorb(ui, repo, stack=None, targetc'
     """
     if stack is None:
         limit = ui.configint('absorb', 'max-stack-size')
-        stack = getdraftstack(repo['.'], limit)
+        headctx = repo['.']
+        if len(headctx.parents()) > 1:
+            raise error.Abort(_('cannot absorb into a merge'))
+        stack = getdraftstack(headctx, limit)
         if limit and len(stack) >= limit:
             ui.warn(_('absorb: only the recent %d changesets will '
                       'be analysed\n')
@@ -932,7 +935,7 b' def absorb(ui, repo, stack=None, targetc'
     if opts.get('interactive'):
         diff = patch.diff(repo, stack[-1].node(), targetctx.node(), matcher)
         origchunks = patch.parsepatch(diff)
-        chunks = cmdutil.recordfilter(ui, origchunks)[0]
+        chunks = cmdutil.recordfilter(ui, origchunks, matcher)[0]
         targetctx = overlaydiffcontext(stack[-1], chunks)
     fm = None
     if opts.get('print_changes') or not opts.get('apply_changes'):

hgext/automv.py

@@ -81,10 +81,10 b' def _interestingfiles(repo, matcher):'
 
     """
     stat = repo.status(match=matcher)
-    added = stat[1]
-    removed = stat[2]
+    added = stat.added
+    removed = stat.removed
 
-    copy = copies._forwardcopies(repo['.'], repo[None], matcher)
+    copy = copies.pathcopies(repo['.'], repo[None], matcher)
     # remove the copy files for which we already have copy info
     added = [f for f in added if f not in copy]
 

hgext/blackbox.py

@@ -9,12 +9,14 b''
 """log repository events to a blackbox for debugging
 
 Logs event information to .hg/blackbox.log to help debug and diagnose problems.
-The events that get logged can be configured via the blackbox.track config key.
+The events that get logged can be configured via the blackbox.track and
+blackbox.ignore config keys.
 
 Examples::
 
   [blackbox]
   track = *
+  ignore = pythonhook
   # dirty is *EXPENSIVE* (slow);
   # each log entry indicates `+` if the repository is dirty, like :hg:`id`.
   dirty = True
@@ -84,6 +86,9 b" configitem('blackbox', 'maxfiles',"
 configitem('blackbox', 'track',
     default=lambda: ['*'],
 )
+configitem('blackbox', 'ignore',
+    default=lambda: ['chgserver', 'cmdserver', 'extension'],
+)
 configitem('blackbox', 'date-format',
     default='%Y/%m/%d %H:%M:%S',
 )
@@ -94,12 +99,15 b' class blackboxlogger(object):'
     def __init__(self, ui, repo):
         self._repo = repo
         self._trackedevents = set(ui.configlist('blackbox', 'track'))
+        self._ignoredevents = set(ui.configlist('blackbox', 'ignore'))
         self._maxfiles = ui.configint('blackbox', 'maxfiles')
         self._maxsize = ui.configbytes('blackbox', 'maxsize')
         self._inlog = False
 
     def tracked(self, event):
-        return b'*' in self._trackedevents or event in self._trackedevents
+        return ((b'*' in self._trackedevents
+                 and event not in self._ignoredevents)
+                or event in self._trackedevents)
 
     def log(self, ui, event, msg, opts):
         # self._log() -> ctx.dirty() may create new subrepo instance, which

hgext/convert/__init__.py

@@ -439,6 +439,11 b' def convert(ui, src, dest=None, revmapfi'
     :convert.hg.sourcename: records the given string as a 'convert_source' extra
         value on each commit made in the target repository. The default is None.
 
+    :convert.hg.preserve-hash: only works with mercurial sources. Make convert
+        prevent performance improvement to the list of modified files in commits
+        when such an improvement would cause the hash of a commit to change.
+        The default is False.
+
     All Destinations
     ################
 

hgext/convert/common.py

@@ -114,7 +114,7 b" SKIPREV = 'SKIP'"
 class commit(object):
     def __init__(self, author, date, desc, parents, branch=None, rev=None,
                  extra=None, sortkey=None, saverev=True, phase=phases.draft,
-                 optparents=None):
+                 optparents=None, ctx=None):
         self.author = author or 'unknown'
         self.date = date or '0 0'
         self.desc = desc
@@ -126,6 +126,7 b' class commit(object):'
         self.sortkey = sortkey
         self.saverev = saverev
         self.phase = phase
+        self.ctx = ctx # for hg to hg conversions
 
 class converter_source(object):
     """Conversion source interface"""

hgext/convert/hg.py

@@ -339,7 +339,11 b' class mercurial_sink(common.converter_si'
                                    phases.phasenames[commit.phase], 'convert')
 
             with self.repo.transaction("convert") as tr:
-                node = nodemod.hex(self.repo.commitctx(ctx))
+                if self.repo.ui.config('convert', 'hg.preserve-hash'):
+                    origctx = commit.ctx
+                else:
+                    origctx = None
+                node = nodemod.hex(self.repo.commitctx(ctx, origctx=origctx))
 
                 # If the node value has changed, but the phase is lower than
                 # draft, set it back to draft since it hasn't been exposed
@@ -591,7 +595,8 b' class mercurial_source(common.converter_'
                              extra=ctx.extra(),
                              sortkey=ctx.rev(),
                              saverev=self.saverev,
-                             phase=ctx.phase())
+                             phase=ctx.phase(),
+                             ctx=ctx)
 
     def numcommits(self):
         return len(self.repo)

hgext/convert/monotone.py

@@ -284,9 +284,9 b' class monotone_source(common.converter_s'
                     # d2 => d3
                     ignoremove[tofile] = 1
             for tofile, fromfile in renamed.items():
-                self.ui.debug (_("copying file in renamed directory "
-                                 "from '%s' to '%s'")
-                               % (fromfile, tofile), '\n')
+                self.ui.debug(
+                    "copying file in renamed directory from '%s' to '%s'"
+                    % (fromfile, tofile), '\n')
                 files[tofile] = rev
                 copies[tofile] = fromfile
             for fromfile in renamed.values():
@@ -370,4 +370,3 b' class monotone_source(common.converter_s'
             self.mtnwritefp = None
             self.mtnreadfp.close()
             self.mtnreadfp = None
-

hgext/convert/subversion.py

@@ -1333,7 +1333,7 b' class svn_sink(converter_sink, commandli'
                 rev = self.commit_re.search(output).group(1)
             except AttributeError:
                 if not files:
-                    return parents[0] if parents else None
+                    return parents[0] if parents else 'None'
                 self.ui.warn(_('unexpected svn output:\n'))
                 self.ui.warn(output)
                 raise error.Abort(_('unable to cope with svn output'))

hgext/eol.py

@@ -400,7 +400,7 b' def reposetup(ui, repo):'
                     if wlock is not None:
                         wlock.release()
 
-        def commitctx(self, ctx, error=False):
+        def commitctx(self, ctx, error=False, origctx=None):
             for f in sorted(ctx.added() + ctx.modified()):
                 if not self._eolmatch(f):
                     continue
@@ -416,6 +416,6 b' def reposetup(ui, repo):'
                 if inconsistenteol(data):
                     raise errormod.Abort(_("inconsistent newline style "
                                            "in %s\n") % f)
-            return super(eolrepo, self).commitctx(ctx, error)
+            return super(eolrepo, self).commitctx(ctx, error, origctx)
     repo.__class__ = eolrepo
     repo._hgcleardirstate()

hgext/fastannotate/revmap.py

@@ -8,6 +8,7 b''
 from __future__ import absolute_import
 
 import bisect
+import io
 import os
 import struct
 
@@ -246,7 +247,7 b' def getlastnode(path):'
     hsh = None
     try:
         with open(path, 'rb') as f:
-            f.seek(-_hshlen, 2)
+            f.seek(-_hshlen, io.SEEK_END)
             if f.tell() > len(revmap.HEADER):
                 hsh = f.read(_hshlen)
     except IOError:

hgext/fix.py

@@ -72,12 +72,43 b" in a text file by ensuring that 'sort' r"
 To account for changes made by each tool, the line numbers used for incremental
 formatting are recomputed before executing the next tool. So, each tool may see
 different values for the arguments added by the :linerange suboption.
+
+Each fixer tool is allowed to return some metadata in addition to the fixed file
+content. The metadata must be placed before the file content on stdout,
+separated from the file content by a zero byte. The metadata is parsed as a JSON
+value (so, it should be UTF-8 encoded and contain no zero bytes). A fixer tool
+is expected to produce this metadata encoding if and only if the :metadata
+suboption is true::
+
+  [fix]
+  tool:command = tool --prepend-json-metadata
+  tool:metadata = true
+
+The metadata values are passed to hooks, which can be used to print summaries or
+perform other post-fixing work. The supported hooks are::
+
+  "postfixfile"
+    Run once for each file in each revision where any fixer tools made changes
+    to the file content. Provides "$HG_REV" and "$HG_PATH" to identify the file,
+    and "$HG_METADATA" with a map of fixer names to metadata values from fixer
+    tools that affected the file. Fixer tools that didn't affect the file have a
+    valueof None. Only fixer tools that executed are present in the metadata.
+
+  "postfix"
+    Run once after all files and revisions have been handled. Provides
+    "$HG_REPLACEMENTS" with information about what revisions were created and
+    made obsolete. Provides a boolean "$HG_WDIRWRITTEN" to indicate whether any
+    files in the working copy were updated. Provides a list "$HG_METADATA"
+    mapping fixer tool names to lists of metadata values returned from
+    executions that modified a file. This aggregates the same metadata
+    previously passed to the "postfixfile" hook.
 """
 
 from __future__ import absolute_import
 
 import collections
 import itertools
+import json
 import os
 import re
 import subprocess
@@ -117,13 +148,14 b' command = registrar.command(cmdtable)'
 configtable = {}
 configitem = registrar.configitem(configtable)
 
-# Register the suboptions allowed for each configured fixer.
+# Register the suboptions allowed for each configured fixer, and default values.
 FIXER_ATTRS = {
     'command': None,
     'linerange': None,
     'fileset': None,
     'pattern': None,
     'priority': 0,
+    'metadata': False,
 }
 
 for key, default in FIXER_ATTRS.items():
@@ -201,10 +233,12 b' def fix(ui, repo, *pats, **opts):'
             for rev, path in items:
                 ctx = repo[rev]
                 olddata = ctx[path].data()
-                newdata = fixfile(ui, opts, fixers, ctx, path, basectxs[rev])
+                metadata, newdata = fixfile(ui, opts, fixers, ctx, path,
+                                            basectxs[rev])
                 # Don't waste memory/time passing unchanged content back, but
                 # produce one result per item either way.
-                yield (rev, path, newdata if newdata != olddata else None)
+                yield (rev, path, metadata,
+                       newdata if newdata != olddata else None)
         results = worker.worker(ui, 1.0, getfixes, tuple(), workqueue,
                                 threadsafe=False)
 
@@ -215,15 +249,25 b' def fix(ui, repo, *pats, **opts):'
         # the tests deterministic. It might also be considered a feature since
         # it makes the results more easily reproducible.
         filedata = collections.defaultdict(dict)
+        aggregatemetadata = collections.defaultdict(list)
         replacements = {}
         wdirwritten = False
         commitorder = sorted(revstofix, reverse=True)
         with ui.makeprogress(topic=_('fixing'), unit=_('files'),
                              total=sum(numitems.values())) as progress:
-            for rev, path, newdata in results:
+            for rev, path, filerevmetadata, newdata in results:
                 progress.increment(item=path)
+                for fixername, fixermetadata in filerevmetadata.items():
+                    aggregatemetadata[fixername].append(fixermetadata)
                 if newdata is not None:
                     filedata[rev][path] = newdata
+                    hookargs = {
+                      'rev': rev,
+                      'path': path,
+                      'metadata': filerevmetadata,
+                    }
+                    repo.hook('postfixfile', throw=False,
+                              **pycompat.strkwargs(hookargs))
                 numitems[rev] -= 1
                 # Apply the fixes for this and any other revisions that are
                 # ready and sitting at the front of the queue. Using a loop here
@@ -240,6 +284,12 b' def fix(ui, repo, *pats, **opts):'
                     del filedata[rev]
 
         cleanup(repo, replacements, wdirwritten)
+        hookargs = {
+            'replacements': replacements,
+            'wdirwritten': wdirwritten,
+            'metadata': aggregatemetadata,
+        }
+        repo.hook('postfix', throw=True, **pycompat.strkwargs(hookargs))
 
 def cleanup(repo, replacements, wdirwritten):
     """Calls scmutil.cleanupnodes() with the given replacements.
@@ -491,6 +541,7 b' def fixfile(ui, opts, fixers, fixctx, pa'
     A fixer tool's stdout will become the file's new content if and only if it
     exits with code zero.
     """
+    metadata = {}
     newdata = fixctx[path].data()
     for fixername, fixer in fixers.iteritems():
         if fixer.affects(opts, fixctx, path):
@@ -506,9 +557,20 b' def fixfile(ui, opts, fixers, fixctx, pa'
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)
-            newerdata, stderr = proc.communicate(newdata)
+            stdout, stderr = proc.communicate(newdata)
             if stderr:
                 showstderr(ui, fixctx.rev(), fixername, stderr)
+            newerdata = stdout
+            if fixer.shouldoutputmetadata():
+                try:
+                    metadatajson, newerdata = stdout.split('\0', 1)
+                    metadata[fixername] = json.loads(metadatajson)
+                except ValueError:
+                    ui.warn(_('ignored invalid output from fixer tool: %s\n') %
+                            (fixername,))
+                    continue
+            else:
+                metadata[fixername] = None
             if proc.returncode == 0:
                 newdata = newerdata
             else:
@@ -519,7 +581,7 b' def fixfile(ui, opts, fixers, fixctx, pa'
                     ui, _('no fixes will be applied'),
                     hint=_('use --config fix.failure=continue to apply any '
                            'successful fixes anyway'))
-    return newdata
+    return metadata, newdata
 
 def showstderr(ui, rev, fixername, stderr):
     """Writes the lines of the stderr string as warnings on the ui
@@ -667,6 +729,10 b' class Fixer(object):'
         """Should this fixer run on the file at the given path and context?"""
         return scmutil.match(fixctx, [self._pattern], opts)(path)
 
+    def shouldoutputmetadata(self):
+        """Should the stdout of this fixer start with JSON and a null byte?"""
+        return self._metadata
+
     def command(self, ui, path, rangesfn):
         """A shell command to use to invoke this fixer on the given file/lines
 

hgext/githelp.py

@@ -192,12 +192,15 b' def am(ui, repo, *args, **kwargs):'
 def apply(ui, repo, *args, **kwargs):
     cmdoptions = [
         ('p', 'p', int, ''),
+        ('', 'directory', '', ''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
     cmd = Command('import --no-commit')
     if (opts.get('p')):
         cmd['-p'] = opts.get('p')
+    if opts.get('directory'):
+        cmd['--prefix'] = opts.get('directory')
     cmd.extend(args)
 
     ui.status((bytes(cmd)), "\n")
@@ -681,6 +684,7 b' def mergetool(ui, repo, *args, **kwargs)'
 def mv(ui, repo, *args, **kwargs):
     cmdoptions = [
         ('f', 'force', None, ''),
+        ('n', 'dry-run', None, ''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
@@ -689,6 +693,8 b' def mv(ui, repo, *args, **kwargs):'
 
     if opts.get('force'):
         cmd['-f'] = None
+    if opts.get('dry_run'):
+        cmd['-n'] = None
 
     ui.status((bytes(cmd)), "\n")
 
@@ -917,6 +923,7 b' def show(ui, repo, *args, **kwargs):'
 
 def stash(ui, repo, *args, **kwargs):
     cmdoptions = [
+        ('p', 'patch', None, ''),
     ]
     args, opts = parseoptions(ui, cmdoptions, args)
 
@@ -925,6 +932,17 b' def stash(ui, repo, *args, **kwargs):'
 
     if action == 'list':
         cmd['-l'] = None
+        if opts.get('patch'):
+            cmd['-p'] = None
+    elif action == 'show':
+        if opts.get('patch'):
+            cmd['-p'] = None
+        else:
+            cmd['--stat'] = None
+        if len(args) > 1:
+            cmd.append(args[1])
+    elif action == 'clear':
+        cmd['--cleanup'] = None
     elif action == 'drop':
         cmd['-d'] = None
         if len(args) > 1:
@@ -937,10 +955,9 b' def stash(ui, repo, *args, **kwargs):'
             cmd.append(args[1])
         if action == 'apply':
             cmd['--keep'] = None
-    elif (action == 'branch' or action == 'show' or action == 'clear'
-        or action == 'create'):
+    elif action == 'branch' or action == 'create':
         ui.status(_("note: Mercurial doesn't have equivalents to the "
-                    "git stash branch, show, clear, or create actions\n\n"))
+                    "git stash branch or create actions\n\n"))
         return
     else:
         if len(args) > 0:

hgext/gpg.py

@@ -49,6 +49,11 b" configitem('gpg', '.*',"
 
 # Custom help category
 _HELP_CATEGORY = 'gpg'
+help.CATEGORY_ORDER.insert(
+    help.CATEGORY_ORDER.index(registrar.command.CATEGORY_HELP),
+    _HELP_CATEGORY
+)
+help.CATEGORY_NAMES[_HELP_CATEGORY] = 'Signing changes (GPG)'
 
 class gpg(object):
     def __init__(self, path, key=None):

hgext/histedit.py

@@ -1079,6 +1079,8 b' def movecursor(state, oldpos, newpos):'
 def changemode(state, mode):
     curmode, _ = state['mode']
     state['mode'] = (mode, curmode)
+    if mode == MODE_PATCH:
+        state['modes'][MODE_PATCH]['patchcontents'] = patchcontents(state)
 
 def makeselection(state, pos):
     state['selected'] = pos
@@ -1134,7 +1136,7 b' def changeview(state, delta, unit):'
     if mode != MODE_PATCH:
         return
     mode_state = state['modes'][mode]
-    num_lines = len(patchcontents(state))
+    num_lines = len(mode_state['patchcontents'])
     page_height = state['page_height']
     unit = page_height if unit == 'page' else 1
     num_pages = 1 + (num_lines - 1) / page_height
@@ -1227,15 +1229,25 b' def addln(win, y, x, line, color=None):'
     else:
         win.addstr(y, x, line)
 
+def _trunc_head(line, n):
+    if len(line) <= n:
+        return line
+    return '> ' + line[-(n - 2):]
+def _trunc_tail(line, n):
+    if len(line) <= n:
+        return line
+    return line[:n - 2] + ' >'
+
 def patchcontents(state):
     repo = state['repo']
     rule = state['rules'][state['pos']]
-    repo.ui.verbose = True
     displayer = logcmdutil.changesetdisplayer(repo.ui, repo, {
         "patch": True,  "template": "status"
     }, buffered=True)
-    displayer.show(rule.ctx)
-    displayer.close()
+    overrides = {('ui',  'verbose'): True}
+    with repo.ui.configoverride(overrides, source='histedit'):
+        displayer.show(rule.ctx)
+        displayer.close()
     return displayer.hunk[rule.ctx.rev()].splitlines()
 
 def _chisteditmain(repo, rules, stdscr):
@@ -1283,11 +1295,23 b' def _chisteditmain(repo, rules, stdscr):'
         line = "bookmark:  {0}".format(' '.join(bms))
         win.addstr(3, 1, line[:length])
 
-        line = "files:     {0}".format(','.join(ctx.files()))
+        line = "summary:   {0}".format(ctx.description().splitlines()[0])
         win.addstr(4, 1, line[:length])
 
-        line = "summary:   {0}".format(ctx.description().splitlines()[0])
-        win.addstr(5, 1, line[:length])
+        line = "files:     "
+        win.addstr(5, 1, line)
+        fnx = 1 + len(line)
+        fnmaxx = length - fnx + 1
+        y = 5
+        fnmaxn = maxy - (1 + y) - 1
+        files = ctx.files()
+        for i, line1 in enumerate(files):
+            if len(files) > fnmaxn and i == fnmaxn - 1:
+                win.addstr(y, fnx, _trunc_tail(','.join(files[i:]), fnmaxx))
+                y = y + 1
+                break
+            win.addstr(y, fnx, _trunc_head(line1, fnmaxx))
+            y = y + 1
 
         conflicts = rule.conflicts
         if len(conflicts) > 0:
@@ -1296,7 +1320,7 b' def _chisteditmain(repo, rules, stdscr):'
         else:
             conflictstr = 'no overlap'
 
-        win.addstr(6, 1, conflictstr[:length])
+        win.addstr(y, 1, conflictstr[:length])
         win.noutrefresh()
 
     def helplines(mode):
@@ -1372,15 +1396,16 b' pgup/K: move patch up, pgdn/J: move patc'
 
     def renderpatch(win, state):
         start = state['modes'][MODE_PATCH]['line_offset']
-        renderstring(win, state, patchcontents(state)[start:], diffcolors=True)
+        content = state['modes'][MODE_PATCH]['patchcontents']
+        renderstring(win, state, content[start:], diffcolors=True)
 
     def layout(mode):
         maxy, maxx = stdscr.getmaxyx()
         helplen = len(helplines(mode))
         return {
-            'commit': (8, maxx),
+            'commit': (12, maxx),
             'help': (helplen, maxx),
-            'main': (maxy - helplen - 8, maxx),
+            'main': (maxy - helplen - 12, maxx),
         }
 
     def drawvertwin(size, y, x):
@@ -1894,6 +1919,14 b' def _aborthistedit(ui, repo, state, noba'
     finally:
             state.clear()
 
+def hgaborthistedit(ui, repo):
+    state = histeditstate(repo)
+    nobackup = not ui.configbool('rewrite', 'backup-bundle')
+    with repo.wlock() as wlock, repo.lock() as lock:
+        state.wlock = wlock
+        state.lock = lock
+        _aborthistedit(ui, repo, state, nobackup=nobackup)
+
 def _edithisteditplan(ui, repo, state, rules):
     state.read()
     if not rules:
@@ -2288,8 +2321,6 b' def summaryhook(ui, repo):'
 
 def extsetup(ui):
     cmdutil.summaryhooks.add('histedit', summaryhook)
-    cmdutil.unfinishedstates.append(
-        ['histedit-state', False, True, _('histedit in progress'),
-         _("use 'hg histedit --continue' or 'hg histedit --abort'")])
-    cmdutil.afterresolvedstates.append(
-        ['histedit-state', _('hg histedit --continue')])
+    statemod.addunfinished('histedit', fname='histedit-state', allowcommit=True,
+                            continueflag=True, abortfunc=hgaborthistedit)
+

hgext/keyword.py

@@ -785,8 +785,8 b' def reposetup(ui, repo):'
             finally:
                 del self.commitctx
 
-        def kwcommitctx(self, ctx, error=False):
-            n = super(kwrepo, self).commitctx(ctx, error)
+        def kwcommitctx(self, ctx, error=False, origctx=None):
+            n = super(kwrepo, self).commitctx(ctx, error, origctx)
             # no lock needed, only called from repo.commit() which already locks
             if not kwt.postcommit:
                 restrict = kwt.restrict

hgext/largefiles/overrides.py

@@ -515,7 +515,7 b' def overridecalculateupdates(origfn, rep'
     return actions, diverge, renamedelete
 
 @eh.wrapfunction(merge, 'recordupdates')
-def mergerecordupdates(orig, repo, actions, branchmerge):
+def mergerecordupdates(orig, repo, actions, branchmerge, getfiledata):
     if 'lfmr' in actions:
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
         for lfile, args, msg in actions['lfmr']:
@@ -526,7 +526,7 b' def mergerecordupdates(orig, repo, actio'
             lfdirstate.add(lfile)
         lfdirstate.write()
 
-    return orig(repo, actions, branchmerge)
+    return orig(repo, actions, branchmerge, getfiledata)
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits without prompting the user.
@@ -545,7 +545,8 b' def overridefilemerge(origfn, premerge, '
         (dhash == ahash or
          repo.ui.promptchoice(
              _('largefile %s has a merge conflict\nancestor was %s\n'
-               'keep (l)ocal %s or\ntake (o)ther %s?'
+               'you can keep (l)ocal %s or take (o)ther %s.\n'
+               'what do you want to do?'
                '$$ &Local $$ &Other') %
                (lfutil.splitstandin(orig), ahash, dhash, ohash),
              0) == 1)):

hgext/lfs/__init__.py

@@ -227,9 +227,9 b' def _reposetup(ui, repo):'
 
     class lfsrepo(repo.__class__):
         @localrepo.unfilteredmethod
-        def commitctx(self, ctx, error=False):
+        def commitctx(self, ctx, error=False, origctx=None):
             repo.svfs.options['lfstrack'] = _trackedmatcher(self)
-            return super(lfsrepo, self).commitctx(ctx, error)
+            return super(lfsrepo, self).commitctx(ctx, error, origctx=origctx)
 
     repo.__class__ = lfsrepo
 

hgext/mq.py

@@ -144,9 +144,21 b' except KeyError:'
     stripext = extensions.load(dummyui(), 'strip', '')
 
 strip = stripext.strip
-checksubstate = stripext.checksubstate
-checklocalchanges = stripext.checklocalchanges
-
+
+def checksubstate(repo, baserev=None):
+    '''return list of subrepos at a different revision than substate.
+    Abort if any subrepos have uncommitted changes.'''
+    inclsubs = []
+    wctx = repo[None]
+    if baserev:
+        bctx = repo[baserev]
+    else:
+        bctx = wctx.p1()
+    for s in sorted(wctx.substate):
+        wctx.sub(s).bailifchanged(True)
+        if s not in bctx.substate or bctx.sub(s).dirty():
+            inclsubs.append(s)
+    return inclsubs
 
 # Patch names looks like unix-file names.
 # They must be joinable with queue directory and result in the patch path.
@@ -1149,7 +1161,19 b' class queue(object):'
             # plain versions for i18n tool to detect them
             _("local changes found, qrefresh first")
             _("local changed subrepos found, qrefresh first")
-        return checklocalchanges(repo, force, excsuffix)
+
+        s = repo.status()
+        if not force:
+            cmdutil.checkunfinished(repo)
+            if s.modified or s.added or s.removed or s.deleted:
+                _("local changes found") # i18n tool detection
+                raise error.Abort(_("local changes found" + excsuffix))
+            if checksubstate(repo):
+                _("local changed subrepos found") # i18n tool detection
+                raise error.Abort(_("local changed subrepos found" + excsuffix))
+        else:
+            cmdutil.checkunfinished(repo, skipmerge=True)
+        return s
 
     _reserved = ('series', 'status', 'guards', '.', '..')
     def checkreservedname(self, name):

hgext/narrow/narrowbundle2.py

@@ -51,21 +51,25 b' def getbundlechangegrouppart_narrow(bund'
     assert repo.ui.configbool('experimental', 'narrowservebrokenellipses')
 
     cgversions = b2caps.get('changegroup')
-    if cgversions:  # 3.1 and 3.2 ship with an empty value
-        cgversions = [v for v in cgversions
-                      if v in changegroup.supportedoutgoingversions(repo)]
-        if not cgversions:
-            raise ValueError(_('no common changegroup version'))
-        version = max(cgversions)
-    else:
-        raise ValueError(_("server does not advertise changegroup version,"
-                           " can't negotiate support for ellipsis nodes"))
+    cgversions = [v for v in cgversions
+                  if v in changegroup.supportedoutgoingversions(repo)]
+    if not cgversions:
+        raise ValueError(_('no common changegroup version'))
+    version = max(cgversions)
 
-    include = sorted(filter(bool, kwargs.get(r'includepats', [])))
-    exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
-    newmatch = narrowspec.match(repo.root, include=include, exclude=exclude)
+    oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
+    oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
+    newinclude = sorted(filter(bool, kwargs.get(r'includepats', [])))
+    newexclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
+    known = {bin(n) for n in kwargs.get(r'known', [])}
+    generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
+                            newexclude, version, common, heads, known,
+                            kwargs.get(r'depth', None))
 
-    depth = kwargs.get(r'depth', None)
+def generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
+                            newexclude, version, common, heads, known, depth):
+    newmatch = narrowspec.match(repo.root, include=newinclude,
+                                exclude=newexclude)
     if depth is not None:
         depth = int(depth)
         if depth < 1:
@@ -73,10 +77,7 b' def getbundlechangegrouppart_narrow(bund'
 
     heads = set(heads or repo.heads())
     common = set(common or [nullid])
-    oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
-    oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
-    known = {bin(n) for n in kwargs.get(r'known', [])}
-    if known and (oldinclude != include or oldexclude != exclude):
+    if known and (oldinclude != newinclude or oldexclude != newexclude):
         # Steps:
         # 1. Send kill for "$known & ::common"
         #

hgext/narrow/narrowcommands.py

@@ -146,7 +146,7 b' def pullbundle2extraprepare(orig, pullop'
         kwargs['excludepats'] = exclude
     # calculate known nodes only in ellipses cases because in non-ellipses cases
     # we have all the nodes
-    if wireprototypes.ELLIPSESCAP in pullop.remote.capabilities():
+    if wireprototypes.ELLIPSESCAP1 in pullop.remote.capabilities():
         kwargs['known'] = [node.hex(ctx.node()) for ctx in
                            repo.set('::%ln', pullop.common)
                            if ctx.node() != node.nullid]
@@ -216,7 +216,7 b' def _narrow(ui, repo, remote, commoninc,'
                     todelete.append(f)
             elif f.startswith('meta/'):
                 dir = f[5:-13]
-                dirs = ['.'] + sorted(util.dirs({dir})) + [dir]
+                dirs = sorted(util.dirs({dir})) + [dir]
                 include = True
                 for d in dirs:
                     visit = newmatch.visitdir(d)
@@ -253,7 +253,14 b' def _widen(ui, repo, remote, commoninc, '
     # then send that information to server whether we want ellipses or not.
     # Theoretically a non-ellipses repo should be able to use narrow
     # functionality from an ellipses enabled server
-    ellipsesremote = wireprototypes.ELLIPSESCAP in remote.capabilities()
+    remotecap = remote.capabilities()
+    ellipsesremote = any(cap in remotecap
+                         for cap in wireprototypes.SUPPORTED_ELLIPSESCAP)
+
+    # check whether we are talking to a server which supports old version of
+    # ellipses capabilities
+    isoldellipses = (ellipsesremote and wireprototypes.ELLIPSESCAP1 in
+                     remotecap and wireprototypes.ELLIPSESCAP not in remotecap)
 
     def pullbundle2extraprepare_widen(orig, pullop, kwargs):
         orig(pullop, kwargs)
@@ -271,19 +278,22 b' def _widen(ui, repo, remote, commoninc, '
     # silence the devel-warning of applying an empty changegroup
     overrides = {('devel', 'all-warnings'): False}
 
+    common = commoninc[0]
     with ui.uninterruptible():
-        common = commoninc[0]
         if ellipsesremote:
             ds = repo.dirstate
             p1, p2 = ds.p1(), ds.p2()
             with ds.parentchange():
                 ds.setparents(node.nullid, node.nullid)
+        if isoldellipses:
             with wrappedextraprepare:
-                with repo.ui.configoverride(overrides, 'widen'):
-                    exchange.pull(repo, remote, heads=common)
-            with ds.parentchange():
-                ds.setparents(p1, p2)
+                exchange.pull(repo, remote, heads=common)
         else:
+            known = []
+            if ellipsesremote:
+                known = [node.hex(ctx.node()) for ctx in
+                         repo.set('::%ln', common)
+                         if ctx.node() != node.nullid]
             with remote.commandexecutor() as e:
                 bundle = e.callcommand('narrow_widen', {
                     'oldincludes': oldincludes,
@@ -292,15 +302,20 b' def _widen(ui, repo, remote, commoninc, '
                     'newexcludes': newexcludes,
                     'cgversion': '03',
                     'commonheads': common,
-                    'known': [],
-                    'ellipses': False,
+                    'known': known,
+                    'ellipses': ellipsesremote,
                 }).result()
 
-            with repo.transaction('widening') as tr:
-                with repo.ui.configoverride(overrides, 'widen'):
-                    tgetter = lambda: tr
-                    bundle2.processbundle(repo, bundle,
-                            transactiongetter=tgetter)
+            trmanager = exchange.transactionmanager(repo, 'widen', remote.url())
+            with trmanager, repo.ui.configoverride(overrides, 'widen'):
+                op = bundle2.bundleoperation(repo, trmanager.transaction,
+                                             source='widen')
+                # TODO: we should catch error.Abort here
+                bundle2.processbundle(repo, bundle, op=op)
+
+        if ellipsesremote:
+            with ds.parentchange():
+                ds.setparents(p1, p2)
 
         with repo.transaction('widening'):
             repo.setnewnarrowpats()

hgext/narrow/narrowdirstate.py

@@ -16,21 +16,21 b' def wrapdirstate(repo, dirstate):'
     """Add narrow spec dirstate ignore, block changes outside narrow spec."""
 
     def _editfunc(fn):
-        def _wrapper(self, *args):
+        def _wrapper(self, *args, **kwargs):
             narrowmatch = repo.narrowmatch()
             for f in args:
                 if f is not None and not narrowmatch(f) and f not in self:
                     raise error.Abort(_("cannot track '%s' - it is outside " +
                         "the narrow clone") % f)
-            return fn(self, *args)
+            return fn(self, *args, **kwargs)
         return _wrapper
 
     class narrowdirstate(dirstate.__class__):
         # Prevent adding/editing/copying/deleting files that are outside the
         # sparse checkout
         @_editfunc
-        def normal(self, *args):
-            return super(narrowdirstate, self).normal(*args)
+        def normal(self, *args, **kwargs):
+            return super(narrowdirstate, self).normal(*args, **kwargs)
 
         @_editfunc
         def add(self, *args):

hgext/narrow/narrowtemplates.py

@@ -37,7 +37,7 b' def outsidenarrow(context, mapping):'
     repo = context.resource(mapping, 'repo')
     ctx = context.resource(mapping, 'ctx')
     m = repo.narrowmatch()
-    if not m.always():
+    if ctx.files() and not m.always():
         if not any(m(f) for f in ctx.files()):
             return 'outsidenarrow'
     return ''

hgext/narrow/narrowwirepeer.py

@@ -13,12 +13,15 b' from mercurial import ('
     extensions,
     hg,
     narrowspec,
+    node as nodemod,
     pycompat,
     wireprototypes,
     wireprotov1peer,
     wireprotov1server,
 )
 
+from . import narrowbundle2
+
 def uisetup():
     wireprotov1peer.wirepeer.narrow_widen = peernarrowwiden
 
@@ -69,21 +72,26 b' def narrow_widen(repo, proto, oldinclude'
         narrowspec.validatepatterns(set(newexcludes))
 
         common = wireprototypes.decodelist(commonheads)
-        known = None
-        if known:
-            known = wireprototypes.decodelist(known)
+        known = wireprototypes.decodelist(known)
+        known = {nodemod.bin(n) for n in known}
         if ellipses == '0':
             ellipses = False
         else:
             ellipses = bool(ellipses)
         cgversion = cgversion
-        newmatch = narrowspec.match(repo.root, include=newincludes,
-                                    exclude=newexcludes)
-        oldmatch = narrowspec.match(repo.root, include=oldincludes,
-                                    exclude=oldexcludes)
 
-        bundler = bundle2.widen_bundle(repo, oldmatch, newmatch, common, known,
-                                             cgversion, ellipses)
+        bundler = bundle2.bundle20(repo.ui)
+        if not ellipses:
+            newmatch = narrowspec.match(repo.root, include=newincludes,
+                                        exclude=newexcludes)
+            oldmatch = narrowspec.match(repo.root, include=oldincludes,
+                                        exclude=oldexcludes)
+            bundle2.widen_bundle(bundler, repo, oldmatch, newmatch, common,
+                                 known, cgversion, ellipses)
+        else:
+            narrowbundle2.generateellipsesbundle2(bundler, repo, oldincludes,
+                    oldexcludes, newincludes, newexcludes, cgversion, common,
+                    list(common), known, None)
     except error.Abort as exc:
         bundler = bundle2.bundle20(repo.ui)
         manargs = [('message', pycompat.bytestr(exc))]

hgext/phabricator.py

@@ -65,6 +65,7 b' from mercurial import ('
     scmutil,
     smartset,
     tags,
+    templatefilters,
     templateutil,
     url as urlmod,
     util,
@@ -124,8 +125,28 b' colortable = {'
      )),
 ]
 
-def vcrcommand(name, flags, spec, helpcategory=None):
+def vcrcommand(name, flags, spec, helpcategory=None, optionalrepo=False):
     fullflags = flags + _VCR_FLAGS
+    def hgmatcher(r1, r2):
+        if r1.uri != r2.uri or r1.method != r2.method:
+            return False
+        r1params = r1.body.split(b'&')
+        r2params = r2.body.split(b'&')
+        return set(r1params) == set(r2params)
+
+    def sanitiserequest(request):
+        request.body = re.sub(
+            r'cli-[a-z0-9]+',
+            r'cli-hahayouwish',
+            request.body
+        )
+        return request
+
+    def sanitiseresponse(response):
+        if r'set-cookie' in response[r'headers']:
+            del response[r'headers'][r'set-cookie']
+        return response
+
     def decorate(fn):
         def inner(*args, **kwargs):
             cassette = pycompat.fsdecode(kwargs.pop(r'test_vcr', None))
@@ -136,18 +157,22 b' def vcrcommand(name, flags, spec, helpca'
                     import vcr.stubs as stubs
                     vcr = vcrmod.VCR(
                         serializer=r'json',
+                        before_record_request=sanitiserequest,
+                        before_record_response=sanitiseresponse,
                         custom_patches=[
                             (urlmod, r'httpconnection',
                              stubs.VCRHTTPConnection),
                             (urlmod, r'httpsconnection',
                              stubs.VCRHTTPSConnection),
                         ])
-                    with vcr.use_cassette(cassette):
+                    vcr.register_matcher(r'hgmatcher', hgmatcher)
+                    with vcr.use_cassette(cassette, match_on=[r'hgmatcher']):
                         return fn(*args, **kwargs)
             return fn(*args, **kwargs)
         inner.__name__ = fn.__name__
         inner.__doc__ = fn.__doc__
-        return command(name, fullflags, spec, helpcategory=helpcategory)(inner)
+        return command(name, fullflags, spec, helpcategory=helpcategory,
+                       optionalrepo=optionalrepo)(inner)
     return decorate
 
 def urlencodenested(params):
@@ -174,24 +199,24 b' def urlencodenested(params):'
     process(b'', params)
     return util.urlreq.urlencode(flatparams)
 
-def readurltoken(repo):
+def readurltoken(ui):
     """return conduit url, token and make sure they exist
 
     Currently read from [auth] config section. In the future, it might
     make sense to read from .arcconfig and .arcrc as well.
     """
-    url = repo.ui.config(b'phabricator', b'url')
+    url = ui.config(b'phabricator', b'url')
     if not url:
         raise error.Abort(_(b'config %s.%s is required')
                           % (b'phabricator', b'url'))
 
-    res = httpconnectionmod.readauthforuri(repo.ui, url, util.url(url).user)
+    res = httpconnectionmod.readauthforuri(ui, url, util.url(url).user)
     token = None
 
     if res:
         group, auth = res
 
-        repo.ui.debug(b"using auth.%s.* for authentication\n" % group)
+        ui.debug(b"using auth.%s.* for authentication\n" % group)
 
         token = auth.get(b'phabtoken')
 
@@ -201,15 +226,15 b' def readurltoken(repo):'
 
     return url, token
 
-def callconduit(repo, name, params):
+def callconduit(ui, name, params):
     """call Conduit API, params is a dict. return json.loads result, or None"""
-    host, token = readurltoken(repo)
+    host, token = readurltoken(ui)
     url, authinfo = util.url(b'/'.join([host, b'api', name])).authinfo()
-    repo.ui.debug(b'Conduit Call: %s %s\n' % (url, pycompat.byterepr(params)))
+    ui.debug(b'Conduit Call: %s %s\n' % (url, pycompat.byterepr(params)))
     params = params.copy()
     params[b'api.token'] = token
     data = urlencodenested(params)
-    curlcmd = repo.ui.config(b'phabricator', b'curlcmd')
+    curlcmd = ui.config(b'phabricator', b'curlcmd')
     if curlcmd:
         sin, sout = procutil.popen2(b'%s -d @- %s'
                                     % (curlcmd, procutil.shellquote(url)))
@@ -217,11 +242,11 b' def callconduit(repo, name, params):'
         sin.close()
         body = sout.read()
     else:
-        urlopener = urlmod.opener(repo.ui, authinfo)
+        urlopener = urlmod.opener(ui, authinfo)
         request = util.urlreq.request(pycompat.strurl(url), data=data)
         with contextlib.closing(urlopener.open(request)) as rsp:
             body = rsp.read()
-    repo.ui.debug(b'Conduit Response: %s\n' % body)
+    ui.debug(b'Conduit Response: %s\n' % body)
     parsed = pycompat.rapply(
         lambda x: encoding.unitolocal(x) if isinstance(x, pycompat.unicode)
         else x,
@@ -233,7 +258,7 b' def callconduit(repo, name, params):'
         raise error.Abort(msg)
     return parsed[b'result']
 
-@vcrcommand(b'debugcallconduit', [], _(b'METHOD'))
+@vcrcommand(b'debugcallconduit', [], _(b'METHOD'), optionalrepo=True)
 def debugcallconduit(ui, repo, name):
     """call Conduit API
 
@@ -250,7 +275,7 b' def debugcallconduit(ui, repo, name):'
     # json.dumps only accepts unicode strings
     result = pycompat.rapply(lambda x:
         encoding.unifromlocal(x) if isinstance(x, bytes) else x,
-        callconduit(repo, name, params)
+        callconduit(ui, name, params)
     )
     s = json.dumps(result, sort_keys=True, indent=2, separators=(u',', u': '))
     ui.write(b'%s\n' % encoding.unitolocal(s))
@@ -264,7 +289,7 b' def getrepophid(repo):'
     callsign = repo.ui.config(b'phabricator', b'callsign')
     if not callsign:
         return None
-    query = callconduit(repo, b'diffusion.repository.search',
+    query = callconduit(repo.ui, b'diffusion.repository.search',
                         {b'constraints': {b'callsigns': [callsign]}})
     if len(query[b'data']) == 0:
         return None
@@ -320,7 +345,7 b' def getoldnodedrevmap(repo, nodelist):'
     # Phabricator, and expect precursors overlap with it.
     if toconfirm:
         drevs = [drev for force, precs, drev in toconfirm.values()]
-        alldiffs = callconduit(unfi, b'differential.querydiffs',
+        alldiffs = callconduit(unfi.ui, b'differential.querydiffs',
                                {b'revisionIDs': drevs})
         getnode = lambda d: bin(
             getdiffmeta(d).get(b'node', b'')) or None
@@ -370,7 +395,7 b' def creatediff(ctx):'
     params = {b'diff': getdiff(ctx, mdiff.diffopts(git=True, context=32767))}
     if repophid:
         params[b'repositoryPHID'] = repophid
-    diff = callconduit(repo, b'differential.createrawdiff', params)
+    diff = callconduit(repo.ui, b'differential.createrawdiff', params)
     if not diff:
         raise error.Abort(_(b'cannot create diff for %s') % ctx)
     return diff
@@ -380,35 +405,39 b' def writediffproperties(ctx, diff):'
     params = {
         b'diff_id': diff[b'id'],
         b'name': b'hg:meta',
-        b'data': json.dumps({
-            u'user': encoding.unifromlocal(ctx.user()),
-            u'date': u'{:.0f} {}'.format(*ctx.date()),
-            u'node': encoding.unifromlocal(ctx.hex()),
-            u'parent': encoding.unifromlocal(ctx.p1().hex()),
+        b'data': templatefilters.json({
+            b'user': ctx.user(),
+            b'date': b'%d %d' % ctx.date(),
+            b'branch': ctx.branch(),
+            b'node': ctx.hex(),
+            b'parent': ctx.p1().hex(),
         }),
     }
-    callconduit(ctx.repo(), b'differential.setdiffproperty', params)
+    callconduit(ctx.repo().ui, b'differential.setdiffproperty', params)
 
     params = {
         b'diff_id': diff[b'id'],
         b'name': b'local:commits',
-        b'data': json.dumps({
-            encoding.unifromlocal(ctx.hex()): {
-                u'author': encoding.unifromlocal(stringutil.person(ctx.user())),
-                u'authorEmail': encoding.unifromlocal(
-                    stringutil.email(ctx.user())),
-                u'time': u'{:.0f}'.format(ctx.date()[0]),
+        b'data': templatefilters.json({
+            ctx.hex(): {
+                b'author': stringutil.person(ctx.user()),
+                b'authorEmail': stringutil.email(ctx.user()),
+                b'time': int(ctx.date()[0]),
+                b'commit': ctx.hex(),
+                b'parents': [ctx.p1().hex()],
+                b'branch': ctx.branch(),
             },
         }),
     }
-    callconduit(ctx.repo(), b'differential.setdiffproperty', params)
+    callconduit(ctx.repo().ui, b'differential.setdiffproperty', params)
 
-def createdifferentialrevision(ctx, revid=None, parentrevid=None, oldnode=None,
-                               olddiff=None, actions=None):
+def createdifferentialrevision(ctx, revid=None, parentrevphid=None,
+                               oldnode=None, olddiff=None, actions=None,
+                               comment=None):
     """create or update a Differential Revision
 
     If revid is None, create a new Differential Revision, otherwise update
-    revid. If parentrevid is not None, set it as a dependency.
+    revid. If parentrevphid is not None, set it as a dependency.
 
     If oldnode is not None, check if the patch content (without commit message
     and metadata) has changed before creating another diff.
@@ -427,6 +456,8 b' def createdifferentialrevision(ctx, revi'
     if neednewdiff:
         diff = creatediff(ctx)
         transactions.append({b'type': b'update', b'value': diff[b'phid']})
+        if comment:
+            transactions.append({b'type': b'comment', b'value': comment})
     else:
         # Even if we don't need to upload a new diff because the patch content
         # does not change. We might still need to update its metadata so
@@ -435,21 +466,17 b' def createdifferentialrevision(ctx, revi'
         diff = olddiff
     writediffproperties(ctx, diff)
 
-    # Use a temporary summary to set dependency. There might be better ways but
-    # I cannot find them for now. But do not do that if we are updating an
-    # existing revision (revid is not None) since that introduces visible
-    # churns (someone edited "Summary" twice) on the web page.
-    if parentrevid and revid is None:
-        summary = b'Depends on D%d' % parentrevid
-        transactions += [{b'type': b'summary', b'value': summary},
-                         {b'type': b'summary', b'value': b' '}]
+    # Set the parent Revision every time, so commit re-ordering is picked-up
+    if parentrevphid:
+        transactions.append({b'type': b'parents.set',
+                             b'value': [parentrevphid]})
 
     if actions:
         transactions += actions
 
     # Parse commit message and update related fields.
     desc = ctx.description()
-    info = callconduit(repo, b'differential.parsecommitmessage',
+    info = callconduit(repo.ui, b'differential.parsecommitmessage',
                        {b'corpus': desc})
     for k, v in info[b'fields'].items():
         if k in [b'title', b'summary', b'testPlan']:
@@ -460,7 +487,7 b' def createdifferentialrevision(ctx, revi'
         # Update an existing Differential Revision
         params[b'objectIdentifier'] = revid
 
-    revision = callconduit(repo, b'differential.revision.edit', params)
+    revision = callconduit(repo.ui, b'differential.revision.edit', params)
     if not revision:
         raise error.Abort(_(b'cannot create revision for %s') % ctx)
 
@@ -470,7 +497,7 b' def userphids(repo, names):'
     """convert user names to PHIDs"""
     names = [name.lower() for name in names]
     query = {b'constraints': {b'usernames': names}}
-    result = callconduit(repo, b'user.search', query)
+    result = callconduit(repo.ui, b'user.search', query)
     # username not found is not an error of the API. So check if we have missed
     # some names here.
     data = result[b'data']
@@ -485,6 +512,9 b' def userphids(repo, names):'
          [(b'r', b'rev', [], _(b'revisions to send'), _(b'REV')),
           (b'', b'amend', True, _(b'update commit messages')),
           (b'', b'reviewer', [], _(b'specify reviewers')),
+          (b'', b'blocker', [], _(b'specify blocking reviewers')),
+          (b'm', b'comment', b'',
+           _(b'add a comment to Revisions with new/updated Diffs')),
           (b'', b'confirm', None, _(b'ask for confirmation before sending'))],
          _(b'REV [OPTIONS]'),
          helpcategory=command.CATEGORY_IMPORT_EXPORT)
@@ -536,16 +566,23 b' def phabsend(ui, repo, *revs, **opts):'
 
     actions = []
     reviewers = opts.get(b'reviewer', [])
+    blockers = opts.get(b'blocker', [])
+    phids = []
     if reviewers:
-        phids = userphids(repo, reviewers)
+        phids.extend(userphids(repo, reviewers))
+    if blockers:
+        phids.extend(map(
+            lambda phid: b'blocking(%s)' % phid, userphids(repo, blockers)
+        ))
+    if phids:
         actions.append({b'type': b'reviewers.add', b'value': phids})
 
     drevids = [] # [int]
     diffmap = {} # {newnode: diff}
 
-    # Send patches one by one so we know their Differential Revision IDs and
+    # Send patches one by one so we know their Differential Revision PHIDs and
     # can provide dependency relationship
-    lastrevid = None
+    lastrevphid = None
     for rev in revs:
         ui.debug(b'sending rev %d\n' % rev)
         ctx = repo[rev]
@@ -555,9 +592,11 b' def phabsend(ui, repo, *revs, **opts):'
         if oldnode != ctx.node() or opts.get(b'amend'):
             # Create or update Differential Revision
             revision, diff = createdifferentialrevision(
-                ctx, revid, lastrevid, oldnode, olddiff, actions)
+                ctx, revid, lastrevphid, oldnode, olddiff, actions,
+                opts.get(b'comment'))
             diffmap[ctx.node()] = diff
             newrevid = int(revision[b'object'][b'id'])
+            newrevphid = revision[b'object'][b'phid']
             if revid:
                 action = b'updated'
             else:
@@ -571,8 +610,9 b' def phabsend(ui, repo, *revs, **opts):'
                 tags.tag(repo, tagname, ctx.node(), message=None, user=None,
                          date=None, local=True)
         else:
-            # Nothing changed. But still set "newrevid" so the next revision
-            # could depend on this one.
+            # Nothing changed. But still set "newrevphid" so the next revision
+            # could depend on this one and "newrevid" for the summary line.
+            newrevphid = querydrev(repo, str(revid))[0][b'phid']
             newrevid = revid
             action = b'skipped'
 
@@ -587,12 +627,12 b' def phabsend(ui, repo, *revs, **opts):'
         ui.write(_(b'%s - %s - %s: %s\n') % (drevdesc, actiondesc, nodedesc,
                                              desc))
         drevids.append(newrevid)
-        lastrevid = newrevid
+        lastrevphid = newrevphid
 
     # Update commit messages and remove tags
     if opts.get(b'amend'):
         unfi = repo.unfiltered()
-        drevs = callconduit(repo, b'differential.query', {b'ids': drevids})
+        drevs = callconduit(ui, b'differential.query', {b'ids': drevids})
         with repo.wlock(), repo.lock(), repo.transaction(b'phabsend'):
             wnode = unfi[b'.'].node()
             mapping = {} # {oldnode: [newnode]}
@@ -632,10 +672,11 b' def phabsend(ui, repo, *revs, **opts):'
 # Map from "hg:meta" keys to header understood by "hg import". The order is
 # consistent with "hg export" output.
 _metanamemap = util.sortdict([(b'user', b'User'), (b'date', b'Date'),
-                              (b'node', b'Node ID'), (b'parent', b'Parent ')])
+                              (b'branch', b'Branch'), (b'node', b'Node ID'),
+                              (b'parent', b'Parent ')])
 
 def _confirmbeforesend(repo, revs, oldmap):
-    url, token = readurltoken(repo)
+    url, token = readurltoken(repo.ui)
     ui = repo.ui
     for rev in revs:
         ctx = repo[rev]
@@ -777,7 +818,7 b' def querydrev(repo, spec):'
         key = (params.get(b'ids') or params.get(b'phids') or [None])[0]
         if key in prefetched:
             return prefetched[key]
-        drevs = callconduit(repo, b'differential.query', params)
+        drevs = callconduit(repo.ui, b'differential.query', params)
         # Fill prefetched with the result
         for drev in drevs:
             prefetched[drev[b'phid']] = drev
@@ -901,16 +942,31 b' def getdiffmeta(diff):'
     """
     props = diff.get(b'properties') or {}
     meta = props.get(b'hg:meta')
-    if not meta and props.get(b'local:commits'):
-        commit = sorted(props[b'local:commits'].values())[0]
-        meta = {
-            b'date': b'%d 0' % commit[b'time'],
-            b'node': commit[b'rev'],
-            b'user': b'%s <%s>' % (commit[b'author'], commit[b'authorEmail']),
-        }
-        if len(commit.get(b'parents', ())) >= 1:
-            meta[b'parent'] = commit[b'parents'][0]
-    return meta or {}
+    if not meta:
+        if props.get(b'local:commits'):
+            commit = sorted(props[b'local:commits'].values())[0]
+            meta = {}
+            if b'author' in commit and b'authorEmail' in commit:
+                meta[b'user'] = b'%s <%s>' % (commit[b'author'],
+                                              commit[b'authorEmail'])
+            if b'time' in commit:
+                meta[b'date'] = b'%d 0' % int(commit[b'time'])
+            if b'branch' in commit:
+                meta[b'branch'] = commit[b'branch']
+            node = commit.get(b'commit', commit.get(b'rev'))
+            if node:
+                meta[b'node'] = node
+            if len(commit.get(b'parents', ())) >= 1:
+                meta[b'parent'] = commit[b'parents'][0]
+        else:
+            meta = {}
+    if b'date' not in meta and b'dateCreated' in diff:
+        meta[b'date'] = b'%s 0' % diff[b'dateCreated']
+    if b'branch' not in meta and diff.get(b'branch'):
+        meta[b'branch'] = diff[b'branch']
+    if b'parent' not in meta and diff.get(b'sourceControlBaseRevision'):
+        meta[b'parent'] = diff[b'sourceControlBaseRevision']
+    return meta
 
 def readpatch(repo, drevs, write):
     """generate plain-text patch readable by 'hg import'
@@ -920,14 +976,14 b' def readpatch(repo, drevs, write):'
     """
     # Prefetch hg:meta property for all diffs
     diffids = sorted(set(max(int(v) for v in drev[b'diffs']) for drev in drevs))
-    diffs = callconduit(repo, b'differential.querydiffs', {b'ids': diffids})
+    diffs = callconduit(repo.ui, b'differential.querydiffs', {b'ids': diffids})
 
     # Generate patch for each drev
     for drev in drevs:
         repo.ui.note(_(b'reading D%s\n') % drev[b'id'])
 
         diffid = max(int(v) for v in drev[b'diffs'])
-        body = callconduit(repo, b'differential.getrawdiff',
+        body = callconduit(repo.ui, b'differential.getrawdiff',
                            {b'diffID': diffid})
         desc = getdescfromdrev(drev)
         header = b'# HG changeset patch\n'
@@ -1001,7 +1057,7 b' def phabupdate(ui, repo, spec, **opts):'
         if actions:
             params = {b'objectIdentifier': drev[b'phid'],
                       b'transactions': actions}
-            callconduit(repo, b'differential.revision.edit', params)
+            callconduit(ui, b'differential.revision.edit', params)
 
 templatekeyword = registrar.templatekeyword()
 

hgext/rebase.py

@@ -108,7 +108,9 b' def _revsetdestrebase(repo, subset, x):'
 
 @revsetpredicate('_destautoorphanrebase')
 def _revsetdestautoorphanrebase(repo, subset, x):
-    """automatic rebase destination for a single orphan revision"""
+    # ``_destautoorphanrebase()``
+
+    # automatic rebase destination for a single orphan revision.
     unfi = repo.unfiltered()
     obsoleted = unfi.revs('obsolete()')
 
@@ -848,8 +850,9 b' def rebase(ui, repo, **opts):'
       singletransaction = True
 
     By default, rebase writes to the working copy, but you can configure it to
-    run in-memory for for better performance, and to allow it to run if the
-    working copy is dirty::
+    run in-memory for better performance. When the rebase is not moving the
+    parent(s) of the working copy (AKA the "currently checked out changesets"),
+    this may also allow it to run even if the working copy is dirty::
 
       [rebase]
       experimental.inmemory = True
@@ -1819,7 +1822,7 b' def pullrebase(orig, ui, repo, *args, **'
                 ui.debug('--update and --rebase are not compatible, ignoring '
                          'the update flag\n')
 
-            cmdutil.checkunfinished(repo)
+            cmdutil.checkunfinished(repo, skipmerge=True)
             cmdutil.bailifchanged(repo, hint=_('cannot pull with rebase: '
                 'please commit or shelve your changes first'))
 
@@ -1920,6 +1923,22 b' def _computeobsoletenotrebased(repo, reb'
         obsoleteextinctsuccessors,
     )
 
+def abortrebase(ui, repo):
+    with repo.wlock(), repo.lock():
+        rbsrt = rebaseruntime(repo, ui)
+        rbsrt._prepareabortorcontinue(isabort=True)
+
+def continuerebase(ui, repo):
+    with repo.wlock(), repo.lock():
+        rbsrt = rebaseruntime(repo, ui)
+        ms = mergemod.mergestate.read(repo)
+        mergeutil.checkunresolved(ms)
+        retcode = rbsrt._prepareabortorcontinue(isabort=False)
+        if retcode is not None:
+            return retcode
+        rbsrt._performrebase(None)
+        rbsrt._finishrebase()
+
 def summaryhook(ui, repo):
     if not repo.vfs.exists('rebasestate'):
         return
@@ -1947,8 +1966,6 b' def uisetup(ui):'
     entry[1].append(('t', 'tool', '',
                      _("specify merge tool for rebase")))
     cmdutil.summaryhooks.add('rebase', summaryhook)
-    cmdutil.unfinishedstates.append(
-        ['rebasestate', False, False, _('rebase in progress'),
-         _("use 'hg rebase --continue' or 'hg rebase --abort'")])
-    cmdutil.afterresolvedstates.append(
-        ['rebasestate', _('hg rebase --continue')])
+    statemod.addunfinished('rebase', fname='rebasestate', stopflag=True,
+                            continueflag=True, abortfunc=abortrebase,
+                            continuefunc=continuerebase)

hgext/remotefilelog/__init__.py

@@ -293,6 +293,35 b' def uisetup(ui):'
     # debugdata needs remotefilelog.len to work
     extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow)
 
+    changegroup.cgpacker = shallowbundle.shallowcg1packer
+
+    extensions.wrapfunction(changegroup, '_addchangegroupfiles',
+                            shallowbundle.addchangegroupfiles)
+    extensions.wrapfunction(
+        changegroup, 'makechangegroup', shallowbundle.makechangegroup)
+    extensions.wrapfunction(localrepo, 'makestore', storewrapper)
+    extensions.wrapfunction(exchange, 'pull', exchangepull)
+    extensions.wrapfunction(merge, 'applyupdates', applyupdates)
+    extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles)
+    extensions.wrapfunction(context.workingctx, '_checklookup', checklookup)
+    extensions.wrapfunction(scmutil, '_findrenames', findrenames)
+    extensions.wrapfunction(copies, '_computeforwardmissing',
+                            computeforwardmissing)
+    extensions.wrapfunction(dispatch, 'runcommand', runcommand)
+    extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets)
+    extensions.wrapfunction(context.changectx, 'filectx', filectx)
+    extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx)
+    extensions.wrapfunction(patch, 'trydiff', trydiff)
+    extensions.wrapfunction(hg, 'verify', _verify)
+    scmutil.fileprefetchhooks.add('remotefilelog', _fileprefetchhook)
+
+    # disappointing hacks below
+    extensions.wrapfunction(scmutil, 'getrenamedfn', getrenamedfn)
+    extensions.wrapfunction(revset, 'filelog', filelogrevset)
+    revset.symbols['filelog'] = revset.filelog
+    extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs)
+
+
 def cloneshallow(orig, ui, repo, *args, **opts):
     if opts.get(r'shallow'):
         repos = []
@@ -405,6 +434,158 b' def setupclient(ui, repo):'
     shallowrepo.wraprepo(repo)
     repo.store = shallowstore.wrapstore(repo.store)
 
+def storewrapper(orig, requirements, path, vfstype):
+    s = orig(requirements, path, vfstype)
+    if constants.SHALLOWREPO_REQUIREMENT in requirements:
+        s = shallowstore.wrapstore(s)
+
+    return s
+
+# prefetch files before update
+def applyupdates(orig, repo, actions, wctx, mctx, overwrite, wantfiledata,
+                 labels=None):
+    if isenabled(repo):
+        manifest = mctx.manifest()
+        files = []
+        for f, args, msg in actions['g']:
+            files.append((f, hex(manifest[f])))
+        # batch fetch the needed files from the server
+        repo.fileservice.prefetch(files)
+    return orig(repo, actions, wctx, mctx, overwrite, wantfiledata,
+                labels=labels)
+
+# Prefetch merge checkunknownfiles
+def checkunknownfiles(orig, repo, wctx, mctx, force, actions,
+    *args, **kwargs):
+    if isenabled(repo):
+        files = []
+        sparsematch = repo.maybesparsematch(mctx.rev())
+        for f, (m, actionargs, msg) in actions.iteritems():
+            if sparsematch and not sparsematch(f):
+                continue
+            if m in ('c', 'dc', 'cm'):
+                files.append((f, hex(mctx.filenode(f))))
+            elif m == 'dg':
+                f2 = actionargs[0]
+                files.append((f2, hex(mctx.filenode(f2))))
+        # batch fetch the needed files from the server
+        repo.fileservice.prefetch(files)
+    return orig(repo, wctx, mctx, force, actions, *args, **kwargs)
+
+# Prefetch files before status attempts to look at their size and contents
+def checklookup(orig, self, files):
+    repo = self._repo
+    if isenabled(repo):
+        prefetchfiles = []
+        for parent in self._parents:
+            for f in files:
+                if f in parent:
+                    prefetchfiles.append((f, hex(parent.filenode(f))))
+        # batch fetch the needed files from the server
+        repo.fileservice.prefetch(prefetchfiles)
+    return orig(self, files)
+
+# Prefetch the logic that compares added and removed files for renames
+def findrenames(orig, repo, matcher, added, removed, *args, **kwargs):
+    if isenabled(repo):
+        files = []
+        pmf = repo['.'].manifest()
+        for f in removed:
+            if f in pmf:
+                files.append((f, hex(pmf[f])))
+        # batch fetch the needed files from the server
+        repo.fileservice.prefetch(files)
+    return orig(repo, matcher, added, removed, *args, **kwargs)
+
+# prefetch files before pathcopies check
+def computeforwardmissing(orig, a, b, match=None):
+    missing = orig(a, b, match=match)
+    repo = a._repo
+    if isenabled(repo):
+        mb = b.manifest()
+
+        files = []
+        sparsematch = repo.maybesparsematch(b.rev())
+        if sparsematch:
+            sparsemissing = set()
+            for f in missing:
+                if sparsematch(f):
+                    files.append((f, hex(mb[f])))
+                    sparsemissing.add(f)
+            missing = sparsemissing
+
+        # batch fetch the needed files from the server
+        repo.fileservice.prefetch(files)
+    return missing
+
+# close cache miss server connection after the command has finished
+def runcommand(orig, lui, repo, *args, **kwargs):
+    fileservice = None
+    # repo can be None when running in chg:
+    # - at startup, reposetup was called because serve is not norepo
+    # - a norepo command like "help" is called
+    if repo and isenabled(repo):
+        fileservice = repo.fileservice
+    try:
+        return orig(lui, repo, *args, **kwargs)
+    finally:
+        if fileservice:
+            fileservice.close()
+
+# prevent strip from stripping remotefilelogs
+def _collectbrokencsets(orig, repo, files, striprev):
+    if isenabled(repo):
+        files = list([f for f in files if not repo.shallowmatch(f)])
+    return orig(repo, files, striprev)
+
+# changectx wrappers
+def filectx(orig, self, path, fileid=None, filelog=None):
+    if fileid is None:
+        fileid = self.filenode(path)
+    if (isenabled(self._repo) and self._repo.shallowmatch(path)):
+        return remotefilectx.remotefilectx(self._repo, path, fileid=fileid,
+                                           changectx=self, filelog=filelog)
+    return orig(self, path, fileid=fileid, filelog=filelog)
+
+def workingfilectx(orig, self, path, filelog=None):
+    if (isenabled(self._repo) and self._repo.shallowmatch(path)):
+        return remotefilectx.remoteworkingfilectx(self._repo, path,
+                                                  workingctx=self,
+                                                  filelog=filelog)
+    return orig(self, path, filelog=filelog)
+
+# prefetch required revisions before a diff
+def trydiff(orig, repo, revs, ctx1, ctx2, modified, added, removed,
+    copy, getfilectx, *args, **kwargs):
+    if isenabled(repo):
+        prefetch = []
+        mf1 = ctx1.manifest()
+        for fname in modified + added + removed:
+            if fname in mf1:
+                fnode = getfilectx(fname, ctx1).filenode()
+                # fnode can be None if it's a edited working ctx file
+                if fnode:
+                    prefetch.append((fname, hex(fnode)))
+            if fname not in removed:
+                fnode = getfilectx(fname, ctx2).filenode()
+                if fnode:
+                    prefetch.append((fname, hex(fnode)))
+
+        repo.fileservice.prefetch(prefetch)
+
+    return orig(repo, revs, ctx1, ctx2, modified, added, removed, copy,
+                getfilectx, *args, **kwargs)
+
+# Prevent verify from processing files
+# a stub for mercurial.hg.verify()
+def _verify(orig, repo, level=None):
+    lock = repo.lock()
+    try:
+        return shallowverifier.shallowverifier(repo).verify()
+    finally:
+        lock.release()
+
+
 clientonetime = False
 def onetimeclientsetup(ui):
     global clientonetime
@@ -412,163 +593,6 b' def onetimeclientsetup(ui):'
         return
     clientonetime = True
 
-    changegroup.cgpacker = shallowbundle.shallowcg1packer
-
-    extensions.wrapfunction(changegroup, '_addchangegroupfiles',
-                            shallowbundle.addchangegroupfiles)
-    extensions.wrapfunction(
-        changegroup, 'makechangegroup', shallowbundle.makechangegroup)
-
-    def storewrapper(orig, requirements, path, vfstype):
-        s = orig(requirements, path, vfstype)
-        if constants.SHALLOWREPO_REQUIREMENT in requirements:
-            s = shallowstore.wrapstore(s)
-
-        return s
-    extensions.wrapfunction(localrepo, 'makestore', storewrapper)
-
-    extensions.wrapfunction(exchange, 'pull', exchangepull)
-
-    # prefetch files before update
-    def applyupdates(orig, repo, actions, wctx, mctx, overwrite, labels=None):
-        if isenabled(repo):
-            manifest = mctx.manifest()
-            files = []
-            for f, args, msg in actions['g']:
-                files.append((f, hex(manifest[f])))
-            # batch fetch the needed files from the server
-            repo.fileservice.prefetch(files)
-        return orig(repo, actions, wctx, mctx, overwrite, labels=labels)
-    extensions.wrapfunction(merge, 'applyupdates', applyupdates)
-
-    # Prefetch merge checkunknownfiles
-    def checkunknownfiles(orig, repo, wctx, mctx, force, actions,
-                          *args, **kwargs):
-        if isenabled(repo):
-            files = []
-            sparsematch = repo.maybesparsematch(mctx.rev())
-            for f, (m, actionargs, msg) in actions.iteritems():
-                if sparsematch and not sparsematch(f):
-                    continue
-                if m in ('c', 'dc', 'cm'):
-                    files.append((f, hex(mctx.filenode(f))))
-                elif m == 'dg':
-                    f2 = actionargs[0]
-                    files.append((f2, hex(mctx.filenode(f2))))
-            # batch fetch the needed files from the server
-            repo.fileservice.prefetch(files)
-        return orig(repo, wctx, mctx, force, actions, *args, **kwargs)
-    extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles)
-
-    # Prefetch files before status attempts to look at their size and contents
-    def checklookup(orig, self, files):
-        repo = self._repo
-        if isenabled(repo):
-            prefetchfiles = []
-            for parent in self._parents:
-                for f in files:
-                    if f in parent:
-                        prefetchfiles.append((f, hex(parent.filenode(f))))
-            # batch fetch the needed files from the server
-            repo.fileservice.prefetch(prefetchfiles)
-        return orig(self, files)
-    extensions.wrapfunction(context.workingctx, '_checklookup', checklookup)
-
-    # Prefetch the logic that compares added and removed files for renames
-    def findrenames(orig, repo, matcher, added, removed, *args, **kwargs):
-        if isenabled(repo):
-            files = []
-            pmf = repo['.'].manifest()
-            for f in removed:
-                if f in pmf:
-                    files.append((f, hex(pmf[f])))
-            # batch fetch the needed files from the server
-            repo.fileservice.prefetch(files)
-        return orig(repo, matcher, added, removed, *args, **kwargs)
-    extensions.wrapfunction(scmutil, '_findrenames', findrenames)
-
-    # prefetch files before mergecopies check
-    def computenonoverlap(orig, repo, c1, c2, *args, **kwargs):
-        u1, u2 = orig(repo, c1, c2, *args, **kwargs)
-        if isenabled(repo):
-            m1 = c1.manifest()
-            m2 = c2.manifest()
-            files = []
-
-            sparsematch1 = repo.maybesparsematch(c1.rev())
-            if sparsematch1:
-                sparseu1 = set()
-                for f in u1:
-                    if sparsematch1(f):
-                        files.append((f, hex(m1[f])))
-                        sparseu1.add(f)
-                u1 = sparseu1
-
-            sparsematch2 = repo.maybesparsematch(c2.rev())
-            if sparsematch2:
-                sparseu2 = set()
-                for f in u2:
-                    if sparsematch2(f):
-                        files.append((f, hex(m2[f])))
-                        sparseu2.add(f)
-                u2 = sparseu2
-
-            # batch fetch the needed files from the server
-            repo.fileservice.prefetch(files)
-        return u1, u2
-    extensions.wrapfunction(copies, '_computenonoverlap', computenonoverlap)
-
-    # prefetch files before pathcopies check
-    def computeforwardmissing(orig, a, b, match=None):
-        missing = orig(a, b, match=match)
-        repo = a._repo
-        if isenabled(repo):
-            mb = b.manifest()
-
-            files = []
-            sparsematch = repo.maybesparsematch(b.rev())
-            if sparsematch:
-                sparsemissing = set()
-                for f in missing:
-                    if sparsematch(f):
-                        files.append((f, hex(mb[f])))
-                        sparsemissing.add(f)
-                missing = sparsemissing
-
-            # batch fetch the needed files from the server
-            repo.fileservice.prefetch(files)
-        return missing
-    extensions.wrapfunction(copies, '_computeforwardmissing',
-                            computeforwardmissing)
-
-    # close cache miss server connection after the command has finished
-    def runcommand(orig, lui, repo, *args, **kwargs):
-        fileservice = None
-        # repo can be None when running in chg:
-        # - at startup, reposetup was called because serve is not norepo
-        # - a norepo command like "help" is called
-        if repo and isenabled(repo):
-            fileservice = repo.fileservice
-        try:
-            return orig(lui, repo, *args, **kwargs)
-        finally:
-            if fileservice:
-                fileservice.close()
-    extensions.wrapfunction(dispatch, 'runcommand', runcommand)
-
-    # disappointing hacks below
-    scmutil.getrenamedfn = getrenamedfn
-    extensions.wrapfunction(revset, 'filelog', filelogrevset)
-    revset.symbols['filelog'] = revset.filelog
-    extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs)
-
-    # prevent strip from stripping remotefilelogs
-    def _collectbrokencsets(orig, repo, files, striprev):
-        if isenabled(repo):
-            files = list([f for f in files if not repo.shallowmatch(f)])
-        return orig(repo, files, striprev)
-    extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets)
-
     # Don't commit filelogs until we know the commit hash, since the hash
     # is present in the filelog blob.
     # This violates Mercurial's filelog->manifest->changelog write order,
@@ -611,60 +635,10 b' def onetimeclientsetup(ui):'
         return node
     extensions.wrapfunction(changelog.changelog, 'add', changelogadd)
 
-    # changectx wrappers
-    def filectx(orig, self, path, fileid=None, filelog=None):
-        if fileid is None:
-            fileid = self.filenode(path)
-        if (isenabled(self._repo) and self._repo.shallowmatch(path)):
-            return remotefilectx.remotefilectx(self._repo, path,
-                fileid=fileid, changectx=self, filelog=filelog)
-        return orig(self, path, fileid=fileid, filelog=filelog)
-    extensions.wrapfunction(context.changectx, 'filectx', filectx)
-
-    def workingfilectx(orig, self, path, filelog=None):
-        if (isenabled(self._repo) and self._repo.shallowmatch(path)):
-            return remotefilectx.remoteworkingfilectx(self._repo,
-                path, workingctx=self, filelog=filelog)
-        return orig(self, path, filelog=filelog)
-    extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx)
+def getrenamedfn(orig, repo, endrev=None):
+    if not isenabled(repo) or copies.usechangesetcentricalgo(repo):
+        return orig(repo, endrev)
 
-    # prefetch required revisions before a diff
-    def trydiff(orig, repo, revs, ctx1, ctx2, modified, added, removed,
-                copy, getfilectx, *args, **kwargs):
-        if isenabled(repo):
-            prefetch = []
-            mf1 = ctx1.manifest()
-            for fname in modified + added + removed:
-                if fname in mf1:
-                    fnode = getfilectx(fname, ctx1).filenode()
-                    # fnode can be None if it's a edited working ctx file
-                    if fnode:
-                        prefetch.append((fname, hex(fnode)))
-                if fname not in removed:
-                    fnode = getfilectx(fname, ctx2).filenode()
-                    if fnode:
-                        prefetch.append((fname, hex(fnode)))
-
-            repo.fileservice.prefetch(prefetch)
-
-        return orig(repo, revs, ctx1, ctx2, modified, added, removed,
-            copy, getfilectx, *args, **kwargs)
-    extensions.wrapfunction(patch, 'trydiff', trydiff)
-
-    # Prevent verify from processing files
-    # a stub for mercurial.hg.verify()
-    def _verify(orig, repo):
-        lock = repo.lock()
-        try:
-            return shallowverifier.shallowverifier(repo).verify()
-        finally:
-            lock.release()
-
-    extensions.wrapfunction(hg, 'verify', _verify)
-
-    scmutil.fileprefetchhooks.add('remotefilelog', _fileprefetchhook)
-
-def getrenamedfn(repo, endrev=None):
     rcache = {}
 
     def getrenamed(fn, rev):
@@ -1019,9 +993,6 b' def _fileprefetchhook(repo, revs, match)'
             mf = ctx.manifest()
             sparsematch = repo.maybesparsematch(ctx.rev())
             for path in ctx.walk(match):
-                if path.endswith('/'):
-                    # Tree manifest that's being excluded as part of narrow
-                    continue
                 if (not sparsematch or sparsematch(path)) and path in mf:
                     allfiles.append((path, hex(mf[path])))
         repo.fileservice.prefetch(allfiles)

hgext/remotefilelog/fileserverclient.py

@@ -396,6 +396,9 b' class fileserverclient(object):'
                                 batchdefault = 10
                             batchsize = self.ui.configint(
                                 'remotefilelog', 'batchsize', batchdefault)
+                            self.ui.debug(
+                                b'requesting %d files from '
+                                b'remotefilelog server...\n' % len(missed))
                             _getfilesbatch(
                                 remote, self.receivemissing, progress.increment,
                                 missed, idmap, batchsize)

hgext/remotefilelog/repack.py

@@ -43,7 +43,8 b' def backgroundrepack(repo, incremental=T'
     if packsonly:
         cmd.append('--packsonly')
     repo.ui.warn(msg)
-    procutil.runbgcommand(cmd, encoding.environ)
+    # We know this command will find a binary, so don't block on it starting.
+    procutil.runbgcommand(cmd, encoding.environ, ensurestart=False)
 
 def fullrepack(repo, options=None):
     """If ``packsonly`` is True, stores creating only loose objects are skipped.

hgext/remotefilelog/shallowrepo.py

@@ -33,13 +33,6 b' from . import ('
     shallowutil,
 )
 
-if util.safehasattr(util, '_hgexecutable'):
-    # Before 5be286db
-    _hgexecutable = util.hgexecutable
-else:
-    from mercurial.utils import procutil
-    _hgexecutable = procutil.hgexecutable
-
 # These make*stores functions are global so that other extensions can replace
 # them.
 def makelocalstores(repo):
@@ -168,7 +161,7 b' def wraprepo(repo):'
                                                               **kwargs)
 
         @localrepo.unfilteredmethod
-        def commitctx(self, ctx, error=False):
+        def commitctx(self, ctx, error=False, origctx=None):
             """Add a new revision to current repository.
             Revision information is passed via the context argument.
             """
@@ -186,18 +179,21 b' def wraprepo(repo):'
                         files.append((f, hex(fparent1)))
                 self.fileservice.prefetch(files)
             return super(shallowrepository, self).commitctx(ctx,
-                                                            error=error)
+                                                            error=error,
+                                                            origctx=origctx)
 
         def backgroundprefetch(self, revs, base=None, repack=False, pats=None,
                                opts=None):
             """Runs prefetch in background with optional repack
             """
-            cmd = [_hgexecutable(), '-R', repo.origroot, 'prefetch']
+            cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'prefetch']
             if repack:
                 cmd.append('--repack')
             if revs:
                 cmd += ['-r', revs]
-            procutil.runbgcommand(cmd, encoding.environ)
+            # We know this command will find a binary, so don't block
+            # on it starting.
+            procutil.runbgcommand(cmd, encoding.environ, ensurestart=False)
 
         def prefetch(self, revs, base=None, pats=None, opts=None):
             """Prefetches all the necessary file revisions for the given revs

hgext/remotenames.py

@@ -167,6 +167,8 b' class lazyremotenamedict(mutablemapping)'
         for k, vtup in self.potentialentries.iteritems():
             yield (k, [bin(vtup[0])])
 
+    items = iteritems
+
 class remotenames(object):
     """
     This class encapsulates all the remotenames state. It also contains

hgext/share.py

@@ -125,6 +125,10 b' def extsetup(ui):'
 
 def _hassharedbookmarks(repo):
     """Returns whether this repo has shared bookmarks"""
+    if bookmarks.bookmarksinstore(repo):
+        # Kind of a lie, but it means that we skip our custom reads and writes
+        # from/to the source repo.
+        return False
     try:
         shared = repo.vfs.read('shared').splitlines()
     except IOError as inst:

hgext/show.py

@@ -460,8 +460,8 b' def _updatedocstring():'
     longest = max(map(len, showview._table.keys()))
     entries = []
     for key in sorted(showview._table.keys()):
-        entries.append(pycompat.sysstr('    %s   %s' % (
-            key.ljust(longest), showview._table[key]._origdoc)))
+        entries.append(r'    %s   %s' % (
+            pycompat.sysstr(key.ljust(longest)), showview._table[key]._origdoc))
 
     cmdtable['show'][0].__doc__ = pycompat.sysstr('%s\n\n%s\n    ') % (
         cmdtable['show'][0].__doc__.rstrip(),

hgext/sparse.py

@@ -228,7 +228,7 b' def _setupdirstate(ui):'
     hint = _('include file with `hg debugsparse --include <pattern>` or use ' +
              '`hg add -s <file>` to include file directory while adding')
     for func in editfuncs:
-        def _wrapper(orig, self, *args):
+        def _wrapper(orig, self, *args, **kwargs):
             sparsematch = self._sparsematcher
             if not sparsematch.always():
                 for f in args:
@@ -237,7 +237,7 b' def _setupdirstate(ui):'
                         raise error.Abort(_("cannot add '%s' - it is outside "
                                             "the sparse checkout") % f,
                                           hint=hint)
-            return orig(self, *args)
+            return orig(self, *args, **kwargs)
         extensions.wrapfunction(dirstate.dirstate, func, _wrapper)
 
 @command('debugsparse', [

hgext/strip.py

@@ -31,31 +31,13 b' command = registrar.command(cmdtable)'
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
-def checksubstate(repo, baserev=None):
-    '''return list of subrepos at a different revision than substate.
-    Abort if any subrepos have uncommitted changes.'''
-    inclsubs = []
-    wctx = repo[None]
-    if baserev:
-        bctx = repo[baserev]
-    else:
-        bctx = wctx.p1()
-    for s in sorted(wctx.substate):
-        wctx.sub(s).bailifchanged(True)
-        if s not in bctx.substate or bctx.sub(s).dirty():
-            inclsubs.append(s)
-    return inclsubs
-
-def checklocalchanges(repo, force=False, excsuffix=''):
-    cmdutil.checkunfinished(repo)
+def checklocalchanges(repo, force=False):
     s = repo.status()
     if not force:
-        if s.modified or s.added or s.removed or s.deleted:
-            _("local changes found") # i18n tool detection
-            raise error.Abort(_("local changes found" + excsuffix))
-        if checksubstate(repo):
-            _("local changed subrepos found") # i18n tool detection
-            raise error.Abort(_("local changed subrepos found" + excsuffix))
+        cmdutil.checkunfinished(repo)
+        cmdutil.bailifchanged(repo)
+    else:
+        cmdutil.checkunfinished(repo, skipmerge=True)
     return s
 
 def _findupdatetarget(repo, nodes):

hgext/transplant.py

@@ -35,6 +35,7 b' from mercurial import ('
     revset,
     scmutil,
     smartset,
+    state as statemod,
     util,
     vfs as vfsmod,
 )
@@ -757,9 +758,12 b' def kwtransplanted(context, mapping):'
     return n and nodemod.hex(n) or ''
 
 def extsetup(ui):
-    cmdutil.unfinishedstates.append(
-        ['transplant/journal', True, False, _('transplant in progress'),
-         _("use 'hg transplant --continue' or 'hg update' to abort")])
+    statemod.addunfinished (
+        'transplant', fname='transplant/journal', clearable=True,
+        statushint=_('To continue:    hg transplant --continue\n'
+                     'To abort:       hg update'),
+        cmdhint=_("use 'hg transplant --continue' or 'hg update' to abort")
+    )
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = [revsettransplanted, kwtransplanted]

hgext/zeroconf/Zeroconf.py

@@ -89,6 +89,8 b' import threading'
 import time
 import traceback
 
+from mercurial import pycompat
+
 __all__ = ["Zeroconf", "ServiceInfo", "ServiceBrowser"]
 
 # hook for threads
@@ -270,6 +272,8 b' class DNSQuestion(DNSEntry):'
     """A DNS question entry"""
 
     def __init__(self, name, type, clazz):
+        if pycompat.ispy3 and isinstance(name, str):
+            name = name.encode('ascii')
         if not name.endswith(".local."):
             raise NonLocalNameException(name)
         DNSEntry.__init__(self, name, type, clazz)
@@ -535,7 +539,7 b' class DNSIncoming(object):'
 
     def readString(self, len):
         """Reads a string of a given length from the packet"""
-        format = '!' + str(len) + 's'
+        format = '!%ds' % len
         length = struct.calcsize(format)
         info = struct.unpack(format,
                              self.data[self.offset:self.offset + length])
@@ -613,7 +617,7 b' class DNSIncoming(object):'
 
     def readName(self):
         """Reads a domain name from the packet"""
-        result = ''
+        result = r''
         off = self.offset
         next = -1
         first = off
@@ -625,7 +629,7 b' class DNSIncoming(object):'
                 break
             t = len & 0xC0
             if t == 0x00:
-                result = ''.join((result, self.readUTF(off, len) + '.'))
+                result = r''.join((result, self.readUTF(off, len) + r'.'))
                 off += len
             elif t == 0xC0:
                 if next < 0:

hgext/zeroconf/__init__.py

@@ -34,6 +34,7 b' from mercurial import ('
     encoding,
     extensions,
     hg,
+    pycompat,
     ui as uimod,
 )
 from mercurial.hgweb import (
@@ -55,7 +56,7 b' def getip():'
     # finds external-facing interface without sending any packets (Linux)
     try:
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect(('1.0.0.1', 0))
+        s.connect((r'1.0.0.1', 0))
         ip = s.getsockname()[0]
         return ip
     except socket.error:
@@ -64,17 +65,17 b' def getip():'
     # Generic method, sometimes gives useless results
     try:
         dumbip = socket.gethostbyaddr(socket.gethostname())[2][0]
-        if ':' in dumbip:
-            dumbip = '127.0.0.1'
-        if not dumbip.startswith('127.'):
+        if r':' in dumbip:
+            dumbip = r'127.0.0.1'
+        if not dumbip.startswith(r'127.'):
             return dumbip
     except (socket.gaierror, socket.herror):
-        dumbip = '127.0.0.1'
+        dumbip = r'127.0.0.1'
 
     # works elsewhere, but actually sends a packet
     try:
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect(('1.0.0.1', 1))
+        s.connect((r'1.0.0.1', 1))
         ip = s.getsockname()[0]
         return ip
     except socket.error:
@@ -86,19 +87,19 b' def publish(name, desc, path, port):'
     global server, localip
     if not server:
         ip = getip()
-        if ip.startswith('127.'):
+        if ip.startswith(r'127.'):
             # if we have no internet connection, this can happen.
             return
         localip = socket.inet_aton(ip)
         server = Zeroconf.Zeroconf(ip)
 
-    hostname = socket.gethostname().split('.')[0]
-    host = hostname + ".local"
-    name = "%s-%s" % (hostname, name)
+    hostname = socket.gethostname().split(r'.')[0]
+    host = hostname + r".local"
+    name = r"%s-%s" % (hostname, name)
 
     # advertise to browsers
     svc = Zeroconf.ServiceInfo('_http._tcp.local.',
-                               name + '._http._tcp.local.',
+                               pycompat.bytestr(name + r'._http._tcp.local.'),
                                server = host,
                                port = port,
                                properties = {'description': desc,
@@ -108,7 +109,7 b' def publish(name, desc, path, port):'
 
     # advertise to Mercurial clients
     svc = Zeroconf.ServiceInfo('_hg._tcp.local.',
-                               name + '._hg._tcp.local.',
+                               pycompat.bytestr(name + r'._hg._tcp.local.'),
                                server = host,
                                port = port,
                                properties = {'description': desc,
@@ -158,7 +159,7 b' class listener(object):'
 
 def getzcpaths():
     ip = getip()
-    if ip.startswith('127.'):
+    if ip.startswith(r'127.'):
         return
     server = Zeroconf.Zeroconf(ip)
     l = listener()
@@ -166,10 +167,10 b' def getzcpaths():'
     time.sleep(1)
     server.close()
     for value in l.found.values():
-        name = value.name[:value.name.index('.')]
-        url = "http://%s:%s%s" % (socket.inet_ntoa(value.address), value.port,
-                                  value.properties.get("path", "/"))
-        yield "zc-" + name, url
+        name = value.name[:value.name.index(b'.')]
+        url = r"http://%s:%s%s" % (socket.inet_ntoa(value.address), value.port,
+                                   value.properties.get(r"path", r"/"))
+        yield b"zc-" + name, pycompat.bytestr(url)
 
 def config(orig, self, section, key, *args, **kwargs):
     if section == "paths" and key.startswith("zc-"):

mercurial/__init__.py

@@ -29,7 +29,7 b' if sys.version_info[0] >= 3:'
         """A sys.meta_path finder that uses a custom module loader."""
         def find_spec(self, fullname, path, target=None):
             # Only handle Mercurial-related modules.
-            if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')):
+            if not fullname.startswith(('mercurial.', 'hgext.')):
                 return None
             # don't try to parse binary
             if fullname.startswith('mercurial.cext.'):
@@ -54,7 +54,16 b' if sys.version_info[0] >= 3:'
                 if finder == self:
                     continue
 
-                spec = finder.find_spec(fullname, path, target=target)
+                # Originally the API was a `find_module` method, but it was
+                # renamed to `find_spec` in python 3.4, with a new `target`
+                # argument.
+                find_spec_method = getattr(finder, 'find_spec', None)
+                if find_spec_method:
+                    spec = find_spec_method(fullname, path, target=target)
+                else:
+                    spec = finder.find_module(fullname)
+                    if spec is not None:
+                        spec = importlib.util.spec_from_loader(fullname, spec)
                 if spec:
                     break
 
@@ -216,7 +225,9 b' if sys.version_info[0] >= 3:'
 
                 # It changes iteritems/values to items/values as they are not
                 # present in Python 3 world.
-                elif fn in ('iteritems', 'itervalues'):
+                elif (fn in ('iteritems', 'itervalues') and
+                      not (tokens[i - 1].type == token.NAME and
+                           tokens[i - 1].string == 'def')):
                     yield t._replace(string=fn[4:])
                     continue
 
@@ -227,7 +238,7 b' if sys.version_info[0] >= 3:'
     # ``replacetoken`` or any mechanism that changes semantics of module
     # loading is changed. Otherwise cached bytecode may get loaded without
     # the new transformation mechanisms applied.
-    BYTECODEHEADER = b'HG\x00\x0b'
+    BYTECODEHEADER = b'HG\x00\x0c'
 
     class hgloader(importlib.machinery.SourceFileLoader):
         """Custom module loader that transforms source code.

mercurial/bookmarks.py

@@ -33,6 +33,14 b' from . import ('
 # custom styles
 activebookmarklabel = 'bookmarks.active bookmarks.current'
 
+BOOKMARKS_IN_STORE_REQUIREMENT = 'bookmarksinstore'
+
+def bookmarksinstore(repo):
+    return BOOKMARKS_IN_STORE_REQUIREMENT in repo.requirements
+
+def bookmarksvfs(repo):
+    return repo.svfs if bookmarksinstore(repo) else repo.vfs
+
 def _getbkfile(repo):
     """Hook so that extensions that mess with the store can hook bm storage.
 
@@ -40,7 +48,7 b' def _getbkfile(repo):'
     bookmarks or the committed ones. Other extensions (like share)
     may need to tweak this behavior further.
     """
-    fp, pending = txnutil.trypending(repo.root, repo.vfs, 'bookmarks')
+    fp, pending = txnutil.trypending(repo.root, bookmarksvfs(repo), 'bookmarks')
     return fp
 
 class bmstore(object):
@@ -91,8 +99,11 b' class bmstore(object):'
                         # ValueError:
                         # - node in nm, for non-20-bytes entry
                         # - split(...), for string without ' '
-                        repo.ui.warn(_('malformed line in .hg/bookmarks: %r\n')
-                                     % pycompat.bytestr(line))
+                        bookmarkspath = '.hg/bookmarks'
+                        if bookmarksinstore(repo):
+                            bookmarkspath = '.hg/store/bookmarks'
+                        repo.ui.warn(_('malformed line in %s: %r\n')
+                                     % (bookmarkspath, pycompat.bytestr(line)))
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
@@ -192,8 +203,9 b' class bmstore(object):'
         """record that bookmarks have been changed in a transaction
 
         The transaction is then responsible for updating the file content."""
+        location = '' if bookmarksinstore(self._repo) else 'plain'
         tr.addfilegenerator('bookmarks', ('bookmarks',), self._write,
-                            location='plain')
+                            location=location)
         tr.hookargs['bookmark_moved'] = '1'
 
     def _writerepo(self, repo):
@@ -203,28 +215,24 b' class bmstore(object):'
             rbm.active = None
             rbm._writeactive()
 
-        with repo.wlock():
-            file_ = repo.vfs('bookmarks', 'w', atomictemp=True,
-                             checkambig=True)
-            try:
-                self._write(file_)
-            except: # re-raises
-                file_.discard()
-                raise
-            finally:
-                file_.close()
+        if bookmarksinstore(repo):
+            vfs = repo.svfs
+            lock = repo.lock()
+        else:
+            vfs = repo.vfs
+            lock = repo.wlock()
+        with lock:
+            with vfs('bookmarks', 'w', atomictemp=True, checkambig=True) as f:
+                self._write(f)
 
     def _writeactive(self):
         if self._aclean:
             return
         with self._repo.wlock():
             if self._active is not None:
-                f = self._repo.vfs('bookmarks.current', 'w', atomictemp=True,
-                                   checkambig=True)
-                try:
+                with self._repo.vfs('bookmarks.current', 'w', atomictemp=True,
+                                   checkambig=True) as f:
                     f.write(encoding.fromlocal(self._active))
-                finally:
-                    f.close()
             else:
                 self._repo.vfs.tryunlink('bookmarks.current')
         self._aclean = True
@@ -306,28 +314,12 b' def _readactive(repo, marks):'
     itself as we commit. This function returns the name of that bookmark.
     It is stored in .hg/bookmarks.current
     """
-    try:
-        file = repo.vfs('bookmarks.current')
-    except IOError as inst:
-        if inst.errno != errno.ENOENT:
-            raise
-        return None
-    try:
-        # No readline() in osutil.posixfile, reading everything is
-        # cheap.
-        # Note that it's possible for readlines() here to raise
-        # IOError, since we might be reading the active mark over
-        # static-http which only tries to load the file when we try
-        # to read from it.
-        mark = encoding.tolocal((file.readlines() or [''])[0])
-        if mark == '' or mark not in marks:
-            mark = None
-    except IOError as inst:
-        if inst.errno != errno.ENOENT:
-            raise
-        return None
-    finally:
-        file.close()
+    # No readline() in osutil.posixfile, reading everything is
+    # cheap.
+    content = repo.vfs.tryread('bookmarks.current')
+    mark = encoding.tolocal((content.splitlines() or [''])[0])
+    if mark == '' or mark not in marks:
+        mark = None
     return mark
 
 def activate(repo, mark):
@@ -453,7 +445,11 b' def listbookmarks(repo):'
     return d
 
 def pushbookmark(repo, key, old, new):
-    with repo.wlock(), repo.lock(), repo.transaction('bookmarks') as tr:
+    if bookmarksinstore(repo):
+        wlock = util.nullcontextmanager()
+    else:
+        wlock = repo.wlock()
+    with wlock, repo.lock(), repo.transaction('bookmarks') as tr:
         marks = repo._bookmarks
         existing = hex(marks.get(key, ''))
         if existing != old and existing != new:

mercurial/branchmap.py

@@ -121,6 +121,12 b' def _unknownnode(node):'
     """
     raise ValueError(r'node %s does not exist' % pycompat.sysstr(hex(node)))
 
+def _branchcachedesc(repo):
+    if repo.filtername is not None:
+        return 'branch cache (%s)' % repo.filtername
+    else:
+        return 'branch cache'
+
 class branchcache(object):
     """A dict like object that hold branches heads cache.
 
@@ -212,6 +218,8 b' class branchcache(object):'
             self._verifybranch(k)
             yield k, v
 
+    items = iteritems
+
     def hasbranch(self, label):
         """ checks whether a branch of this name exists or not """
         self._verifybranch(label)
@@ -241,11 +249,9 b' class branchcache(object):'
 
         except Exception as inst:
             if repo.ui.debugflag:
-                msg = 'invalid branchheads cache'
-                if repo.filtername is not None:
-                    msg += ' (%s)' % repo.filtername
-                msg += ': %s\n'
-                repo.ui.debug(msg % pycompat.bytestr(inst))
+                msg = 'invalid %s: %s\n'
+                repo.ui.debug(msg % (_branchcachedesc(repo),
+                                     pycompat.bytestr(inst)))
             bcache = None
 
         finally:
@@ -351,9 +357,8 b' class branchcache(object):'
                         state = 'o'
                     f.write("%s %s %s\n" % (hex(node), state, label))
             f.close()
-            repo.ui.log('branchcache',
-                        'wrote %s branch cache with %d labels and %d nodes\n',
-                        repo.filtername, len(self._entries), nodecount)
+            repo.ui.log('branchcache', 'wrote %s with %d labels and %d nodes\n',
+                        _branchcachedesc(repo), len(self._entries), nodecount)
         except (IOError, OSError, error.Abort) as inst:
             # Abort may be raised by read only opener, so log and continue
             repo.ui.debug("couldn't write branch cache: %s\n" %
@@ -378,6 +383,10 b' class branchcache(object):'
         # fetch current topological heads to speed up filtering
         topoheads = set(cl.headrevs())
 
+        # new tip revision which we found after iterating items from new
+        # branches
+        ntiprev = self.tiprev
+
         # if older branchheads are reachable from new ones, they aren't
         # really branchheads. Note checking parents is insufficient:
         # 1 (branch a) -> 2 (branch b) -> 3 (branch a)
@@ -401,9 +410,12 b' class branchcache(object):'
             bheadrevs = sorted(bheadset)
             self[branch] = [cl.node(rev) for rev in bheadrevs]
             tiprev = bheadrevs[-1]
-            if tiprev > self.tiprev:
-                self.tipnode = cl.node(tiprev)
-                self.tiprev = tiprev
+            if tiprev > ntiprev:
+                ntiprev = tiprev
+
+        if ntiprev > self.tiprev:
+            self.tiprev = ntiprev
+            self.tipnode = cl.node(ntiprev)
 
         if not self.validfor(repo):
             # cache key are not valid anymore
@@ -417,8 +429,8 b' class branchcache(object):'
         self.filteredhash = scmutil.filteredhash(repo, self.tiprev)
 
         duration = util.timer() - starttime
-        repo.ui.log('branchcache', 'updated %s branch cache in %.4f seconds\n',
-                    repo.filtername or b'None', duration)
+        repo.ui.log('branchcache', 'updated %s in %.4f seconds\n',
+                    _branchcachedesc(repo), duration)
 
         self.write(repo)
 
@@ -608,51 +620,59 b' class revbranchcache(object):'
         wlock = None
         step = ''
         try:
+            # write the new names
             if self._rbcnamescount < len(self._names):
-                step = ' names'
                 wlock = repo.wlock(wait=False)
-                if self._rbcnamescount != 0:
-                    f = repo.cachevfs.open(_rbcnames, 'ab')
-                    if f.tell() == self._rbcsnameslen:
-                        f.write('\0')
-                    else:
-                        f.close()
-                        repo.ui.debug("%s changed - rewriting it\n" % _rbcnames)
-                        self._rbcnamescount = 0
-                        self._rbcrevslen = 0
-                if self._rbcnamescount == 0:
-                    # before rewriting names, make sure references are removed
-                    repo.cachevfs.unlinkpath(_rbcrevs, ignoremissing=True)
-                    f = repo.cachevfs.open(_rbcnames, 'wb')
-                f.write('\0'.join(encoding.fromlocal(b)
-                                  for b in self._names[self._rbcnamescount:]))
-                self._rbcsnameslen = f.tell()
-                f.close()
-                self._rbcnamescount = len(self._names)
+                step = ' names'
+                self._writenames(repo)
 
+            # write the new revs
             start = self._rbcrevslen * _rbcrecsize
             if start != len(self._rbcrevs):
                 step = ''
                 if wlock is None:
                     wlock = repo.wlock(wait=False)
-                revs = min(len(repo.changelog),
-                           len(self._rbcrevs) // _rbcrecsize)
-                f = repo.cachevfs.open(_rbcrevs, 'ab')
-                if f.tell() != start:
-                    repo.ui.debug("truncating cache/%s to %d\n"
-                                  % (_rbcrevs, start))
-                    f.seek(start)
-                    if f.tell() != start:
-                        start = 0
-                        f.seek(start)
-                    f.truncate()
-                end = revs * _rbcrecsize
-                f.write(self._rbcrevs[start:end])
-                f.close()
-                self._rbcrevslen = revs
+                self._writerevs(repo, start)
+
         except (IOError, OSError, error.Abort, error.LockError) as inst:
             repo.ui.debug("couldn't write revision branch cache%s: %s\n"
                           % (step, stringutil.forcebytestr(inst)))
         finally:
             if wlock is not None:
                 wlock.release()
+
+    def _writenames(self, repo):
+        """ write the new branch names to revbranchcache """
+        if self._rbcnamescount != 0:
+            f = repo.cachevfs.open(_rbcnames, 'ab')
+            if f.tell() == self._rbcsnameslen:
+                f.write('\0')
+            else:
+                f.close()
+                repo.ui.debug("%s changed - rewriting it\n" % _rbcnames)
+                self._rbcnamescount = 0
+                self._rbcrevslen = 0
+        if self._rbcnamescount == 0:
+            # before rewriting names, make sure references are removed
+            repo.cachevfs.unlinkpath(_rbcrevs, ignoremissing=True)
+            f = repo.cachevfs.open(_rbcnames, 'wb')
+        f.write('\0'.join(encoding.fromlocal(b)
+                          for b in self._names[self._rbcnamescount:]))
+        self._rbcsnameslen = f.tell()
+        f.close()
+        self._rbcnamescount = len(self._names)
+
+    def _writerevs(self, repo, start):
+        """ write the new revs to revbranchcache """
+        revs = min(len(repo.changelog), len(self._rbcrevs) // _rbcrecsize)
+        with repo.cachevfs.open(_rbcrevs, 'ab') as f:
+            if f.tell() != start:
+                repo.ui.debug("truncating cache/%s to %d\n" % (_rbcrevs, start))
+                f.seek(start)
+                if f.tell() != start:
+                    start = 0
+                    f.seek(start)
+                f.truncate()
+            end = revs * _rbcrecsize
+            f.write(self._rbcrevs[start:end])
+        self._rbcrevslen = revs

mercurial/bundle2.py

@@ -2298,10 +2298,11 b' def handlestreamv2bundle(op, part):'
     streamclone.applybundlev2(repo, part, filecount, bytecount,
                               requirements)
 
-def widen_bundle(repo, oldmatcher, newmatcher, common, known, cgversion,
-                 ellipses):
+def widen_bundle(bundler, repo, oldmatcher, newmatcher, common,
+                 known, cgversion, ellipses):
     """generates bundle2 for widening a narrow clone
 
+    bundler is the bundle to which data should be added
     repo is the localrepository instance
     oldmatcher matches what the client already has
     newmatcher matches what the client needs (including what it already has)
@@ -2312,7 +2313,6 b' def widen_bundle(repo, oldmatcher, newma'
 
     returns bundle2 of the data required for extending
     """
-    bundler = bundle20(repo.ui)
     commonnodes = set()
     cl = repo.changelog
     for r in repo.revs("::%ln", common):

mercurial/cext/dirs.c

@@ -42,6 +42,9 b' static inline Py_ssize_t _finddir(const '
 			break;
 		pos -= 1;
 	}
+	if (pos == -1) {
+	  return 0;
+	}
 
 	return pos;
 }

mercurial/cext/parsers.c

@@ -667,10 +667,11 b' void dirs_module_init(PyObject *mod);'
 void manifest_module_init(PyObject *mod);
 void revlog_module_init(PyObject *mod);
 
-static const int version = 12;
+static const int version = 13;
 
 static void module_init(PyObject *mod)
 {
+	PyObject *capsule = NULL;
 	PyModule_AddIntConstant(mod, "version", version);
 
 	/* This module constant has two purposes.  First, it lets us unit test
@@ -687,6 +688,12 b' static void module_init(PyObject *mod)'
 	manifest_module_init(mod);
 	revlog_module_init(mod);
 
+	capsule = PyCapsule_New(
+	    make_dirstate_tuple,
+	    "mercurial.cext.parsers.make_dirstate_tuple_CAPI", NULL);
+	if (capsule != NULL)
+		PyModule_AddObject(mod, "make_dirstate_tuple_CAPI", capsule);
+
 	if (PyType_Ready(&dirstateTupleType) < 0) {
 		return;
 	}

mercurial/changegroup.py

@@ -1061,7 +1061,7 b' class cgpacker(object):'
         while tmfnodes:
             tree, nodes = tmfnodes.popitem()
 
-            should_visit = self._matcher.visitdir(tree[:-1] or '.')
+            should_visit = self._matcher.visitdir(tree[:-1])
             if tree and not should_visit:
                 continue
 
@@ -1093,7 +1093,7 b' class cgpacker(object):'
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis)
 
-            if not self._oldmatcher.visitdir(store.tree[:-1] or '.'):
+            if not self._oldmatcher.visitdir(store.tree[:-1]):
                 yield tree, deltas
             else:
                 # 'deltas' is a generator and we need to consume it even if

mercurial/changelog.py

@@ -80,25 +80,55 b' def encodeextra(d):'
     ]
     return "\0".join(items)
 
-def encodecopies(copies):
-    items = [
-        '%s\0%s' % (k, copies[k])
-        for k in sorted(copies)
-    ]
+def encodecopies(files, copies):
+    items = []
+    for i, dst in enumerate(files):
+        if dst in copies:
+            items.append('%d\0%s' % (i, copies[dst]))
+    if len(items) != len(copies):
+        raise error.ProgrammingError('some copy targets missing from file list')
     return "\n".join(items)
 
-def decodecopies(data):
+def decodecopies(files, data):
     try:
         copies = {}
+        if not data:
+            return copies
         for l in data.split('\n'):
-            k, v = l.split('\0')
-            copies[k] = v
+            strindex, src = l.split('\0')
+            i = int(strindex)
+            dst = files[i]
+            copies[dst] = src
         return copies
-    except ValueError:
+    except (ValueError, IndexError):
         # Perhaps someone had chosen the same key name (e.g. "p1copies") and
         # used different syntax for the value.
         return None
 
+def encodefileindices(files, subset):
+    subset = set(subset)
+    indices = []
+    for i, f in enumerate(files):
+        if f in subset:
+            indices.append('%d' % i)
+    return '\n'.join(indices)
+
+def decodefileindices(files, data):
+    try:
+        subset = []
+        if not data:
+            return subset
+        for strindex in data.split('\n'):
+            i = int(strindex)
+            if i < 0 or i >= len(files):
+                return None
+            subset.append(files[i])
+        return subset
+    except (ValueError, IndexError):
+        # Perhaps someone had chosen the same key name (e.g. "added") and
+        # used different syntax for the value.
+        return None
+
 def stripdesc(desc):
     """strip trailing whitespace and leading and trailing empty lines"""
     return '\n'.join([l.rstrip() for l in desc.splitlines()]).strip('\n')
@@ -194,6 +224,10 b' class _changelogrevision(object):'
     user = attr.ib(default='')
     date = attr.ib(default=(0, 0))
     files = attr.ib(default=attr.Factory(list))
+    filesadded = attr.ib(default=None)
+    filesremoved = attr.ib(default=None)
+    p1copies = attr.ib(default=None)
+    p2copies = attr.ib(default=None)
     description = attr.ib(default='')
 
 class changelogrevision(object):
@@ -298,14 +332,24 b' class changelogrevision(object):'
         return self._text[off[2] + 1:off[3]].split('\n')
 
     @property
+    def filesadded(self):
+        rawindices = self.extra.get('filesadded')
+        return rawindices and decodefileindices(self.files, rawindices)
+
+    @property
+    def filesremoved(self):
+        rawindices = self.extra.get('filesremoved')
+        return rawindices and decodefileindices(self.files, rawindices)
+
+    @property
     def p1copies(self):
         rawcopies = self.extra.get('p1copies')
-        return rawcopies and decodecopies(rawcopies)
+        return rawcopies and decodecopies(self.files, rawcopies)
 
     @property
     def p2copies(self):
         rawcopies = self.extra.get('p2copies')
-        return rawcopies and decodecopies(rawcopies)
+        return rawcopies and decodecopies(self.files, rawcopies)
 
     @property
     def description(self):
@@ -380,9 +424,6 b' class changelog(revlog.revlog):'
             if i not in self.filteredrevs:
                 yield i
 
-    def reachableroots(self, minroot, heads, roots, includepath=False):
-        return self.index.reachableroots2(minroot, heads, roots, includepath)
-
     def _checknofilteredinrevs(self, revs):
         """raise the appropriate error if 'revs' contains a filtered revision
 
@@ -562,7 +603,8 b' class changelog(revlog.revlog):'
         return l[3:]
 
     def add(self, manifest, files, desc, transaction, p1, p2,
-                  user, date=None, extra=None, p1copies=None, p2copies=None):
+                  user, date=None, extra=None, p1copies=None, p2copies=None,
+                  filesadded=None, filesremoved=None):
         # Convert to UTF-8 encoded bytestrings as the very first
         # thing: calling any method on a localstr object will turn it
         # into a str object and the cached UTF-8 string is thus lost.
@@ -591,17 +633,23 b' class changelog(revlog.revlog):'
             elif branch in (".", "null", "tip"):
                 raise error.StorageError(_('the name \'%s\' is reserved')
                                          % branch)
-        if (p1copies or p2copies) and extra is None:
+        extrasentries = p1copies, p2copies, filesadded, filesremoved
+        if extra is None and any(x is not None for x in extrasentries):
             extra = {}
-        if p1copies:
-            extra['p1copies'] = encodecopies(p1copies)
-        if p2copies:
-            extra['p2copies'] = encodecopies(p2copies)
+        sortedfiles = sorted(files)
+        if p1copies is not None:
+            extra['p1copies'] = encodecopies(sortedfiles, p1copies)
+        if p2copies is not None:
+            extra['p2copies'] = encodecopies(sortedfiles, p2copies)
+        if filesadded is not None:
+            extra['filesadded'] = encodefileindices(sortedfiles, filesadded)
+        if filesremoved is not None:
+            extra['filesremoved'] = encodefileindices(sortedfiles, filesremoved)
 
         if extra:
             extra = encodeextra(extra)
             parseddate = "%s %s" % (parseddate, extra)
-        l = [hex(manifest), user, parseddate] + sorted(files) + ["", desc]
+        l = [hex(manifest), user, parseddate] + sortedfiles + ["", desc]
         text = "\n".join(l)
         return self.addrevision(text, transaction, len(self), p1, p2)
 

mercurial/chgserver.py

@@ -138,7 +138,9 b' def _getmtimepaths(ui):'
         modules.append(__version__)
     except ImportError:
         pass
-    files = [pycompat.sysexecutable]
+    files = []
+    if pycompat.sysexecutable:
+        files.append(pycompat.sysexecutable)
     for m in modules:
         try:
             files.append(pycompat.fsencode(inspect.getabsfile(m)))

mercurial/cmdutil.py

@@ -38,10 +38,12 b' from . import ('
     pathutil,
     phases,
     pycompat,
+    repair,
     revlog,
     rewriteutil,
     scmutil,
     smartset,
+    state as statemod,
     subrepoutil,
     templatekw,
     templater,
@@ -264,8 +266,8 b' def dorecord(ui, repo, commitfunc, cmdsu'
         In the end we'll record interesting changes, and everything else
         will be left in place, so the user can continue working.
         """
-
-        checkunfinished(repo, commit=True)
+        if not opts.get('interactive-unshelve'):
+            checkunfinished(repo, commit=True)
         wctx = repo[None]
         merge = len(wctx.parents()) > 1
         if merge:
@@ -278,8 +280,8 b' def dorecord(ui, repo, commitfunc, cmdsu'
         force = opts.get('force')
         if not force:
             vdirs = []
+            match = matchmod.badmatch(match, fail)
             match.explicitdir = vdirs.append
-            match.bad = fail
 
         status = repo.status(match=match)
 
@@ -618,74 +620,18 b' To mark files as resolved: hg resolve -'
 
     return _commentlines(msg)
 
-def _helpmessage(continuecmd, abortcmd):
-    msg = _('To continue:    %s\n'
-            'To abort:       %s') % (continuecmd, abortcmd)
-    return _commentlines(msg)
-
-def _rebasemsg():
-    return _helpmessage('hg rebase --continue', 'hg rebase --abort')
-
-def _histeditmsg():
-    return _helpmessage('hg histedit --continue', 'hg histedit --abort')
-
-def _unshelvemsg():
-    return _helpmessage('hg unshelve --continue', 'hg unshelve --abort')
-
-def _graftmsg():
-    return _helpmessage('hg graft --continue', 'hg graft --abort')
-
-def _mergemsg():
-    return _helpmessage('hg commit', 'hg merge --abort')
-
-def _bisectmsg():
-    msg = _('To mark the changeset good:    hg bisect --good\n'
-            'To mark the changeset bad:     hg bisect --bad\n'
-            'To abort:                      hg bisect --reset\n')
-    return _commentlines(msg)
-
-def fileexistspredicate(filename):
-    return lambda repo: repo.vfs.exists(filename)
-
-def _mergepredicate(repo):
-    return len(repo[None].parents()) > 1
-
-STATES = (
-    # (state, predicate to detect states, helpful message function)
-    ('histedit', fileexistspredicate('histedit-state'), _histeditmsg),
-    ('bisect', fileexistspredicate('bisect.state'), _bisectmsg),
-    ('graft', fileexistspredicate('graftstate'), _graftmsg),
-    ('unshelve', fileexistspredicate('shelvedstate'), _unshelvemsg),
-    ('rebase', fileexistspredicate('rebasestate'), _rebasemsg),
-    # The merge state is part of a list that will be iterated over.
-    # They need to be last because some of the other unfinished states may also
-    # be in a merge or update state (eg. rebase, histedit, graft, etc).
-    # We want those to have priority.
-    ('merge', _mergepredicate, _mergemsg),
-)
-
-def _getrepostate(repo):
-    # experimental config: commands.status.skipstates
-    skip = set(repo.ui.configlist('commands', 'status.skipstates'))
-    for state, statedetectionpredicate, msgfn in STATES:
-        if state in skip:
-            continue
-        if statedetectionpredicate(repo):
-            return (state, statedetectionpredicate, msgfn)
-
 def morestatus(repo, fm):
-    statetuple = _getrepostate(repo)
+    statetuple = statemod.getrepostate(repo)
     label = 'status.morestatus'
     if statetuple:
-        state, statedetectionpredicate, helpfulmsg = statetuple
+        state, helpfulmsg = statetuple
         statemsg = _('The repository is in an unfinished *%s* state.') % state
         fm.plain('%s\n' % _commentlines(statemsg), label=label)
         conmsg = _conflictsmsg(repo)
         if conmsg:
             fm.plain('%s\n' % conmsg, label=label)
         if helpfulmsg:
-            helpmsg = helpfulmsg()
-            fm.plain('%s\n' % helpmsg, label=label)
+            fm.plain('%s\n' % _commentlines(helpfulmsg), label=label)
 
 def findpossible(cmd, table, strict=False):
     """
@@ -1668,6 +1614,14 b' def _exportfntemplate(repo, revs, basefm'
                 _exportsingle(repo, ctx, fm, match, switch_parent, seqno,
                               diffopts)
 
+def _prefetchchangedfiles(repo, revs, match):
+    allfiles = set()
+    for rev in revs:
+        for file in repo[rev].files():
+            if not match or match(file):
+                allfiles.add(file)
+    scmutil.prefetchfiles(repo, revs, scmutil.matchfiles(repo, allfiles))
+
 def export(repo, revs, basefm, fntemplate='hg-%h.patch', switch_parent=False,
            opts=None, match=None):
     '''export changesets as hg patches
@@ -1692,7 +1646,7 b' def export(repo, revs, basefm, fntemplat'
                             the given template.
         Otherwise: All revs will be written to basefm.
     '''
-    scmutil.prefetchfiles(repo, revs, match)
+    _prefetchchangedfiles(repo, revs, match)
 
     if not fntemplate:
         _exportfile(repo, revs, basefm, '<unnamed>', switch_parent, opts, match)
@@ -1702,7 +1656,7 b' def export(repo, revs, basefm, fntemplat'
 
 def exportfile(repo, revs, fp, switch_parent=False, opts=None, match=None):
     """Export changesets to the given file stream"""
-    scmutil.prefetchfiles(repo, revs, match)
+    _prefetchchangedfiles(repo, revs, match)
 
     dest = getattr(fp, 'name', '<unnamed>')
     with formatter.formatter(repo.ui, fp, 'export', {}) as fm:
@@ -2345,14 +2299,22 b' def remove(ui, repo, m, prefix, uipathfn'
 
     return ret
 
+def _catfmtneedsdata(fm):
+    return not fm.datahint() or 'data' in fm.datahint()
+
 def _updatecatformatter(fm, ctx, matcher, path, decode):
     """Hook for adding data to the formatter used by ``hg cat``.
 
     Extensions (e.g., lfs) can wrap this to inject keywords/data, but must call
     this method first."""
-    data = ctx[path].data()
-    if decode:
-        data = ctx.repo().wwritedata(path, data)
+
+    # data() can be expensive to fetch (e.g. lfs), so don't fetch it if it
+    # wasn't requested.
+    data = b''
+    if _catfmtneedsdata(fm):
+        data = ctx[path].data()
+        if decode:
+            data = ctx.repo().wwritedata(path, data)
     fm.startitem()
     fm.context(ctx=ctx)
     fm.write('data', '%s', data)
@@ -2383,13 +2345,15 b' def cat(ui, repo, ctx, matcher, basefm, '
         mfnode = ctx.manifestnode()
         try:
             if mfnode and mfl[mfnode].find(file)[0]:
-                scmutil.prefetchfiles(repo, [ctx.rev()], matcher)
+                if _catfmtneedsdata(basefm):
+                    scmutil.prefetchfiles(repo, [ctx.rev()], matcher)
                 write(file)
                 return 0
         except KeyError:
             pass
 
-    scmutil.prefetchfiles(repo, [ctx.rev()], matcher)
+    if _catfmtneedsdata(basefm):
+        scmutil.prefetchfiles(repo, [ctx.rev()], matcher)
 
     for abs in ctx.walk(matcher):
         write(abs)
@@ -2583,12 +2547,18 b' def amend(ui, repo, old, extra, pats, op'
         message = logmessage(ui, opts)
 
         editform = mergeeditform(old, 'commit.amend')
-        editor = getcommiteditor(editform=editform,
-                                 **pycompat.strkwargs(opts))
 
         if not message:
-            editor = getcommiteditor(edit=True, editform=editform)
             message = old.description()
+            # Default if message isn't provided and --edit is not passed is to
+            # invoke editor, but allow --no-edit. If somehow we don't have any
+            # description, let's always start the editor.
+            doedit = not message or opts.get('edit') in [True, None]
+        else:
+            # Default if message is provided is to not invoke editor, but allow
+            # --edit.
+            doedit = opts.get('edit') is True
+        editor = getcommiteditor(edit=doedit, editform=editform)
 
         pureextra = extra.copy()
         extra['amend_source'] = old.hex()
@@ -3289,66 +3259,69 b' summaryhooks = util.hooks()'
 #  - (desturl,   destbranch,   destpeer,   outgoing)
 summaryremotehooks = util.hooks()
 
-# A list of state files kept by multistep operations like graft.
-# Since graft cannot be aborted, it is considered 'clearable' by update.
-# note: bisect is intentionally excluded
-# (state file, clearable, allowcommit, error, hint)
-unfinishedstates = [
-    ('graftstate', True, False, _('graft in progress'),
-     _("use 'hg graft --continue' or 'hg graft --stop' to stop")),
-    ('updatestate', True, False, _('last update was interrupted'),
-     _("use 'hg update' to get a consistent checkout"))
-    ]
-
-def checkunfinished(repo, commit=False):
+
+def checkunfinished(repo, commit=False, skipmerge=False):
     '''Look for an unfinished multistep operation, like graft, and abort
     if found. It's probably good to check this right before
     bailifchanged().
     '''
     # Check for non-clearable states first, so things like rebase will take
     # precedence over update.
-    for f, clearable, allowcommit, msg, hint in unfinishedstates:
-        if clearable or (commit and allowcommit):
+    for state in statemod._unfinishedstates:
+        if (state._clearable or (commit and state._allowcommit) or
+            state._reportonly):
             continue
-        if repo.vfs.exists(f):
-            raise error.Abort(msg, hint=hint)
-
-    for f, clearable, allowcommit, msg, hint in unfinishedstates:
-        if not clearable or (commit and allowcommit):
+        if state.isunfinished(repo):
+            raise error.Abort(state.msg(), hint=state.hint())
+
+    for s in statemod._unfinishedstates:
+        if (not s._clearable or (commit and s._allowcommit) or
+            (s._opname == 'merge' and skipmerge) or s._reportonly):
             continue
-        if repo.vfs.exists(f):
-            raise error.Abort(msg, hint=hint)
+        if s.isunfinished(repo):
+            raise error.Abort(s.msg(), hint=s.hint())
 
 def clearunfinished(repo):
     '''Check for unfinished operations (as above), and clear the ones
     that are clearable.
     '''
-    for f, clearable, allowcommit, msg, hint in unfinishedstates:
-        if not clearable and repo.vfs.exists(f):
-            raise error.Abort(msg, hint=hint)
-    for f, clearable, allowcommit, msg, hint in unfinishedstates:
-        if clearable and repo.vfs.exists(f):
-            util.unlink(repo.vfs.join(f))
-
-afterresolvedstates = [
-    ('graftstate',
-     _('hg graft --continue')),
-    ]
+    for state in statemod._unfinishedstates:
+        if state._reportonly:
+            continue
+        if not state._clearable and state.isunfinished(repo):
+            raise error.Abort(state.msg(), hint=state.hint())
+
+    for s in statemod._unfinishedstates:
+        if s._opname == 'merge' or state._reportonly:
+            continue
+        if s._clearable and s.isunfinished(repo):
+            util.unlink(repo.vfs.join(s._fname))
+
+def getunfinishedstate(repo):
+    ''' Checks for unfinished operations and returns statecheck object
+        for it'''
+    for state in statemod._unfinishedstates:
+        if state.isunfinished(repo):
+            return state
+    return None
 
 def howtocontinue(repo):
     '''Check for an unfinished operation and return the command to finish
     it.
 
-    afterresolvedstates tuples define a .hg/{file} and the corresponding
-    command needed to finish it.
+    statemod._unfinishedstates list is checked for an unfinished operation
+    and the corresponding message to finish it is generated if a method to
+    continue is supported by the operation.
 
     Returns a (msg, warning) tuple. 'msg' is a string and 'warning' is
     a boolean.
     '''
     contmsg = _("continue: %s")
-    for f, msg in afterresolvedstates:
-        if repo.vfs.exists(f):
-            return contmsg % msg, True
+    for state in statemod._unfinishedstates:
+        if not state._continueflag:
+            continue
+        if state.isunfinished(repo):
+            return contmsg % state.continuemsg(), True
     if repo[None].dirty(missing=True, merge=False, branch=False):
         return contmsg % _("hg commit"), False
     return None, None
@@ -3356,8 +3329,8 b' def howtocontinue(repo):'
 def checkafterresolved(repo):
     '''Inform the user about the next action after completing hg resolve
 
-    If there's a matching afterresolvedstates, howtocontinue will yield
-    repo.ui.warn as the reporter.
+    If there's a an unfinished operation that supports continue flag,
+    howtocontinue will yield repo.ui.warn as the reporter.
 
     Otherwise, it will yield repo.ui.note.
     '''
@@ -3382,3 +3355,73 b' def wrongtooltocontinue(repo, task):'
     if after[1]:
         hint = after[0]
     raise error.Abort(_('no %s in progress') % task, hint=hint)
+
+def abortgraft(ui, repo, graftstate):
+    """abort the interrupted graft and rollbacks to the state before interrupted
+    graft"""
+    if not graftstate.exists():
+        raise error.Abort(_("no interrupted graft to abort"))
+    statedata = readgraftstate(repo, graftstate)
+    newnodes = statedata.get('newnodes')
+    if newnodes is None:
+        # and old graft state which does not have all the data required to abort
+        # the graft
+        raise error.Abort(_("cannot abort using an old graftstate"))
+
+    # changeset from which graft operation was started
+    if len(newnodes) > 0:
+        startctx = repo[newnodes[0]].p1()
+    else:
+        startctx = repo['.']
+    # whether to strip or not
+    cleanup = False
+    from . import hg
+    if newnodes:
+        newnodes = [repo[r].rev() for r in newnodes]
+        cleanup = True
+        # checking that none of the newnodes turned public or is public
+        immutable = [c for c in newnodes if not repo[c].mutable()]
+        if immutable:
+            repo.ui.warn(_("cannot clean up public changesets %s\n")
+                         % ', '.join(bytes(repo[r]) for r in immutable),
+                         hint=_("see 'hg help phases' for details"))
+            cleanup = False
+
+        # checking that no new nodes are created on top of grafted revs
+        desc = set(repo.changelog.descendants(newnodes))
+        if desc - set(newnodes):
+            repo.ui.warn(_("new changesets detected on destination "
+                           "branch, can't strip\n"))
+            cleanup = False
+
+        if cleanup:
+            with repo.wlock(), repo.lock():
+                hg.updaterepo(repo, startctx.node(), overwrite=True)
+                # stripping the new nodes created
+                strippoints = [c.node() for c in repo.set("roots(%ld)",
+                                                          newnodes)]
+                repair.strip(repo.ui, repo, strippoints, backup=False)
+
+    if not cleanup:
+        # we don't update to the startnode if we can't strip
+        startctx = repo['.']
+        hg.updaterepo(repo, startctx.node(), overwrite=True)
+
+    ui.status(_("graft aborted\n"))
+    ui.status(_("working directory is now at %s\n") % startctx.hex()[:12])
+    graftstate.delete()
+    return 0
+
+def readgraftstate(repo, graftstate):
+    """read the graft state file and return a dict of the data stored in it"""
+    try:
+        return graftstate.read()
+    except error.CorruptedState:
+        nodes = repo.vfs.read('graftstate').splitlines()
+        return {'nodes': nodes}
+
+def hgabortgraft(ui, repo):
+    """ abort logic for aborting graft using 'hg abort'"""
+    with repo.wlock():
+        graftstate = statemod.cmdstate(repo, 'graftstate')
+        return abortgraft(ui, repo, graftstate)

mercurial/commands.py

@@ -53,16 +53,17 b' from . import ('
     pycompat,
     rcutil,
     registrar,
-    repair,
     revsetlang,
     rewriteutil,
     scmutil,
     server,
+    shelve as shelvemod,
     state as statemod,
     streamclone,
     tags as tagsmod,
     ui as uimod,
     util,
+    verify as verifymod,
     wireprotoserver,
 )
 from .utils import (
@@ -130,6 +131,29 b' debugrevlogopts = cmdutil.debugrevlogopt'
 
 # Commands start here, listed alphabetically
 
+@command('abort',
+    dryrunopts, helpcategory=command.CATEGORY_CHANGE_MANAGEMENT,
+    helpbasic=True)
+def abort(ui, repo, **opts):
+    """abort an unfinished operation (EXPERIMENTAL)
+
+    Aborts a multistep operation like graft, histedit, rebase, merge,
+    and unshelve if they are in an unfinished state.
+
+    use --dry-run/-n to dry run the command.
+    """
+    dryrun = opts.get(r'dry_run')
+    abortstate = cmdutil.getunfinishedstate(repo)
+    if not abortstate:
+        raise error.Abort(_('no operation in progress'))
+    if not abortstate.abortfunc:
+        raise error.Abort((_("%s in progress but does not support 'hg abort'") %
+                            (abortstate._opname)), hint=abortstate.hint())
+    if dryrun:
+        ui.status(_('%s in progress, will be aborted\n') % (abortstate._opname))
+        return
+    return abortstate.abortfunc(ui, repo)
+
 @command('add',
     walkopts + subrepoopts + dryrunopts,
     _('[OPTION]... [FILE]...'),
@@ -1582,6 +1606,8 b' def clone(ui, source, dest=None, **opts)'
     ('', 'amend', None, _('amend the parent of the working directory')),
     ('s', 'secret', None, _('use the secret phase for committing')),
     ('e', 'edit', None, _('invoke editor on commit messages')),
+    ('', 'force-close-branch', None,
+     _('forcibly close branch from a non-head changeset (ADVANCED)')),
     ('i', 'interactive', None, _('use interactive mode')),
     ] + walkopts + commitopts + commitopts2 + subrepoopts,
     _('[OPTION]... [FILE]...'),
@@ -1669,11 +1695,19 b' def _docommit(ui, repo, *pats, **opts):'
     bheads = repo.branchheads(branch)
 
     extra = {}
-    if opts.get('close_branch'):
+    if opts.get('close_branch') or opts.get('force_close_branch'):
         extra['close'] = '1'
 
-        if not bheads:
-            raise error.Abort(_('can only close branch heads'))
+        if repo['.'].closesbranch():
+            raise error.Abort(_('current revision is already a branch closing'
+                                ' head'))
+        elif not bheads:
+            raise error.Abort(_('branch "%s" has no heads to close') % branch)
+        elif (branch == repo['.'].branch() and repo['.'].node() not in bheads
+              and not opts.get('force_close_branch')):
+            hint = _('use --force-close-branch to close branch from a non-head'
+                     ' changeset')
+            raise error.Abort(_('can only close branch heads'), hint=hint)
         elif opts.get('amend'):
             if (repo['.'].p1().branch() != branch and
                 repo['.'].p2().branch() != branch):
@@ -1732,6 +1766,10 b' def _docommit(ui, repo, *pats, **opts):'
 
     cmdutil.commitstatus(repo, node, branch, bheads, opts)
 
+    if not ui.quiet and ui.configbool('commands', 'commit.post-status'):
+        status(ui, repo, modified=True, added=True, removed=True, deleted=True,
+               unknown=True, subrepos=opts.get('subrepos'))
+
 @command('config|showconfig|debugconfig',
     [('u', 'untrusted', None, _('show untrusted configuration options')),
      ('e', 'edit', None, _('edit user config')),
@@ -1853,6 +1891,30 b' def config(ui, repo, *values, **opts):'
         return 0
     return 1
 
+@command('continue',
+    dryrunopts, helpcategory=command.CATEGORY_CHANGE_MANAGEMENT,
+    helpbasic=True)
+def continuecmd(ui, repo, **opts):
+    """resumes an interrupted operation (EXPERIMENTAL)
+
+    Finishes a multistep operation like graft, histedit, rebase, merge,
+    and unshelve if they are in an interrupted state.
+
+    use --dry-run/-n to dry run the command.
+    """
+    dryrun = opts.get(r'dry_run')
+    contstate = cmdutil.getunfinishedstate(repo)
+    if not contstate:
+        raise error.Abort(_('no operation in progress'))
+    if not contstate.continuefunc:
+        raise error.Abort((_("%s in progress but does not support "
+                             "'hg continue'") % (contstate._opname)),
+                             hint=contstate.continuemsg())
+    if dryrun:
+        ui.status(_('%s in progress, will be resumed\n') % (contstate._opname))
+        return
+    return contstate.continuefunc(ui, repo)
+
 @command('copy|cp',
     [('A', 'after', None, _('record a copy that has already occurred')),
     ('f', 'force', None, _('forcibly copy over an existing managed file')),
@@ -2449,14 +2511,14 b' def _dograft(ui, repo, *revs, **opts):'
                 opts.get('currentuser'), opts.get('rev'))):
             raise error.Abort(_("cannot specify any other flag with '--abort'"))
 
-        return _abortgraft(ui, repo, graftstate)
+        return cmdutil.abortgraft(ui, repo, graftstate)
     elif opts.get('continue'):
         cont = True
         if revs:
             raise error.Abort(_("can't specify --continue and revisions"))
         # read in unfinished revisions
         if graftstate.exists():
-            statedata = _readgraftstate(repo, graftstate)
+            statedata = cmdutil.readgraftstate(repo, graftstate)
             if statedata.get('date'):
                 opts['date'] = statedata['date']
             if statedata.get('user'):
@@ -2626,69 +2688,6 b' def _dograft(ui, repo, *revs, **opts):'
 
     return 0
 
-def _abortgraft(ui, repo, graftstate):
-    """abort the interrupted graft and rollbacks to the state before interrupted
-    graft"""
-    if not graftstate.exists():
-        raise error.Abort(_("no interrupted graft to abort"))
-    statedata = _readgraftstate(repo, graftstate)
-    newnodes = statedata.get('newnodes')
-    if newnodes is None:
-        # and old graft state which does not have all the data required to abort
-        # the graft
-        raise error.Abort(_("cannot abort using an old graftstate"))
-
-    # changeset from which graft operation was started
-    if len(newnodes) > 0:
-        startctx = repo[newnodes[0]].p1()
-    else:
-        startctx = repo['.']
-    # whether to strip or not
-    cleanup = False
-    if newnodes:
-        newnodes = [repo[r].rev() for r in newnodes]
-        cleanup = True
-        # checking that none of the newnodes turned public or is public
-        immutable = [c for c in newnodes if not repo[c].mutable()]
-        if immutable:
-            repo.ui.warn(_("cannot clean up public changesets %s\n")
-                         % ', '.join(bytes(repo[r]) for r in immutable),
-                         hint=_("see 'hg help phases' for details"))
-            cleanup = False
-
-        # checking that no new nodes are created on top of grafted revs
-        desc = set(repo.changelog.descendants(newnodes))
-        if desc - set(newnodes):
-            repo.ui.warn(_("new changesets detected on destination "
-                           "branch, can't strip\n"))
-            cleanup = False
-
-        if cleanup:
-            with repo.wlock(), repo.lock():
-                hg.updaterepo(repo, startctx.node(), overwrite=True)
-                # stripping the new nodes created
-                strippoints = [c.node() for c in repo.set("roots(%ld)",
-                                                          newnodes)]
-                repair.strip(repo.ui, repo, strippoints, backup=False)
-
-    if not cleanup:
-        # we don't update to the startnode if we can't strip
-        startctx = repo['.']
-        hg.updaterepo(repo, startctx.node(), overwrite=True)
-
-    ui.status(_("graft aborted\n"))
-    ui.status(_("working directory is now at %s\n") % startctx.hex()[:12])
-    graftstate.delete()
-    return 0
-
-def _readgraftstate(repo, graftstate):
-    """read the graft state file and return a dict of the data stored in it"""
-    try:
-        return graftstate.read()
-    except error.CorruptedState:
-        nodes = repo.vfs.read('graftstate').splitlines()
-        return {'nodes': nodes}
-
 def _stopgraft(ui, repo, graftstate):
     """stop the interrupted graft"""
     if not graftstate.exists():
@@ -2700,6 +2699,12 b' def _stopgraft(ui, repo, graftstate):'
     ui.status(_("working directory is now at %s\n") % pctx.hex()[:12])
     return 0
 
+statemod.addunfinished(
+    'graft', fname='graftstate', clearable=True, stopflag=True,
+    continueflag=True, abortfunc=cmdutil.hgabortgraft,
+    cmdhint=_("use 'hg graft --continue' or 'hg graft --stop' to stop")
+)
+
 @command('grep',
     [('0', 'print0', None, _('end fields with NUL')),
     ('', 'all', None, _('print all revisions that match (DEPRECATED) ')),
@@ -3715,7 +3720,8 b' def locate(ui, repo, *pats, **opts):'
      _('follow line range of specified file (EXPERIMENTAL)'),
      _('FILE,RANGE')),
     ('', 'removed', None, _('include revisions where files were removed')),
-    ('m', 'only-merges', None, _('show only merges (DEPRECATED)')),
+    ('m', 'only-merges', None,
+     _('show only merges (DEPRECATED) (use -r "merge()" instead)')),
     ('u', 'user', [], _('revisions committed by user'), _('USER')),
     ('', 'only-branch', [],
      _('show only changesets within the given named branch (DEPRECATED)'),
@@ -3876,12 +3882,12 b' def log(ui, repo, *pats, **opts):'
         # then filter the result by logcmdutil._makerevset() and --limit
         revs, differ = logcmdutil.getlinerangerevs(repo, revs, opts)
 
-    getrenamed = None
+    getcopies = None
     if opts.get('copies'):
         endrev = None
         if revs:
             endrev = revs.max() + 1
-        getrenamed = scmutil.getrenamedfn(repo, endrev=endrev)
+        getcopies = scmutil.getcopiesfn(repo, endrev=endrev)
 
     ui.pager('log')
     displayer = logcmdutil.changesetdisplayer(ui, repo, opts, differ,
@@ -3890,7 +3896,7 b' def log(ui, repo, *pats, **opts):'
         displayfn = logcmdutil.displaygraphrevs
     else:
         displayfn = logcmdutil.displayrevs
-    displayfn(ui, repo, revs, displayer, getrenamed)
+    displayfn(ui, repo, revs, displayer, getcopies)
 
 @command('manifest',
     [('r', 'rev', '', _('revision to display'), _('REV')),
@@ -3983,7 +3989,7 b' def merge(ui, repo, node=None, **opts):'
     If no revision is specified, the working directory's parent is a
     head revision, and the current branch contains exactly one other
     head, the other head is merged with by default. Otherwise, an
-    explicit revision with which to merge with must be provided.
+    explicit revision with which to merge must be provided.
 
     See :hg:`help resolve` for information on handling file conflicts.
 
@@ -3999,6 +4005,10 b' def merge(ui, repo, node=None, **opts):'
     if abort and repo.dirstate.p2() == nullid:
         cmdutil.wrongtooltocontinue(repo, _('merge'))
     if abort:
+        state = cmdutil.getunfinishedstate(repo)
+        if state and state._opname != 'merge':
+            raise error.Abort(_('cannot abort merge with %s in progress') %
+                                (state._opname), hint=state.hint())
         if node:
             raise error.Abort(_("cannot specify a node with --abort"))
         if opts.get('rev'):
@@ -4036,6 +4046,14 b' def merge(ui, repo, node=None, **opts):'
         return hg.merge(repo, node, force=force, mergeforce=force,
                         labels=labels, abort=abort)
 
+statemod.addunfinished(
+    'merge', fname=None, clearable=True, allowcommit=True,
+    cmdmsg=_('outstanding uncommitted merge'), abortfunc=hg.abortmerge,
+    statushint=_('To continue:    hg commit\n'
+                 'To abort:       hg merge --abort'),
+    cmdhint=_("use 'hg commit' or 'hg merge --abort'")
+)
+
 @command('outgoing|out',
     [('f', 'force', None, _('run even when the destination is unrelated')),
     ('r', 'rev', [],
@@ -4672,7 +4690,7 b' def recover(ui, repo, **opts):'
     """
     ret = repo.recover()
     if ret:
-        if opts['verify']:
+        if opts[r'verify']:
             return hg.verify(repo)
         else:
             msg = _("(verify step skipped, run  `hg verify` to check your "
@@ -5217,16 +5235,30 b' def rollback(ui, repo, **opts):'
                          force=opts.get(r'force'))
 
 @command(
-    'root', [], intents={INTENT_READONLY},
+    'root', [] + formatteropts, intents={INTENT_READONLY},
     helpcategory=command.CATEGORY_WORKING_DIRECTORY)
-def root(ui, repo):
+def root(ui, repo, **opts):
     """print the root (top) of the current working directory
 
     Print the root directory of the current repository.
 
+    .. container:: verbose
+
+      Template:
+
+      The following keywords are supported in addition to the common template
+      keywords and functions. See also :hg:`help templates`.
+
+      :hgpath:    String. Path to the .hg directory.
+      :storepath: String. Path to the directory holding versioned data.
+
     Returns 0 on success.
     """
-    ui.write(repo.root + "\n")
+    opts = pycompat.byteskwargs(opts)
+    with ui.formatter('root', opts) as fm:
+        fm.startitem()
+        fm.write('reporoot', '%s\n', repo.root)
+        fm.data(hgpath=repo.path, storepath=repo.spath)
 
 @command('serve',
     [('A', 'accesslog', '', _('name of access log file to write to'),
@@ -5299,6 +5331,106 b' def serve(ui, repo, **opts):'
     service = server.createservice(ui, repo, opts)
     return server.runservice(opts, initfn=service.init, runfn=service.run)
 
+@command('shelve',
+         [('A', 'addremove', None,
+           _('mark new/missing files as added/removed before shelving')),
+          ('u', 'unknown', None,
+           _('store unknown files in the shelve')),
+          ('', 'cleanup', None,
+           _('delete all shelved changes')),
+          ('', 'date', '',
+           _('shelve with the specified commit date'), _('DATE')),
+          ('d', 'delete', None,
+           _('delete the named shelved change(s)')),
+          ('e', 'edit', False,
+           _('invoke editor on commit messages')),
+          ('k', 'keep', False,
+           _('shelve, but keep changes in the working directory')),
+          ('l', 'list', None,
+           _('list current shelves')),
+          ('m', 'message', '',
+           _('use text as shelve message'), _('TEXT')),
+          ('n', 'name', '',
+           _('use the given name for the shelved commit'), _('NAME')),
+          ('p', 'patch', None,
+           _('output patches for changes (provide the names of the shelved '
+             'changes as positional arguments)')),
+          ('i', 'interactive', None,
+           _('interactive mode')),
+          ('', 'stat', None,
+           _('output diffstat-style summary of changes (provide the names of '
+             'the shelved changes as positional arguments)')
+           )] + cmdutil.walkopts,
+         _('hg shelve [OPTION]... [FILE]...'),
+         helpcategory=command.CATEGORY_WORKING_DIRECTORY)
+def shelve(ui, repo, *pats, **opts):
+    '''save and set aside changes from the working directory
+
+    Shelving takes files that "hg status" reports as not clean, saves
+    the modifications to a bundle (a shelved change), and reverts the
+    files so that their state in the working directory becomes clean.
+
+    To restore these changes to the working directory, using "hg
+    unshelve"; this will work even if you switch to a different
+    commit.
+
+    When no files are specified, "hg shelve" saves all not-clean
+    files. If specific files or directories are named, only changes to
+    those files are shelved.
+
+    In bare shelve (when no files are specified, without interactive,
+    include and exclude option), shelving remembers information if the
+    working directory was on newly created branch, in other words working
+    directory was on different branch than its first parent. In this
+    situation unshelving restores branch information to the working directory.
+
+    Each shelved change has a name that makes it easier to find later.
+    The name of a shelved change defaults to being based on the active
+    bookmark, or if there is no active bookmark, the current named
+    branch.  To specify a different name, use ``--name``.
+
+    To see a list of existing shelved changes, use the ``--list``
+    option. For each shelved change, this will print its name, age,
+    and description; use ``--patch`` or ``--stat`` for more details.
+
+    To delete specific shelved changes, use ``--delete``. To delete
+    all shelved changes, use ``--cleanup``.
+    '''
+    opts = pycompat.byteskwargs(opts)
+    allowables = [
+        ('addremove', {'create'}), # 'create' is pseudo action
+        ('unknown', {'create'}),
+        ('cleanup', {'cleanup'}),
+#       ('date', {'create'}), # ignored for passing '--date "0 0"' in tests
+        ('delete', {'delete'}),
+        ('edit', {'create'}),
+        ('keep', {'create'}),
+        ('list', {'list'}),
+        ('message', {'create'}),
+        ('name', {'create'}),
+        ('patch', {'patch', 'list'}),
+        ('stat', {'stat', 'list'}),
+    ]
+    def checkopt(opt):
+        if opts.get(opt):
+            for i, allowable in allowables:
+                if opts[i] and opt not in allowable:
+                    raise error.Abort(_("options '--%s' and '--%s' may not be "
+                                       "used together") % (opt, i))
+            return True
+    if checkopt('cleanup'):
+        if pats:
+            raise error.Abort(_("cannot specify names when using '--cleanup'"))
+        return shelvemod.cleanupcmd(ui, repo)
+    elif checkopt('delete'):
+        return shelvemod.deletecmd(ui, repo, pats)
+    elif checkopt('list'):
+        return shelvemod.listcmd(ui, repo, pats, opts)
+    elif checkopt('patch') or checkopt('stat'):
+        return shelvemod.patchcmds(ui, repo, pats, opts)
+    else:
+        return shelvemod.createcmd(ui, repo, pats, opts)
+
 _NOTTERSE = 'nothing'
 
 @command('status|st',
@@ -6027,6 +6159,68 b' def unbundle(ui, repo, fname1, *fnames, '
 
     return postincoming(ui, repo, modheads, opts.get(r'update'), None, None)
 
+@command('unshelve',
+         [('a', 'abort', None,
+           _('abort an incomplete unshelve operation')),
+          ('c', 'continue', None,
+           _('continue an incomplete unshelve operation')),
+          ('i', 'interactive', None,
+           _('use interactive mode (EXPERIMENTAL)')),
+          ('k', 'keep', None,
+           _('keep shelve after unshelving')),
+          ('n', 'name', '',
+           _('restore shelved change with given name'), _('NAME')),
+          ('t', 'tool', '', _('specify merge tool')),
+          ('', 'date', '',
+           _('set date for temporary commits (DEPRECATED)'), _('DATE'))],
+         _('hg unshelve [OPTION]... [FILE]... [-n SHELVED]'),
+         helpcategory=command.CATEGORY_WORKING_DIRECTORY)
+def unshelve(ui, repo, *shelved, **opts):
+    """restore a shelved change to the working directory
+
+    This command accepts an optional name of a shelved change to
+    restore. If none is given, the most recent shelved change is used.
+
+    If a shelved change is applied successfully, the bundle that
+    contains the shelved changes is moved to a backup location
+    (.hg/shelve-backup).
+
+    Since you can restore a shelved change on top of an arbitrary
+    commit, it is possible that unshelving will result in a conflict
+    between your changes and the commits you are unshelving onto. If
+    this occurs, you must resolve the conflict, then use
+    ``--continue`` to complete the unshelve operation. (The bundle
+    will not be moved until you successfully complete the unshelve.)
+
+    (Alternatively, you can use ``--abort`` to abandon an unshelve
+    that causes a conflict. This reverts the unshelved changes, and
+    leaves the bundle in place.)
+
+    If bare shelved change (when no files are specified, without interactive,
+    include and exclude option) was done on newly created branch it would
+    restore branch information to the working directory.
+
+    After a successful unshelve, the shelved changes are stored in a
+    backup directory. Only the N most recent backups are kept. N
+    defaults to 10 but can be overridden using the ``shelve.maxbackups``
+    configuration option.
+
+    .. container:: verbose
+
+       Timestamp in seconds is used to decide order of backups. More
+       than ``maxbackups`` backups are kept, if same timestamp
+       prevents from deciding exact order of them, for safety.
+    """
+    with repo.wlock():
+        return shelvemod.dounshelve(ui, repo, *shelved, **opts)
+
+statemod.addunfinished(
+    'unshelve', fname='shelvedstate', continueflag=True,
+    abortfunc=shelvemod.hgabortunshelve,
+    continuefunc=shelvemod.hgcontinueunshelve,
+    cmdmsg=_('unshelve already in progress'),
+)
+
 @command('update|up|checkout|co',
     [('C', 'clean', None, _('discard uncommitted changes (no backup)')),
     ('c', 'check', None, _('require clean working directory')),
@@ -6123,7 +6317,6 b' def update(ui, repo, node=None, **opts):'
 
     with repo.wlock():
         cmdutil.clearunfinished(repo)
-
         if date:
             rev = cmdutil.finddate(ui, repo, date)
 
@@ -6147,8 +6340,10 b' def update(ui, repo, node=None, **opts):'
                 ui.warn("(%s)\n" % obsfatemsg)
         return ret
 
-@command('verify', [], helpcategory=command.CATEGORY_MAINTENANCE)
-def verify(ui, repo):
+@command('verify',
+         [('', 'full', False, 'perform more checks (EXPERIMENTAL)')],
+         helpcategory=command.CATEGORY_MAINTENANCE)
+def verify(ui, repo, **opts):
     """verify the integrity of the repository
 
     Verify the integrity of the current repository.
@@ -6164,7 +6359,12 b' def verify(ui, repo):'
 
     Returns 0 on success, 1 if errors are encountered.
     """
-    return hg.verify(repo)
+    opts = pycompat.byteskwargs(opts)
+
+    level = None
+    if opts['full']:
+        level = verifymod.VERIFY_FULL
+    return hg.verify(repo, level)
 
 @command(
     'version', [] + formatteropts, helpcategory=command.CATEGORY_HELP,
@@ -6233,16 +6433,6 b' def version_(ui, **opts):'
 def loadcmdtable(ui, name, cmdtable):
     """Load command functions from specified cmdtable
     """
-    cmdtable = cmdtable.copy()
-    for cmd in list(cmdtable):
-        if not cmd.startswith('^'):
-            continue
-        ui.deprecwarn("old-style command registration '%s' in extension '%s'"
-                      % (cmd, name), '4.8')
-        entry = cmdtable.pop(cmd)
-        entry[0].helpbasic = True
-        cmdtable[cmd[1:]] = entry
-
     overrides = [cmd for cmd in cmdtable if cmd in table]
     if overrides:
         ui.warn(_("extension '%s' overrides commands: %s\n")

mercurial/configitems.py

@@ -202,6 +202,9 b" coreconfigitem('color', 'pagermode',"
     default=dynamicdefault,
 )
 _registerdiffopts(section='commands', configprefix='commit.interactive.')
+coreconfigitem('commands', 'commit.post-status',
+    default=False,
+)
 coreconfigitem('commands', 'grep.all-files',
     default=False,
 )
@@ -288,6 +291,9 b" coreconfigitem('convert', 'hg.clonebranc"
 coreconfigitem('convert', 'hg.ignoreerrors',
     default=False,
 )
+coreconfigitem('convert', 'hg.preserve-hash',
+    default=False,
+)
 coreconfigitem('convert', 'hg.revs',
     default=None,
 )
@@ -526,12 +532,22 b" coreconfigitem('experimental', 'evolutio"
 coreconfigitem('experimental', 'evolution.bundle-obsmarker',
     default=False,
 )
+coreconfigitem('experimental', 'log.topo',
+    default=False,
+)
 coreconfigitem('experimental', 'evolution.report-instabilities',
     default=True,
 )
 coreconfigitem('experimental', 'evolution.track-operation',
     default=True,
 )
+# repo-level config to exclude a revset visibility
+#
+# The target use case is to use `share` to expose different subset of the same
+# repository, especially server side. See also `server.view`.
+coreconfigitem('experimental', 'extra-filter-revs',
+    default=None,
+)
 coreconfigitem('experimental', 'maxdeltachainspan',
     default=-1,
 )
@@ -663,6 +679,9 b" coreconfigitem('extdata', '.*',"
     default=None,
     generic=True,
 )
+coreconfigitem('format', 'bookmarks-in-store',
+    default=False,
+)
 coreconfigitem('format', 'chunkcachesize',
     default=None,
 )
@@ -931,6 +950,9 b" coreconfigitem('profiling', 'showmax',"
 coreconfigitem('profiling', 'showmin',
     default=dynamicdefault,
 )
+coreconfigitem('profiling', 'showtime',
+    default=True,
+)
 coreconfigitem('profiling', 'sort',
     default='inlinetime',
 )
@@ -1072,6 +1094,9 b" coreconfigitem('share', 'pool',"
 coreconfigitem('share', 'poolnaming',
     default='identity',
 )
+coreconfigitem('shelve','maxbackups',
+    default=10,
+)
 coreconfigitem('smtp', 'host',
     default=None,
 )

mercurial/context.py

@@ -272,6 +272,30 b' class basectx(object):'
         except error.LookupError:
             return ''
 
+    @propertycache
+    def _copies(self):
+        p1copies = {}
+        p2copies = {}
+        p1 = self.p1()
+        p2 = self.p2()
+        narrowmatch = self._repo.narrowmatch()
+        for dst in self.files():
+            if not narrowmatch(dst) or dst not in self:
+                continue
+            copied = self[dst].renamed()
+            if not copied:
+                continue
+            src, srcnode = copied
+            if src in p1 and p1[src].filenode() == srcnode:
+                p1copies[dst] = src
+            elif src in p2 and p2[src].filenode() == srcnode:
+                p2copies[dst] = src
+        return p1copies, p2copies
+    def p1copies(self):
+        return self._copies[0]
+    def p2copies(self):
+        return self._copies[1]
+
     def sub(self, path, allowcreate=True):
         '''return a subrepo for the stored revision of path, never wdir()'''
         return subrepo.subrepo(self, path, allowcreate=allowcreate)
@@ -439,6 +463,36 b' class changectx(basectx):'
         return self._changeset.date
     def files(self):
         return self._changeset.files
+    def filesmodified(self):
+        modified = set(self.files())
+        modified.difference_update(self.filesadded())
+        modified.difference_update(self.filesremoved())
+        return sorted(modified)
+    def filesadded(self):
+        source = self._repo.ui.config('experimental', 'copies.read-from')
+        if (source == 'changeset-only' or
+            (source == 'compatibility' and
+             self._changeset.filesadded is not None)):
+            return self._changeset.filesadded or []
+
+        added = []
+        for f in self.files():
+            if not any(f in p for p in self.parents()):
+                added.append(f)
+        return added
+    def filesremoved(self):
+        source = self._repo.ui.config('experimental', 'copies.read-from')
+        if (source == 'changeset-only' or
+            (source == 'compatibility' and
+             self._changeset.filesremoved is not None)):
+            return self._changeset.filesremoved or []
+
+        removed = []
+        for f in self.files():
+            if f not in self:
+                removed.append(f)
+        return removed
+
     @propertycache
     def _copies(self):
         source = self._repo.ui.config('experimental', 'copies.read-from')
@@ -456,27 +510,7 b' class changectx(basectx):'
         # Otherwise (config said to read only from filelog, or we are in
         # compatiblity mode and there is not data in the changeset), we get
         # the copy metadata from the filelogs.
-        p1copies = {}
-        p2copies = {}
-        p1 = self.p1()
-        p2 = self.p2()
-        narrowmatch = self._repo.narrowmatch()
-        for dst in self.files():
-            if not narrowmatch(dst) or dst not in self:
-                continue
-            copied = self[dst].renamed()
-            if not copied:
-                continue
-            src, srcnode = copied
-            if src in p1 and p1[src].filenode() == srcnode:
-                p1copies[dst] = src
-            elif src in p2 and p2[src].filenode() == srcnode:
-                p2copies[dst] = src
-        return p1copies, p2copies
-    def p1copies(self):
-        return self._copies[0]
-    def p2copies(self):
-        return self._copies[1]
+        return super(changectx, self)._copies
     def description(self):
         return self._changeset.description
     def branch(self):
@@ -1098,7 +1132,7 b' class committablectx(basectx):'
     """A committablectx object provides common functionality for a context that
     wants the ability to commit, e.g. workingctx or memctx."""
     def __init__(self, repo, text="", user=None, date=None, extra=None,
-                 changes=None):
+                 changes=None, branch=None):
         super(committablectx, self).__init__(repo)
         self._rev = None
         self._node = None
@@ -1113,13 +1147,9 b' class committablectx(basectx):'
         self._extra = {}
         if extra:
             self._extra = extra.copy()
-        if 'branch' not in self._extra:
-            try:
-                branch = encoding.fromlocal(self._repo.dirstate.branch())
-            except UnicodeDecodeError:
-                raise error.Abort(_('branch name not in UTF-8!'))
-            self._extra['branch'] = branch
-        if self._extra['branch'] == '':
+        if branch is not None:
+            self._extra['branch'] = encoding.fromlocal(branch)
+        if not self._extra.get('branch'):
             self._extra['branch'] = 'default'
 
     def __bytes__(self):
@@ -1132,42 +1162,6 b' class committablectx(basectx):'
 
     __bool__ = __nonzero__
 
-    def _buildflagfunc(self):
-        # Create a fallback function for getting file flags when the
-        # filesystem doesn't support them
-
-        copiesget = self._repo.dirstate.copies().get
-        parents = self.parents()
-        if len(parents) < 2:
-            # when we have one parent, it's easy: copy from parent
-            man = parents[0].manifest()
-            def func(f):
-                f = copiesget(f, f)
-                return man.flags(f)
-        else:
-            # merges are tricky: we try to reconstruct the unstored
-            # result from the merge (issue1802)
-            p1, p2 = parents
-            pa = p1.ancestor(p2)
-            m1, m2, ma = p1.manifest(), p2.manifest(), pa.manifest()
-
-            def func(f):
-                f = copiesget(f, f) # may be wrong for merges with copies
-                fl1, fl2, fla = m1.flags(f), m2.flags(f), ma.flags(f)
-                if fl1 == fl2:
-                    return fl1
-                if fl1 == fla:
-                    return fl2
-                if fl2 == fla:
-                    return fl1
-                return '' # punt for conflicts
-
-        return func
-
-    @propertycache
-    def _flagfunc(self):
-        return self._repo.dirstate.flagfunc(self._buildflagfunc)
-
     @propertycache
     def _status(self):
         return self._repo.status()
@@ -1206,26 +1200,10 b' class committablectx(basectx):'
         return self._status.removed
     def deleted(self):
         return self._status.deleted
-    @propertycache
-    def _copies(self):
-        p1copies = {}
-        p2copies = {}
-        parents = self._repo.dirstate.parents()
-        p1manifest = self._repo[parents[0]].manifest()
-        p2manifest = self._repo[parents[1]].manifest()
-        narrowmatch = self._repo.narrowmatch()
-        for dst, src in self._repo.dirstate.copies().items():
-            if not narrowmatch(dst):
-                continue
-            if src in p1manifest:
-                p1copies[dst] = src
-            elif src in p2manifest:
-                p2copies[dst] = src
-        return p1copies, p2copies
-    def p1copies(self):
-        return self._copies[0]
-    def p2copies(self):
-        return self._copies[1]
+    filesmodified = modified
+    filesadded = added
+    filesremoved = removed
+
     def branch(self):
         return encoding.tolocal(self._extra['branch'])
     def closesbranch(self):
@@ -1257,33 +1235,10 b' class committablectx(basectx):'
     def children(self):
         return []
 
-    def flags(self, path):
-        if r'_manifest' in self.__dict__:
-            try:
-                return self._manifest.flags(path)
-            except KeyError:
-                return ''
-
-        try:
-            return self._flagfunc(path)
-        except OSError:
-            return ''
-
     def ancestor(self, c2):
         """return the "best" ancestor context of self and c2"""
         return self._parents[0].ancestor(c2) # punt on two parents for now
 
-    def walk(self, match):
-        '''Generates matching file names.'''
-        return sorted(self._repo.dirstate.walk(self._repo.narrowmatch(match),
-                                               subrepos=sorted(self.substate),
-                                               unknown=True, ignored=False))
-
-    def matches(self, match):
-        match = self._repo.narrowmatch(match)
-        ds = self._repo.dirstate
-        return sorted(f for f in ds.matches(match) if ds[f] != 'r')
-
     def ancestors(self):
         for p in self._parents:
             yield p
@@ -1301,18 +1256,6 b' class committablectx(basectx):'
 
         """
 
-        with self._repo.dirstate.parentchange():
-            for f in self.modified() + self.added():
-                self._repo.dirstate.normal(f)
-            for f in self.removed():
-                self._repo.dirstate.drop(f)
-            self._repo.dirstate.setparents(node)
-
-        # write changes out explicitly, because nesting wlock at
-        # runtime may prevent 'wlock.release()' in 'repo.commit()'
-        # from immediately doing so for subsequent changing files
-        self._repo.dirstate.write(self._repo.currenttransaction())
-
     def dirty(self, missing=False, merge=True, branch=True):
         return False
 
@@ -1327,7 +1270,14 b' class workingctx(committablectx):'
     """
     def __init__(self, repo, text="", user=None, date=None, extra=None,
                  changes=None):
-        super(workingctx, self).__init__(repo, text, user, date, extra, changes)
+        branch = None
+        if not extra or 'branch' not in extra:
+            try:
+                branch = repo.dirstate.branch()
+            except UnicodeDecodeError:
+                raise error.Abort(_('branch name not in UTF-8!'))
+        super(workingctx, self).__init__(repo, text, user, date, extra, changes,
+                                         branch=branch)
 
     def __iter__(self):
         d = self._repo.dirstate
@@ -1355,6 +1305,54 b' class workingctx(committablectx):'
         self._manifest
         return super(workingctx, self)._fileinfo(path)
 
+    def _buildflagfunc(self):
+        # Create a fallback function for getting file flags when the
+        # filesystem doesn't support them
+
+        copiesget = self._repo.dirstate.copies().get
+        parents = self.parents()
+        if len(parents) < 2:
+            # when we have one parent, it's easy: copy from parent
+            man = parents[0].manifest()
+            def func(f):
+                f = copiesget(f, f)
+                return man.flags(f)
+        else:
+            # merges are tricky: we try to reconstruct the unstored
+            # result from the merge (issue1802)
+            p1, p2 = parents
+            pa = p1.ancestor(p2)
+            m1, m2, ma = p1.manifest(), p2.manifest(), pa.manifest()
+
+            def func(f):
+                f = copiesget(f, f) # may be wrong for merges with copies
+                fl1, fl2, fla = m1.flags(f), m2.flags(f), ma.flags(f)
+                if fl1 == fl2:
+                    return fl1
+                if fl1 == fla:
+                    return fl2
+                if fl2 == fla:
+                    return fl1
+                return '' # punt for conflicts
+
+        return func
+
+    @propertycache
+    def _flagfunc(self):
+        return self._repo.dirstate.flagfunc(self._buildflagfunc)
+
+    def flags(self, path):
+        if r'_manifest' in self.__dict__:
+            try:
+                return self._manifest.flags(path)
+            except KeyError:
+                return ''
+
+        try:
+            return self._flagfunc(path)
+        except OSError:
+            return ''
+
     def filectx(self, path, filelog=None):
         """get a file context from the working directory"""
         return workingfilectx(self._repo, path, workingctx=self,
@@ -1579,6 +1577,23 b' class workingctx(committablectx):'
         return s
 
     @propertycache
+    def _copies(self):
+        p1copies = {}
+        p2copies = {}
+        parents = self._repo.dirstate.parents()
+        p1manifest = self._repo[parents[0]].manifest()
+        p2manifest = self._repo[parents[1]].manifest()
+        narrowmatch = self._repo.narrowmatch()
+        for dst, src in self._repo.dirstate.copies().items():
+            if not narrowmatch(dst):
+                continue
+            if src in p1manifest:
+                p1copies[dst] = src
+            elif src in p2manifest:
+                p2copies[dst] = src
+        return p1copies, p2copies
+
+    @propertycache
     def _manifest(self):
         """generate a manifest corresponding to the values in self._status
 
@@ -1651,8 +1666,29 b' class workingctx(committablectx):'
             match.bad = bad
         return match
 
+    def walk(self, match):
+        '''Generates matching file names.'''
+        return sorted(self._repo.dirstate.walk(self._repo.narrowmatch(match),
+                                               subrepos=sorted(self.substate),
+                                               unknown=True, ignored=False))
+
+    def matches(self, match):
+        match = self._repo.narrowmatch(match)
+        ds = self._repo.dirstate
+        return sorted(f for f in ds.matches(match) if ds[f] != 'r')
+
     def markcommitted(self, node):
-        super(workingctx, self).markcommitted(node)
+        with self._repo.dirstate.parentchange():
+            for f in self.modified() + self.added():
+                self._repo.dirstate.normal(f)
+            for f in self.removed():
+                self._repo.dirstate.drop(f)
+            self._repo.dirstate.setparents(node)
+
+        # write changes out explicitly, because nesting wlock at
+        # runtime may prevent 'wlock.release()' in 'repo.commit()'
+        # from immediately doing so for subsequent changing files
+        self._repo.dirstate.write(self._repo.currenttransaction())
 
         sparse.aftercommit(self._repo, node)
 
@@ -1726,6 +1762,8 b' class workingfilectx(committablefilectx)'
 
     def size(self):
         return self._repo.wvfs.lstat(self._path).st_size
+    def lstat(self):
+        return self._repo.wvfs.lstat(self._path)
     def date(self):
         t, tz = self._changectx.date()
         try:
@@ -1761,14 +1799,13 b' class workingfilectx(committablefilectx)'
 
     def write(self, data, flags, backgroundclose=False, **kwargs):
         """wraps repo.wwrite"""
-        self._repo.wwrite(self._path, data, flags,
-                          backgroundclose=backgroundclose,
-                          **kwargs)
+        return self._repo.wwrite(self._path, data, flags,
+                                 backgroundclose=backgroundclose,
+                                 **kwargs)
 
     def markcopied(self, src):
         """marks this file a copy of `src`"""
-        if self._repo.dirstate[self._path] in "nma":
-            self._repo.dirstate.copy(src, self._path)
+        self._repo.dirstate.copy(src, self._path)
 
     def clearunknown(self):
         """Removes conflicting items in the working directory so that
@@ -1913,7 +1950,7 b' class overlayworkingctx(committablectx):'
         if self.isdirty(path):
             return self._cache[path]['copied']
         else:
-            raise error.ProgrammingError('copydata() called on clean context')
+            return None
 
     def flags(self, path):
         if self.isdirty(path):
@@ -2055,7 +2092,7 b' class overlayworkingctx(committablectx):'
         else:
             parents = (self._repo[parents[0]], self._repo[parents[1]])
 
-        files = self._cache.keys()
+        files = self.files()
         def getfile(repo, memctx, path):
             if self._cache[path]['exists']:
                 return memfilectx(repo, memctx, path,
@@ -2118,7 +2155,9 b' class overlayworkingctx(committablectx):'
         # the file is marked as existing.
         if exists and data is None:
             oldentry = self._cache.get(path) or {}
-            data = oldentry.get('data') or self._wrappedctx[path].data()
+            data = oldentry.get('data')
+            if data is None:
+                data = self._wrappedctx[path].data()
 
         self._cache[path] = {
             'exists': exists,
@@ -2305,7 +2344,8 b' class memctx(committablectx):'
 
     def __init__(self, repo, parents, text, files, filectxfn, user=None,
                  date=None, extra=None, branch=None, editor=False):
-        super(memctx, self).__init__(repo, text, user, date, extra)
+        super(memctx, self).__init__(repo, text, user, date, extra,
+                                     branch=branch)
         self._rev = None
         self._node = None
         parents = [(p or nullid) for p in parents]
@@ -2313,8 +2353,6 b' class memctx(committablectx):'
         self._parents = [self._repo[p] for p in (p1, p2)]
         files = sorted(set(files))
         self._files = files
-        if branch is not None:
-            self._extra['branch'] = encoding.fromlocal(branch)
         self.substate = {}
 
         if isinstance(filectxfn, patch.filestore):

mercurial/copies.py

@@ -107,40 +107,60 b' def _findlimit(repo, ctxa, ctxb):'
     # This only occurs when a is a descendent of b or visa-versa.
     return min(limit, a, b)
 
-def _chain(src, dst, a, b):
-    """chain two sets of copies a->b"""
-    t = a.copy()
-    for k, v in b.iteritems():
-        if v in t:
-            # found a chain
-            if t[v] != k:
-                # file wasn't renamed back to itself
-                t[k] = t[v]
-            if v not in dst:
-                # chain was a rename, not a copy
-                del t[v]
-        if v in src:
-            # file is a copy of an existing file
-            t[k] = v
+def _filter(src, dst, t):
+    """filters out invalid copies after chaining"""
+
+    # When _chain()'ing copies in 'a' (from 'src' via some other commit 'mid')
+    # with copies in 'b' (from 'mid' to 'dst'), we can get the different cases
+    # in the following table (not including trivial cases). For example, case 2
+    # is where a file existed in 'src' and remained under that name in 'mid' and
+    # then was renamed between 'mid' and 'dst'.
+    #
+    # case src mid dst result
+    #   1   x   y   -    -
+    #   2   x   y   y   x->y
+    #   3   x   y   x    -
+    #   4   x   y   z   x->z
+    #   5   -   x   y    -
+    #   6   x   x   y   x->y
+    #
+    # _chain() takes care of chaining the copies in 'a' and 'b', but it
+    # cannot tell the difference between cases 1 and 2, between 3 and 4, or
+    # between 5 and 6, so it includes all cases in its result.
+    # Cases 1, 3, and 5 are then removed by _filter().
 
     for k, v in list(t.items()):
+        # remove copies from files that didn't exist
+        if v not in src:
+            del t[k]
         # remove criss-crossed copies
-        if k in src and v in dst:
+        elif k in src and v in dst:
             del t[k]
         # remove copies to files that were then removed
         elif k not in dst:
             del t[k]
 
+def _chain(a, b):
+    """chain two sets of copies 'a' and 'b'"""
+    t = a.copy()
+    for k, v in b.iteritems():
+        if v in t:
+            t[k] = t[v]
+        else:
+            t[k] = v
     return t
 
-def _tracefile(fctx, am, limit=node.nullrev):
+def _tracefile(fctx, am, basemf, limit):
     """return file context that is the ancestor of fctx present in ancestor
     manifest am, stopping after the first ancestor lower than limit"""
 
     for f in fctx.ancestors():
-        if am.get(f.path(), None) == f.filenode():
-            return f
-        if limit >= 0 and not f.isintroducedafter(limit):
+        path = f.path()
+        if am.get(path, None) == f.filenode():
+            return path
+        if basemf and basemf.get(path, None) == f.filenode():
+            return path
+        if not f.isintroducedafter(limit):
             return None
 
 def _dirstatecopies(repo, match=None):
@@ -165,7 +185,7 b' def usechangesetcentricalgo(repo):'
     return (repo.ui.config('experimental', 'copies.read-from') in
             ('changeset-only', 'compatibility'))
 
-def _committedforwardcopies(a, b, match):
+def _committedforwardcopies(a, b, base, match):
     """Like _forwardcopies(), but b.rev() cannot be None (working copy)"""
     # files might have to be traced back to the fctx parent of the last
     # one-side-only changeset, but not further back than that
@@ -183,6 +203,7 b' def _committedforwardcopies(a, b, match)'
     if debug:
         dbg('debug.copies:      search limit: %d\n' % limit)
     am = a.manifest()
+    basemf = None if base is None else base.manifest()
 
     # find where new files came from
     # we currently don't try to find where old files went, too expensive
@@ -204,9 +225,9 b' def _committedforwardcopies(a, b, match)'
     ancestrycontext = a._repo.changelog.ancestors([b.rev()], inclusive=True)
 
     if debug:
-        dbg('debug.copies:      missing file to search: %d\n' % len(missing))
+        dbg('debug.copies:      missing files to search: %d\n' % len(missing))
 
-    for f in missing:
+    for f in sorted(missing):
         if debug:
             dbg('debug.copies:        tracing file: %s\n' % f)
         fctx = b[f]
@@ -214,11 +235,11 b' def _committedforwardcopies(a, b, match)'
 
         if debug:
             start = util.timer()
-        ofctx = _tracefile(fctx, am, limit)
-        if ofctx:
+        opath = _tracefile(fctx, am, basemf, limit)
+        if opath:
             if debug:
-                dbg('debug.copies:          rename of: %s\n' % ofctx._path)
-            cm[f] = ofctx.path()
+                dbg('debug.copies:          rename of: %s\n' % opath)
+            cm[f] = opath
         if debug:
             dbg('debug.copies:          time: %f seconds\n'
                 % (util.timer() - start))
@@ -245,40 +266,30 b' def _changesetforwardcopies(a, b, match)'
     # 'work' contains 3-tuples of a (revision number, parent number, copies).
     # The parent number is only used for knowing which parent the copies dict
     # came from.
+    # NOTE: To reduce costly copying the 'copies' dicts, we reuse the same
+    # instance for *one* of the child nodes (the last one). Once an instance
+    # has been put on the queue, it is thus no longer safe to modify it.
+    # Conversely, it *is* safe to modify an instance popped off the queue.
     work = [(r, 1, {}) for r in roots]
     heapq.heapify(work)
+    alwaysmatch = match.always()
     while work:
-        r, i1, copies1 = heapq.heappop(work)
+        r, i1, copies = heapq.heappop(work)
         if work and work[0][0] == r:
             # We are tracing copies from both parents
             r, i2, copies2 = heapq.heappop(work)
-            copies = {}
-            ctx = repo[r]
-            p1man, p2man = ctx.p1().manifest(), ctx.p2().manifest()
-            allcopies = set(copies1) | set(copies2)
-            # TODO: perhaps this filtering should be done as long as ctx
-            # is merge, whether or not we're tracing from both parent.
-            for dst in allcopies:
-                if not match(dst):
-                    continue
-                if dst not in copies2:
-                    # Copied on p1 side: mark as copy from p1 side if it didn't
-                    # already exist on p2 side
-                    if dst not in p2man:
-                        copies[dst] = copies1[dst]
-                elif dst not in copies1:
-                    # Copied on p2 side: mark as copy from p2 side if it didn't
-                    # already exist on p1 side
-                    if dst not in p1man:
-                        copies[dst] = copies2[dst]
-                else:
-                    # Copied on both sides: mark as copy from p1 side
-                    copies[dst] = copies1[dst]
-        else:
-            copies = copies1
+            for dst, src in copies2.items():
+                # Unlike when copies are stored in the filelog, we consider
+                # it a copy even if the destination already existed on the
+                # other branch. It's simply too expensive to check if the
+                # file existed in the manifest.
+                if dst not in copies:
+                    # If it was copied on the p1 side, leave it as copied from
+                    # that side, even if it was also copied on the p2 side.
+                    copies[dst] = copies2[dst]
         if r == b.rev():
             return copies
-        for c in children[r]:
+        for i, c in enumerate(children[r]):
             childctx = repo[c]
             if r == childctx.p1().rev():
                 parent = 1
@@ -287,27 +298,36 b' def _changesetforwardcopies(a, b, match)'
                 assert r == childctx.p2().rev()
                 parent = 2
                 childcopies = childctx.p2copies()
-            if not match.always():
+            if not alwaysmatch:
                 childcopies = {dst: src for dst, src in childcopies.items()
                                if match(dst)}
-            childcopies = _chain(a, childctx, copies, childcopies)
-            heapq.heappush(work, (c, parent, childcopies))
+            # Copy the dict only if later iterations will also need it
+            if i != len(children[r]) - 1:
+                newcopies = copies.copy()
+            else:
+                newcopies = copies
+            if childcopies:
+                newcopies = _chain(newcopies, childcopies)
+            for f in childctx.filesremoved():
+                if f in newcopies:
+                    del newcopies[f]
+            heapq.heappush(work, (c, parent, newcopies))
     assert False
 
-def _forwardcopies(a, b, match=None):
+def _forwardcopies(a, b, base=None, match=None):
     """find {dst@b: src@a} copy mapping where a is an ancestor of b"""
 
+    if base is None:
+        base = a
     match = a.repo().narrowmatch(match)
     # check for working copy
     if b.rev() is None:
-        if a == b.p1():
-            # short-circuit to avoid issues with merge states
-            return _dirstatecopies(b._repo, match)
-
-        cm = _committedforwardcopies(a, b.p1(), match)
+        cm = _committedforwardcopies(a, b.p1(), base, match)
         # combine copies from dirstate if necessary
-        return _chain(a, b, cm, _dirstatecopies(b._repo, match))
-    return _committedforwardcopies(a, b, match)
+        copies = _chain(cm, _dirstatecopies(b._repo, match))
+    else:
+        copies  = _committedforwardcopies(a, b, base, match)
+    return copies
 
 def _backwardrenames(a, b, match):
     if a._repo.ui.config('experimental', 'copytrace') == 'off':
@@ -343,90 +363,24 b' def pathcopies(x, y, match=None):'
     if a == x:
         if debug:
             repo.ui.debug('debug.copies: search mode: forward\n')
-        return _forwardcopies(x, y, match=match)
-    if a == y:
+        if y.rev() is None and x == y.p1():
+            # short-circuit to avoid issues with merge states
+            return _dirstatecopies(repo, match)
+        copies = _forwardcopies(x, y, match=match)
+    elif a == y:
         if debug:
             repo.ui.debug('debug.copies: search mode: backward\n')
-        return _backwardrenames(x, y, match=match)
-    if debug:
-        repo.ui.debug('debug.copies: search mode: combined\n')
-    return _chain(x, y, _backwardrenames(x, a, match=match),
-                  _forwardcopies(a, y, match=match))
-
-def _computenonoverlap(repo, c1, c2, addedinm1, addedinm2, baselabel=''):
-    """Computes, based on addedinm1 and addedinm2, the files exclusive to c1
-    and c2. This is its own function so extensions can easily wrap this call
-    to see what files mergecopies is about to process.
-
-    Even though c1 and c2 are not used in this function, they are useful in
-    other extensions for being able to read the file nodes of the changed files.
-
-    "baselabel" can be passed to help distinguish the multiple computations
-    done in the graft case.
-    """
-    u1 = sorted(addedinm1 - addedinm2)
-    u2 = sorted(addedinm2 - addedinm1)
-
-    header = "  unmatched files in %s"
-    if baselabel:
-        header += ' (from %s)' % baselabel
-    if u1:
-        repo.ui.debug("%s:\n   %s\n" % (header % 'local', "\n   ".join(u1)))
-    if u2:
-        repo.ui.debug("%s:\n   %s\n" % (header % 'other', "\n   ".join(u2)))
-
-    return u1, u2
-
-def _makegetfctx(ctx):
-    """return a 'getfctx' function suitable for _checkcopies usage
-
-    We have to re-setup the function building 'filectx' for each
-    '_checkcopies' to ensure the linkrev adjustment is properly setup for
-    each. Linkrev adjustment is important to avoid bug in rename
-    detection. Moreover, having a proper '_ancestrycontext' setup ensures
-    the performance impact of this adjustment is kept limited. Without it,
-    each file could do a full dag traversal making the time complexity of
-    the operation explode (see issue4537).
-
-    This function exists here mostly to limit the impact on stable. Feel
-    free to refactor on default.
-    """
-    rev = ctx.rev()
-    repo = ctx._repo
-    ac = getattr(ctx, '_ancestrycontext', None)
-    if ac is None:
-        revs = [rev]
-        if rev is None:
-            revs = [p.rev() for p in ctx.parents()]
-        ac = repo.changelog.ancestors(revs, inclusive=True)
-        ctx._ancestrycontext = ac
-    def makectx(f, n):
-        if n in node.wdirfilenodeids:  # in a working context?
-            if ctx.rev() is None:
-                return ctx.filectx(f)
-            return repo[None][f]
-        fctx = repo.filectx(f, fileid=n)
-        # setup only needed for filectx not create from a changectx
-        fctx._ancestrycontext = ac
-        fctx._descendantrev = rev
-        return fctx
-    return util.lrucachefunc(makectx)
-
-def _combinecopies(copyfrom, copyto, finalcopy, diverge, incompletediverge):
-    """combine partial copy paths"""
-    remainder = {}
-    for f in copyfrom:
-        if f in copyto:
-            finalcopy[copyto[f]] = copyfrom[f]
-            del copyto[f]
-    for f in incompletediverge:
-        assert f not in diverge
-        ic = incompletediverge[f]
-        if ic[0] in copyto:
-            diverge[f] = [copyto[ic[0]], ic[1]]
-        else:
-            remainder[f] = ic
-    return remainder
+        copies = _backwardrenames(x, y, match=match)
+    else:
+        if debug:
+            repo.ui.debug('debug.copies: search mode: combined\n')
+        base = None
+        if a.rev() != node.nullrev:
+            base = x
+        copies = _chain(_backwardrenames(x, a, match=match),
+                        _forwardcopies(a, y, base, match=match))
+    _filter(x, y, copies)
+    return copies
 
 def mergecopies(repo, c1, c2, base):
     """
@@ -485,7 +439,14 b' def mergecopies(repo, c1, c2, base):'
         return _dirstatecopies(repo, narrowmatch), {}, {}, {}, {}
 
     copytracing = repo.ui.config('experimental', 'copytrace')
-    boolctrace = stringutil.parsebool(copytracing)
+    if stringutil.parsebool(copytracing) is False:
+        # stringutil.parsebool() returns None when it is unable to parse the
+        # value, so we should rely on making sure copytracing is on such cases
+        return {}, {}, {}, {}, {}
+
+    if usechangesetcentricalgo(repo):
+        # The heuristics don't make sense when we need changeset-centric algos
+        return _fullcopytracing(repo, c1, c2, base)
 
     # Copy trace disabling is explicitly below the node == p1 logic above
     # because the logic above is required for a simple copy to be kept across a
@@ -497,10 +458,6 b' def mergecopies(repo, c1, c2, base):'
         if _isfullcopytraceable(repo, c1, base):
             return _fullcopytracing(repo, c1, c2, base)
         return _heuristicscopytracing(repo, c1, c2, base)
-    elif boolctrace is False:
-        # stringutil.parsebool() returns None when it is unable to parse the
-        # value, so we should rely on making sure copytracing is on such cases
-        return {}, {}, {}, {}, {}
     else:
         return _fullcopytracing(repo, c1, c2, base)
 
@@ -522,6 +479,23 b' def _isfullcopytraceable(repo, c1, base)'
         return commits < sourcecommitlimit
     return False
 
+def _checksinglesidecopies(src, dsts1, m1, m2, mb, c2, base,
+                           copy, renamedelete):
+    if src not in m2:
+        # deleted on side 2
+        if src not in m1:
+            # renamed on side 1, deleted on side 2
+            renamedelete[src] = dsts1
+    elif m2[src] != mb[src]:
+        if not _related(c2[src], base[src]):
+            return
+        # modified on side 2
+        for dst in dsts1:
+            if dst not in m2:
+                # dst not added on side 2 (handle as regular
+                # "both created" case in manifestmerge otherwise)
+                copy[dst] = src
+
 def _fullcopytracing(repo, c1, c2, base):
     """ The full copytracing algorithm which finds all the new files that were
     added from merge base up to the top commit and for each file it checks if
@@ -530,159 +504,84 b' def _fullcopytracing(repo, c1, c2, base)'
     This is pretty slow when a lot of changesets are involved but will track all
     the copies.
     """
-    # In certain scenarios (e.g. graft, update or rebase), base can be
-    # overridden We still need to know a real common ancestor in this case We
-    # can't just compute _c1.ancestor(_c2) and compare it to ca, because there
-    # can be multiple common ancestors, e.g. in case of bidmerge.  Because our
-    # caller may not know if the revision passed in lieu of the CA is a genuine
-    # common ancestor or not without explicitly checking it, it's better to
-    # determine that here.
-    #
-    # base.isancestorof(wc) is False, work around that
-    _c1 = c1.p1() if c1.rev() is None else c1
-    _c2 = c2.p1() if c2.rev() is None else c2
-    # an endpoint is "dirty" if it isn't a descendant of the merge base
-    # if we have a dirty endpoint, we need to trigger graft logic, and also
-    # keep track of which endpoint is dirty
-    dirtyc1 = not base.isancestorof(_c1)
-    dirtyc2 = not base.isancestorof(_c2)
-    graft = dirtyc1 or dirtyc2
-    tca = base
-    if graft:
-        tca = _c1.ancestor(_c2)
-
-    limit = _findlimit(repo, c1, c2)
-    repo.ui.debug("  searching for copies back to rev %d\n" % limit)
-
     m1 = c1.manifest()
     m2 = c2.manifest()
     mb = base.manifest()
 
-    # gather data from _checkcopies:
-    # - diverge = record all diverges in this dict
-    # - copy = record all non-divergent copies in this dict
-    # - fullcopy = record all copies in this dict
-    # - incomplete = record non-divergent partial copies here
-    # - incompletediverge = record divergent partial copies here
-    diverge = {} # divergence data is shared
-    incompletediverge  = {}
-    data1 = {'copy': {},
-             'fullcopy': {},
-             'incomplete': {},
-             'diverge': diverge,
-             'incompletediverge': incompletediverge,
-            }
-    data2 = {'copy': {},
-             'fullcopy': {},
-             'incomplete': {},
-             'diverge': diverge,
-             'incompletediverge': incompletediverge,
-            }
+    copies1 = pathcopies(base, c1)
+    copies2 = pathcopies(base, c2)
+
+    inversecopies1 = {}
+    inversecopies2 = {}
+    for dst, src in copies1.items():
+        inversecopies1.setdefault(src, []).append(dst)
+    for dst, src in copies2.items():
+        inversecopies2.setdefault(src, []).append(dst)
+
+    copy = {}
+    diverge = {}
+    renamedelete = {}
+    allsources = set(inversecopies1) | set(inversecopies2)
+    for src in allsources:
+        dsts1 = inversecopies1.get(src)
+        dsts2 = inversecopies2.get(src)
+        if dsts1 and dsts2:
+            # copied/renamed on both sides
+            if src not in m1 and src not in m2:
+                # renamed on both sides
+                dsts1 = set(dsts1)
+                dsts2 = set(dsts2)
+                # If there's some overlap in the rename destinations, we
+                # consider it not divergent. For example, if side 1 copies 'a'
+                # to 'b' and 'c' and deletes 'a', and side 2 copies 'a' to 'c'
+                # and 'd' and deletes 'a'.
+                if dsts1 & dsts2:
+                    for dst in (dsts1 & dsts2):
+                        copy[dst] = src
+                else:
+                    diverge[src] = sorted(dsts1 | dsts2)
+            elif src in m1 and src in m2:
+                # copied on both sides
+                dsts1 = set(dsts1)
+                dsts2 = set(dsts2)
+                for dst in (dsts1 & dsts2):
+                    copy[dst] = src
+            # TODO: Handle cases where it was renamed on one side and copied
+            # on the other side
+        elif dsts1:
+            # copied/renamed only on side 1
+            _checksinglesidecopies(src, dsts1, m1, m2, mb, c2, base,
+                                   copy, renamedelete)
+        elif dsts2:
+            # copied/renamed only on side 2
+            _checksinglesidecopies(src, dsts2, m2, m1, mb, c1, base,
+                                   copy, renamedelete)
+
+    renamedeleteset = set()
+    divergeset = set()
+    for dsts in diverge.values():
+        divergeset.update(dsts)
+    for dsts in renamedelete.values():
+        renamedeleteset.update(dsts)
 
     # find interesting file sets from manifests
     addedinm1 = m1.filesnotin(mb, repo.narrowmatch())
     addedinm2 = m2.filesnotin(mb, repo.narrowmatch())
-    bothnew = sorted(addedinm1 & addedinm2)
-    if tca == base:
-        # unmatched file from base
-        u1r, u2r = _computenonoverlap(repo, c1, c2, addedinm1, addedinm2)
-        u1u, u2u = u1r, u2r
-    else:
-        # unmatched file from base (DAG rotation in the graft case)
-        u1r, u2r = _computenonoverlap(repo, c1, c2, addedinm1, addedinm2,
-                                      baselabel='base')
-        # unmatched file from topological common ancestors (no DAG rotation)
-        # need to recompute this for directory move handling when grafting
-        mta = tca.manifest()
-        u1u, u2u = _computenonoverlap(repo, c1, c2,
-                                      m1.filesnotin(mta, repo.narrowmatch()),
-                                      m2.filesnotin(mta, repo.narrowmatch()),
-                                      baselabel='topological common ancestor')
-
-    for f in u1u:
-        _checkcopies(c1, c2, f, base, tca, dirtyc1, limit, data1)
-
-    for f in u2u:
-        _checkcopies(c2, c1, f, base, tca, dirtyc2, limit, data2)
-
-    copy = dict(data1['copy'])
-    copy.update(data2['copy'])
-    fullcopy = dict(data1['fullcopy'])
-    fullcopy.update(data2['fullcopy'])
-
-    if dirtyc1:
-        _combinecopies(data2['incomplete'], data1['incomplete'], copy, diverge,
-                       incompletediverge)
-    if dirtyc2:
-        _combinecopies(data1['incomplete'], data2['incomplete'], copy, diverge,
-                       incompletediverge)
-
-    renamedelete = {}
-    renamedeleteset = set()
-    divergeset = set()
-    for of, fl in list(diverge.items()):
-        if len(fl) == 1 or of in c1 or of in c2:
-            del diverge[of] # not actually divergent, or not a rename
-            if of not in c1 and of not in c2:
-                # renamed on one side, deleted on the other side, but filter
-                # out files that have been renamed and then deleted
-                renamedelete[of] = [f for f in fl if f in c1 or f in c2]
-                renamedeleteset.update(fl) # reverse map for below
-        else:
-            divergeset.update(fl) # reverse map for below
+    u1 = sorted(addedinm1 - addedinm2)
+    u2 = sorted(addedinm2 - addedinm1)
 
-    if bothnew:
-        repo.ui.debug("  unmatched files new in both:\n   %s\n"
-                      % "\n   ".join(bothnew))
-    bothdiverge = {}
-    bothincompletediverge = {}
-    remainder = {}
-    both1 = {'copy': {},
-             'fullcopy': {},
-             'incomplete': {},
-             'diverge': bothdiverge,
-             'incompletediverge': bothincompletediverge
-            }
-    both2 = {'copy': {},
-             'fullcopy': {},
-             'incomplete': {},
-             'diverge': bothdiverge,
-             'incompletediverge': bothincompletediverge
-            }
-    for f in bothnew:
-        _checkcopies(c1, c2, f, base, tca, dirtyc1, limit, both1)
-        _checkcopies(c2, c1, f, base, tca, dirtyc2, limit, both2)
-    if dirtyc1 and dirtyc2:
-        remainder = _combinecopies(both2['incomplete'], both1['incomplete'],
-                                   copy, bothdiverge, bothincompletediverge)
-        remainder1 = _combinecopies(both1['incomplete'], both2['incomplete'],
-                                   copy, bothdiverge, bothincompletediverge)
-        remainder.update(remainder1)
-    elif dirtyc1:
-        # incomplete copies may only be found on the "dirty" side for bothnew
-        assert not both2['incomplete']
-        remainder = _combinecopies({}, both1['incomplete'], copy, bothdiverge,
-                                   bothincompletediverge)
-    elif dirtyc2:
-        assert not both1['incomplete']
-        remainder = _combinecopies({}, both2['incomplete'], copy, bothdiverge,
-                                   bothincompletediverge)
-    else:
-        # incomplete copies and divergences can't happen outside grafts
-        assert not both1['incomplete']
-        assert not both2['incomplete']
-        assert not bothincompletediverge
-    for f in remainder:
-        assert f not in bothdiverge
-        ic = remainder[f]
-        if ic[0] in (m1 if dirtyc1 else m2):
-            # backed-out rename on one side, but watch out for deleted files
-            bothdiverge[f] = ic
-    for of, fl in bothdiverge.items():
-        if len(fl) == 2 and fl[0] == fl[1]:
-            copy[fl[0]] = of # not actually divergent, just matching renames
+    header = "  unmatched files in %s"
+    if u1:
+        repo.ui.debug("%s:\n   %s\n" % (header % 'local', "\n   ".join(u1)))
+    if u2:
+        repo.ui.debug("%s:\n   %s\n" % (header % 'other', "\n   ".join(u2)))
 
-    if fullcopy and repo.ui.debugflag:
+    fullcopy = copies1.copy()
+    fullcopy.update(copies2)
+    if not fullcopy:
+        return copy, {}, diverge, renamedelete, {}
+
+    if repo.ui.debugflag:
         repo.ui.debug("  all copies found (* = to merge, ! = divergent, "
                       "% = renamed and deleted):\n")
         for f in sorted(fullcopy):
@@ -697,16 +596,10 b' def _fullcopytracing(repo, c1, c2, base)'
                                                               note))
     del divergeset
 
-    if not fullcopy:
-        return copy, {}, diverge, renamedelete, {}
-
     repo.ui.debug("  checking for directory renames\n")
 
     # generate a directory move map
     d1, d2 = c1.dirs(), c2.dirs()
-    # Hack for adding '', which is not otherwise added, to d1 and d2
-    d1.addpath('/')
-    d2.addpath('/')
     invalid = set()
     dirmove = {}
 
@@ -746,7 +639,7 b' def _fullcopytracing(repo, c1, c2, base)'
 
     movewithdir = {}
     # check unaccounted nonoverlapping files against directory moves
-    for f in u1r + u2r:
+    for f in u1 + u2:
         if f not in fullcopy:
             for d in dirmove:
                 if f.startswith(d):
@@ -893,99 +786,6 b' def _related(f1, f2):'
     except StopIteration:
         return False
 
-def _checkcopies(srcctx, dstctx, f, base, tca, remotebase, limit, data):
-    """
-    check possible copies of f from msrc to mdst
-
-    srcctx = starting context for f in msrc
-    dstctx = destination context for f in mdst
-    f = the filename to check (as in msrc)
-    base = the changectx used as a merge base
-    tca = topological common ancestor for graft-like scenarios
-    remotebase = True if base is outside tca::srcctx, False otherwise
-    limit = the rev number to not search beyond
-    data = dictionary of dictionary to store copy data. (see mergecopies)
-
-    note: limit is only an optimization, and provides no guarantee that
-    irrelevant revisions will not be visited
-    there is no easy way to make this algorithm stop in a guaranteed way
-    once it "goes behind a certain revision".
-    """
-
-    msrc = srcctx.manifest()
-    mdst = dstctx.manifest()
-    mb = base.manifest()
-    mta = tca.manifest()
-    # Might be true if this call is about finding backward renames,
-    # This happens in the case of grafts because the DAG is then rotated.
-    # If the file exists in both the base and the source, we are not looking
-    # for a rename on the source side, but on the part of the DAG that is
-    # traversed backwards.
-    #
-    # In the case there is both backward and forward renames (before and after
-    # the base) this is more complicated as we must detect a divergence.
-    # We use 'backwards = False' in that case.
-    backwards = not remotebase and base != tca and f in mb
-    getsrcfctx = _makegetfctx(srcctx)
-    getdstfctx = _makegetfctx(dstctx)
-
-    if msrc[f] == mb.get(f) and not remotebase:
-        # Nothing to merge
-        return
-
-    of = None
-    seen = {f}
-    for oc in getsrcfctx(f, msrc[f]).ancestors():
-        of = oc.path()
-        if of in seen:
-            # check limit late - grab last rename before
-            if oc.linkrev() < limit:
-                break
-            continue
-        seen.add(of)
-
-        # remember for dir rename detection
-        if backwards:
-            data['fullcopy'][of] = f # grafting backwards through renames
-        else:
-            data['fullcopy'][f] = of
-        if of not in mdst:
-            continue # no match, keep looking
-        if mdst[of] == mb.get(of):
-            return # no merge needed, quit early
-        c2 = getdstfctx(of, mdst[of])
-        # c2 might be a plain new file on added on destination side that is
-        # unrelated to the droids we are looking for.
-        cr = _related(oc, c2)
-        if cr and (of == f or of == c2.path()): # non-divergent
-            if backwards:
-                data['copy'][of] = f
-            elif of in mb:
-                data['copy'][f] = of
-            elif remotebase: # special case: a <- b <- a -> b "ping-pong" rename
-                data['copy'][of] = f
-                del data['fullcopy'][f]
-                data['fullcopy'][of] = f
-            else: # divergence w.r.t. graft CA on one side of topological CA
-                for sf in seen:
-                    if sf in mb:
-                        assert sf not in data['diverge']
-                        data['diverge'][sf] = [f, of]
-                        break
-            return
-
-    if of in mta:
-        if backwards or remotebase:
-            data['incomplete'][of] = f
-        else:
-            for sf in seen:
-                if sf in mb:
-                    if tca == base:
-                        data['diverge'].setdefault(sf, []).append(f)
-                    else:
-                        data['incompletediverge'][sf] = [of, f]
-                    return
-
 def duplicatecopies(repo, wctx, rev, fromrev, skiprev=None):
     """reproduce copies from fromrev to rev in the dirstate
 
@@ -1005,8 +805,7 b' def duplicatecopies(repo, wctx, rev, fro'
         # metadata across the rebase anyway).
         exclude = pathcopies(repo[fromrev], repo[skiprev])
     for dst, src in pathcopies(repo[fromrev], repo[rev]).iteritems():
-        # copies.pathcopies returns backward renames, so dst might not
-        # actually be in the dirstate
         if dst in exclude:
             continue
-        wctx[dst].markcopied(src)
+        if dst in wctx:
+            wctx[dst].markcopied(src)

mercurial/crecord.py

@@ -608,6 +608,7 b' class curseschunkselector(object):'
 
         # the currently selected header, hunk, or hunk-line
         self.currentselecteditem = self.headerlist[0]
+        self.lastapplieditem = None
 
         # updated when printing out patch-display -- the 'lines' here are the
         # line positions *in the pad*, not on the screen.
@@ -723,7 +724,7 b' class curseschunkselector(object):'
         self.currentselecteditem = nextitem
         self.recenterdisplayedarea()
 
-    def nextsametype(self):
+    def nextsametype(self, test=False):
         currentitem = self.currentselecteditem
         sametype = lambda item: isinstance(item, type(currentitem))
         nextitem = currentitem.nextitem()
@@ -739,7 +740,8 b' class curseschunkselector(object):'
                 self.togglefolded(parent)
 
         self.currentselecteditem = nextitem
-        self.recenterdisplayedarea()
+        if not test:
+            self.recenterdisplayedarea()
 
     def rightarrowevent(self):
         """
@@ -838,6 +840,8 b' class curseschunkselector(object):'
         """
         if item is None:
             item = self.currentselecteditem
+            # Only set this when NOT using 'toggleall'
+            self.lastapplieditem = item
 
         item.applied = not item.applied
 
@@ -931,6 +935,45 b' class curseschunkselector(object):'
                     self.toggleapply(item)
         self.waslasttoggleallapplied = not self.waslasttoggleallapplied
 
+    def toggleallbetween(self):
+        "toggle applied on or off for all items in range [lastapplied,current]."
+        if (not self.lastapplieditem or
+            self.currentselecteditem == self.lastapplieditem):
+            # Treat this like a normal 'x'/' '
+            self.toggleapply()
+            return
+
+        startitem = self.lastapplieditem
+        enditem = self.currentselecteditem
+        # Verify that enditem is "after" startitem, otherwise swap them.
+        for direction in ['forward', 'reverse']:
+            nextitem = startitem.nextitem()
+            while nextitem and nextitem != enditem:
+                nextitem = nextitem.nextitem()
+            if nextitem:
+                break
+            # Looks like we went the wrong direction :)
+            startitem, enditem = enditem, startitem
+
+        if not nextitem:
+            # We didn't find a path going either forward or backward? Don't know
+            # how this can happen, let's not crash though.
+            return
+
+        nextitem = startitem
+        # Switch all items to be the opposite state of the currently selected
+        # item. Specifically:
+        #  [ ] startitem
+        #  [x] middleitem
+        #  [ ] enditem  <-- currently selected
+        # This will turn all three on, since the currently selected item is off.
+        # This does *not* invert each item (i.e. middleitem stays marked/on)
+        desiredstate = not self.currentselecteditem.applied
+        while nextitem != enditem.nextitem():
+            if nextitem.applied != desiredstate:
+                self.toggleapply(item=nextitem)
+            nextitem = nextitem.nextitem()
+
     def togglefolded(self, item=None, foldparent=False):
         "toggle folded flag of specified item (defaults to currently selected)"
         if item is None:
@@ -1460,9 +1503,10 b' changes, the unselected changes are stil'
 can use crecord multiple times to split large changes into smaller changesets.
 the following are valid keystrokes:
 
-                [space] : (un-)select item ([~]/[x] = partly/fully applied)
+              x [space] : (un-)select item ([~]/[x] = partly/fully applied)
                 [enter] : (un-)select item and go to next item of same type
                       A : (un-)select all items
+                      X : (un-)select all items between current and most-recent
     up/down-arrow [k/j] : go to previous/next unfolded item
         pgup/pgdn [K/J] : go to previous/next item of same type
  right/left-arrow [l/h] : go to child item / parent item
@@ -1724,7 +1768,7 b' are you sure you want to review/edit and'
         keypressed = pycompat.bytestr(keypressed)
         if keypressed in ["k", "KEY_UP"]:
             self.uparrowevent()
-        if keypressed in ["K", "KEY_PPAGE"]:
+        elif keypressed in ["K", "KEY_PPAGE"]:
             self.uparrowshiftevent()
         elif keypressed in ["j", "KEY_DOWN"]:
             self.downarrowevent()
@@ -1742,8 +1786,6 b' are you sure you want to review/edit and'
             self.toggleamend(self.opts, test)
         elif keypressed in ["c"]:
             return True
-        elif test and keypressed in ['X']:
-            return True
         elif keypressed in ["r"]:
             if self.reviewcommit():
                 self.opts['review'] = True
@@ -1751,11 +1793,13 b' are you sure you want to review/edit and'
         elif test and keypressed in ['R']:
             self.opts['review'] = True
             return True
-        elif keypressed in [' '] or (test and keypressed in ["TOGGLE"]):
+        elif keypressed in [' ', 'x']:
             self.toggleapply()
         elif keypressed in ['\n', 'KEY_ENTER']:
             self.toggleapply()
-            self.nextsametype()
+            self.nextsametype(test=test)
+        elif keypressed in ['X']:
+            self.toggleallbetween()
         elif keypressed in ['A']:
             self.toggleall()
         elif keypressed in ['e']:

mercurial/dagop.py

@@ -259,13 +259,10 b' def descendantrevs(revs, revsfn, parentr'
                 yield rev
                 break
 
-def _reachablerootspure(repo, minroot, roots, heads, includepath):
-    """return (heads(::<roots> and ::<heads>))
-
-    If includepath is True, return (<roots>::<heads>)."""
+def _reachablerootspure(pfunc, minroot, roots, heads, includepath):
+    """See revlog.reachableroots"""
     if not roots:
         return []
-    parentrevs = repo.changelog.parentrevs
     roots = set(roots)
     visit = list(heads)
     reachable = set()
@@ -282,7 +279,7 b' def _reachablerootspure(repo, minroot, r'
             reached(rev)
             if not includepath:
                 continue
-        parents = parentrevs(rev)
+        parents = pfunc(rev)
         seen[rev] = parents
         for parent in parents:
             if parent >= minroot and parent not in seen:
@@ -298,18 +295,13 b' def _reachablerootspure(repo, minroot, r'
     return reachable
 
 def reachableroots(repo, roots, heads, includepath=False):
-    """return (heads(::<roots> and ::<heads>))
-
-    If includepath is True, return (<roots>::<heads>)."""
+    """See revlog.reachableroots"""
     if not roots:
         return baseset()
     minroot = roots.min()
     roots = list(roots)
     heads = list(heads)
-    try:
-        revs = repo.changelog.reachableroots(minroot, heads, roots, includepath)
-    except AttributeError:
-        revs = _reachablerootspure(repo, minroot, roots, heads, includepath)
+    revs = repo.changelog.reachableroots(minroot, heads, roots, includepath)
     revs = baseset(revs)
     revs.sort()
     return revs

mercurial/debugcommands.py

@@ -1240,7 +1240,7 b' def debuginstall(ui, **opts):'
 
     # Python
     fm.write('pythonexe', _("checking Python executable (%s)\n"),
-             pycompat.sysexecutable)
+             pycompat.sysexecutable or _("unknown"))
     fm.write('pythonver', _("checking Python version (%s)\n"),
              ("%d.%d.%d" % sys.version_info[:3]))
     fm.write('pythonlib', _("checking Python lib (%s)...\n"),
@@ -1278,16 +1278,28 b' def debuginstall(ui, **opts):'
     fm.write('hgmodules', _("checking installed modules (%s)...\n"),
              os.path.dirname(pycompat.fsencode(__file__)))
 
-    if policy.policy in ('c', 'allow'):
+    rustandc = policy.policy in ('rust+c', 'rust+c-allow')
+    rustext = rustandc  # for now, that's the only case
+    cext = policy.policy in ('c', 'allow') or rustandc
+    nopure = cext or rustext
+    if nopure:
         err = None
         try:
-            from .cext import (
-                base85,
-                bdiff,
-                mpatch,
-                osutil,
-            )
-            dir(bdiff), dir(mpatch), dir(base85), dir(osutil) # quiet pyflakes
+            if cext:
+                from .cext import (
+                    base85,
+                    bdiff,
+                    mpatch,
+                    osutil,
+                )
+                # quiet pyflakes
+                dir(bdiff), dir(mpatch), dir(base85), dir(osutil)
+            if rustext:
+                from .rustext import (
+                    ancestor,
+                    dirstate,
+                )
+                dir(ancestor), dir(dirstate) # quiet pyflakes
         except Exception as inst:
             err = stringutil.forcebytestr(inst)
             problems += 1

mercurial/dirstate.py

@@ -28,6 +28,7 b' from . import ('
 )
 
 parsers = policy.importmod(r'parsers')
+dirstatemod = policy.importrust(r'dirstate', default=parsers)
 
 propertycache = util.propertycache
 filecache = scmutil.filecache
@@ -390,12 +391,24 b' class dirstate(object):'
         self._updatedfiles.add(f)
         self._map.addfile(f, oldstate, state, mode, size, mtime)
 
-    def normal(self, f):
-        '''Mark a file normal and clean.'''
-        s = os.lstat(self._join(f))
-        mtime = s[stat.ST_MTIME]
-        self._addpath(f, 'n', s.st_mode,
-                      s.st_size & _rangemask, mtime & _rangemask)
+    def normal(self, f, parentfiledata=None):
+        '''Mark a file normal and clean.
+
+        parentfiledata: (mode, size, mtime) of the clean file
+
+        parentfiledata should be computed from memory (for mode,
+        size), as or close as possible from the point where we
+        determined the file was clean, to limit the risk of the
+        file having been changed by an external process between the
+        moment where the file was determined to be clean and now.'''
+        if parentfiledata:
+            (mode, size, mtime) = parentfiledata
+        else:
+            s = os.lstat(self._join(f))
+            mode = s.st_mode
+            size = s.st_size
+            mtime = s[stat.ST_MTIME]
+        self._addpath(f, 'n', mode, size & _rangemask, mtime & _rangemask)
         self._map.copymap.pop(f, None)
         if f in self._map.nonnormalset:
             self._map.nonnormalset.remove(f)
@@ -656,8 +669,6 b' class dirstate(object):'
         self._dirty = False
 
     def _dirignore(self, f):
-        if f == '.':
-            return False
         if self._ignore(f):
             return True
         for p in util.finddirs(f):
@@ -751,15 +762,16 b' class dirstate(object):'
                 del files[i]
             j += 1
 
-        if not files or '.' in files:
-            files = ['.']
+        if not files or '' in files:
+            files = ['']
+            # constructing the foldmap is expensive, so don't do it for the
+            # common case where files is ['']
+            normalize = None
         results = dict.fromkeys(subrepos)
         results['.hg'] = None
 
         for ff in files:
-            # constructing the foldmap is expensive, so don't do it for the
-            # common case where files is ['.']
-            if normalize and ff != '.':
+            if normalize:
                 nf = normalize(ff, False, True)
             else:
                 nf = ff
@@ -903,9 +915,7 b' class dirstate(object):'
                 if visitentries == 'this' or visitentries == 'all':
                     visitentries = None
                 skip = None
-                if nd == '.':
-                    nd = ''
-                else:
+                if nd != '':
                     skip = '.hg'
                 try:
                     entries = listdir(join(nd), stat=True, skip=skip)
@@ -1465,7 +1475,7 b' class dirstatemap(object):'
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
-        parse_dirstate = util.nogc(parsers.parse_dirstate)
+        parse_dirstate = util.nogc(dirstatemod.parse_dirstate)
         p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
@@ -1476,8 +1486,8 b' class dirstatemap(object):'
         self.get = self._map.get
 
     def write(self, st, now):
-        st.write(parsers.pack_dirstate(self._map, self.copymap,
-                                       self.parents(), now))
+        st.write(dirstatemod.pack_dirstate(self._map, self.copymap,
+                                           self.parents(), now))
         st.close()
         self._dirtyparents = False
         self.nonnormalset, self.otherparentset = self.nonnormalentries()

mercurial/discovery.py

@@ -343,10 +343,19 b' def checkheads(pushop):'
     # 1. Check for new branches on the remote.
     if newbranches and not newbranch:  # new branch requires --new-branch
         branchnames = ', '.join(sorted(newbranches))
-        raise error.Abort(_("push creates new remote branches: %s!")
-                           % branchnames,
-                         hint=_("use 'hg push --new-branch' to create"
-                                " new remote branches"))
+        # Calculate how many of the new branches are closed branches
+        closedbranches = set()
+        for tag, heads, tip, isclosed in repo.branchmap().iterbranches():
+            if isclosed:
+                closedbranches.add(tag)
+        closedbranches = (closedbranches & set(newbranches))
+        if closedbranches:
+            errmsg = (_("push creates new remote branches: %s (%d closed)!")
+                        % (branchnames, len(closedbranches)))
+        else:
+            errmsg = (_("push creates new remote branches: %s!")% branchnames)
+        hint=_("use 'hg push --new-branch' to create new remote branches")
+        raise error.Abort(errmsg, hint=hint)
 
     # 2. Find heads that we need not warn about
     nowarnheads = _nowarnheads(pushop)

mercurial/exchange.py

@@ -539,10 +539,12 b' def push(repo, remote, force=False, revs'
     # get lock as we might write phase data
     wlock = lock = None
     try:
-        # bundle2 push may receive a reply bundle touching bookmarks or other
-        # things requiring the wlock. Take it now to ensure proper ordering.
+        # bundle2 push may receive a reply bundle touching bookmarks
+        # requiring the wlock. Take it now to ensure proper ordering.
         maypushback = pushop.ui.configbool('experimental', 'bundle2.pushback')
-        if (not _forcebundle1(pushop)) and maypushback:
+        if ((not _forcebundle1(pushop)) and
+            maypushback and
+            not bookmod.bookmarksinstore(repo)):
             wlock = pushop.repo.wlock()
         lock = pushop.repo.lock()
         pushop.trmanager = transactionmanager(pushop.repo,
@@ -1548,7 +1550,10 b' def pull(repo, remote, heads=None, force'
             raise error.Abort(msg)
 
     pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
-    with repo.wlock(), repo.lock(), pullop.trmanager:
+    wlock = util.nullcontextmanager()
+    if not bookmod.bookmarksinstore(repo):
+        wlock = repo.wlock()
+    with wlock, repo.lock(), pullop.trmanager:
         # Use the modern wire protocol, if available.
         if remote.capable('command-changesetdata'):
             exchangev2.pull(pullop)
@@ -2395,7 +2400,8 b' def unbundle(repo, cg, heads, source, ur'
             try:
                 def gettransaction():
                     if not lockandtr[2]:
-                        lockandtr[0] = repo.wlock()
+                        if not bookmod.bookmarksinstore(repo):
+                            lockandtr[0] = repo.wlock()
                         lockandtr[1] = repo.lock()
                         lockandtr[2] = repo.transaction(source)
                         lockandtr[2].hookargs['source'] = source

mercurial/extensions.py

@@ -43,7 +43,8 b' from .utils import ('
     'progress',
     'interhg',
     'inotify',
-    'hgcia'
+    'hgcia',
+    'shelve',
 }
 
 def extensions(ui=None):
@@ -221,14 +222,7 b' def _runextsetup(name, ui):'
     extsetup = getattr(_extensions[name], 'extsetup', None)
     if extsetup:
         try:
-            try:
-                extsetup(ui)
-            except TypeError:
-                if pycompat.getargspec(extsetup).args:
-                    raise
-                ui.deprecwarn("extsetup for '%s' must take a ui argument"
-                              % name, "4.9")
-                extsetup() # old extsetup with no ui argument
+            extsetup(ui)
         except Exception as inst:
             ui.traceback(force=True)
             msg = stringutil.forcebytestr(inst)

mercurial/exthelper.py

@@ -15,9 +15,12 b' from . import ('
     commands,
     error,
     extensions,
+    pycompat,
     registrar,
 )
 
+from hgdemandimport import tracing
+
 class exthelper(object):
     """Helper for modular extension setup
 
@@ -135,7 +138,8 b' class exthelper(object):'
         for cont, funcname, wrapper in self._functionwrappers:
             extensions.wrapfunction(cont, funcname, wrapper)
         for c in self._uicallables:
-            c(ui)
+            with tracing.log(b'finaluisetup: %s', pycompat.sysbytes(repr(c))):
+                c(ui)
 
     def finaluipopulate(self, ui):
         """Method to be used as the extension uipopulate
@@ -175,7 +179,8 b' class exthelper(object):'
                     entry[1].append(opt)
 
         for c in self._extcallables:
-            c(ui)
+            with tracing.log(b'finalextsetup: %s', pycompat.sysbytes(repr(c))):
+                c(ui)
 
     def finalreposetup(self, ui, repo):
         """Method to be used as the extension reposetup
@@ -187,7 +192,8 b' class exthelper(object):'
         - Changes to repo.__class__, repo.dirstate.__class__
         """
         for c in self._repocallables:
-            c(ui, repo)
+            with tracing.log(b'finalreposetup: %s', pycompat.sysbytes(repr(c))):
+                c(ui, repo)
 
     def uisetup(self, call):
         """Decorated function will be executed during uisetup

mercurial/filemerge.py

@@ -60,17 +60,20 b' nomerge = internaltool.nomerge'
 mergeonly = internaltool.mergeonly # just the full merge, no premerge
 fullmerge = internaltool.fullmerge # both premerge and merge
 
+# IMPORTANT: keep the last line of this prompt very short ("What do you want to
+# do?") because of issue6158, ideally to <40 English characters (to allow other
+# languages that may take more columns to still have a chance to fit in an
+# 80-column screen).
 _localchangedotherdeletedmsg = _(
     "file '%(fd)s' was deleted in other%(o)s but was modified in local%(l)s.\n"
-    "What do you want to do?\n"
-    "use (c)hanged version, (d)elete, or leave (u)nresolved?"
+    "You can use (c)hanged version, (d)elete, or leave (u)nresolved.\n"
+    "What do you want to do?"
     "$$ &Changed $$ &Delete $$ &Unresolved")
 
 _otherchangedlocaldeletedmsg = _(
     "file '%(fd)s' was deleted in local%(l)s but was modified in other%(o)s.\n"
-    "What do you want to do?\n"
-    "use (c)hanged version, leave (d)eleted, or "
-    "leave (u)nresolved?"
+    "You can use (c)hanged version, leave (d)eleted, or leave (u)nresolved.\n"
+    "What do you want to do?"
     "$$ &Changed $$ &Deleted $$ &Unresolved")
 
 class absentfilectx(object):
@@ -299,9 +302,14 b' def _iprompt(repo, mynode, orig, fcd, fc'
                 _otherchangedlocaldeletedmsg % prompts, 2)
             choice = ['other', 'local', 'unresolved'][index]
         else:
+            # IMPORTANT: keep the last line of this prompt ("What do you want to
+            # do?") very short, see comment next to _localchangedotherdeletedmsg
+            # at the top of the file for details.
             index = ui.promptchoice(
-                _("keep (l)ocal%(l)s, take (o)ther%(o)s, or leave (u)nresolved"
-                  " for %(fd)s?"
+                _("file '%(fd)s' needs to be resolved.\n"
+                  "You can keep (l)ocal%(l)s, take (o)ther%(o)s, or leave "
+                  "(u)nresolved.\n"
+                  "What do you want to do?"
                   "$$ &Local $$ &Other $$ &Unresolved") % prompts, 2)
             choice = ['local', 'other', 'unresolved'][index]
 

mercurial/graphmod.py

@@ -469,22 +469,6 b' def ascii(ui, state, type, char, text, c'
     while len(text) < len(lines):
         text.append("")
 
-    if any(len(char) > 1 for char in edgemap.values()):
-        # limit drawing an edge to the first or last N lines of the current
-        # section the rest of the edge is drawn like a parent line.
-        parent = state['styles'][PARENT][-1:]
-        def _drawgp(char, i):
-            # should a grandparent character be drawn for this line?
-            if len(char) < 2:
-                return True
-            num = int(char[:-1])
-            # either skip first num lines or take last num lines, based on sign
-            return -num <= i if num < 0 else (len(lines) - i) <= num
-        for i, line in enumerate(lines):
-            line[:] = [c[-1:] if _drawgp(c, i) else parent for c in line]
-        edgemap.update(
-            (e, (c if len(c) < 2 else parent)) for e, c in edgemap.items())
-
     # print lines
     indentation_level = max(ncols, ncols + coldiff)
     lines = ["%-*s " % (2 * indentation_level, "".join(line)) for line in lines]

mercurial/hbisect.py

@@ -32,6 +32,7 b' def bisect(repo, state):'
     if searching for a first bad one.
     """
 
+    repo = repo.unfiltered()
     changelog = repo.changelog
     clparents = changelog.parentrevs
     skip = {changelog.rev(n) for n in state['skip']}
@@ -139,7 +140,7 b' def load_state(repo):'
     state = {'current': [], 'good': [], 'bad': [], 'skip': []}
     for l in repo.vfs.tryreadlines("bisect.state"):
         kind, node = l[:-1].split()
-        node = repo.lookup(node)
+        node = repo.unfiltered().lookup(node)
         if kind not in state:
             raise error.Abort(_("unknown bisect kind %s") % kind)
         state[kind].append(node)
@@ -184,7 +185,7 b' def get(repo, status):'
     """
     state = load_state(repo)
     if status in ('good', 'bad', 'skip', 'current'):
-        return map(repo.changelog.rev, state[status])
+        return map(repo.unfiltered().changelog.rev, state[status])
     else:
         # In the following sets, we do *not* call 'bisect()' with more
         # than one level of recursion, because that can be very, very
@@ -268,6 +269,7 b' def label(repo, node):'
     return None
 
 def printresult(ui, repo, state, displayer, nodes, good):
+    repo = repo.unfiltered()
     if len(nodes) == 1:
         # narrowed it down to a single revision
         if good:

mercurial/help.py

@@ -320,6 +320,8 b' internalstable = sorted(['
      loaddoc('config', subdir='internals')),
     (['extensions', 'extension'], _('Extension API'),
      loaddoc('extensions', subdir='internals')),
+    (['mergestate'], _('Mergestate'),
+     loaddoc('mergestate', subdir='internals')),
     (['requirements'], _('Repository Requirements'),
      loaddoc('requirements', subdir='internals')),
     (['revlogs'], _('Revision Logs'),
@@ -453,7 +455,7 b' def inserttweakrc(ui, topic, doc):'
 addtopichook('config', inserttweakrc)
 
 def help_(ui, commands, name, unknowncmd=False, full=True, subtopic=None,
-          **opts):
+          fullname=None, **opts):
     '''
     Generate the help for 'name' as unformatted restructured text. If
     'name' is None, describe the commands available.
@@ -689,6 +691,8 b' def help_(ui, commands, name, unknowncmd'
             for names, header, doc in subtopics[name]:
                 if subtopic in names:
                     break
+            if not any(subtopic in s[0] for s in subtopics[name]):
+                raise error.UnknownCommand(name)
 
         if not header:
             for topic in helptable:
@@ -812,8 +816,16 b' def help_(ui, commands, name, unknowncmd'
             if unknowncmd:
                 raise error.UnknownCommand(name)
             else:
-                msg = _('no such help topic: %s') % name
-                hint = _("try 'hg help --keyword %s'") % name
+                if fullname:
+                    formatname = fullname
+                else:
+                    formatname = name
+                if subtopic:
+                    hintname = subtopic
+                else:
+                    hintname = name
+                msg = _('no such help topic: %s') % formatname
+                hint = _("try 'hg help --keyword %s'") % hintname
                 raise error.Abort(msg, hint=hint)
     else:
         # program name
@@ -848,7 +860,7 b' def formattedhelp(ui, commands, fullname'
     termwidth = ui.termwidth() - 2
     if textwidth <= 0 or termwidth < textwidth:
         textwidth = termwidth
-    text = help_(ui, commands, name,
+    text = help_(ui, commands, name, fullname=fullname,
                  subtopic=subtopic, unknowncmd=unknowncmd, full=full, **opts)
 
     blocks, pruned = minirst.parse(text, keep=keep)

mercurial/help/config.txt

@@ -438,6 +438,10 b' effect and style see :hg:`help color`.'
 ``commands``
 ------------
 
+``commit.post-status``
+    Show status of files in the working directory after successful commit.
+    (default: False)
+
 ``resolve.confirm``
     Confirm before performing action if no filename is passed.
     (default: False)
@@ -875,6 +879,15 b' https://www.mercurial-scm.org/wiki/Missi'
 
     On some system, Mercurial installation may lack `zstd` supports. Default is `zlib`.
 
+``bookmarks-in-store``
+    Store bookmarks in .hg/store/. This means that bookmarks are shared when
+    using `hg share` regardless of the `-B` option.
+
+    Repositories with this on-disk format require Mercurial version 5.1.
+
+    Disabled by default.
+
+
 ``graph``
 ---------
 
@@ -1767,6 +1780,11 b' statistical text report generated from t'
 
     The option is unused on other formats.
 
+``showtime``
+    Show time taken as absolute durations, in addition to percentages.
+    Only used by the ``hotpath`` format.
+    (default: true)
+
 ``progress``
 ------------
 

mercurial/help/internals/requirements.txt

@@ -129,3 +129,16 b' August 2017). This requirement and featu'
 disappear in a future Mercurial release. The requirement will only
 be present on repositories that have opted in to a sparse working
 directory.
+
+bookmarksinstore
+==================
+
+Bookmarks are stored in ``.hg/store/`` instead of directly in ``.hg/``
+where they used to be stored. The active bookmark is still stored
+directly in ``.hg/``. This makes them always shared by ``hg share``,
+whether or not ``-B`` was passed.
+
+Support for this requirement was added in Mercurial 5.1 (released
+August 2019). The requirement will only be present on repositories
+that have opted in to this format (by having
+``format.bookmarks-in-store=true`` set when they were created).

mercurial/help/internals/revlogs.txt

@@ -28,8 +28,8 b' File Format'
 ===========
 
 A revlog begins with a 32-bit big endian integer holding version info
-and feature flags. This integer is shared with the first revision
-entry.
+and feature flags. This integer overlaps with the first four bytes of
+the first revision entry.
 
 This integer is logically divided into 2 16-bit shorts. The least
 significant half of the integer is the format/version short. The other
@@ -78,10 +78,10 b' 00 02 00 01'
 00 03 00 01
    v1 + inline + generaldelta
 
-Following the 32-bit header is the remainder of the first index entry.
-Following that are remaining *index* data. Inlined revision data is
-possibly located between index entries. More on this layout is described
-below.
+Following the 32-bit header is the remaining 60 bytes of the first index
+entry. Following that are additional *index* entries. Inlined revision
+data is possibly located between index entries. More on this inlined
+layout is described below.
 
 Version 1 Format
 ================
@@ -149,8 +149,12 b' If revision data is not inline, then raw'
 separate byte container. The offsets from bytes 0-5 and the compressed
 length from bytes 8-11 define how to access this data.
 
-The first 4 bytes of the revlog are shared between the revlog header
-and the 6 byte absolute offset field from the first revlog entry.
+The 6 byte absolute offset field from the first revlog entry overlaps
+with the revlog header. That is, the first 6 bytes of the first revlog
+entry can be split into four bytes containing the header for the revlog
+file and an additional two bytes containing the offset for the first
+entry. Since this is the offset from the beginning of the file for the
+first revision entry, the two bytes will always be set to zero.
 
 Version 2 Format
 ================

mercurial/hg.py

@@ -956,31 +956,34 b' def merge(repo, node, force=None, remind'
           abort=False):
     """Branch merge with node, resolving changes. Return true if any
     unresolved conflicts."""
-    if not abort:
-        stats = mergemod.update(repo, node, branchmerge=True, force=force,
-                                mergeforce=mergeforce, labels=labels)
-    else:
-        ms = mergemod.mergestate.read(repo)
-        if ms.active():
-            # there were conflicts
-            node = ms.localctx.hex()
-        else:
-            # there were no conficts, mergestate was not stored
-            node = repo['.'].hex()
+    if abort:
+        return abortmerge(repo.ui, repo)
 
-        repo.ui.status(_("aborting the merge, updating back to"
-                         " %s\n") % node[:12])
-        stats = mergemod.update(repo, node, branchmerge=False, force=True,
-                                labels=labels)
-
+    stats = mergemod.update(repo, node, branchmerge=True, force=force,
+                            mergeforce=mergeforce, labels=labels)
     _showstats(repo, stats)
     if stats.unresolvedcount:
         repo.ui.status(_("use 'hg resolve' to retry unresolved file merges "
                          "or 'hg merge --abort' to abandon\n"))
-    elif remind and not abort:
+    elif remind:
         repo.ui.status(_("(branch merge, don't forget to commit)\n"))
     return stats.unresolvedcount > 0
 
+def abortmerge(ui, repo):
+    ms = mergemod.mergestate.read(repo)
+    if ms.active():
+        # there were conflicts
+        node = ms.localctx.hex()
+    else:
+        # there were no conficts, mergestate was not stored
+        node = repo['.'].hex()
+
+    repo.ui.status(_("aborting the merge, updating back to"
+                     " %s\n") % node[:12])
+    stats = mergemod.update(repo, node, branchmerge=False, force=True)
+    _showstats(repo, stats)
+    return stats.unresolvedcount > 0
+
 def _incoming(displaychlist, subreporecurse, ui, repo, source,
         opts, buffered=False):
     """
@@ -1092,9 +1095,9 b' def outgoing(ui, repo, dest, opts):'
     recurse()
     return 0 # exit code is zero since we found outgoing changes
 
-def verify(repo):
+def verify(repo, level=None):
     """verify the consistency of a repository"""
-    ret = verifymod.verify(repo)
+    ret = verifymod.verify(repo, level=level)
 
     # Broken subrepo references in hidden csets don't seem worth worrying about,
     # since they can't be pushed/pulled, and --hidden can be used if they are a

mercurial/hgweb/__init__.py

@@ -38,6 +38,9 b' def hgweb(config, name=None, baseui=None'
     - list of virtual:real tuples (multi-repo view)
     '''
 
+    if isinstance(config, pycompat.unicode):
+        raise error.ProgrammingError(
+            'Mercurial only supports encoded strings: %r' % config)
     if ((isinstance(config, bytes) and not os.path.isdir(config)) or
         isinstance(config, dict) or isinstance(config, list)):
         # create a multi-dir interface

mercurial/hgweb/hgwebdir_mod.py

@@ -414,14 +414,10 b' class hgwebdir(object):'
                     return self.makeindex(req, res, tmpl, subdir)
 
             def _virtualdirs():
-                # Check the full virtual path, each parent, and the root ('')
-                if virtual != '':
-                    yield virtual
-
-                    for p in util.finddirs(virtual):
-                        yield p
-
-                yield ''
+                # Check the full virtual path, and each parent
+                yield virtual
+                for p in util.finddirs(virtual):
+                    yield p
 
             for virtualrepo in _virtualdirs():
                 real = repos.get(virtualrepo)

mercurial/hgweb/webutil.py

@@ -409,12 +409,6 b' def whyunstable(context, mapping):'
 
 whyunstable._requires = {'repo', 'ctx'}
 
-# helper to mark a function as a new-style template keyword; can be removed
-# once old-style function gets unsupported and new-style becomes the default
-def _kwfunc(f):
-    f._requires = ()
-    return f
-
 def commonentry(repo, ctx):
     node = scmutil.binnode(ctx)
     return {
@@ -439,8 +433,8 b' def commonentry(repo, ctx):'
         'branches': nodebranchdict(repo, ctx),
         'tags': nodetagsdict(repo, node),
         'bookmarks': nodebookmarksdict(repo, node),
-        'parent': _kwfunc(lambda context, mapping: parents(ctx)),
-        'child': _kwfunc(lambda context, mapping: children(ctx)),
+        'parent': lambda context, mapping: parents(ctx),
+        'child': lambda context, mapping: children(ctx),
     }
 
 def changelistentry(web, ctx):
@@ -457,9 +451,9 b' def changelistentry(web, ctx):'
 
     entry = commonentry(repo, ctx)
     entry.update({
-        'allparents': _kwfunc(lambda context, mapping: parents(ctx)),
-        'parent': _kwfunc(lambda context, mapping: parents(ctx, rev - 1)),
-        'child': _kwfunc(lambda context, mapping: children(ctx, rev + 1)),
+        'allparents': lambda context, mapping: parents(ctx),
+        'parent': lambda context, mapping: parents(ctx, rev - 1),
+        'child': lambda context, mapping: children(ctx, rev + 1),
         'changelogtag': showtags,
         'files': files,
     })
@@ -529,7 +523,7 b' def changesetentry(web, ctx):'
         changesetbranch=showbranch,
         files=templateutil.mappedgenerator(_listfilesgen,
                                            args=(ctx, web.stripecount)),
-        diffsummary=_kwfunc(lambda context, mapping: diffsummary(diffstatsgen)),
+        diffsummary=lambda context, mapping: diffsummary(diffstatsgen),
         diffstat=diffstats,
         archives=web.archivelist(ctx.hex()),
         **pycompat.strkwargs(commonentry(web.repo, ctx)))

mercurial/httppeer.py

@@ -382,6 +382,7 b' class httppeer(wireprotov1peer.wirepeer)'
         self._path = path
         self._url = url
         self._caps = caps
+        self.limitedarguments = caps is not None and 'httppostargs' not in caps
         self._urlopener = opener
         self._requestbuilder = requestbuilder
 
@@ -750,6 +751,9 b' class httpv2executor(object):'
 
 @interfaceutil.implementer(repository.ipeerv2)
 class httpv2peer(object):
+
+    limitedarguments = False
+
     def __init__(self, ui, repourl, apipath, opener, requestbuilder,
                  apidescriptor):
         self.ui = ui

mercurial/localrepo.py

@@ -128,8 +128,7 b' class mixedrepostorecache(_basefilecache'
         # scmutil.filecache only uses the path for passing back into our
         # join(), so we can safely pass a list of paths and locations
         super(mixedrepostorecache, self).__init__(*pathsandlocations)
-        for path, location in pathsandlocations:
-            _cachedfiles.update(pathsandlocations)
+        _cachedfiles.update(pathsandlocations)
 
     def join(self, obj, fnameandlocation):
         fname, location = fnameandlocation
@@ -910,6 +909,7 b' class localrepository(object):'
         'treemanifest',
         REVLOGV2_REQUIREMENT,
         SPARSEREVLOG_REQUIREMENT,
+        bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT,
     }
     _basesupported = supportedformats | {
         'store',
@@ -1069,6 +1069,8 b' class localrepository(object):'
         # Signature to cached matcher instance.
         self._sparsematchercache = {}
 
+        self._extrafilterid = repoview.extrafilter(ui)
+
     def _getvfsward(self, origfunc):
         """build a ward for self.vfs"""
         rref = weakref.ref(self)
@@ -1216,11 +1218,14 b' class localrepository(object):'
 
         In other word, there is always only one level of `repoview` "filtering".
         """
+        if self._extrafilterid is not None and '%' not in name:
+            name = name + '%'  + self._extrafilterid
+
         cls = repoview.newtype(self.unfiltered().__class__)
         return cls(self, name, visibilityexceptions)
 
     @mixedrepostorecache(('bookmarks', 'plain'), ('bookmarks.current', 'plain'),
-                         ('00changelog.i', ''))
+                         ('bookmarks', ''), ('00changelog.i', ''))
     def _bookmarks(self):
         return bookmarks.bmstore(self)
 
@@ -1982,7 +1987,7 b' class localrepository(object):'
                 (self.vfs, 'journal.dirstate'),
                 (self.vfs, 'journal.branch'),
                 (self.vfs, 'journal.desc'),
-                (self.vfs, 'journal.bookmarks'),
+                (bookmarks.bookmarksvfs(self), 'journal.bookmarks'),
                 (self.svfs, 'journal.phaseroots'))
 
     def undofiles(self):
@@ -1997,8 +2002,9 b' class localrepository(object):'
                           encoding.fromlocal(self.dirstate.branch()))
         self.vfs.write("journal.desc",
                           "%d\n%s\n" % (len(self), desc))
-        self.vfs.write("journal.bookmarks",
-                          self.vfs.tryread("bookmarks"))
+        bookmarksvfs = bookmarks.bookmarksvfs(self)
+        bookmarksvfs.write("journal.bookmarks",
+                           bookmarksvfs.tryread("bookmarks"))
         self.svfs.write("journal.phaseroots",
                            self.svfs.tryread("phaseroots"))
 
@@ -2068,8 +2074,9 b' class localrepository(object):'
         vfsmap = {'plain': self.vfs, '': self.svfs}
         transaction.rollback(self.svfs, vfsmap, 'undo', ui.warn,
                              checkambigfiles=_cachedfiles)
-        if self.vfs.exists('undo.bookmarks'):
-            self.vfs.rename('undo.bookmarks', 'bookmarks', checkambig=True)
+        bookmarksvfs = bookmarks.bookmarksvfs(self)
+        if bookmarksvfs.exists('undo.bookmarks'):
+            bookmarksvfs.rename('undo.bookmarks', 'bookmarks', checkambig=True)
         if self.svfs.exists('undo.phaseroots'):
             self.svfs.rename('undo.phaseroots', 'phaseroots', checkambig=True)
         self.invalidate()
@@ -2152,6 +2159,8 b' class localrepository(object):'
             for ctx in self['.'].parents():
                 ctx.manifest()  # accessing the manifest is enough
 
+            # accessing fnode cache warms the cache
+            tagsmod.fnoderevs(self.ui, unfi, unfi.changelog.revs())
             # accessing tags warm the cache
             self.tags()
             self.filtered('served').tags()
@@ -2362,7 +2371,10 b' class localrepository(object):'
             node = fctx.filenode()
             if node in [fparent1, fparent2]:
                 self.ui.debug('reusing %s filelog entry\n' % fname)
-                if manifest1.flags(fname) != fctx.flags():
+                if ((fparent1 != nullid and
+                     manifest1.flags(fname) != fctx.flags()) or
+                    (fparent2 != nullid and
+                     manifest2.flags(fname) != fctx.flags())):
                     changelist.append(fname)
                 return node
 
@@ -2556,17 +2568,17 b' class localrepository(object):'
                         _('note: commit message saved in %s\n') % msgfn)
                 raise
 
-        def commithook(node=hex(ret), parent1=hookp1, parent2=hookp2):
+        def commithook():
             # hack for command that use a temporary commit (eg: histedit)
             # temporary commit got stripped before hook release
             if self.changelog.hasnode(ret):
-                self.hook("commit", node=node, parent1=parent1,
-                          parent2=parent2)
+                self.hook("commit", node=hex(ret), parent1=hookp1,
+                          parent2=hookp2)
         self._afterlock(commithook)
         return ret
 
     @unfilteredmethod
-    def commitctx(self, ctx, error=False):
+    def commitctx(self, ctx, error=False, origctx=None):
         """Add a new revision to current repository.
         Revision information is passed via the context argument.
 
@@ -2574,6 +2586,12 b' class localrepository(object):'
         modified/added/removed files. On merge, it may be wider than the
         ctx.files() to be committed, since any file nodes derived directly
         from p1 or p2 are excluded from the committed ctx.files().
+
+        origctx is for convert to work around the problem that bug
+        fixes to the files list in changesets change hashes. For
+        convert to be the identity, it can pass an origctx and this
+        function will use the same files list when it makes sense to
+        do so.
         """
 
         p1, p2 = ctx.p1(), ctx.p2()
@@ -2581,10 +2599,13 b' class localrepository(object):'
 
         writecopiesto = self.ui.config('experimental', 'copies.write-to')
         writefilecopymeta = writecopiesto != 'changeset-only'
+        writechangesetcopy = (writecopiesto in
+                              ('changeset-only', 'compatibility'))
         p1copies, p2copies = None, None
-        if writecopiesto in ('changeset-only', 'compatibility'):
+        if writechangesetcopy:
             p1copies = ctx.p1copies()
             p2copies = ctx.p2copies()
+        filesadded, filesremoved = None, None
         with self.lock(), self.transaction("commit") as tr:
             trp = weakref.proxy(tr)
 
@@ -2593,6 +2614,9 b' class localrepository(object):'
                 self.ui.debug('reusing known manifest\n')
                 mn = ctx.manifestnode()
                 files = ctx.files()
+                if writechangesetcopy:
+                    filesadded = ctx.filesadded()
+                    filesremoved = ctx.filesremoved()
             elif ctx.files():
                 m1ctx = p1.manifestctx()
                 m2ctx = p2.manifestctx()
@@ -2633,10 +2657,51 b' class localrepository(object):'
                         raise
 
                 # update manifest
-                removed = [f for f in sorted(removed) if f in m1 or f in m2]
-                drop = [f for f in removed if f in m]
+                removed = [f for f in removed if f in m1 or f in m2]
+                drop = sorted([f for f in removed if f in m])
                 for f in drop:
                     del m[f]
+                if p2.rev() != nullrev:
+                    @util.cachefunc
+                    def mas():
+                        p1n = p1.node()
+                        p2n = p2.node()
+                        cahs = self.changelog.commonancestorsheads(p1n, p2n)
+                        if not cahs:
+                            cahs = [nullrev]
+                        return [self[r].manifest() for r in cahs]
+                    def deletionfromparent(f):
+                        # When a file is removed relative to p1 in a merge, this
+                        # function determines whether the absence is due to a
+                        # deletion from a parent, or whether the merge commit
+                        # itself deletes the file. We decide this by doing a
+                        # simplified three way merge of the manifest entry for
+                        # the file. There are two ways we decide the merge
+                        # itself didn't delete a file:
+                        # - neither parent (nor the merge) contain the file
+                        # - exactly one parent contains the file, and that
+                        #   parent has the same filelog entry as the merge
+                        #   ancestor (or all of them if there two). In other
+                        #   words, that parent left the file unchanged while the
+                        #   other one deleted it.
+                        # One way to think about this is that deleting a file is
+                        # similar to emptying it, so the list of changed files
+                        # should be similar either way. The computation
+                        # described above is not done directly in _filecommit
+                        # when creating the list of changed files, however
+                        # it does something very similar by comparing filelog
+                        # nodes.
+                        if f in m1:
+                            return (f not in m2
+                                    and all(f in ma and ma.find(f) == m1.find(f)
+                                            for ma in mas()))
+                        elif f in m2:
+                            return all(f in ma and ma.find(f) == m2.find(f)
+                                       for ma in mas())
+                        else:
+                            return True
+                    removed = [f for f in removed if not deletionfromparent(f)]
+
                 files = changed + removed
                 md = None
                 if not files:
@@ -2659,8 +2724,13 b' class localrepository(object):'
                     mn = mctx.write(trp, linkrev,
                                     p1.manifestnode(), p2.manifestnode(),
                                     added, drop, match=self.narrowmatch())
+
+                    if writechangesetcopy:
+                        filesadded = [f for f in changed
+                                      if not (f in m1 or f in m2)]
+                        filesremoved = removed
                 else:
-                    self.ui.debug('reusing manifest form p1 (listed files '
+                    self.ui.debug('reusing manifest from p1 (listed files '
                                   'actually unchanged)\n')
                     mn = p1.manifestnode()
             else:
@@ -2668,13 +2738,26 b' class localrepository(object):'
                 mn = p1.manifestnode()
                 files = []
 
+            if writecopiesto == 'changeset-only':
+                # If writing only to changeset extras, use None to indicate that
+                # no entry should be written. If writing to both, write an empty
+                # entry to prevent the reader from falling back to reading
+                # filelogs.
+                p1copies = p1copies or None
+                p2copies = p2copies or None
+                filesadded = filesadded or None
+                filesremoved = filesremoved or None
+
+            if origctx and origctx.manifestnode() == mn:
+                files = origctx.files()
+
             # update changelog
             self.ui.note(_("committing changelog\n"))
             self.changelog.delayupdate(tr)
             n = self.changelog.add(mn, files, ctx.description(),
                                    trp, p1.node(), p2.node(),
                                    user, ctx.date(), ctx.extra().copy(),
-                                   p1copies, p2copies)
+                                   p1copies, p2copies, filesadded, filesremoved)
             xp1, xp2 = p1.hex(), p2 and p2.hex() or ''
             self.hook('pretxncommit', throw=True, node=hex(n), parent1=xp1,
                       parent2=xp2)
@@ -3013,6 +3096,9 b' def newreporequirements(ui, createopts):'
     if createopts.get('lfs'):
         requirements.add('lfs')
 
+    if ui.configbool('format', 'bookmarks-in-store'):
+        requirements.add(bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT)
+
     return requirements
 
 def filterknowncreateopts(ui, createopts):

mercurial/logcmdutil.py

@@ -743,10 +743,15 b' def getrevs(repo, pats, opts):'
             return match
 
     expr = _makerevset(repo, match, pats, slowpath, opts)
-    if opts.get('graph') and opts.get('rev'):
+    if opts.get('graph'):
         # User-specified revs might be unsorted, but don't sort before
         # _makerevset because it might depend on the order of revs
-        if not (revs.isdescending() or revs.istopo()):
+        if repo.ui.configbool('experimental', 'log.topo'):
+            if not revs.istopo():
+                revs = dagop.toposort(revs, repo.changelog.parentrevs)
+                # TODO: try to iterate the set lazily
+                revs = revset.baseset(list(revs), istopo=True)
+        elif not (revs.isdescending() or revs.istopo()):
             revs.sort(reverse=True)
     if expr:
         matcher = revset.match(None, expr)
@@ -857,7 +862,7 b' def _graphnodeformatter(ui, displayer):'
         return templ.renderdefault(props)
     return formatnode
 
-def displaygraph(ui, repo, dag, displayer, edgefn, getrenamed=None, props=None):
+def displaygraph(ui, repo, dag, displayer, edgefn, getcopies=None, props=None):
     props = props or {}
     formatnode = _graphnodeformatter(ui, displayer)
     state = graphmod.asciistate()
@@ -885,13 +890,7 b' def displaygraph(ui, repo, dag, displaye'
 
     for rev, type, ctx, parents in dag:
         char = formatnode(repo, ctx)
-        copies = None
-        if getrenamed and ctx.rev():
-            copies = []
-            for fn in ctx.files():
-                rename = getrenamed(fn, ctx.rev())
-                if rename:
-                    copies.append((fn, rename))
+        copies = getcopies(ctx) if getcopies else None
         edges = edgefn(type, char, state, rev, parents)
         firstedge = next(edges)
         width = firstedge[2]
@@ -910,16 +909,10 b' def displaygraphrevs(ui, repo, revs, dis'
     revdag = graphmod.dagwalker(repo, revs)
     displaygraph(ui, repo, revdag, displayer, graphmod.asciiedges, getrenamed)
 
-def displayrevs(ui, repo, revs, displayer, getrenamed):
+def displayrevs(ui, repo, revs, displayer, getcopies):
     for rev in revs:
         ctx = repo[rev]
-        copies = None
-        if getrenamed is not None and rev:
-            copies = []
-            for fn in ctx.files():
-                rename = getrenamed(fn, rev)
-                if rename:
-                    copies.append((fn, rename))
+        copies = getcopies(ctx) if getcopies else None
         displayer.show(ctx, copies=copies)
         displayer.flush(ctx)
     displayer.close()

mercurial/manifest.py

@@ -506,9 +506,9 b' class manifestdict(object):'
             if match(fn):
                 yield fn
 
-        # for dirstate.walk, files=['.'] means "walk the whole tree".
+        # for dirstate.walk, files=[''] means "walk the whole tree".
         # follow that here, too
-        fset.discard('.')
+        fset.discard('')
 
         for fn in sorted(fset):
             if not self.hasdir(fn):
@@ -1078,9 +1078,9 b' class treemanifest(object):'
                 fset.remove(fn)
             yield fn
 
-        # for dirstate.walk, files=['.'] means "walk the whole tree".
+        # for dirstate.walk, files=[''] means "walk the whole tree".
         # follow that here, too
-        fset.discard('.')
+        fset.discard('')
 
         for fn in sorted(fset):
             if not self.hasdir(fn):
@@ -1088,7 +1088,7 b' class treemanifest(object):'
 
     def _walk(self, match):
         '''Recursively generates matching file names for walk().'''
-        visit = match.visitchildrenset(self._dir[:-1] or '.')
+        visit = match.visitchildrenset(self._dir[:-1])
         if not visit:
             return
 
@@ -1116,7 +1116,7 b' class treemanifest(object):'
         '''recursively generate a new manifest filtered by the match argument.
         '''
 
-        visit = match.visitchildrenset(self._dir[:-1] or '.')
+        visit = match.visitchildrenset(self._dir[:-1])
         if visit == 'all':
             return self.copy()
         ret = treemanifest(self._dir)
@@ -1275,7 +1275,7 b' class treemanifest(object):'
             return m._dirs.get(d, emptytree)._node
 
         # let's skip investigating things that `match` says we do not need.
-        visit = match.visitchildrenset(self._dir[:-1] or '.')
+        visit = match.visitchildrenset(self._dir[:-1])
         visit = self._loadchildrensetlazy(visit)
         if visit == 'this' or visit == 'all':
             visit = None
@@ -1294,7 +1294,7 b' class treemanifest(object):'
 
         If `matcher` is provided, it only returns subtrees that match.
         """
-        if matcher and not matcher.visitdir(self._dir[:-1] or '.'):
+        if matcher and not matcher.visitdir(self._dir[:-1]):
             return
         if not matcher or matcher(self._dir[:-1]):
             yield self
@@ -1417,6 +1417,10 b' class manifestfulltextcache(util.lrucach'
             self.write()
         self._read = False
 
+# and upper bound of what we expect from compression
+# (real live value seems to be "3")
+MAXCOMPRESSION = 3
+
 @interfaceutil.implementer(repository.imanifeststorage)
 class manifestrevlog(object):
     '''A revlog that stores manifest texts. This is responsible for caching the
@@ -1467,7 +1471,8 b' class manifestrevlog(object):'
         self._revlog = revlog.revlog(opener, indexfile,
                                      # only root indexfile is cached
                                      checkambig=not bool(tree),
-                                     mmaplargeindex=True)
+                                     mmaplargeindex=True,
+                                     upperboundcomp=MAXCOMPRESSION)
 
         self.index = self._revlog.index
         self.version = self._revlog.version
@@ -1526,8 +1531,8 b' class manifestrevlog(object):'
 
             _checkforbidden(added)
             # combine the changed lists into one sorted iterator
-            work = heapq.merge([(x, False) for x in added],
-                               [(x, True) for x in removed])
+            work = heapq.merge([(x, False) for x in sorted(added)],
+                               [(x, True) for x in sorted(removed)])
 
             arraytext, deltatext = m.fastdelta(self.fulltextcache[p1], work)
             cachedelta = self._revlog.rev(p1), deltatext
@@ -1725,7 +1730,7 b' class manifestlog(object):'
             return self._dirmancache[tree][node]
 
         if not self._narrowmatch.always():
-            if not self._narrowmatch.visitdir(tree[:-1] or '.'):
+            if not self._narrowmatch.visitdir(tree[:-1]):
                 return excludeddirmanifestctx(tree, node)
         if tree:
             if self._rootstore._treeondisk:
@@ -1918,7 +1923,7 b' class treemanifestctx(object):'
     def _storage(self):
         narrowmatch = self._manifestlog._narrowmatch
         if not narrowmatch.always():
-            if not narrowmatch.visitdir(self._dir[:-1] or '.'):
+            if not narrowmatch.visitdir(self._dir[:-1]):
                 return excludedmanifestrevlog(self._dir)
         return self._manifestlog.getstorage(self._dir)
 

mercurial/match.py

@@ -17,6 +17,7 b' from . import ('
     encoding,
     error,
     pathutil,
+    policy,
     pycompat,
     util,
 )
@@ -24,6 +25,8 b' from .utils import ('
     stringutil,
 )
 
+rustmod = policy.importrust('filepatterns')
+
 allpatternkinds = ('re', 'glob', 'path', 'relglob', 'relpath', 'relre',
                    'rootglob',
                    'listfile', 'listfile0', 'set', 'include', 'subinclude',
@@ -305,9 +308,6 b' class basematcher(object):'
 
     def __call__(self, fn):
         return self.matchfn(fn)
-    def __iter__(self):
-        for f in self._files:
-            yield f
     # Callbacks related to how the matcher is used by dirstate.walk.
     # Subscribers to these events must monkeypatch the matcher object.
     def bad(self, f, msg):
@@ -377,7 +377,7 b' class basematcher(object):'
           the following values (assuming the implementation of visitchildrenset
           is capable of recognizing this; some implementations are not).
 
-          '.' -> {'foo', 'qux'}
+          '' -> {'foo', 'qux'}
           'baz' -> set()
           'foo' -> {'bar'}
           # Ideally this would be 'all', but since the prefix nature of matchers
@@ -480,11 +480,19 b' class predicatematcher(basematcher):'
              or pycompat.byterepr(self.matchfn))
         return '<predicatenmatcher pred=%s>' % s
 
+def normalizerootdir(dir, funcname):
+    if dir == '.':
+        util.nouideprecwarn("match.%s() no longer accepts "
+                            "'.', use '' instead." % funcname, '5.1')
+        return ''
+    return dir
+
+
 class patternmatcher(basematcher):
     """Matches a set of (kind, pat, source) against a 'root' directory.
 
     >>> kindpats = [
-    ...     (b're', b'.*\.c$', b''),
+    ...     (b're', br'.*\.c$', b''),
     ...     (b'path', b'foo/a', b''),
     ...     (b'relpath', b'b', b''),
     ...     (b'glob', b'*.h', b''),
@@ -504,7 +512,7 b' class patternmatcher(basematcher):'
     True
 
     >>> m.files()
-    ['.', 'foo/a', 'b', '.']
+    ['', 'foo/a', 'b', '']
     >>> m.exact(b'foo/a')
     True
     >>> m.exact(b'b')
@@ -522,13 +530,13 b' class patternmatcher(basematcher):'
 
     @propertycache
     def _dirs(self):
-        return set(util.dirs(self._fileset)) | {'.'}
+        return set(util.dirs(self._fileset))
 
     def visitdir(self, dir):
+        dir = normalizerootdir(dir, 'visitdir')
         if self._prefix and dir in self._fileset:
             return 'all'
-        return ('.' in self._fileset or
-                dir in self._fileset or
+        return (dir in self._fileset or
                 dir in self._dirs or
                 any(parentdir in self._fileset
                     for parentdir in util.finddirs(dir)))
@@ -561,7 +569,7 b' class _dirchildren(object):'
             addpath(f)
 
     def addpath(self, path):
-        if path == '.':
+        if path == '':
             return
         dirs = self._dirs
         findsplitdirs = _dirchildren._findsplitdirs
@@ -575,16 +583,15 b' class _dirchildren(object):'
         # yields (dirname, basename) tuples, walking back to the root.  This is
         # very similar to util.finddirs, except:
         #  - produces a (dirname, basename) tuple, not just 'dirname'
-        #  - includes root dir
         # Unlike manifest._splittopdir, this does not suffix `dirname` with a
-        # slash, and produces '.' for the root instead of ''.
+        # slash.
         oldpos = len(path)
         pos = path.rfind('/')
         while pos != -1:
             yield path[:pos], path[pos + 1:oldpos]
             oldpos = pos
             pos = path.rfind('/', 0, pos)
-        yield '.', path[:oldpos]
+        yield '', path[:oldpos]
 
     def get(self, path):
         return self._dirs.get(path, set())
@@ -603,13 +610,13 b' class includematcher(basematcher):'
         self._dirs = set(dirs)
         # parents are directories which are non-recursively included because
         # they are needed to get to items in _dirs or _roots.
-        self._parents = set(parents)
+        self._parents = parents
 
     def visitdir(self, dir):
+        dir = normalizerootdir(dir, 'visitdir')
         if self._prefix and dir in self._roots:
             return 'all'
-        return ('.' in self._roots or
-                dir in self._roots or
+        return (dir in self._roots or
                 dir in self._dirs or
                 dir in self._parents or
                 any(parentdir in self._roots
@@ -632,7 +639,7 b' class includematcher(basematcher):'
             return 'all'
         # Note: this does *not* include the 'dir in self._parents' case from
         # visitdir, that's handled below.
-        if ('.' in self._roots or
+        if ('' in self._roots or
             dir in self._roots or
             dir in self._dirs or
             any(parentdir in self._roots
@@ -651,7 +658,7 b' class exactmatcher(basematcher):'
     r'''Matches the input files exactly. They are interpreted as paths, not
     patterns (so no kind-prefixes).
 
-    >>> m = exactmatcher([b'a.txt', b're:.*\.c$'])
+    >>> m = exactmatcher([b'a.txt', br're:.*\.c$'])
     >>> m(b'a.txt')
     True
     >>> m(b'b.txt')
@@ -664,7 +671,7 b' class exactmatcher(basematcher):'
     So pattern 're:.*\.c$' is not considered as a regex, but as a file name
     >>> m(b'main.c')
     False
-    >>> m(b're:.*\.c$')
+    >>> m(br're:.*\.c$')
     True
     '''
 
@@ -680,22 +687,25 b' class exactmatcher(basematcher):'
 
     @propertycache
     def _dirs(self):
-        return set(util.dirs(self._fileset)) | {'.'}
+        return set(util.dirs(self._fileset))
 
     def visitdir(self, dir):
+        dir = normalizerootdir(dir, 'visitdir')
         return dir in self._dirs
 
     def visitchildrenset(self, dir):
+        dir = normalizerootdir(dir, 'visitchildrenset')
+
         if not self._fileset or dir not in self._dirs:
             return set()
 
-        candidates = self._fileset | self._dirs - {'.'}
-        if dir != '.':
+        candidates = self._fileset | self._dirs - {''}
+        if dir != '':
             d = dir + '/'
             candidates = set(c[len(d):] for c in candidates if
                              c.startswith(d))
         # self._dirs includes all of the directories, recursively, so if
-        # we're attempting to match foo/bar/baz.txt, it'll have '.', 'foo',
+        # we're attempting to match foo/bar/baz.txt, it'll have '', 'foo',
         # 'foo/bar' in it. Thus we can safely ignore a candidate that has a
         # '/' in it, indicating a it's for a subdir-of-a-subdir; the
         # immediate subdir will be in there without a slash.
@@ -769,7 +779,7 b' class differencematcher(basematcher):'
         # Possible values for m1:         set(...), set()
         # Possible values for m2: 'this', set(...)
         # We ignore m2's set results. They're possibly incorrect:
-        #  m1 = path:dir/subdir, m2=rootfilesin:dir, visitchildrenset('.'):
+        #  m1 = path:dir/subdir, m2=rootfilesin:dir, visitchildrenset(''):
         #    m1 returns {'dir'}, m2 returns {'dir'}, if we subtracted we'd
         #    return set(), which is *not* correct, we still need to visit 'dir'!
         return m1_set
@@ -915,14 +925,16 b' class subdirmatcher(basematcher):'
         return self._matcher.matchfn(self._path + "/" + f)
 
     def visitdir(self, dir):
-        if dir == '.':
+        dir = normalizerootdir(dir, 'visitdir')
+        if dir == '':
             dir = self._path
         else:
             dir = self._path + "/" + dir
         return self._matcher.visitdir(dir)
 
     def visitchildrenset(self, dir):
-        if dir == '.':
+        dir = normalizerootdir(dir, 'visitchildrenset')
+        if dir == '':
             dir = self._path
         else:
             dir = self._path + "/" + dir
@@ -991,18 +1003,18 b' class prefixdirmatcher(basematcher):'
 
     @propertycache
     def _pathdirs(self):
-        return set(util.finddirs(self._path)) | {'.'}
+        return set(util.finddirs(self._path))
 
     def visitdir(self, dir):
         if dir == self._path:
-            return self._matcher.visitdir('.')
+            return self._matcher.visitdir('')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitdir(dir[len(self._pathprefix):])
         return dir in self._pathdirs
 
     def visitchildrenset(self, dir):
         if dir == self._path:
-            return self._matcher.visitchildrenset('.')
+            return self._matcher.visitchildrenset('')
         if dir.startswith(self._pathprefix):
             return self._matcher.visitchildrenset(dir[len(self._pathprefix):])
         if dir in self._pathdirs:
@@ -1075,7 +1087,7 b' class unionmatcher(basematcher):'
 def patkind(pattern, default=None):
     '''If pattern is 'kind:pat' with a known kind, return kind.
 
-    >>> patkind(b're:.*\.c$')
+    >>> patkind(br're:.*\.c$')
     're'
     >>> patkind(b'glob:*.c')
     'glob'
@@ -1178,9 +1190,23 b' def _globre(pat):'
     return res
 
 def _regex(kind, pat, globsuffix):
-    '''Convert a (normalized) pattern of any kind into a regular expression.
+    '''Convert a (normalized) pattern of any kind into a
+    regular expression.
     globsuffix is appended to the regexp of globs.'''
-    if not pat:
+
+    if rustmod is not None:
+        try:
+            return rustmod.build_single_regex(
+                kind,
+                pat,
+                globsuffix
+            )
+        except rustmod.PatternError:
+            raise error.ProgrammingError(
+                'not a regex pattern: %s:%s' % (kind, pat)
+            )
+
+    if not pat and kind in ('glob', 'relpath'):
         return ''
     if kind == 're':
         return pat
@@ -1324,13 +1350,17 b' def _patternrootsanddirs(kindpats):'
                 if '[' in p or '{' in p or '*' in p or '?' in p:
                     break
                 root.append(p)
-            r.append('/'.join(root) or '.')
+            r.append('/'.join(root))
         elif kind in ('relpath', 'path'):
-            r.append(pat or '.')
+            if pat == '.':
+                pat = ''
+            r.append(pat)
         elif kind in ('rootfilesin',):
-            d.append(pat or '.')
+            if pat == '.':
+                pat = ''
+            d.append(pat)
         else: # relglob, re, relre
-            r.append('.')
+            r.append('')
     return r, d
 
 def _roots(kindpats):
@@ -1347,31 +1377,33 b' def _rootsdirsandparents(kindpats):'
 
     Returns a tuple of (roots, dirs, parents).
 
-    >>> _rootsdirsandparents(
+    >>> r = _rootsdirsandparents(
     ...     [(b'glob', b'g/h/*', b''), (b'glob', b'g/h', b''),
     ...      (b'glob', b'g*', b'')])
-    (['g/h', 'g/h', '.'], [], ['g', '.'])
-    >>> _rootsdirsandparents(
+    >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
+    (['g/h', 'g/h', ''], []) ['', 'g']
+    >>> r = _rootsdirsandparents(
     ...     [(b'rootfilesin', b'g/h', b''), (b'rootfilesin', b'', b'')])
-    ([], ['g/h', '.'], ['g', '.'])
-    >>> _rootsdirsandparents(
+    >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
+    ([], ['g/h', '']) ['', 'g']
+    >>> r = _rootsdirsandparents(
     ...     [(b'relpath', b'r', b''), (b'path', b'p/p', b''),
     ...      (b'path', b'', b'')])
-    (['r', 'p/p', '.'], [], ['p', '.'])
-    >>> _rootsdirsandparents(
+    >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
+    (['r', 'p/p', ''], []) ['', 'p']
+    >>> r = _rootsdirsandparents(
     ...     [(b'relglob', b'rg*', b''), (b're', b're/', b''),
     ...      (b'relre', b'rr', b'')])
-    (['.', '.', '.'], [], ['.'])
+    >>> print(r[0:2], sorted(r[2])) # the set has an unstable output
+    (['', '', ''], []) ['']
     '''
     r, d = _patternrootsanddirs(kindpats)
 
-    p = []
-    # Append the parents as non-recursive/exact directories, since they must be
+    p = set()
+    # Add the parents as non-recursive/exact directories, since they must be
     # scanned to get to either the roots or the other exact directories.
-    p.extend(util.dirs(d))
-    p.extend(util.dirs(r))
-    # util.dirs() does not include the root directory, so add it manually
-    p.append('.')
+    p.update(util.dirs(d))
+    p.update(util.dirs(r))
 
     # FIXME: all uses of this function convert these to sets, do so before
     # returning.
@@ -1421,9 +1453,24 b' def readpatternfile(filepath, warn, sour'
     pattern        # pattern of the current default type
 
     if sourceinfo is set, returns a list of tuples:
-    (pattern, lineno, originalline). This is useful to debug ignore patterns.
+    (pattern, lineno, originalline).
+    This is useful to debug ignore patterns.
     '''
 
+    if rustmod is not None:
+        result, warnings = rustmod.read_pattern_file(
+            filepath,
+            bool(warn),
+            sourceinfo,
+        )
+
+        for warning_params in warnings:
+            # Can't be easily emitted from Rust, because it would require
+            # a mechanism for both gettext and calling the `warn` function.
+            warn(_("%s: ignoring invalid syntax '%s'\n") % warning_params)
+
+        return result
+
     syntaxes = {
         're': 'relre:',
         'regexp': 'relre:',

mercurial/merge.py

@@ -10,6 +10,7 b' from __future__ import absolute_import'
 import errno
 import hashlib
 import shutil
+import stat
 import struct
 
 from .i18n import _
@@ -683,7 +684,7 b' class mergestate(object):'
     def recordactions(self):
         """record remove/add/get actions in the dirstate"""
         branchmerge = self._repo.dirstate.p2() != nullid
-        recordupdates(self._repo, self.actions(), branchmerge)
+        recordupdates(self._repo, self.actions(), branchmerge, None)
 
     def queueremove(self, f):
         """queues a file to be removed from the dirstate
@@ -1380,7 +1381,6 b' def calculateupdates(repo, wctx, mctx, a'
         # Pick the best bid for each file
         repo.ui.note(_('\nauction for merging merge bids\n'))
         actions = {}
-        dms = [] # filenames that have dm actions
         for f, bids in sorted(fbids.items()):
             # bids is a mapping from action method to list af actions
             # Consensus?
@@ -1389,8 +1389,6 b' def calculateupdates(repo, wctx, mctx, a'
                 if all(a == l[0] for a in l[1:]): # len(bids) is > 1
                     repo.ui.note(_(" %s: consensus for %s\n") % (f, m))
                     actions[f] = l[0]
-                    if m == ACTION_DIR_RENAME_MOVE_LOCAL:
-                        dms.append(f)
                     continue
             # If keep is an option, just do it.
             if ACTION_KEEP in bids:
@@ -1415,18 +1413,7 b' def calculateupdates(repo, wctx, mctx, a'
             repo.ui.warn(_(' %s: ambiguous merge - picked %s action\n') %
                          (f, m))
             actions[f] = l[0]
-            if m == ACTION_DIR_RENAME_MOVE_LOCAL:
-                dms.append(f)
             continue
-        # Work around 'dm' that can cause multiple actions for the same file
-        for f in dms:
-            dm, (f0, flags), msg = actions[f]
-            assert dm == ACTION_DIR_RENAME_MOVE_LOCAL, dm
-            if f0 in actions and actions[f0][0] == ACTION_REMOVE:
-                # We have one bid for removing a file and another for moving it.
-                # These two could be merged as first move and then delete ...
-                # but instead drop moving and just delete.
-                del actions[f]
         repo.ui.note(_('end of auction\n\n'))
 
     if wctx.rev() is None:
@@ -1478,13 +1465,17 b' def batchremove(repo, wctx, actions):'
         repo.ui.warn(_("current directory was removed\n"
                        "(consider changing to repo root: %s)\n") % repo.root)
 
-def batchget(repo, mctx, wctx, actions):
+def batchget(repo, mctx, wctx, wantfiledata, actions):
     """apply gets to the working directory
 
     mctx is the context to get from
 
-    yields tuples for progress updates
+    Yields arbitrarily many (False, tuple) for progress updates, followed by
+    exactly one (True, filedata). When wantfiledata is false, filedata is an
+    empty dict. When wantfiledata is true, filedata[f] is a triple (mode, size,
+    mtime) of the file f written for each action.
     """
+    filedata = {}
     verbose = repo.ui.verbose
     fctx = mctx.filectx
     ui = repo.ui
@@ -1508,16 +1499,24 b' def batchget(repo, mctx, wctx, actions):'
                 if repo.wvfs.lexists(conflicting):
                     orig = scmutil.backuppath(ui, repo, conflicting)
                     util.rename(repo.wjoin(conflicting), orig)
-            wctx[f].clearunknown()
+            wfctx = wctx[f]
+            wfctx.clearunknown()
             atomictemp = ui.configbool("experimental", "update.atomic-file")
-            wctx[f].write(fctx(f).data(), flags, backgroundclose=True,
-                          atomictemp=atomictemp)
+            size = wfctx.write(fctx(f).data(), flags,
+                               backgroundclose=True,
+                               atomictemp=atomictemp)
+            if wantfiledata:
+                s = wfctx.lstat()
+                mode = s.st_mode
+                mtime = s[stat.ST_MTIME]
+                filedata[f] = ((mode, size, mtime)) # for dirstate.normal
             if i == 100:
-                yield i, f
+                yield False, (i, f)
                 i = 0
             i += 1
     if i > 0:
-        yield i, f
+        yield False, (i, f)
+    yield True, filedata
 
 def _prefetchfiles(repo, ctx, actions):
     """Invoke ``scmutil.prefetchfiles()`` for the files relevant to the dict
@@ -1564,14 +1563,17 b' def emptyactions():'
                     ACTION_PATH_CONFLICT,
                     ACTION_PATH_CONFLICT_RESOLVE))
 
-def applyupdates(repo, actions, wctx, mctx, overwrite, labels=None):
+def applyupdates(repo, actions, wctx, mctx, overwrite, wantfiledata,
+                 labels=None):
     """apply the merge action list to the working directory
 
     wctx is the working copy context
     mctx is the context to be merged into the working copy
 
-    Return a tuple of counts (updated, merged, removed, unresolved) that
-    describes how many files were affected by the update.
+    Return a tuple of (counts, filedata), where counts is a tuple
+    (updated, merged, removed, unresolved) that describes how many
+    files were affected by the update, and filedata is as described in
+    batchget.
     """
 
     _prefetchfiles(repo, mctx, actions)
@@ -1663,11 +1665,18 b' def applyupdates(repo, actions, wctx, mc'
     # get in parallel.
     threadsafe = repo.ui.configbool('experimental',
                                     'worker.wdir-get-thread-safe')
-    prog = worker.worker(repo.ui, cost, batchget, (repo, mctx, wctx),
+    prog = worker.worker(repo.ui, cost, batchget,
+                         (repo, mctx, wctx, wantfiledata),
                          actions[ACTION_GET],
-                         threadsafe=threadsafe)
-    for i, item in prog:
-        progress.increment(step=i, item=item)
+                         threadsafe=threadsafe,
+                         hasretval=True)
+    getfiledata = {}
+    for final, res in prog:
+        if final:
+            getfiledata = res
+        else:
+            i, item = res
+            progress.increment(step=i, item=item)
     updated = len(actions[ACTION_GET])
 
     if [a for a in actions[ACTION_GET] if a[0] == '.hgsubstate']:
@@ -1792,6 +1801,10 b' def applyupdates(repo, actions, wctx, mc'
         mfiles = set(a[0] for a in actions[ACTION_MERGE])
         for k, acts in extraactions.iteritems():
             actions[k].extend(acts)
+            if k == ACTION_GET and wantfiledata:
+                # no filedata until mergestate is updated to provide it
+                for a in acts:
+                    getfiledata[a[0]] = None
             # Remove these files from actions[ACTION_MERGE] as well. This is
             # important because in recordupdates, files in actions[ACTION_MERGE]
             # are processed after files in other actions, and the merge driver
@@ -1814,9 +1827,10 b' def applyupdates(repo, actions, wctx, mc'
                                  if a[0] in mfiles]
 
     progress.complete()
-    return updateresult(updated, merged, removed, unresolved)
+    assert len(getfiledata) == (len(actions[ACTION_GET]) if wantfiledata else 0)
+    return updateresult(updated, merged, removed, unresolved), getfiledata
 
-def recordupdates(repo, actions, branchmerge):
+def recordupdates(repo, actions, branchmerge, getfiledata):
     "record merge actions to the dirstate"
     # remove (must come first)
     for f, args, msg in actions.get(ACTION_REMOVE, []):
@@ -1864,7 +1878,8 b' def recordupdates(repo, actions, branchm'
         if branchmerge:
             repo.dirstate.otherparent(f)
         else:
-            repo.dirstate.normal(f)
+            parentfiledata = getfiledata[f] if getfiledata else None
+            repo.dirstate.normal(f, parentfiledata=parentfiledata)
 
     # merge
     for f, args, msg in actions.get(ACTION_MERGE, []):
@@ -1991,14 +2006,10 b' def update(repo, node, branchmerge, forc'
             wc = repo[None]
         pl = wc.parents()
         p1 = pl[0]
-        pas = [None]
+        p2 = repo[node]
         if ancestor is not None:
             pas = [repo[ancestor]]
-
-        overwrite = force and not branchmerge
-
-        p2 = repo[node]
-        if pas[0] is None:
+        else:
             if repo.ui.configlist('merge', 'preferancestor') == ['*']:
                 cahs = repo.changelog.commonancestorsheads(p1.node(), p2.node())
                 pas = [repo[anc] for anc in (sorted(cahs) or [nullid])]
@@ -2007,6 +2018,7 b' def update(repo, node, branchmerge, forc'
 
         fp1, fp2, xp1, xp2 = p1.node(), p2.node(), bytes(p1), bytes(p2)
 
+        overwrite = force and not branchmerge
         ### check phase
         if not overwrite:
             if len(pl) > 1:
@@ -2183,12 +2195,15 b' def update(repo, node, branchmerge, forc'
                   'fsmonitor enabled; enable fsmonitor to improve performance; '
                   'see "hg help -e fsmonitor")\n'))
 
-        stats = applyupdates(repo, actions, wc, p2, overwrite, labels=labels)
+        updatedirstate = not partial and not wc.isinmemory()
+        wantfiledata = updatedirstate and not branchmerge
+        stats, getfiledata = applyupdates(repo, actions, wc, p2, overwrite,
+                                          wantfiledata, labels=labels)
 
-        if not partial and not wc.isinmemory():
+        if updatedirstate:
             with repo.dirstate.parentchange():
                 repo.setparents(fp1, fp2)
-                recordupdates(repo, actions, branchmerge)
+                recordupdates(repo, actions, branchmerge, getfiledata)
                 # update completed, clear state
                 util.unlink(repo.vfs.join('updatestate'))
 
@@ -2219,7 +2234,7 b' def graft(repo, ctx, pctx, labels=None, '
     pctx - merge base, usually ctx.p1()
     labels - merge labels eg ['local', 'graft']
     keepparent - keep second parent if any
-    keepparent - if unresolved, keep parent used for the merge
+    keepconflictparent - if unresolved, keep parent used for the merge
 
     """
     # If we're grafting a descendant onto an ancestor, be sure to pass

mercurial/minirst.py

@@ -44,6 +44,9 b' def subsubsection(s):'
 def subsubsubsection(s):
     return "%s\n%s\n\n" % (s, "." * encoding.colwidth(s))
 
+def subsubsubsubsection(s):
+    return "%s\n%s\n\n" % (s, "'" * encoding.colwidth(s))
+
 def replace(text, substs):
     '''
     Apply a list of (find, replace) pairs to a text.

mercurial/narrowspec.py

@@ -7,14 +7,13 b''
 
 from __future__ import absolute_import
 
-import errno
-
 from .i18n import _
 from . import (
     error,
     match as matchmod,
     merge,
     repository,
+    scmutil,
     sparse,
     util,
 )
@@ -144,15 +143,9 b' def parseconfig(ui, spec):'
     return includepats, excludepats
 
 def load(repo):
-    try:
-        spec = repo.svfs.read(FILENAME)
-    except IOError as e:
-        # Treat "narrowspec does not exist" the same as "narrowspec file exists
-        # and is empty".
-        if e.errno == errno.ENOENT:
-            return set(), set()
-        raise
-
+    # Treat "narrowspec does not exist" the same as "narrowspec file exists
+    # and is empty".
+    spec = repo.svfs.tryread(FILENAME)
     return parseconfig(repo.ui, spec)
 
 def save(repo, includepats, excludepats):
@@ -266,9 +259,12 b' def _writeaddedfiles(repo, pctx, files):'
         if not repo.wvfs.exists(f):
             addgaction((f, (mf.flags(f), False), "narrowspec updated"))
     merge.applyupdates(repo, actions, wctx=repo[None],
-                       mctx=repo['.'], overwrite=False)
+                       mctx=repo['.'], overwrite=False, wantfiledata=False)
 
 def checkworkingcopynarrowspec(repo):
+    # Avoid infinite recursion when updating the working copy
+    if getattr(repo, '_updatingnarrowspec', False):
+        return
     storespec = repo.svfs.tryread(FILENAME)
     wcspec = repo.vfs.tryread(DIRSTATE_FILENAME)
     if wcspec != storespec:
@@ -283,6 +279,7 b' def updateworkingcopy(repo, assumeclean='
     """
     oldspec = repo.vfs.tryread(DIRSTATE_FILENAME)
     newspec = repo.svfs.tryread(FILENAME)
+    repo._updatingnarrowspec = True
 
     oldincludes, oldexcludes = parseconfig(repo.ui, oldspec)
     newincludes, newexcludes = parseconfig(repo.ui, newspec)
@@ -292,8 +289,8 b' def updateworkingcopy(repo, assumeclean='
     removedmatch = matchmod.differencematcher(oldmatch, newmatch)
 
     ds = repo.dirstate
-    lookup, status = ds.status(removedmatch, subrepos=[], ignored=False,
-                               clean=True, unknown=False)
+    lookup, status = ds.status(removedmatch, subrepos=[], ignored=True,
+                               clean=True, unknown=True)
     trackeddirty = status.modified + status.added
     clean = status.clean
     if assumeclean:
@@ -302,15 +299,19 b' def updateworkingcopy(repo, assumeclean='
     else:
         trackeddirty.extend(lookup)
     _deletecleanfiles(repo, clean)
+    uipathfn = scmutil.getuipathfn(repo)
     for f in sorted(trackeddirty):
-        repo.ui.status(_('not deleting possibly dirty file %s\n') % f)
+        repo.ui.status(_('not deleting possibly dirty file %s\n') % uipathfn(f))
+    for f in sorted(status.unknown):
+        repo.ui.status(_('not deleting unknown file %s\n') % uipathfn(f))
+    for f in sorted(status.ignored):
+        repo.ui.status(_('not deleting ignored file %s\n') % uipathfn(f))
     for f in clean + trackeddirty:
         ds.drop(f)
 
-    repo.narrowpats = newincludes, newexcludes
-    repo._narrowmatch = newmatch
     pctx = repo['.']
     newfiles = [f for f in pctx.manifest().walk(addedmatch) if f not in ds]
     for f in newfiles:
         ds.normallookup(f)
     _writeaddedfiles(repo, pctx, newfiles)
+    repo._updatingnarrowspec = False

mercurial/obsolete.py

@@ -93,10 +93,6 b" parsers = policy.importmod(r'parsers')"
 _calcsize = struct.calcsize
 propertycache = util.propertycache
 
-# the obsolete feature is not mature enough to be enabled by default.
-# you have to rely on third party extension extension to enable this.
-_enabled = False
-
 # Options for obsolescence
 createmarkersopt = 'createmarkers'
 allowunstableopt = 'allowunstable'
@@ -124,11 +120,6 b' def _getoptionvalue(repo, option):'
         if 'all' in result:
             return True
 
-        # For migration purposes, temporarily return true if the config hasn't
-        # been set but _enabled is true.
-        if len(result) == 0 and _enabled:
-            return True
-
         # Temporary hack for next check
         newconfig = repo.ui.config('experimental', 'evolution.createmarkers')
         if newconfig:

mercurial/patch.py

@@ -1089,7 +1089,9 b' def filterpatch(ui, headers, match, oper'
             return skipfile, skipfile, skipall, newpatches
         while True:
             resps = messages['help'][operation]
-            r = ui.promptchoice("%s %s" % (query, resps))
+            # IMPORTANT: keep the last line of this prompt short (<40 english
+            # chars is a good target) because of issue6158.
+            r = ui.promptchoice("%s\n(enter ? for help) %s" % (query, resps))
             ui.write("\n")
             if r == 8: # ?
                 for c, t in ui.extractchoices(resps)[1]:

mercurial/policy.py

@@ -13,6 +13,9 b' import sys'
 # Rules for how modules can be loaded. Values are:
 #
 #    c - require C extensions
+#    rust+c - require Rust and C extensions
+#    rust+c-allow - allow Rust and C extensions with fallback to pure Python
+#                   for each
 #    allow - allow pure Python implementation when C loading fails
 #    cffi - required cffi versions (implemented within pure module)
 #    cffi-allow - allow pure Python implementation if cffi version is missing
@@ -29,6 +32,9 b" policy = b'allow'"
     b'cffi': (r'cffi', None),
     b'cffi-allow': (r'cffi', r'pure'),
     b'py': (None, r'pure'),
+    # For now, rust policies impact importrust only
+    b'rust+c': (r'cext', None),
+    b'rust+c-allow': (r'cext', r'pure'),
 }
 
 try:
@@ -69,7 +75,7 b' def _importfrom(pkgname, modname):'
     (r'cext', r'bdiff'): 3,
     (r'cext', r'mpatch'): 1,
     (r'cext', r'osutil'): 4,
-    (r'cext', r'parsers'): 12,
+    (r'cext', r'parsers'): 13,
 }
 
 # map import request to other package or module
@@ -107,3 +113,34 b' def importmod(modname):'
                 raise
     pn, mn = _modredirects.get((purepkg, modname), (purepkg, modname))
     return _importfrom(pn, mn)
+
+def _isrustpermissive():
+    """Assuming the policy is a Rust one, tell if it's permissive."""
+    return policy.endswith(b'-allow')
+
+def importrust(modname, member=None, default=None):
+    """Import Rust module according to policy and availability.
+
+    If policy isn't a Rust one, this returns `default`.
+
+    If either the module or its member is not available, this returns `default`
+    if policy is permissive and raises `ImportError` if not.
+    """
+    if not policy.startswith(b'rust'):
+        return default
+
+    try:
+        mod = _importfrom(r'rustext', modname)
+    except ImportError:
+        if _isrustpermissive():
+            return default
+        raise
+    if member is None:
+        return mod
+
+    try:
+        return getattr(mod, member)
+    except AttributeError:
+        if _isrustpermissive():
+            return default
+        raise ImportError(r"Cannot import name %s" % member)

mercurial/profiling.py

@@ -147,6 +147,8 b' def statprofile(ui, fp):'
             # inconsistent config: profiling.showmin
             limit = ui.configwith(fraction, 'profiling', 'showmin', 0.05)
             kwargs[r'limit'] = limit
+            showtime = ui.configbool('profiling', 'showtime')
+            kwargs[r'showtime'] = showtime
 
         statprof.display(fp, data=data, format=displayformat, **kwargs)
 

mercurial/pure/osutil.py

@@ -5,7 +5,7 b''
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division
 
 import ctypes
 import ctypes.util
@@ -149,7 +149,7 b' if not pycompat.iswindows:'
             cmsg.cmsg_type != _SCM_RIGHTS):
             return []
         rfds = ctypes.cast(cmsg.cmsg_data, ctypes.POINTER(ctypes.c_int))
-        rfdscount = ((cmsg.cmsg_len - _cmsghdr.cmsg_data.offset) /
+        rfdscount = ((cmsg.cmsg_len - _cmsghdr.cmsg_data.offset) //
                      ctypes.sizeof(ctypes.c_int))
         return [rfds[i] for i in pycompat.xrange(rfdscount)]
 

mercurial/registrar.py

@@ -64,8 +64,8 b' class _funcregistrarbase(object):'
             raise error.ProgrammingError(msg)
 
         if func.__doc__ and not util.safehasattr(func, '_origdoc'):
-            doc = pycompat.sysbytes(func.__doc__).strip()
-            func._origdoc = doc
+            func._origdoc = func.__doc__.strip()
+            doc = pycompat.sysbytes(func._origdoc)
             func.__doc__ = pycompat.sysstr(self._formatdoc(decl, doc))
 
         self._table[name] = func
@@ -338,19 +338,10 b' class templatekeyword(_templateregistrar'
             '''
             pass
 
-        # old API (DEPRECATED)
-        @templatekeyword('mykeyword')
-        def mykeywordfunc(repo, ctx, templ, cache, revcache, **args):
-            '''Explanation of this template keyword ....
-            '''
-            pass
-
     The first string argument is used also in online help.
 
     Optional argument 'requires' should be a collection of resource names
-    which the template keyword depends on. This also serves as a flag to
-    switch to the new API. If 'requires' is unspecified, all template
-    keywords and resources are expanded to the function arguments.
+    which the template keyword depends on.
 
     'templatekeyword' instance in example above can be used to
     decorate multiple functions.
@@ -362,7 +353,7 b' class templatekeyword(_templateregistrar'
     Otherwise, explicit 'templatekw.loadkeyword()' is needed.
     """
 
-    def _extrasetup(self, name, func, requires=None):
+    def _extrasetup(self, name, func, requires=()):
         func._requires = requires
 
 class templatefilter(_templateregistrarbase):

mercurial/repair.py

@@ -279,7 +279,9 b' def _bookmarkmovements(repo, tostrip):'
         if rev in tostrip:
             updatebm.append(m)
     newbmtarget = None
-    if updatebm: # don't compute anything is there is no bookmark to move anyway
+    # If we need to move bookmarks, compute bookmark
+    # targets. Otherwise we can skip doing this logic.
+    if updatebm:
         # For a set s, max(parents(s) - s) is the same as max(heads(::s - s)),
         # but is much faster
         newbmtarget = repo.revs('max(parents(%ld) - (%ld))', tostrip, tostrip)
@@ -364,8 +366,9 b' def stripmanifest(repo, striprev, tr, fi'
     striptrees(repo, tr, striprev, files)
 
 def striptrees(repo, tr, striprev, files):
-    if 'treemanifest' in repo.requirements: # safe but unnecessary
-                                            # otherwise
+    if 'treemanifest' in repo.requirements:
+        # This logic is safe if treemanifest isn't enabled, but also
+        # pointless, so we skip it if treemanifest isn't enabled.
         for unencoded, encoded, size in repo.store.datafiles():
             if (unencoded.startswith('meta/') and
                 unencoded.endswith('00manifest.i')):
@@ -416,7 +419,9 b' def rebuildfncache(ui, repo):'
 
         progress.complete()
 
-        if 'treemanifest' in repo.requirements: # safe but unnecessary otherwise
+        if 'treemanifest' in repo.requirements:
+            # This logic is safe if treemanifest isn't enabled, but also
+            # pointless, so we skip it if treemanifest isn't enabled.
             for dir in util.dirs(seenfiles):
                 i = 'meta/%s/00manifest.i' % dir
                 d = 'meta/%s/00manifest.d' % dir

mercurial/repository.py

@@ -291,6 +291,10 b' class ipeercommandexecutor(interfaceutil'
 class ipeerrequests(interfaceutil.Interface):
     """Interface for executing commands on a peer."""
 
+    limitedarguments = interfaceutil.Attribute(
+        """True if the peer cannot receive large argument value for commands."""
+    )
+
     def commandexecutor():
         """A context manager that resolves to an ipeercommandexecutor.
 
@@ -329,6 +333,8 b' class ipeerv2(ipeerconnection, ipeercapa'
 class peer(object):
     """Base class for peer repositories."""
 
+    limitedarguments = False
+
     def capable(self, name):
         caps = self.capabilities()
         if name in caps:
@@ -1650,7 +1656,7 b' class ilocalrepositorymain(interfaceutil'
                editor=False, extra=None):
         """Add a new revision to the repository."""
 
-    def commitctx(ctx, error=False):
+    def commitctx(ctx, error=False, origctx=None):
         """Commit a commitctx instance to the repository."""
 
     def destroying():

mercurial/repoview.py

@@ -17,6 +17,10 b' from . import ('
     phases,
     pycompat,
     tags as tagsmod,
+    util,
+)
+from .utils import (
+    repoviewutil,
 )
 
 def hideablerevs(repo):
@@ -154,6 +158,35 b" filtertable = {'visible': computehidden,"
                'immutable':  computemutable,
                'base':  computeimpactable}
 
+_basefiltername = list(filtertable)
+
+def extrafilter(ui):
+    """initialize extra filter and return its id
+
+    If extra filtering is configured, we make sure the associated filtered view
+    are declared and return the associated id.
+    """
+    frevs = ui.config('experimental', 'extra-filter-revs')
+    if frevs is None:
+        return None
+
+    fid = pycompat.sysbytes(util.DIGESTS['sha1'](frevs).hexdigest())[:12]
+
+    combine = lambda fname: fname + '%' + fid
+
+    subsettable = repoviewutil.subsettable
+
+    if combine('base') not in filtertable:
+        for name in _basefiltername:
+            def extrafilteredrevs(repo, *args, **kwargs):
+                baserevs = filtertable[name](repo, *args, **kwargs)
+                extrarevs = frozenset(repo.revs(frevs))
+                return baserevs | extrarevs
+            filtertable[combine(name)] = extrafilteredrevs
+            if name in subsettable:
+                subsettable[combine(name)] = combine(subsettable[name])
+    return fid
+
 def filterrevs(repo, filtername, visibilityexceptions=None):
     """returns set of filtered revision for this filter name
 

mercurial/revlog.py

@@ -16,6 +16,7 b' from __future__ import absolute_import'
 import collections
 import contextlib
 import errno
+import io
 import os
 import struct
 import zlib
@@ -97,11 +98,8 b' REVIDX_KNOWN_FLAGS'
 REVIDX_RAWTEXT_CHANGING_FLAGS
 
 parsers = policy.importmod(r'parsers')
-try:
-    from . import rustext
-    rustext.__name__  # force actual import (see hgdemandimport)
-except ImportError:
-    rustext = None
+rustancestor = policy.importrust(r'ancestor')
+rustdagop = policy.importrust(r'dagop')
 
 # Aliased for performance.
 _zlibdecompress = zlib.decompress
@@ -337,15 +335,21 b' class revlog(object):'
     configured threshold.
 
     If censorable is True, the revlog can have censored revisions.
+
+    If `upperboundcomp` is not None, this is the expected maximal gain from
+    compression for the data content.
     """
     def __init__(self, opener, indexfile, datafile=None, checkambig=False,
-                 mmaplargeindex=False, censorable=False):
+                 mmaplargeindex=False, censorable=False,
+                 upperboundcomp=None):
         """
         create a revlog object
 
         opener is a function that abstracts the file opening operation
         and can be used to implement COW semantics or the like.
+
         """
+        self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + ".d")
         self.opener = opener
@@ -825,8 +829,8 b' class revlog(object):'
             checkrev(r)
         # and we're sure ancestors aren't filtered as well
 
-        if rustext is not None:
-            lazyancestors = rustext.ancestor.LazyAncestors
+        if rustancestor is not None:
+            lazyancestors = rustancestor.LazyAncestors
             arg = self.index
         elif util.safehasattr(parsers, 'rustlazyancestors'):
             lazyancestors = ancestor.rustlazyancestors
@@ -915,8 +919,8 b' class revlog(object):'
         if common is None:
             common = [nullrev]
 
-        if rustext is not None:
-            return rustext.ancestor.MissingAncestors(self.index, common)
+        if rustancestor is not None:
+            return rustancestor.MissingAncestors(self.index, common)
         return ancestor.incrementalmissingancestors(self.parentrevs, common)
 
     def findmissingrevs(self, common=None, heads=None):
@@ -1130,8 +1134,8 b' class revlog(object):'
                 return self.index.headrevs()
             except AttributeError:
                 return self._headrevs()
-        if rustext is not None:
-            return rustext.dagop.headrevs(self.index, revs)
+        if rustdagop is not None:
+            return rustdagop.headrevs(self.index, revs)
         return dagop.headrevs(revs, self._uncheckedparentrevs)
 
     def computephases(self, roots):
@@ -1216,14 +1220,25 b' class revlog(object):'
         A revision is considered an ancestor of itself.
 
         The implementation of this is trivial but the use of
-        commonancestorsheads is not."""
+        reachableroots is not."""
         if a == nullrev:
             return True
         elif a == b:
             return True
         elif a > b:
             return False
-        return a in self._commonancestorsheads(a, b)
+        return bool(self.reachableroots(a, [b], [a], includepath=False))
+
+    def reachableroots(self, minroot, heads, roots, includepath=False):
+        """return (heads(::<roots> and <roots>::<heads>))
+
+        If includepath is True, return (<roots>::<heads>)."""
+        try:
+            return self.index.reachableroots2(minroot, heads, roots,
+                                              includepath)
+        except AttributeError:
+            return dagop._reachablerootspure(self.parentrevs,
+                                             minroot, roots, heads, includepath)
 
     def ancestor(self, a, b):
         """calculate the "best" common ancestor of nodes a and b"""
@@ -1340,13 +1355,13 b' class revlog(object):'
         """Find the shortest unambiguous prefix that matches node."""
         def isvalid(prefix):
             try:
-                node = self._partialmatch(prefix)
+                matchednode = self._partialmatch(prefix)
             except error.AmbiguousPrefixLookupError:
                 return False
             except error.WdirUnsupported:
                 # single 'ff...' match
                 return True
-            if node is None:
+            if matchednode is None:
                 raise error.LookupError(node, self.indexfile, _('no node'))
             return True
 
@@ -2292,7 +2307,7 b' class revlog(object):'
 
         try:
             with self._datafp() as f:
-                f.seek(0, 2)
+                f.seek(0, io.SEEK_END)
                 actual = f.tell()
             dd = actual - expected
         except IOError as inst:
@@ -2302,7 +2317,7 b' class revlog(object):'
 
         try:
             f = self.opener(self.indexfile)
-            f.seek(0, 2)
+            f.seek(0, io.SEEK_END)
             actual = f.tell()
             f.close()
             s = self._io.size

mercurial/revlogutils/deltas.py

@@ -679,6 +679,31 b' def _candidategroups(revlog, textlen, p1'
             # if chain already have too much data, skip base
             if deltas_limit < chainsize:
                 continue
+            if sparse and revlog.upperboundcomp is not None:
+                maxcomp = revlog.upperboundcomp
+                basenotsnap = (p1, p2, nullrev)
+                if rev not in basenotsnap and revlog.issnapshot(rev):
+                    snapshotdepth = revlog.snapshotdepth(rev)
+                    # If text is significantly larger than the base, we can
+                    # expect the resulting delta to be proportional to the size
+                    # difference
+                    revsize = revlog.rawsize(rev)
+                    rawsizedistance = max(textlen - revsize, 0)
+                    # use an estimate of the compression upper bound.
+                    lowestrealisticdeltalen = rawsizedistance // maxcomp
+
+                    # check the absolute constraint on the delta size
+                    snapshotlimit = textlen >> snapshotdepth
+                    if snapshotlimit < lowestrealisticdeltalen:
+                        # delta lower bound is larger than accepted upper bound
+                        continue
+
+                    # check the relative constraint on the delta size
+                    revlength = revlog.length(rev)
+                    if revlength < lowestrealisticdeltalen:
+                        # delta probable lower bound is larger than target base
+                        continue
+
             group.append(rev)
         if group:
             # XXX: in the sparse revlog case, group can become large,
@@ -907,6 +932,21 b' class deltacomputer(object):'
 
     def _builddeltainfo(self, revinfo, base, fh):
         # can we use the cached delta?
+        revlog = self.revlog
+        chainbase = revlog.chainbase(base)
+        if revlog._generaldelta:
+            deltabase = base
+        else:
+            deltabase = chainbase
+        snapshotdepth = None
+        if revlog._sparserevlog and deltabase == nullrev:
+            snapshotdepth = 0
+        elif revlog._sparserevlog and revlog.issnapshot(deltabase):
+            # A delta chain should always be one full snapshot,
+            # zero or more semi-snapshots, and zero or more deltas
+            p1, p2 = revlog.rev(revinfo.p1), revlog.rev(revinfo.p2)
+            if deltabase not in (p1, p2) and revlog.issnapshot(deltabase):
+                snapshotdepth = len(revlog._deltachain(deltabase)[0])
         delta = None
         if revinfo.cachedelta:
             cachebase, cachediff = revinfo.cachedelta
@@ -920,31 +960,22 b' class deltacomputer(object):'
                 delta = revinfo.cachedelta[1]
         if delta is None:
             delta = self._builddeltadiff(base, revinfo, fh)
-        revlog = self.revlog
+        # snapshotdept need to be neither None nor 0 level snapshot
+        if revlog.upperboundcomp is not None and snapshotdepth:
+            lowestrealisticdeltalen = len(delta) // revlog.upperboundcomp
+            snapshotlimit = revinfo.textlen >> snapshotdepth
+            if snapshotlimit < lowestrealisticdeltalen:
+                return None
+            if revlog.length(base) < lowestrealisticdeltalen:
+                return None
         header, data = revlog.compress(delta)
         deltalen = len(header) + len(data)
-        chainbase = revlog.chainbase(base)
         offset = revlog.end(len(revlog) - 1)
         dist = deltalen + offset - revlog.start(chainbase)
-        if revlog._generaldelta:
-            deltabase = base
-        else:
-            deltabase = chainbase
         chainlen, compresseddeltalen = revlog._chaininfo(base)
         chainlen += 1
         compresseddeltalen += deltalen
 
-        revlog = self.revlog
-        snapshotdepth = None
-        if deltabase == nullrev:
-            snapshotdepth = 0
-        elif revlog._sparserevlog and revlog.issnapshot(deltabase):
-            # A delta chain should always be one full snapshot,
-            # zero or more semi-snapshots, and zero or more deltas
-            p1, p2 = revlog.rev(revinfo.p1), revlog.rev(revinfo.p2)
-            if deltabase not in (p1, p2) and revlog.issnapshot(deltabase):
-                snapshotdepth = len(revlog._deltachain(deltabase)[0])
-
         return _deltainfo(dist, deltalen, (header, data), deltabase,
                           chainbase, chainlen, compresseddeltalen,
                           snapshotdepth)
@@ -1002,8 +1033,9 b' class deltacomputer(object):'
                 nominateddeltas.append(deltainfo)
             for candidaterev in candidaterevs:
                 candidatedelta = self._builddeltainfo(revinfo, candidaterev, fh)
-                if isgooddeltainfo(self.revlog, candidatedelta, revinfo):
-                    nominateddeltas.append(candidatedelta)
+                if candidatedelta is not None:
+                    if isgooddeltainfo(self.revlog, candidatedelta, revinfo):
+                        nominateddeltas.append(candidatedelta)
             if nominateddeltas:
                 deltainfo = min(nominateddeltas, key=lambda x: x.deltalen)
             if deltainfo is not None:

mercurial/revset.py

@@ -52,6 +52,9 b' generatorset = smartset.generatorset'
 spanset = smartset.spanset
 fullreposet = smartset.fullreposet
 
+# revisions not included in all(), but populated if specified
+_virtualrevs = (node.nullrev, node.wdirrev)
+
 # Constants for ordering requirement, used in getset():
 #
 # If 'define', any nested functions and operations MAY change the ordering of
@@ -120,8 +123,7 b' def stringset(repo, subset, x, order):'
     if not x:
         raise error.ParseError(_("empty string is not a valid revision"))
     x = scmutil.intrev(scmutil.revsymbol(repo, x))
-    if (x in subset
-        or x == node.nullrev and isinstance(subset, fullreposet)):
+    if x in subset or x in _virtualrevs and isinstance(subset, fullreposet):
         return baseset([x])
     return baseset()
 
@@ -1359,8 +1361,13 b' def merge(repo, subset, x):'
     # i18n: "merge" is a keyword
     getargs(x, 0, 0, _("merge takes no arguments"))
     cl = repo.changelog
-    return subset.filter(lambda r: cl.parentrevs(r)[1] != -1,
-                         condrepr='<merge>')
+    nullrev = node.nullrev
+    def ismerge(r):
+        try:
+            return cl.parentrevs(r)[1] != nullrev
+        except error.WdirUnsupported:
+            return bool(repo[r].p2())
+    return subset.filter(ismerge, condrepr='<merge>')
 
 @predicate('branchpoint()', safe=True)
 def branchpoint(repo, subset, x):
@@ -1847,7 +1854,7 b' def rev(repo, subset, x):'
     except (TypeError, ValueError):
         # i18n: "rev" is a keyword
         raise error.ParseError(_("rev expects a number"))
-    if l not in repo.changelog and l not in (node.nullrev, node.wdirrev):
+    if l not in repo.changelog and l not in _virtualrevs:
         return baseset()
     return subset & baseset([l])
 
@@ -2262,7 +2269,7 b' def _orderedlist(repo, subset, x):'
             if r in seen:
                 continue
             if (r in subset
-                or r == node.nullrev and isinstance(subset, fullreposet)):
+                or r in _virtualrevs and isinstance(subset, fullreposet)):
                 ls.append(r)
             seen.add(r)
     return baseset(ls)

mercurial/scmutil.py

@@ -1247,6 +1247,28 b' def getrenamedfn(repo, endrev=None):'
 
     return getrenamed
 
+def getcopiesfn(repo, endrev=None):
+    if copiesmod.usechangesetcentricalgo(repo):
+        def copiesfn(ctx):
+            if ctx.p2copies():
+                allcopies = ctx.p1copies().copy()
+                # There should be no overlap
+                allcopies.update(ctx.p2copies())
+                return sorted(allcopies.items())
+            else:
+                return sorted(ctx.p1copies().items())
+    else:
+        getrenamed = getrenamedfn(repo, endrev)
+        def copiesfn(ctx):
+            copies = []
+            for fn in ctx.files():
+                rename = getrenamed(fn, ctx.rev())
+                if rename:
+                    copies.append((fn, rename))
+            return copies
+
+    return copiesfn
+
 def dirstatecopy(ui, repo, wctx, src, dst, dryrun=False, cwd=None):
     """Update the dirstate to reflect the intent of copying src to dst. For
     different reasons it might not end with dst being marked as copied from src.
@@ -1519,7 +1541,12 b' def extdatasource(repo, source):'
                 pass # we ignore data for nodes that don't exist locally
     finally:
         if proc:
-            proc.communicate()
+            try:
+                proc.communicate()
+            except ValueError:
+                # This happens if we started iterating src and then
+                # get a parse error on a line. It should be safe to ignore.
+                pass
         if src:
             src.close()
     if proc and proc.returncode != 0:

mercurial/setdiscovery.py

@@ -110,22 +110,23 b' class partialdiscovery(object):'
     (all tracked revisions are known locally)
     """
 
-    def __init__(self, repo, targetheads):
+    def __init__(self, repo, targetheads, respectsize):
         self._repo = repo
         self._targetheads = targetheads
         self._common = repo.changelog.incrementalmissingrevs()
         self._undecided = None
         self.missing = set()
         self._childrenmap = None
+        self._respectsize = respectsize
 
     def addcommons(self, commons):
-        """registrer nodes known as common"""
+        """register nodes known as common"""
         self._common.addbases(commons)
         if self._undecided is not None:
             self._common.removeancestorsfrom(self._undecided)
 
     def addmissings(self, missings):
-        """registrer some nodes as missing"""
+        """register some nodes as missing"""
         newmissing = self._repo.revs('%ld::%ld', missings, self.undecided)
         if newmissing:
             self.missing.update(newmissing)
@@ -241,11 +242,13 b' class partialdiscovery(object):'
 
         # update from roots
         revsroots = set(repo.revs('roots(%ld)', revs))
-
         childrenrevs = self._childrengetter()
-
         _updatesample(revs, revsroots, sample, childrenrevs)
         assert sample
+
+        if not self._respectsize:
+            size = max(size, min(len(revsroots), len(revsheads)))
+
         sample = _limitsample(sample, size)
         if len(sample) < size:
             more = size - len(sample)
@@ -256,7 +259,8 b' def findcommonheads(ui, local, remote,'
                     initialsamplesize=100,
                     fullsamplesize=200,
                     abortwhenunrelated=True,
-                    ancestorsof=None):
+                    ancestorsof=None,
+                    samplegrowth=1.05):
     '''Return a tuple (common, anyincoming, remoteheads) used to identify
     missing nodes from or in remote.
     '''
@@ -275,9 +279,63 b' def findcommonheads(ui, local, remote,'
     # early exit if we know all the specified remote heads already
     ui.debug("query 1; heads\n")
     roundtrips += 1
-    sample = _limitsample(ownheads, initialsamplesize)
-    # indices between sample and externalized version must match
-    sample = list(sample)
+    # We also ask remote about all the local heads. That set can be arbitrarily
+    # large, so we used to limit it size to `initialsamplesize`. We no longer
+    # do as it proved counter productive. The skipped heads could lead to a
+    # large "undecided" set, slower to be clarified than if we asked the
+    # question for all heads right away.
+    #
+    # We are already fetching all server heads using the `heads` commands,
+    # sending a equivalent number of heads the other way should not have a
+    # significant impact.  In addition, it is very likely that we are going to
+    # have to issue "known" request for an equivalent amount of revisions in
+    # order to decide if theses heads are common or missing.
+    #
+    # find a detailled analysis below.
+    #
+    # Case A: local and server both has few heads
+    #
+    #     Ownheads is below initialsamplesize, limit would not have any effect.
+    #
+    # Case B: local has few heads and server has many
+    #
+    #     Ownheads is below initialsamplesize, limit would not have any effect.
+    #
+    # Case C: local and server both has many heads
+    #
+    #     We now transfert some more data, but not significantly more than is
+    #     already transfered to carry the server heads.
+    #
+    # Case D: local has many heads, server has few
+    #
+    #   D.1 local heads are mostly known remotely
+    #
+    #     All the known head will have be part of a `known` request at some
+    #     point for the discovery to finish. Sending them all earlier is
+    #     actually helping.
+    #
+    #     (This case is fairly unlikely, it requires the numerous heads to all
+    #     be merged server side in only a few heads)
+    #
+    #   D.2 local heads are mostly missing remotely
+    #
+    #     To determine that the heads are missing, we'll have to issue `known`
+    #     request for them or one of their ancestors. This amount of `known`
+    #     request will likely be in the same order of magnitude than the amount
+    #     of local heads.
+    #
+    #     The only case where we can be more efficient using `known` request on
+    #     ancestors are case were all the "missing" local heads are based on a
+    #     few changeset, also "missing".  This means we would have a "complex"
+    #     graph (with many heads) attached to, but very independant to a the
+    #     "simple" graph on the server. This is a fairly usual case and have
+    #     not been met in the wild so far.
+    if remote.limitedarguments:
+        sample = _limitsample(ownheads, initialsamplesize)
+        # indices between sample and externalized version must match
+        sample = list(sample)
+    else:
+        sample = ownheads
 
     with remote.commandexecutor() as e:
         fheads = e.callcommand('heads', {})
@@ -318,7 +376,7 b' def findcommonheads(ui, local, remote,'
 
     # full blown discovery
 
-    disco = partialdiscovery(local, ownheads)
+    disco = partialdiscovery(local, ownheads, remote.limitedarguments)
     # treat remote heads (and maybe own heads) as a first implicit sample
     # response
     disco.addcommons(knownsrvheads)
@@ -335,6 +393,8 b' def findcommonheads(ui, local, remote,'
                 ui.debug("taking initial sample\n")
             samplefunc = disco.takefullsample
             targetsize = fullsamplesize
+            if not remote.limitedarguments:
+                fullsamplesize = int(fullsamplesize * samplegrowth)
         else:
             # use even cheaper initial sample
             ui.debug("taking quick initial sample\n")

mercurial/shelve.py

@@ -27,8 +27,8 b' import errno'
 import itertools
 import stat
 
-from mercurial.i18n import _
-from mercurial import (
+from .i18n import _
+from . import (
     bookmarks,
     bundle2,
     bundlerepo,
@@ -45,37 +45,17 b' from mercurial import ('
     patch,
     phases,
     pycompat,
-    registrar,
     repair,
     scmutil,
     templatefilters,
     util,
     vfs as vfsmod,
 )
-
-from . import (
-    rebase,
-)
-from mercurial.utils import (
+from .utils import (
     dateutil,
     stringutil,
 )
 
-cmdtable = {}
-command = registrar.command(cmdtable)
-# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
-# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
-# be specifying the version(s) of Mercurial they are tested with, or
-# leave the attribute unspecified.
-testedwith = 'ships-with-hg-core'
-
-configtable = {}
-configitem = registrar.configitem(configtable)
-
-configitem('shelve', 'maxbackups',
-    default=10,
-)
-
 backupdir = 'shelve-backup'
 shelvedir = 'shelved'
 shelvefileextensions = ['hg', 'patch', 'shelve']
@@ -451,8 +431,6 b' def createcmd(ui, repo, pats, opts):'
 def _docreatecmd(ui, repo, pats, opts):
     wctx = repo[None]
     parents = wctx.parents()
-    if len(parents) > 1:
-        raise error.Abort(_('cannot shelve while merging'))
     parent = parents[0]
     origbranch = wctx.branch()
 
@@ -646,7 +624,30 b' def checkparents(repo, state):'
         raise error.Abort(_('working directory parents do not match unshelve '
                            'state'))
 
-def unshelveabort(ui, repo, state, opts):
+def _loadshelvedstate(ui, repo, opts):
+    try:
+        state = shelvedstate.load(repo)
+        if opts.get('keep') is None:
+            opts['keep'] = state.keep
+    except IOError as err:
+        if err.errno != errno.ENOENT:
+            raise
+        cmdutil.wrongtooltocontinue(repo, _('unshelve'))
+    except error.CorruptedState as err:
+        ui.debug(pycompat.bytestr(err) + '\n')
+        if opts.get('continue'):
+            msg = _('corrupted shelved state file')
+            hint = _('please run hg unshelve --abort to abort unshelve '
+                     'operation')
+            raise error.Abort(msg, hint=hint)
+        elif opts.get('abort'):
+            shelvedstate.clear(repo)
+            raise error.Abort(_('could not read shelved state file, your '
+                                'working copy may be in an unexpected state\n'
+                                'please update to some commit\n'))
+    return state
+
+def unshelveabort(ui, repo, state):
     """subcommand that abort an in-progress unshelve"""
     with repo.lock():
         try:
@@ -656,11 +657,6 b' def unshelveabort(ui, repo, state, opts)'
             if (state.activebookmark
                     and state.activebookmark in repo._bookmarks):
                 bookmarks.activate(repo, state.activebookmark)
-
-            if repo.vfs.exists('unshelverebasestate'):
-                repo.vfs.rename('unshelverebasestate', 'rebasestate')
-                rebase.clearstatus(repo)
-
             mergefiles(ui, repo, state.wctx, state.pendingctx)
             if not phases.supportinternal(repo):
                 repair.strip(ui, repo, state.nodestoremove, backup=False,
@@ -669,6 +665,12 b' def unshelveabort(ui, repo, state, opts)'
             shelvedstate.clear(repo)
             ui.warn(_("unshelve of '%s' aborted\n") % state.name)
 
+def hgabortunshelve(ui, repo):
+    """logic to  abort unshelve using 'hg abort"""
+    with repo.wlock():
+        state = _loadshelvedstate(ui, repo, {'abort' : True})
+        return unshelveabort(ui, repo, state)
+
 def mergefiles(ui, repo, wctx, shelvectx):
     """updates to wctx and merges the changes from shelvectx into the
     dirstate."""
@@ -692,11 +694,11 b' def unshelvecleanup(ui, repo, name, opts'
             if shfile.exists():
                 shfile.movetobackup()
         cleanupoldbackups(repo)
-
-def unshelvecontinue(ui, repo, state, opts):
+def unshelvecontinue(ui, repo, state, opts, basename=None):
     """subcommand to continue an in-progress unshelve"""
     # We're finishing off a merge. First parent is our original
     # parent, second is the temporary "fake" commit we're unshelving.
+    interactive = opts.get('interactive')
     with repo.lock():
         checkparents(repo, state)
         ms = merge.mergestate.read(repo)
@@ -719,10 +721,15 b' def unshelvecontinue(ui, repo, state, op'
         with repo.ui.configoverride(overrides, 'unshelve'):
             with repo.dirstate.parentchange():
                 repo.setparents(state.parents[0], nodemod.nullid)
-                newnode = repo.commit(text=shelvectx.description(),
-                                      extra=shelvectx.extra(),
-                                      user=shelvectx.user(),
-                                      date=shelvectx.date())
+                if not interactive:
+                    ispartialunshelve = False
+                    newnode = repo.commit(text=shelvectx.description(),
+                                        extra=shelvectx.extra(),
+                                        user=shelvectx.user(),
+                                        date=shelvectx.date())
+                else:
+                    newnode, ispartialunshelve = _dounshelveinteractive(ui,
+                        repo, shelvectx, basename, opts)
 
         if newnode is None:
             # If it ended up being a no-op commit, then the normal
@@ -739,22 +746,24 b' def unshelvecontinue(ui, repo, state, op'
             shelvectx = repo[newnode]
 
         hg.updaterepo(repo, pendingctx.node(), overwrite=False)
-
-        if repo.vfs.exists('unshelverebasestate'):
-            repo.vfs.rename('unshelverebasestate', 'rebasestate')
-            rebase.clearstatus(repo)
-
         mergefiles(ui, repo, state.wctx, shelvectx)
         restorebranch(ui, repo, state.branchtorestore)
 
-        if not phases.supportinternal(repo):
-            repair.strip(ui, repo, state.nodestoremove, backup=False,
-                         topic='shelve')
+        if not ispartialunshelve:
+            if not phases.supportinternal(repo):
+                repair.strip(ui, repo, state.nodestoremove, backup=False,
+                            topic='shelve')
+            shelvedstate.clear(repo)
+            unshelvecleanup(ui, repo, state.name, opts)
         _restoreactivebookmark(repo, state.activebookmark)
-        shelvedstate.clear(repo)
-        unshelvecleanup(ui, repo, state.name, opts)
         ui.status(_("unshelve of '%s' complete\n") % state.name)
 
+def hgcontinueunshelve(ui, repo):
+    """logic to resume unshelve using 'hg continue'"""
+    with repo.wlock():
+        state = _loadshelvedstate(ui, repo, {'continue' : True})
+        return unshelvecontinue(ui, repo, state, {'keep' : state.keep})
+
 def _commitworkingcopychanges(ui, repo, opts, tmpwctx):
     """Temporarily commit working copy changes before moving unshelve commit"""
     # Store pending changes in a commit and remember added in case a shelve
@@ -795,14 +804,40 b' def _unshelverestorecommit(ui, repo, tr,'
 
     return repo, shelvectx
 
+def _dounshelveinteractive(ui, repo, shelvectx, basename, opts):
+    """The user might want to unshelve certain changes only from the stored
+    shelve. So, we would create two commits. One with requested changes to
+    unshelve at that time and the latter is shelved for future.
+    """
+    opts['message'] = shelvectx.description()
+    opts['interactive-unshelve'] = True
+    pats = []
+    commitfunc = getcommitfunc(shelvectx.extra(), interactive=True,
+                               editor=True)
+    newnode = cmdutil.dorecord(ui, repo, commitfunc, None, False,
+                               cmdutil.recordfilter, *pats,
+                               **pycompat.strkwargs(opts))
+    snode = repo.commit(text=shelvectx.description(),
+                        extra=shelvectx.extra(),
+                        user=shelvectx.user(),
+                        date=shelvectx.date())
+    m = scmutil.matchfiles(repo, repo[snode].files())
+    if snode:
+        _shelvecreatedcommit(repo, snode, basename, m)
+
+    return newnode, bool(snode)
+
 def _rebaserestoredcommit(ui, repo, opts, tr, oldtiprev, basename, pctx,
                           tmpwctx, shelvectx, branchtorestore,
                           activebookmark):
     """Rebase restored commit from its original location to a destination"""
     # If the shelve is not immediately on top of the commit
     # we'll be merging with, rebase it to be on top.
-    if tmpwctx.node() == shelvectx.p1().node():
-        return shelvectx
+    interactive = opts.get('interactive')
+    if tmpwctx.node() == shelvectx.p1().node() and not interactive:
+        # We won't skip on interactive mode because, the user might want to
+        # unshelve certain changes only.
+        return shelvectx, False
 
     overrides = {
         ('ui', 'forcemerge'): opts.get('tool', ''),
@@ -826,10 +861,15 b' def _rebaserestoredcommit(ui, repo, opts'
 
         with repo.dirstate.parentchange():
             repo.setparents(tmpwctx.node(), nodemod.nullid)
-            newnode = repo.commit(text=shelvectx.description(),
-                                  extra=shelvectx.extra(),
-                                  user=shelvectx.user(),
-                                  date=shelvectx.date())
+            if not interactive:
+                ispartialunshelve = False
+                newnode = repo.commit(text=shelvectx.description(),
+                                      extra=shelvectx.extra(),
+                                      user=shelvectx.user(),
+                                      date=shelvectx.date())
+            else:
+                newnode, ispartialunshelve = _dounshelveinteractive(ui, repo,
+                                                shelvectx, basename, opts)
 
         if newnode is None:
             # If it ended up being a no-op commit, then the normal
@@ -844,7 +884,7 b' def _rebaserestoredcommit(ui, repo, opts'
             shelvectx = repo[newnode]
             hg.updaterepo(repo, tmpwctx.node(), False)
 
-    return shelvectx
+    return shelvectx, ispartialunshelve
 
 def _forgetunknownfiles(repo, shelvectx, addedbefore):
     # Forget any files that were unknown before the shelve, unknown before
@@ -877,70 +917,18 b' def _checkunshelveuntrackedproblems(ui, '
         hint = _("run hg status to see which files are missing")
         raise error.Abort(m, hint=hint)
 
-@command('unshelve',
-         [('a', 'abort', None,
-           _('abort an incomplete unshelve operation')),
-          ('c', 'continue', None,
-           _('continue an incomplete unshelve operation')),
-          ('k', 'keep', None,
-           _('keep shelve after unshelving')),
-          ('n', 'name', '',
-           _('restore shelved change with given name'), _('NAME')),
-          ('t', 'tool', '', _('specify merge tool')),
-          ('', 'date', '',
-           _('set date for temporary commits (DEPRECATED)'), _('DATE'))],
-         _('hg unshelve [[-n] SHELVED]'),
-         helpcategory=command.CATEGORY_WORKING_DIRECTORY)
-def unshelve(ui, repo, *shelved, **opts):
-    """restore a shelved change to the working directory
-
-    This command accepts an optional name of a shelved change to
-    restore. If none is given, the most recent shelved change is used.
-
-    If a shelved change is applied successfully, the bundle that
-    contains the shelved changes is moved to a backup location
-    (.hg/shelve-backup).
-
-    Since you can restore a shelved change on top of an arbitrary
-    commit, it is possible that unshelving will result in a conflict
-    between your changes and the commits you are unshelving onto. If
-    this occurs, you must resolve the conflict, then use
-    ``--continue`` to complete the unshelve operation. (The bundle
-    will not be moved until you successfully complete the unshelve.)
-
-    (Alternatively, you can use ``--abort`` to abandon an unshelve
-    that causes a conflict. This reverts the unshelved changes, and
-    leaves the bundle in place.)
-
-    If bare shelved change(when no files are specified, without interactive,
-    include and exclude option) was done on newly created branch it would
-    restore branch information to the working directory.
-
-    After a successful unshelve, the shelved changes are stored in a
-    backup directory. Only the N most recent backups are kept. N
-    defaults to 10 but can be overridden using the ``shelve.maxbackups``
-    configuration option.
-
-    .. container:: verbose
-
-       Timestamp in seconds is used to decide order of backups. More
-       than ``maxbackups`` backups are kept, if same timestamp
-       prevents from deciding exact order of them, for safety.
-    """
-    with repo.wlock():
-        return _dounshelve(ui, repo, *shelved, **opts)
-
-def _dounshelve(ui, repo, *shelved, **opts):
+def dounshelve(ui, repo, *shelved, **opts):
     opts = pycompat.byteskwargs(opts)
     abortf = opts.get('abort')
     continuef = opts.get('continue')
+    interactive = opts.get('interactive')
     if not abortf and not continuef:
         cmdutil.checkunfinished(repo)
     shelved = list(shelved)
     if opts.get("name"):
         shelved.append(opts["name"])
 
-    if abortf or continuef:
+    if abortf or continuef and not interactive:
         if abortf and continuef:
             raise error.Abort(_('cannot use both abort and continue'))
         if shelved:
@@ -949,49 +937,24 b' def _dounshelve(ui, repo, *shelved, **op'
         if abortf and opts.get('tool', False):
             ui.warn(_('tool option will be ignored\n'))
 
-        try:
-            state = shelvedstate.load(repo)
-            if opts.get('keep') is None:
-                opts['keep'] = state.keep
-        except IOError as err:
-            if err.errno != errno.ENOENT:
-                raise
-            cmdutil.wrongtooltocontinue(repo, _('unshelve'))
-        except error.CorruptedState as err:
-            ui.debug(pycompat.bytestr(err) + '\n')
-            if continuef:
-                msg = _('corrupted shelved state file')
-                hint = _('please run hg unshelve --abort to abort unshelve '
-                         'operation')
-                raise error.Abort(msg, hint=hint)
-            elif abortf:
-                msg = _('could not read shelved state file, your working copy '
-                        'may be in an unexpected state\nplease update to some '
-                        'commit\n')
-                ui.warn(msg)
-                shelvedstate.clear(repo)
-            return
-
+        state = _loadshelvedstate(ui, repo, opts)
         if abortf:
-            return unshelveabort(ui, repo, state, opts)
+            return unshelveabort(ui, repo, state)
         elif continuef:
             return unshelvecontinue(ui, repo, state, opts)
     elif len(shelved) > 1:
         raise error.Abort(_('can only unshelve one change at a time'))
-
-    # abort unshelve while merging (issue5123)
-    parents = repo[None].parents()
-    if len(parents) > 1:
-        raise error.Abort(_('cannot unshelve while merging'))
-
     elif not shelved:
         shelved = listshelves(repo)
         if not shelved:
             raise error.Abort(_('no shelved changes to apply!'))
         basename = util.split(shelved[0][1])[1]
         ui.status(_("unshelving change '%s'\n") % basename)
-    else:
+    elif shelved:
         basename = shelved[0]
+    if continuef and interactive:
+        state = _loadshelvedstate(ui, repo, opts)
+        return unshelvecontinue(ui, repo, state, opts, basename)
 
     if not shelvedfile(repo, basename, patchextension).exists():
         raise error.Abort(_("shelved change '%s' not found") % basename)
@@ -1020,128 +983,20 b' def _dounshelve(ui, repo, *shelved, **op'
         if shelvectx.branch() != shelvectx.p1().branch():
             branchtorestore = shelvectx.branch()
 
-        shelvectx = _rebaserestoredcommit(ui, repo, opts, tr, oldtiprev,
-                                          basename, pctx, tmpwctx,
-                                          shelvectx, branchtorestore,
-                                          activebookmark)
+        shelvectx, ispartialunshelve = _rebaserestoredcommit(ui, repo, opts,
+            tr, oldtiprev, basename, pctx, tmpwctx, shelvectx,
+            branchtorestore, activebookmark)
         overrides = {('ui', 'forcemerge'): opts.get('tool', '')}
         with ui.configoverride(overrides, 'unshelve'):
             mergefiles(ui, repo, pctx, shelvectx)
         restorebranch(ui, repo, branchtorestore)
-        _forgetunknownfiles(repo, shelvectx, addedbefore)
+        if not ispartialunshelve:
+            _forgetunknownfiles(repo, shelvectx, addedbefore)
 
-        shelvedstate.clear(repo)
-        _finishunshelve(repo, oldtiprev, tr, activebookmark)
-        unshelvecleanup(ui, repo, basename, opts)
+            shelvedstate.clear(repo)
+            _finishunshelve(repo, oldtiprev, tr, activebookmark)
+            unshelvecleanup(ui, repo, basename, opts)
     finally:
         if tr:
             tr.release()
         lockmod.release(lock)
-
-@command('shelve',
-         [('A', 'addremove', None,
-           _('mark new/missing files as added/removed before shelving')),
-          ('u', 'unknown', None,
-           _('store unknown files in the shelve')),
-          ('', 'cleanup', None,
-           _('delete all shelved changes')),
-          ('', 'date', '',
-           _('shelve with the specified commit date'), _('DATE')),
-          ('d', 'delete', None,
-           _('delete the named shelved change(s)')),
-          ('e', 'edit', False,
-           _('invoke editor on commit messages')),
-          ('k', 'keep', False,
-           _('shelve, but keep changes in the working directory')),
-          ('l', 'list', None,
-           _('list current shelves')),
-          ('m', 'message', '',
-           _('use text as shelve message'), _('TEXT')),
-          ('n', 'name', '',
-           _('use the given name for the shelved commit'), _('NAME')),
-          ('p', 'patch', None,
-           _('output patches for changes (provide the names of the shelved '
-             'changes as positional arguments)')),
-          ('i', 'interactive', None,
-           _('interactive mode, only works while creating a shelve')),
-          ('', 'stat', None,
-           _('output diffstat-style summary of changes (provide the names of '
-             'the shelved changes as positional arguments)')
-           )] + cmdutil.walkopts,
-         _('hg shelve [OPTION]... [FILE]...'),
-         helpcategory=command.CATEGORY_WORKING_DIRECTORY)
-def shelvecmd(ui, repo, *pats, **opts):
-    '''save and set aside changes from the working directory
-
-    Shelving takes files that "hg status" reports as not clean, saves
-    the modifications to a bundle (a shelved change), and reverts the
-    files so that their state in the working directory becomes clean.
-
-    To restore these changes to the working directory, using "hg
-    unshelve"; this will work even if you switch to a different
-    commit.
-
-    When no files are specified, "hg shelve" saves all not-clean
-    files. If specific files or directories are named, only changes to
-    those files are shelved.
-
-    In bare shelve (when no files are specified, without interactive,
-    include and exclude option), shelving remembers information if the
-    working directory was on newly created branch, in other words working
-    directory was on different branch than its first parent. In this
-    situation unshelving restores branch information to the working directory.
-
-    Each shelved change has a name that makes it easier to find later.
-    The name of a shelved change defaults to being based on the active
-    bookmark, or if there is no active bookmark, the current named
-    branch.  To specify a different name, use ``--name``.
-
-    To see a list of existing shelved changes, use the ``--list``
-    option. For each shelved change, this will print its name, age,
-    and description; use ``--patch`` or ``--stat`` for more details.
-
-    To delete specific shelved changes, use ``--delete``. To delete
-    all shelved changes, use ``--cleanup``.
-    '''
-    opts = pycompat.byteskwargs(opts)
-    allowables = [
-        ('addremove', {'create'}), # 'create' is pseudo action
-        ('unknown', {'create'}),
-        ('cleanup', {'cleanup'}),
-#       ('date', {'create'}), # ignored for passing '--date "0 0"' in tests
-        ('delete', {'delete'}),
-        ('edit', {'create'}),
-        ('keep', {'create'}),
-        ('list', {'list'}),
-        ('message', {'create'}),
-        ('name', {'create'}),
-        ('patch', {'patch', 'list'}),
-        ('stat', {'stat', 'list'}),
-    ]
-    def checkopt(opt):
-        if opts.get(opt):
-            for i, allowable in allowables:
-                if opts[i] and opt not in allowable:
-                    raise error.Abort(_("options '--%s' and '--%s' may not be "
-                                       "used together") % (opt, i))
-            return True
-    if checkopt('cleanup'):
-        if pats:
-            raise error.Abort(_("cannot specify names when using '--cleanup'"))
-        return cleanupcmd(ui, repo)
-    elif checkopt('delete'):
-        return deletecmd(ui, repo, pats)
-    elif checkopt('list'):
-        return listcmd(ui, repo, pats, opts)
-    elif checkopt('patch') or checkopt('stat'):
-        return patchcmds(ui, repo, pats, opts)
-    else:
-        return createcmd(ui, repo, pats, opts)
-
-def extsetup(ui):
-    cmdutil.unfinishedstates.append(
-        [shelvedstate._filename, False, False,
-         _('unshelve already in progress'),
-         _("use 'hg unshelve --continue' or 'hg unshelve --abort'")])
-    cmdutil.afterresolvedstates.append(
-        [shelvedstate._filename, _('hg unshelve --continue')])

mercurial/sparse.py

@@ -248,7 +248,8 b' def prunetemporaryincludes(repo):'
 
     typeactions = mergemod.emptyactions()
     typeactions['r'] = actions
-    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False)
+    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False,
+                          wantfiledata=False)
 
     # Fix dirstate
     for file in dropped:
@@ -382,7 +383,7 b' def filterupdatesactions(repo, wctx, mct'
         typeactions = mergemod.emptyactions()
         typeactions['g'] = actions
         mergemod.applyupdates(repo, typeactions, repo[None], repo['.'],
-                              False)
+                              False, wantfiledata=False)
 
         dirstate = repo.dirstate
         for file, flags, msg in actions:
@@ -486,7 +487,8 b' def refreshwdir(repo, origstatus, origsp'
     for f, (m, args, msg) in actions.iteritems():
         typeactions[m].append((f, args, msg))
 
-    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False)
+    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False,
+                          wantfiledata=False)
 
     # Fix dirstate
     for file in added:

mercurial/sslutil.py

@@ -16,6 +16,7 b' import ssl'
 
 from .i18n import _
 from . import (
+    encoding,
     error,
     node,
     pycompat,
@@ -348,6 +349,17 b' def wrapsocket(sock, keyfile, certfile, '
     if not serverhostname:
         raise error.Abort(_('serverhostname argument is required'))
 
+    if b'SSLKEYLOGFILE' in encoding.environ:
+        try:
+            import sslkeylog
+            sslkeylog.set_keylog(pycompat.fsdecode(
+                encoding.environ[b'SSLKEYLOGFILE']))
+            ui.warn(
+                b'sslkeylog enabled by SSLKEYLOGFILE environment variable\n')
+        except ImportError:
+            ui.warn(b'sslkeylog module missing, '
+                    b'but SSLKEYLOGFILE set in environment\n')
+
     for f in (keyfile, certfile):
         if f and not os.path.exists(f):
             raise error.Abort(

mercurial/state.py

@@ -19,6 +19,8 b' the data.'
 
 from __future__ import absolute_import
 
+from .i18n import _
+
 from . import (
     error,
     util,
@@ -85,3 +87,134 b' class cmdstate(object):'
     def exists(self):
         """check whether the state file exists or not"""
         return self._repo.vfs.exists(self.fname)
+
+class _statecheck(object):
+    """a utility class that deals with multistep operations like graft,
+       histedit, bisect, update etc and check whether such commands
+       are in an unfinished conditition or not and return appropriate message
+       and hint.
+       It also has the ability to register and determine the states of any new
+       multistep operation or multistep command extension.
+    """
+
+    def __init__(self, opname, fname, clearable, allowcommit, reportonly,
+                 continueflag, stopflag, cmdmsg, cmdhint, statushint,
+                 abortfunc, continuefunc):
+        self._opname = opname
+        self._fname = fname
+        self._clearable = clearable
+        self._allowcommit = allowcommit
+        self._reportonly = reportonly
+        self._continueflag = continueflag
+        self._stopflag = stopflag
+        self._cmdmsg = cmdmsg
+        self._cmdhint = cmdhint
+        self._statushint = statushint
+        self.abortfunc = abortfunc
+        self.continuefunc = continuefunc
+
+    def statusmsg(self):
+        """returns the hint message corresponding to the command for
+        hg status --verbose
+        """
+        if not self._statushint:
+            hint = (_('To continue:    hg %s --continue\n'
+                      'To abort:       hg %s --abort') % (self._opname,
+                       self._opname))
+            if self._stopflag:
+                hint = hint + (_('\nTo stop:        hg %s --stop') %
+                            (self._opname))
+            return hint
+        return self._statushint
+
+    def hint(self):
+        """returns the hint message corresponding to an interrupted
+        operation
+        """
+        if not self._cmdhint:
+                return (_("use 'hg %s --continue' or 'hg %s --abort'") %
+                        (self._opname, self._opname))
+        return self._cmdhint
+
+    def msg(self):
+        """returns the status message corresponding to the command"""
+        if not self._cmdmsg:
+            return _('%s in progress') % (self._opname)
+        return self._cmdmsg
+
+    def continuemsg(self):
+        """ returns appropriate continue message corresponding to command"""
+        return _('hg %s --continue') % (self._opname)
+
+    def isunfinished(self, repo):
+        """determines whether a multi-step operation is in progress
+        or not
+        """
+        if self._opname == 'merge':
+            return len(repo[None].parents()) > 1
+        else:
+            return repo.vfs.exists(self._fname)
+
+# A list of statecheck objects for multistep operations like graft.
+_unfinishedstates = []
+
+def addunfinished(opname, fname, clearable=False, allowcommit=False,
+                  reportonly=False, continueflag=False, stopflag=False,
+                  cmdmsg="", cmdhint="", statushint="", abortfunc=None,
+                  continuefunc=None):
+    """this registers a new command or operation to unfinishedstates
+    opname is the name the command or operation
+    fname is the file name in which data should be stored in .hg directory.
+    It is None for merge command.
+    clearable boolean determines whether or not interrupted states can be
+    cleared by running `hg update -C .` which in turn deletes the
+    state file.
+    allowcommit boolean decides whether commit is allowed during interrupted
+    state or not.
+    reportonly flag is used for operations like bisect where we just
+    need to detect the operation using 'hg status --verbose'
+    continueflag is a boolean determines whether or not a command supports
+    `--continue` option or not.
+    stopflag is a boolean that determines whether or not a command supports
+    --stop flag
+    cmdmsg is used to pass a different status message in case standard
+    message of the format "abort: cmdname in progress" is not desired.
+    cmdhint is used to pass a different hint message in case standard
+    message of the format "To continue: hg cmdname --continue
+    To abort: hg cmdname --abort" is not desired.
+    statushint is used to pass a different status message in case standard
+    message of the format ('To continue:    hg cmdname --continue'
+    'To abort:       hg cmdname --abort') is not desired
+    abortfunc stores the function required to abort an unfinished state.
+    continuefunc stores the function required to finish an interrupted
+    operation.
+    """
+    statecheckobj = _statecheck(opname, fname, clearable, allowcommit,
+                                reportonly, continueflag, stopflag, cmdmsg,
+                                cmdhint, statushint, abortfunc, continuefunc)
+    if opname == 'merge':
+        _unfinishedstates.append(statecheckobj)
+    else:
+        _unfinishedstates.insert(0, statecheckobj)
+
+addunfinished(
+    'update', fname='updatestate', clearable=True,
+    cmdmsg=_('last update was interrupted'),
+    cmdhint=_("use 'hg update' to get a consistent checkout"),
+    statushint=_("To continue:    hg update")
+)
+addunfinished(
+    'bisect', fname='bisect.state', allowcommit=True, reportonly=True,
+    statushint=_('To mark the changeset good:    hg bisect --good\n'
+                 'To mark the changeset bad:     hg bisect --bad\n'
+                 'To abort:                      hg bisect --reset\n')
+)
+
+def getrepostate(repo):
+    # experimental config: commands.status.skipstates
+    skip = set(repo.ui.configlist('commands', 'status.skipstates'))
+    for state in _unfinishedstates:
+        if state._opname in skip:
+            continue
+        if state.isunfinished(repo):
+            return (state._opname, state.statusmsg())

mercurial/statichttprepo.py

@@ -155,6 +155,7 b' class statichttprepository(localrepo.loc'
 
         self.names = namespaces.namespaces()
         self.filtername = None
+        self._extrafilterid = None
 
         try:
             requirements = set(self.vfs.read(b'requires').splitlines())

mercurial/statprof.py

@@ -678,6 +678,7 b' def display_hotpath(data, fp, limit=0.05'
     for sample in data.samples:
         root.add(sample.stack[::-1], sample.time - lasttime)
         lasttime = sample.time
+    showtime = kwargs.get(r'showtime', True)
 
     def _write(node, depth, multiple_siblings):
         site = node.site
@@ -695,7 +696,9 b' def display_hotpath(data, fp, limit=0.05'
             # lots of string formatting
             listpattern = ''.ljust(indent) +\
                           ('\\' if multiple_siblings else '|') +\
-                          ' %4.1f%%  %s %s'
+                          ' %4.1f%%' +\
+                          (' %5.2fs' % node.count if showtime else '') +\
+                          '  %s %s'
             liststring = listpattern % (node.count / root.count * 100,
                                         filename, function)
             codepattern = '%' + ('%d' % (55 - len(liststring))) + 's %d:  %s'

mercurial/store.py

@@ -40,7 +40,7 b' def _matchtrackedpath(path, matcher):'
     if path.startswith('data/'):
         return matcher(path[len('data/'):-len('.i')])
     elif path.startswith('meta/'):
-        return matcher.visitdir(path[len('meta/'):-len('/00manifest.i')] or '.')
+        return matcher.visitdir(path[len('meta/'):-len('/00manifest.i')])
 
     raise error.ProgrammingError("cannot decode path %s" % path)
 
@@ -337,7 +337,7 b' def _calcmode(vfs):'
         mode = None
     return mode
 
-_data = ('narrowspec data meta 00manifest.d 00manifest.i'
+_data = ('bookmarks narrowspec data meta 00manifest.d 00manifest.i'
          ' 00changelog.d 00changelog.i phaseroots obsstore')
 
 def isrevlog(f, kind, st):
@@ -612,7 +612,7 b' class fncachestore(basicstore):'
                     raise
 
     def copylist(self):
-        d = ('narrowspec data meta dh fncache phaseroots obsstore'
+        d = ('bookmarks narrowspec data meta dh fncache phaseroots obsstore'
              ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i')
         return (['requires', '00changelog.i'] +
                 ['store/' + f for f in d.split()])

mercurial/subrepo.py

@@ -88,13 +88,15 b' def annotatesubrepoerror(func):'
 def _updateprompt(ui, sub, dirty, local, remote):
     if dirty:
         msg = (_(' subrepository sources for %s differ\n'
-                 'use (l)ocal source (%s) or (r)emote source (%s)?'
+                 'you can use (l)ocal source (%s) or (r)emote source (%s).\n'
+                 'what do you want to do?'
                  '$$ &Local $$ &Remote')
                % (subrelpath(sub), local, remote))
     else:
         msg = (_(' subrepository sources for %s differ (in checked out '
                  'version)\n'
-                 'use (l)ocal source (%s) or (r)emote source (%s)?'
+                 'you can use (l)ocal source (%s) or (r)emote source (%s).\n'
+                 'what do you want to do?'
                  '$$ &Local $$ &Remote')
                % (subrelpath(sub), local, remote))
     return ui.promptchoice(msg, 0)

mercurial/subrepoutil.py

@@ -168,8 +168,9 b' def submerge(repo, wctx, mctx, actx, ove'
                 prompts['ro'] = r[0]
                 if repo.ui.promptchoice(
                     _(' subrepository sources for %(s)s differ\n'
-                      'use (l)ocal%(l)s source (%(lo)s)'
-                      ' or (r)emote%(o)s source (%(ro)s)?'
+                      'you can use (l)ocal%(l)s source (%(lo)s)'
+                      ' or (r)emote%(o)s source (%(ro)s).\n'
+                      'what do you want to do?'
                       '$$ &Local $$ &Remote') % prompts, 0):
                     debug(s, "prompt changed, get", r)
                     wctx.sub(s).get(r, overwrite)
@@ -186,7 +187,9 b' def submerge(repo, wctx, mctx, actx, ove'
                 option = repo.ui.promptchoice(
                     _(' subrepository %(s)s diverged (local revision: %(sl)s, '
                       'remote revision: %(sr)s)\n'
-                      '(M)erge, keep (l)ocal%(l)s or keep (r)emote%(o)s?'
+                      'you can (m)erge, keep (l)ocal%(l)s or keep '
+                      '(r)emote%(o)s.\n'
+                      'what do you want to do?'
                       '$$ &Merge $$ &Local $$ &Remote')
                     % prompts, 0)
                 if option == 0:

mercurial/tags.py

@@ -13,11 +13,13 b''
 from __future__ import absolute_import
 
 import errno
+import io
 
 from .node import (
     bin,
     hex,
     nullid,
+    nullrev,
     short,
 )
 from .i18n import _
@@ -89,7 +91,7 b' def fnoderevs(ui, repo, revs):'
     unfi = repo.unfiltered()
     tonode = unfi.changelog.node
     nodes = [tonode(r) for r in revs]
-    fnodes = _getfnodes(ui, repo, nodes[::-1]) # reversed help the cache
+    fnodes = _getfnodes(ui, repo, nodes)
     fnodes = _filterfnodes(fnodes, nodes)
     return fnodes
 
@@ -457,7 +459,8 b' def _readtagcache(ui, repo):'
     # This is the most expensive part of finding tags, so performance
     # depends primarily on the size of newheads.  Worst case: no cache
     # file, so newheads == repoheads.
-    cachefnode = _getfnodes(ui, repo, repoheads)
+    # Reversed order helps the cache ('repoheads' is in descending order)
+    cachefnode = _getfnodes(ui, repo, reversed(repoheads))
 
     # Caller has to iterate over all heads, but can use the filenodes in
     # cachefnode to get to each .hgtags revision quickly.
@@ -472,7 +475,7 b' def _getfnodes(ui, repo, nodes):'
     starttime = util.timer()
     fnodescache = hgtagsfnodescache(repo.unfiltered())
     cachefnode = {}
-    for node in reversed(nodes):
+    for node in nodes:
         fnode = fnodescache.getfnode(node)
         if fnode != nullid:
             cachefnode[node] = fnode
@@ -560,7 +563,7 b' def _tag(repo, names, node, message, loc'
             " branch name\n") % name)
 
     def writetags(fp, names, munge, prevtags):
-        fp.seek(0, 2)
+        fp.seek(0, io.SEEK_END)
         if prevtags and not prevtags.endswith('\n'):
             fp.write('\n')
         for name in names:
@@ -691,6 +694,9 b' class hgtagsfnodescache(object):'
         If an .hgtags does not exist at the specified revision, nullid is
         returned.
         """
+        if node == nullid:
+            return nullid
+
         ctx = self._repo[node]
         rev = ctx.rev()
 
@@ -715,12 +721,33 b' class hgtagsfnodescache(object):'
         if not computemissing:
             return None
 
-        # Populate missing entry.
-        try:
-            fnode = ctx.filenode('.hgtags')
-        except error.LookupError:
-            # No .hgtags file on this revision.
-            fnode = nullid
+        fnode = None
+        cl = self._repo.changelog
+        p1rev, p2rev = cl._uncheckedparentrevs(rev)
+        p1node = cl.node(p1rev)
+        p1fnode = self.getfnode(p1node, computemissing=False)
+        if p2rev != nullrev:
+            # There is some no-merge changeset where p1 is null and p2 is set
+            # Processing them as merge is just slower, but still gives a good
+            # result.
+            p2node = cl.node(p1rev)
+            p2fnode = self.getfnode(p2node, computemissing=False)
+            if p1fnode != p2fnode:
+                # we cannot rely on readfast because we don't know against what
+                # parent the readfast delta is computed
+                p1fnode = None
+        if p1fnode is not None:
+            mctx = ctx.manifestctx()
+            fnode = mctx.readfast().get('.hgtags')
+            if fnode is None:
+                fnode = p1fnode
+        if fnode is None:
+            # Populate missing entry.
+            try:
+                fnode = ctx.filenode('.hgtags')
+            except error.LookupError:
+                # No .hgtags file on this revision.
+                fnode = nullid
 
         self._writeentry(offset, properprefix, fnode)
         return fnode