branching: merge default into stable for 6.1 freeze

contrib/automation/hgautomation/cli.py

@@ -1,576 +1,576 b''
 # cli.py - Command line interface for automation
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import concurrent.futures as futures
 import os
 import pathlib
 import time
 
 from . import (
     aws,
     HGAutomation,
     linux,
     try_server,
     windows,
 )
 
 
 SOURCE_ROOT = pathlib.Path(
     os.path.abspath(__file__)
 ).parent.parent.parent.parent
 DIST_PATH = SOURCE_ROOT / 'dist'
 
 
 def bootstrap_linux_dev(
     hga: HGAutomation, aws_region, distros=None, parallel=False
 ):
     c = hga.aws_connection(aws_region)
 
     if distros:
         distros = distros.split(',')
     else:
         distros = sorted(linux.DISTROS)
 
     # TODO There is a wonky interaction involving KeyboardInterrupt whereby
     # the context manager that is supposed to terminate the temporary EC2
     # instance doesn't run. Until we fix this, make parallel building opt-in
     # so we don't orphan instances.
     if parallel:
         fs = []
 
         with futures.ThreadPoolExecutor(len(distros)) as e:
             for distro in distros:
                 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
 
             for f in fs:
                 f.result()
     else:
         for distro in distros:
             aws.ensure_linux_dev_ami(c, distro=distro)
 
 
 def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     print('Windows development AMI available as %s' % image.id)
 
 
 def build_inno(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     version,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_inno_installer(
                     instance.winrm_client,
                     py_version,
                     a,
                     DIST_PATH,
                     version=version,
                 )
 
 
 def build_wix(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     version,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_wix_installer(
                     instance.winrm_client,
                     py_version,
                     a,
                     DIST_PATH,
                     version=version,
                 )
 
 
 def build_windows_wheel(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_wheel(
                     instance.winrm_client, py_version, a, DIST_PATH
                 )
 
 
 def build_all_windows_packages(
     hga: HGAutomation, aws_region, revision, version, base_image_name
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 'm6i.large') as insts:
         instance = insts[0]
 
         winrm_client = instance.winrm_client
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
-        for py_version in ("2.7", "3.7", "3.8", "3.9"):
+        for py_version in ("2.7", "3.7", "3.8", "3.9", "3.10"):
             for arch in ("x86", "x64"):
                 windows.purge_hg(winrm_client)
                 windows.build_wheel(
                     winrm_client,
                     python_version=py_version,
                     arch=arch,
                     dest_path=DIST_PATH,
                 )
 
         for py_version in (2, 3):
             for arch in ('x86', 'x64'):
                 windows.purge_hg(winrm_client)
                 windows.build_inno_installer(
                     winrm_client, py_version, arch, DIST_PATH, version=version
                 )
                 windows.build_wix_installer(
                     winrm_client, py_version, arch, DIST_PATH, version=version
                 )
 
 
 def terminate_ec2_instances(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.terminate_ec2_instances(c.ec2resource)
 
 
 def purge_ec2_resources(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.remove_resources(c)
 
 
 def run_tests_linux(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     test_flags,
     distro,
     filesystem,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_linux_dev_ami(c, distro=distro)
 
     t_start = time.time()
 
     ensure_extra_volume = filesystem not in ('default', 'tmpfs')
 
     with aws.temporary_linux_dev_instances(
         c, image, instance_type, ensure_extra_volume=ensure_extra_volume
     ) as insts:
 
         instance = insts[0]
 
         linux.prepare_exec_environment(
             instance.ssh_client, filesystem=filesystem
         )
         linux.synchronize_hg(SOURCE_ROOT, instance, '.')
         t_prepared = time.time()
         linux.run_tests(instance.ssh_client, python_version, test_flags)
         t_done = time.time()
 
     t_setup = t_prepared - t_start
     t_all = t_done - t_start
 
     print(
         'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
         % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
     )
 
 
 def run_tests_windows(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     arch,
     test_flags,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
 
     with aws.temporary_windows_dev_instances(
         c, image, instance_type, disable_antivirus=True
     ) as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, '.', instance)
         windows.run_tests(
             instance.winrm_client, python_version, arch, test_flags
         )
 
 
 def publish_windows_artifacts(
     hg: HGAutomation,
     aws_region,
     version: str,
     pypi: bool,
     mercurial_scm_org: bool,
     ssh_username: str,
 ):
     windows.publish_artifacts(
         DIST_PATH,
         version,
         pypi=pypi,
         mercurial_scm_org=mercurial_scm_org,
         ssh_username=ssh_username,
     )
 
 
 def run_try(hga: HGAutomation, aws_region: str, rev: str):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     try_server.trigger_try(c, rev=rev)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
         '--state-path',
         default='~/.hgautomation',
         help='Path for local state files',
     )
     parser.add_argument(
         '--aws-region',
         help='AWS region to use',
         default='us-west-2',
     )
 
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser(
         'bootstrap-linux-dev',
         help='Bootstrap Linux development environments',
     )
     sp.add_argument(
         '--distros',
         help='Comma delimited list of distros to bootstrap',
     )
     sp.add_argument(
         '--parallel',
         action='store_true',
         help='Generate AMIs in parallel (not CTRL-c safe)',
     )
     sp.set_defaults(func=bootstrap_linux_dev)
 
     sp = subparsers.add_parser(
         'bootstrap-windows-dev',
         help='Bootstrap the Windows development environment',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=bootstrap_windows_dev)
 
     sp = subparsers.add_parser(
         'build-all-windows-packages',
         help='Build all Windows packages',
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--version',
         help='Mercurial version string to use',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_all_windows_packages)
 
     sp = subparsers.add_parser(
         'build-inno',
         help='Build Inno Setup installer(s)',
     )
     sp.add_argument(
         '--python-version',
         help='Which version of Python to target',
         choices={2, 3},
         type=int,
         nargs='*',
         default=[3],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--version',
         help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_inno)
 
     sp = subparsers.add_parser(
         'build-windows-wheel',
         help='Build Windows wheel(s)',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to build for',
-        choices={'2.7', '3.7', '3.8', '3.9'},
+        choices={'2.7', '3.7', '3.8', '3.9', '3.10'},
         nargs='*',
         default=['3.8'],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_windows_wheel)
 
     sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
     sp.add_argument(
         '--python-version',
         help='Which version of Python to target',
         choices={2, 3},
         type=int,
         nargs='*',
         default=[3],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--version',
         help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_wix)
 
     sp = subparsers.add_parser(
         'terminate-ec2-instances',
         help='Terminate all active EC2 instances managed by us',
     )
     sp.set_defaults(func=terminate_ec2_instances)
 
     sp = subparsers.add_parser(
         'purge-ec2-resources',
         help='Purge all EC2 resources managed by us',
     )
     sp.set_defaults(func=purge_ec2_resources)
 
     sp = subparsers.add_parser(
         'run-tests-linux',
         help='Run tests on Linux',
     )
     sp.add_argument(
         '--distro',
         help='Linux distribution to run tests on',
         choices=linux.DISTROS,
         default='debian10',
     )
     sp.add_argument(
         '--filesystem',
         help='Filesystem type to use',
         choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
         default='default',
     )
     sp.add_argument(
         '--instance-type',
         help='EC2 instance type to use',
         default='c5.9xlarge',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
         choices={
             'system2',
             'system3',
             '2.7',
             '3.5',
             '3.6',
             '3.7',
             '3.8',
             'pypy',
             'pypy3.5',
             'pypy3.6',
         },
         default='system2',
     )
     sp.add_argument(
         'test_flags',
         help='Extra command line flags to pass to run-tests.py',
         nargs='*',
     )
     sp.set_defaults(func=run_tests_linux)
 
     sp = subparsers.add_parser(
         'run-tests-windows',
         help='Run tests on Windows',
     )
     sp.add_argument(
         '--instance-type',
         help='EC2 instance type to use',
         default='m6i.large',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
-        choices={'2.7', '3.5', '3.6', '3.7', '3.8', '3.9'},
+        choices={'2.7', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10'},
         default='2.7',
     )
     sp.add_argument(
         '--arch',
         help='Architecture to test',
         choices={'x86', 'x64'},
         default='x64',
     )
     sp.add_argument(
         '--test-flags',
         help='Extra command line flags to pass to run-tests.py',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=run_tests_windows)
 
     sp = subparsers.add_parser(
         'publish-windows-artifacts',
         help='Publish built Windows artifacts (wheels, installers, etc)',
     )
     sp.add_argument(
         '--no-pypi',
         dest='pypi',
         action='store_false',
         default=True,
         help='Skip uploading to PyPI',
     )
     sp.add_argument(
         '--no-mercurial-scm-org',
         dest='mercurial_scm_org',
         action='store_false',
         default=True,
         help='Skip uploading to www.mercurial-scm.org',
     )
     sp.add_argument(
         '--ssh-username',
         help='SSH username for mercurial-scm.org',
     )
     sp.add_argument(
         'version',
         help='Mercurial version string to locate local packages',
     )
     sp.set_defaults(func=publish_windows_artifacts)
 
     sp = subparsers.add_parser(
         'try', help='Run CI automation against a custom changeset'
     )
     sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
     sp.set_defaults(func=run_try)
 
     return parser
 
 
 def main():
     parser = get_parser()
     args = parser.parse_args()
 
     local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
     automation = HGAutomation(local_state_path)
 
     if not hasattr(args, 'func'):
         parser.print_help()
         return
 
     kwargs = dict(vars(args))
     del kwargs['func']
     del kwargs['state_path']
 
     args.func(automation, **kwargs)

contrib/automation/hgautomation/windows.py

@@ -1,674 +1,680 b''
 # windows.py - Automation specific to Windows
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import datetime
 import os
 import paramiko
 import pathlib
 import re
 import subprocess
 import tempfile
 
 from .pypi import upload as pypi_upload
 from .winrm import run_powershell
 
 
 # PowerShell commands to activate a Visual Studio 2008 environment.
 # This is essentially a port of vcvarsall.bat to PowerShell.
 ACTIVATE_VC9_AMD64 = r'''
 Write-Output "activating Visual Studio 2008 environment for AMD64"
 $root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
 $Env:VCINSTALLDIR = "${root}\VC\"
 $Env:WindowsSdkDir = "${root}\WinSDK\"
 $Env:PATH = "${root}\VC\Bin\amd64;${root}\WinSDK\Bin\x64;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:PATH"
 $Env:LIB = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIB"
 $Env:LIBPATH = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIBPATH"
 '''.lstrip()
 
 ACTIVATE_VC9_X86 = r'''
 Write-Output "activating Visual Studio 2008 environment for x86"
 $root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
 $Env:VCINSTALLDIR = "${root}\VC\"
 $Env:WindowsSdkDir = "${root}\WinSDK\"
 $Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE"
 $Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB"
 $Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib;$Env:LIBPATH"
 '''.lstrip()
 
 HG_PURGE = r'''
 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
 Set-Location C:\hgdev\src
 hg.exe --config extensions.purge= purge --all
 if ($LASTEXITCODE -ne 0) {
     throw "process exited non-0: $LASTEXITCODE"
 }
 Write-Output "purged Mercurial repo"
 '''
 
 HG_UPDATE_CLEAN = r'''
 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
 Set-Location C:\hgdev\src
 hg.exe --config extensions.purge= purge --all
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 hg.exe update -C {revision}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 hg.exe log -r .
 Write-Output "updated Mercurial working directory to {revision}"
 '''.lstrip()
 
 BUILD_INNO_PYTHON3 = r'''
 $Env:RUSTUP_HOME = "C:\hgdev\rustup"
 $Env:CARGO_HOME = "C:\hgdev\cargo"
 Set-Location C:\hgdev\src
 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --pyoxidizer-target {pyoxidizer_target} --version {version}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 BUILD_INNO_PYTHON2 = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python {extra_args}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''.lstrip()
 
 BUILD_WHEEL = r'''
 Set-Location C:\hgdev\src
 C:\hgdev\python{python_version}-{arch}\python.exe -m pip wheel --wheel-dir dist .
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 BUILD_WIX_PYTHON3 = r'''
 $Env:RUSTUP_HOME = "C:\hgdev\rustup"
 $Env:CARGO_HOME = "C:\hgdev\cargo"
 Set-Location C:\hgdev\src
 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --pyoxidizer-target {pyoxidizer_target} --version {version}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 BUILD_WIX_PYTHON2 = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 RUN_TESTS = r'''
 C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 WHEEL_FILENAME_PYTHON27_X86 = 'mercurial-{version}-cp27-cp27m-win32.whl'
 WHEEL_FILENAME_PYTHON27_X64 = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
 WHEEL_FILENAME_PYTHON37_X86 = 'mercurial-{version}-cp37-cp37m-win32.whl'
 WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl'
 WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl'
 WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl'
 WHEEL_FILENAME_PYTHON39_X86 = 'mercurial-{version}-cp39-cp39-win32.whl'
 WHEEL_FILENAME_PYTHON39_X64 = 'mercurial-{version}-cp39-cp39-win_amd64.whl'
+WHEEL_FILENAME_PYTHON310_X86 = 'mercurial-{version}-cp310-cp310-win32.whl'
+WHEEL_FILENAME_PYTHON310_X64 = 'mercurial-{version}-cp310-cp310-win_amd64.whl'
 
 EXE_FILENAME_PYTHON2_X86 = 'Mercurial-{version}-x86-python2.exe'
 EXE_FILENAME_PYTHON2_X64 = 'Mercurial-{version}-x64-python2.exe'
 EXE_FILENAME_PYTHON3_X86 = 'Mercurial-{version}-x86.exe'
 EXE_FILENAME_PYTHON3_X64 = 'Mercurial-{version}-x64.exe'
 
 MSI_FILENAME_PYTHON2_X86 = 'mercurial-{version}-x86-python2.msi'
 MSI_FILENAME_PYTHON2_X64 = 'mercurial-{version}-x64-python2.msi'
 MSI_FILENAME_PYTHON3_X86 = 'mercurial-{version}-x86.msi'
 MSI_FILENAME_PYTHON3_X64 = 'mercurial-{version}-x64.msi'
 
 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
 
 X86_USER_AGENT_PATTERN = '.*Windows.*'
 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
 
 EXE_PYTHON2_X86_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x86 Windows (Python 2) '
     '- does not require admin rights'
 )
 EXE_PYTHON2_X64_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x64 Windows (Python 2) '
     '- does not require admin rights'
 )
 # TODO remove Python version once Python 2 is dropped.
 EXE_PYTHON3_X86_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x86 Windows (Python 3) '
     '- does not require admin rights'
 )
 EXE_PYTHON3_X64_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x64 Windows (Python 3) '
     '- does not require admin rights'
 )
 MSI_PYTHON2_X86_DESCRIPTION = (
     'Mercurial {version} MSI installer - x86 Windows (Python 2) '
     '- requires admin rights'
 )
 MSI_PYTHON2_X64_DESCRIPTION = (
     'Mercurial {version} MSI installer - x64 Windows (Python 2) '
     '- requires admin rights'
 )
 MSI_PYTHON3_X86_DESCRIPTION = (
     'Mercurial {version} MSI installer - x86 Windows (Python 3) '
     '- requires admin rights'
 )
 MSI_PYTHON3_X64_DESCRIPTION = (
     'Mercurial {version} MSI installer - x64 Windows (Python 3) '
     '- requires admin rights'
 )
 
 
 def get_vc_prefix(arch):
     if arch == 'x86':
         return ACTIVATE_VC9_X86
     elif arch == 'x64':
         return ACTIVATE_VC9_AMD64
     else:
         raise ValueError('illegal arch: %s; must be x86 or x64' % arch)
 
 
 def fix_authorized_keys_permissions(winrm_client, path):
     commands = [
         '$ErrorActionPreference = "Stop"',
         'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
         r'icacls %s /remove:g "NT Service\sshd"' % path,
     ]
 
     run_powershell(winrm_client, '\n'.join(commands))
 
 
 def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
     """Synchronize local Mercurial repo to remote EC2 instance."""
 
     winrm_client = ec2_instance.winrm_client
 
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_dir = pathlib.Path(temp_dir)
 
         ssh_dir = temp_dir / '.ssh'
         ssh_dir.mkdir()
         ssh_dir.chmod(0o0700)
 
         # Generate SSH key to use for communication.
         subprocess.run(
             [
                 'ssh-keygen',
                 '-t',
                 'rsa',
                 '-b',
                 '4096',
                 '-N',
                 '',
                 '-f',
                 str(ssh_dir / 'id_rsa'),
             ],
             check=True,
             capture_output=True,
         )
 
         # Add it to ~/.ssh/authorized_keys on remote.
         # This assumes the file doesn't already exist.
         authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
         winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
         winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
         fix_authorized_keys_permissions(winrm_client, authorized_keys)
 
         public_ip = ec2_instance.public_ip_address
 
         ssh_config = temp_dir / '.ssh' / 'config'
 
         with open(ssh_config, 'w', encoding='utf-8') as fh:
             fh.write('Host %s\n' % public_ip)
             fh.write('  User Administrator\n')
             fh.write('  StrictHostKeyChecking no\n')
             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
             fh.write('  IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
 
         if not (hg_repo / '.hg').is_dir():
             raise Exception(
                 '%s is not a Mercurial repository; '
                 'synchronization not yet supported' % hg_repo
             )
 
         env = dict(os.environ)
         env['HGPLAIN'] = '1'
         env['HGENCODING'] = 'utf-8'
 
         hg_bin = hg_repo / 'hg'
 
         res = subprocess.run(
             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
             cwd=str(hg_repo),
             env=env,
             check=True,
             capture_output=True,
         )
 
         full_revision = res.stdout.decode('ascii')
 
         args = [
             'python2.7',
             hg_bin,
             '--config',
             'ui.ssh=ssh -F %s' % ssh_config,
             '--config',
             'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
             # Also ensure .hgtags changes are present so auto version
             # calculation works.
             'push',
             '-f',
             '-r',
             full_revision,
             '-r',
             'file(.hgtags)',
             'ssh://%s/c:/hgdev/src' % public_ip,
         ]
 
         res = subprocess.run(args, cwd=str(hg_repo), env=env)
 
         # Allow 1 (no-op) to not trigger error.
         if res.returncode not in (0, 1):
             res.check_returncode()
 
         run_powershell(
             winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
         )
 
         # TODO detect dirty local working directory and synchronize accordingly.
 
 
 def purge_hg(winrm_client):
     """Purge the Mercurial source repository on an EC2 instance."""
     run_powershell(winrm_client, HG_PURGE)
 
 
 def find_latest_dist(winrm_client, pattern):
     """Find path to newest file in dist/ directory matching a pattern."""
 
     res = winrm_client.execute_ps(
         r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
         '| Sort-Object LastWriteTime -Descending '
         '| Select-Object -First 1\n'
         '$v.name' % pattern
     )
     return res[0]
 
 
 def copy_latest_dist(winrm_client, pattern, dest_path):
     """Copy latest file matching pattern in dist/ directory.
 
     Given a WinRM client and a file pattern, find the latest file on the remote
     matching that pattern and copy it to the ``dest_path`` directory on the
     local machine.
     """
     latest = find_latest_dist(winrm_client, pattern)
     source = r'C:\hgdev\src\dist\%s' % latest
     dest = dest_path / latest
     print('copying %s to %s' % (source, dest))
     winrm_client.fetch(source, str(dest))
 
 
 def build_inno_installer(
     winrm_client,
     python_version: int,
     arch: str,
     dest_path: pathlib.Path,
     version=None,
 ):
     """Build the Inno Setup installer on a remote machine.
 
     Using a WinRM client, remote commands are executed to build
     a Mercurial Inno Setup installer.
     """
     print(
         'building Inno Setup installer for Python %d %s'
         % (python_version, arch)
     )
 
     if python_version == 3:
         # TODO fix this limitation in packaging code
         if not version:
             raise Exception(
                 "version string is required when building for Python 3"
             )
 
         if arch == "x86":
             target_triple = "i686-pc-windows-msvc"
         elif arch == "x64":
             target_triple = "x86_64-pc-windows-msvc"
         else:
             raise Exception("unhandled arch: %s" % arch)
 
         ps = BUILD_INNO_PYTHON3.format(
             pyoxidizer_target=target_triple,
             version=version,
         )
     else:
         extra_args = []
         if version:
             extra_args.extend(['--version', version])
 
         ps = get_vc_prefix(arch) + BUILD_INNO_PYTHON2.format(
             arch=arch, extra_args=' '.join(extra_args)
         )
 
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.exe', dest_path)
 
 
 def build_wheel(
     winrm_client, python_version: str, arch: str, dest_path: pathlib.Path
 ):
     """Build Python wheels on a remote machine.
 
     Using a WinRM client, remote commands are executed to build a Python wheel
     for Mercurial.
     """
     print('Building Windows wheel for Python %s %s' % (python_version, arch))
 
     ps = BUILD_WHEEL.format(
         python_version=python_version.replace(".", ""), arch=arch
     )
 
     # Python 2.7 requires an activated environment.
     if python_version == "2.7":
         ps = get_vc_prefix(arch) + ps
 
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.whl', dest_path)
 
 
 def build_wix_installer(
     winrm_client,
     python_version: int,
     arch: str,
     dest_path: pathlib.Path,
     version=None,
 ):
     """Build the WiX installer on a remote machine.
 
     Using a WinRM client, remote commands are executed to build a WiX installer.
     """
     print('Building WiX installer for Python %d %s' % (python_version, arch))
 
     if python_version == 3:
         # TODO fix this limitation in packaging code
         if not version:
             raise Exception(
                 "version string is required when building for Python 3"
             )
 
         if arch == "x86":
             target_triple = "i686-pc-windows-msvc"
         elif arch == "x64":
             target_triple = "x86_64-pc-windows-msvc"
         else:
             raise Exception("unhandled arch: %s" % arch)
 
         ps = BUILD_WIX_PYTHON3.format(
             pyoxidizer_target=target_triple,
             version=version,
         )
     else:
         extra_args = []
         if version:
             extra_args.extend(['--version', version])
 
         ps = get_vc_prefix(arch) + BUILD_WIX_PYTHON2.format(
             arch=arch, extra_args=' '.join(extra_args)
         )
 
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.msi', dest_path)
 
 
 def run_tests(winrm_client, python_version, arch, test_flags=''):
     """Run tests on a remote Windows machine.
 
     ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
     ``arch`` is ``x86`` or ``x64``.
     ``test_flags`` is a str representing extra arguments to pass to
     ``run-tests.py``.
     """
     if not re.match(r'\d\.\d', python_version):
         raise ValueError(
             r'python_version must be \d.\d; got %s' % python_version
         )
 
     if arch not in ('x86', 'x64'):
         raise ValueError('arch must be x86 or x64; got %s' % arch)
 
     python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
 
     ps = RUN_TESTS.format(
         python_path=python_path,
         test_flags=test_flags or '',
     )
 
     run_powershell(winrm_client, ps)
 
 
 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
     return (
         dist_path / WHEEL_FILENAME_PYTHON27_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON27_X64.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON39_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON39_X64.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON310_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON310_X64.format(version=version),
     )
 
 
 def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
     return (
         dist_path / WHEEL_FILENAME_PYTHON27_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON27_X64.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON39_X86.format(version=version),
         dist_path / WHEEL_FILENAME_PYTHON39_X64.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON310_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON310_X64.format(version=version),
         dist_path / EXE_FILENAME_PYTHON2_X86.format(version=version),
         dist_path / EXE_FILENAME_PYTHON2_X64.format(version=version),
         dist_path / EXE_FILENAME_PYTHON3_X86.format(version=version),
         dist_path / EXE_FILENAME_PYTHON3_X64.format(version=version),
         dist_path / MSI_FILENAME_PYTHON2_X86.format(version=version),
         dist_path / MSI_FILENAME_PYTHON2_X64.format(version=version),
         dist_path / MSI_FILENAME_PYTHON3_X86.format(version=version),
         dist_path / MSI_FILENAME_PYTHON3_X64.format(version=version),
     )
 
 
 def generate_latest_dat(version: str):
     python2_x86_exe_filename = EXE_FILENAME_PYTHON2_X86.format(version=version)
     python2_x64_exe_filename = EXE_FILENAME_PYTHON2_X64.format(version=version)
     python3_x86_exe_filename = EXE_FILENAME_PYTHON3_X86.format(version=version)
     python3_x64_exe_filename = EXE_FILENAME_PYTHON3_X64.format(version=version)
     python2_x86_msi_filename = MSI_FILENAME_PYTHON2_X86.format(version=version)
     python2_x64_msi_filename = MSI_FILENAME_PYTHON2_X64.format(version=version)
     python3_x86_msi_filename = MSI_FILENAME_PYTHON3_X86.format(version=version)
     python3_x64_msi_filename = MSI_FILENAME_PYTHON3_X64.format(version=version)
 
     entries = (
         (
             '10',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_exe_filename),
             EXE_PYTHON3_X86_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_exe_filename),
             EXE_PYTHON3_X64_DESCRIPTION.format(version=version),
         ),
         (
             '9',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x86_exe_filename),
             EXE_PYTHON2_X86_DESCRIPTION.format(version=version),
         ),
         (
             '9',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x64_exe_filename),
             EXE_PYTHON2_X64_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_msi_filename),
             MSI_PYTHON3_X86_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_msi_filename),
             MSI_PYTHON3_X64_DESCRIPTION.format(version=version),
         ),
         (
             '9',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x86_msi_filename),
             MSI_PYTHON2_X86_DESCRIPTION.format(version=version),
         ),
         (
             '9',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x64_msi_filename),
             MSI_PYTHON2_X64_DESCRIPTION.format(version=version),
         ),
     )
 
     lines = ['\t'.join(e) for e in entries]
 
     return '\n'.join(lines) + '\n'
 
 
 def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
     """Publish Windows release artifacts to PyPI."""
 
     wheel_paths = resolve_wheel_artifacts(dist_path, version)
 
     for p in wheel_paths:
         if not p.exists():
             raise Exception('%s not found' % p)
 
     print('uploading wheels to PyPI (you may be prompted for credentials)')
     pypi_upload(wheel_paths)
 
 
 def publish_artifacts_mercurial_scm_org(
     dist_path: pathlib.Path, version: str, ssh_username=None
 ):
     """Publish Windows release artifacts to mercurial-scm.org."""
     all_paths = resolve_all_artifacts(dist_path, version)
 
     for p in all_paths:
         if not p.exists():
             raise Exception('%s not found' % p)
 
     client = paramiko.SSHClient()
     client.load_system_host_keys()
     # We assume the system SSH configuration knows how to connect.
     print('connecting to mercurial-scm.org via ssh...')
     try:
         client.connect('mercurial-scm.org', username=ssh_username)
     except paramiko.AuthenticationException:
         print('error authenticating; is an SSH key available in an SSH agent?')
         raise
 
     print('SSH connection established')
 
     print('opening SFTP client...')
     sftp = client.open_sftp()
     print('SFTP client obtained')
 
     for p in all_paths:
         dest_path = '/var/www/release/windows/%s' % p.name
         print('uploading %s to %s' % (p, dest_path))
 
         with p.open('rb') as fh:
             data = fh.read()
 
         with sftp.open(dest_path, 'wb') as fh:
             fh.write(data)
             fh.chmod(0o0664)
 
     latest_dat_path = '/var/www/release/windows/latest.dat'
 
     now = datetime.datetime.utcnow()
     backup_path = dist_path / (
         'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
     )
     print('backing up %s to %s' % (latest_dat_path, backup_path))
 
     with sftp.open(latest_dat_path, 'rb') as fh:
         latest_dat_old = fh.read()
 
     with backup_path.open('wb') as fh:
         fh.write(latest_dat_old)
 
     print('writing %s with content:' % latest_dat_path)
     latest_dat_content = generate_latest_dat(version)
     print(latest_dat_content)
 
     with sftp.open(latest_dat_path, 'wb') as fh:
         fh.write(latest_dat_content.encode('ascii'))
 
 
 def publish_artifacts(
     dist_path: pathlib.Path,
     version: str,
     pypi=True,
     mercurial_scm_org=True,
     ssh_username=None,
 ):
     """Publish Windows release artifacts.
 
     Files are found in `dist_path`. We will look for files with version string
     `version`.
 
     `pypi` controls whether we upload to PyPI.
     `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
     """
     if pypi:
         publish_artifacts_pypi(dist_path, version)
 
     if mercurial_scm_org:
         publish_artifacts_mercurial_scm_org(
             dist_path, version, ssh_username=ssh_username
         )

contrib/heptapod-ci.yml

@@ -1,188 +1,193 b''
 stages:
   - tests
   - phabricator
 
 image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG
 
 variables:
     PYTHON: python
     TEST_HGMODULEPOLICY: "allow"
     HG_CI_IMAGE_TAG: "latest"
     TEST_HGTESTS_ALLOW_NETIO: "0"
 
 .all_template: &all
   when: always
 
 .runtests_template: &runtests
     <<: *all
     stage: tests
     # The runner made a clone as root.
     # We make a new clone owned by user used to run the step.
     before_script:
       - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
       - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
       - cd /tmp/mercurial-ci/
       - ls -1 tests/test-check-*.* > /tmp/check-tests.txt
       - black --version
       - clang-format --version
     script:
         - echo "python used, $PYTHON"
         - echo "$RUNTEST_ARGS"
         - HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" "$PYTHON" tests/run-tests.py --color=always $RUNTEST_ARGS
 
 checks-py2:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
 
 checks-py3:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
         PYTHON: python3
 
 rust-cargo-test-py2: &rust_cargo_test
     <<: *all
     stage: tests
     script:
         - echo "python used, $PYTHON"
         - make rust-tests
 
 rust-cargo-test-py3:
     stage: tests
     <<: *rust_cargo_test
     variables:
         PYTHON: python3
 
 phabricator-refresh:
     stage: phabricator
+    rules:
+      - if: '"$PHABRICATOR_TOKEN" != "NO-PHAB"'
+        when: on_success
+      - if: '"$PHABRICATOR_TOKEN" == "NO-PHAB"'
+        when: never
     variables:
       DEFAULT_COMMENT: ":white_check_mark: refresh by Heptapod after a successful CI run (:octopus: :green_heart:)"
       STABLE_COMMENT: ":white_check_mark: refresh by Heptapod after a successful CI run (:octopus: :green_heart:)\n⚠ This patch is intended for stable ⚠\n{image https://media.giphy.com/media/nYI8SmmChYXK0/source.gif}"
     script:
       - |
         if [ `hg branch` == "stable" ]; then
             ./contrib/phab-refresh-stack.sh --comment "$STABLE_COMMENT";
         else
             ./contrib/phab-refresh-stack.sh --comment "$DEFAULT_COMMENT";
         fi
 
 test-py2:
     <<: *runtests
     variables:
         RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "c"
         TEST_HGTESTS_ALLOW_NETIO: "1"
 
 test-py3:
     <<: *runtests
     variables:
         RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
         TEST_HGTESTS_ALLOW_NETIO: "1"
 
 test-py2-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "py"
 
 test-py3-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "py"
 
 test-py2-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py3-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py3-rhg:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --rhg --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py2-chg:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
         TEST_HGMODULEPOLICY: "c"
 
 test-py3-chg:
     <<: *runtests
     variables:
         PYTHON: python3
         RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
         TEST_HGMODULEPOLICY: "c"
 
 check-pytype-py3:
     extends: .runtests_template
     before_script:
       - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
       - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
       - cd /tmp/mercurial-ci/
       - make local PYTHON=$PYTHON
       - $PYTHON -m pip install --user -U pytype==2021.04.15
     variables:
         RUNTEST_ARGS: " --allow-slow-tests tests/test-check-pytype.t"
         HGTEST_SLOWTIMEOUT: "3600"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
 
 # `sh.exe --login` sets a couple of extra environment variables that are defined
 # in the MinGW shell, but switches CWD to /home/$username.  The previous value
 # is stored in OLDPWD.  Of the added variables, MSYSTEM is crucial to running
 # run-tests.py- it is needed to make run-tests.py generate a `python3` script
 # that satisfies the various shebang lines and delegates to `py -3`.
 .window_runtests_template: &windows_runtests
     <<: *all
     when: manual  # we don't have any Windows runners anymore at the moment
     stage: tests
     before_script:
       - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && ls -1 tests/test-check-*.* > C:/Temp/check-tests.txt'
       # TODO: find/install cvs, bzr, perforce, gpg, sqlite3
 
     script:
         - echo "Entering script section"
         - echo "python used, $Env:PYTHON"
         - Invoke-Expression "$Env:PYTHON -V"
         - Invoke-Expression "$Env:PYTHON -m black --version"
         - echo "$Env:RUNTEST_ARGS"
         - echo "$Env:TMP"
         - echo "$Env:TEMP"
 
         - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" $PYTHON tests/run-tests.py --color=always $RUNTEST_ARGS'
 
 windows-py3:
     <<: *windows_runtests
     tags:
       - windows
     variables:
         TEST_HGMODULEPOLICY: "c"
         RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt"
         PYTHON: py -3
 
 windows-py3-pyox:
     <<: *windows_runtests
     tags:
       - windows
     variables:
         TEST_HGMODULEPOLICY: "c"
         RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt --pyoxidized"
         PYTHON: py -3

contrib/install-windows-dependencies.ps1

@@ -1,189 +1,198 b''
 # install-dependencies.ps1 - Install Windows dependencies for building Mercurial
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This script can be used to bootstrap a Mercurial build environment on
 # Windows.
 #
 # The script makes a lot of assumptions about how things should work.
 # For example, the install location of Python is hardcoded to c:\hgdev\*.
 #
 # The script should be executed from a PowerShell with elevated privileges
 # if you don't want to see a UAC prompt for various installers.
 #
 # The script is tested on Windows 10 and Windows Server 2019 (in EC2).
 
 $VS_BUILD_TOOLS_URL = "https://download.visualstudio.microsoft.com/download/pr/a1603c02-8a66-4b83-b821-811e3610a7c4/aa2db8bb39e0cbd23e9940d8951e0bc3/vs_buildtools.exe"
 $VS_BUILD_TOOLS_SHA256 = "911E292B8E6E5F46CBC17003BDCD2D27A70E616E8D5E6E69D5D489A605CAA139"
 
 $PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9.exe"
 $PYTHON37_x86_SHA256 = "769bb7c74ad1df6d7d74071cc16a984ff6182e4016e11b8949b93db487977220"
 $PYTHON37_X64_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe"
 $PYTHON37_x64_SHA256 = "e69ed52afb5a722e5c56f6c21d594e85c17cb29f12f18bb69751cf1714e0f987"
 
 $PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.10/python-3.8.10.exe"
 $PYTHON38_x86_SHA256 = "ad07633a1f0cd795f3bf9da33729f662281df196b4567fa795829f3bb38a30ac"
 $PYTHON38_x64_URL = "https://www.python.org/ftp/python/3.8.10/python-3.8.10-amd64.exe"
 $PYTHON38_x64_SHA256 = "7628244cb53408b50639d2c1287c659f4e29d3dfdb9084b11aed5870c0c6a48a"
 
-$PYTHON39_x86_URL = "https://www.python.org/ftp/python/3.9.5/python-3.9.5.exe"
-$PYTHON39_x86_SHA256 = "505129081a839b699a6ab9064b441ad922ef03767b5dd4241fd0c2166baf64de"
-$PYTHON39_x64_URL = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-amd64.exe"
-$PYTHON39_x64_SHA256 = "84d5243088ba00c11e51905c704dbe041040dfff044f4e1ce5476844ee2e6eac"
+$PYTHON39_x86_URL = "https://www.python.org/ftp/python/3.9.9/python-3.9.9.exe"
+$PYTHON39_x86_SHA256 = "6646a5683adf14d35e8c53aab946895bc0f0b825f7acac3a62cc85ee7d0dc71a"
+$PYTHON39_X64_URL = "https://www.python.org/ftp/python/3.9.9/python-3.9.9-amd64.exe"
+$PYTHON39_x64_SHA256 = "137d59e5c0b01a8f1bdcba08344402ae658c81c6bf03b6602bd8b4e951ad0714"
+
+$PYTHON310_x86_URL = "https://www.python.org/ftp/python/3.10.0/python-3.10.0.exe"
+$PYTHON310_x86_SHA256 = "ea896eeefb1db9e12fb89ec77a6e28c9fe52b4a162a34c85d9688be2ec2392e8"
+$PYTHON310_X64_URL = "https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe"
+$PYTHON310_x64_SHA256 = "cb580eb7dc55f9198e650f016645023e8b2224cf7d033857d12880b46c5c94ef"
 
 # PIP 19.2.3.
 $PIP_URL = "https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py"
 $PIP_SHA256 = "57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe"
 
 $INNO_SETUP_URL = "http://files.jrsoftware.org/is/5/innosetup-5.6.1-unicode.exe"
 $INNO_SETUP_SHA256 = "27D49E9BC769E9D1B214C153011978DB90DC01C2ACD1DDCD9ED7B3FE3B96B538"
 
 $MINGW_BIN_URL = "https://osdn.net/frs/redir.php?m=constant&f=mingw%2F68260%2Fmingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip"
 $MINGW_BIN_SHA256 = "2AB8EFD7C7D1FC8EAF8B2FA4DA4EEF8F3E47768284C021599BC7435839A046DF"
 
 $MERCURIAL_WHEEL_FILENAME = "mercurial-5.8.1-cp39-cp39-win_amd64.whl"
 $MERCURIAL_WHEEL_URL = "https://files.pythonhosted.org/packages/5c/b5/a5fa664761eef29b6c90eb24cb09ab8fe2c9b4b86af41d42c17476aff29b/$MERCURIAL_WHEEL_FILENAME"
 $MERCURIAL_WHEEL_SHA256 = "cbf3efa68fd7ebf94691bd00d2c86bbd47ca73620c8faa4f18b6c394bf5f82b0"
 
 $RUSTUP_INIT_URL = "https://static.rust-lang.org/rustup/archive/1.21.1/x86_64-pc-windows-gnu/rustup-init.exe"
 $RUSTUP_INIT_SHA256 = "d17df34ba974b9b19cf5c75883a95475aa22ddc364591d75d174090d55711c72"
 
 $PYOXIDIZER_URL = "https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.17/PyOxidizer-0.17.0-x64.msi"
 $PYOXIDIZER_SHA256 = "85c3bc21a18eb5e2db4dad87cca29accf725c7d59dd364a853ab5099c272024b"
 
 # Writing progress slows down downloads substantially. So disable it.
 $progressPreference = 'silentlyContinue'
 
 function Secure-Download($url, $path, $sha256) {
     if (Test-Path -Path $path) {
         Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
 
         if ($hash.Hash -eq $sha256) {
             Write-Output "SHA256 of $path verified as $sha256"
             return
         }
 
         Write-Output "hash mismatch on $path; downloading again"
     }
 
     Write-Output "downloading $url to $path"
     Invoke-WebRequest -Uri $url -OutFile $path
     Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
 
     if ($hash.Hash -ne $sha256) {
         Remove-Item -Path $path
         throw "hash mismatch when downloading $url; got $($hash.Hash), expected $sha256"
     }
 }
 
 function Invoke-Process($path, $arguments) {
     echo "$path $arguments"
 
     $p = Start-Process -FilePath $path -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden
 
     if ($p.ExitCode -ne 0) {
         throw "process exited non-0: $($p.ExitCode)"
     }
 }
 
 function Install-Python3($name, $installer, $dest, $pip) {
     Write-Output "installing $name"
 
     # We hit this when running the script as part of Simple Systems Manager in
     # EC2. The Python 3 installer doesn't seem to like per-user installs
     # when running as the SYSTEM user. So enable global installs if executed in
     # this mode.
     if ($env:USERPROFILE -eq "C:\Windows\system32\config\systemprofile") {
         Write-Output "running with SYSTEM account; installing for all users"
         $allusers = "1"
     }
     else {
         $allusers = "0"
     }
 
     Invoke-Process $installer "/quiet TargetDir=${dest} InstallAllUsers=${allusers} AssociateFiles=0 CompileAll=0 PrependPath=0 Include_doc=0 Include_launcher=0 InstallLauncherAllUsers=0 Include_pip=0 Include_test=0"
     Invoke-Process ${dest}\python.exe $pip
 }
 
 function Install-Rust($prefix) {
     Write-Output "installing Rust"
     $Env:RUSTUP_HOME = "${prefix}\rustup"
     $Env:CARGO_HOME = "${prefix}\cargo"
 
     Invoke-Process "${prefix}\assets\rustup-init.exe" "-y --default-host x86_64-pc-windows-msvc"
     Invoke-Process "${prefix}\cargo\bin\rustup.exe" "target add i686-pc-windows-msvc"
     Invoke-Process "${prefix}\cargo\bin\rustup.exe" "install 1.52.0"
     Invoke-Process "${prefix}\cargo\bin\rustup.exe" "component add clippy"
 }
 
 function Install-Dependencies($prefix) {
     if (!(Test-Path -Path $prefix\assets)) {
         New-Item -Path $prefix\assets -ItemType Directory
     }
 
     $pip = "${prefix}\assets\get-pip.py"
 
     Secure-Download $PYTHON37_x86_URL ${prefix}\assets\python37-x86.exe $PYTHON37_x86_SHA256
     Secure-Download $PYTHON37_x64_URL ${prefix}\assets\python37-x64.exe $PYTHON37_x64_SHA256
     Secure-Download $PYTHON38_x86_URL ${prefix}\assets\python38-x86.exe $PYTHON38_x86_SHA256
     Secure-Download $PYTHON38_x64_URL ${prefix}\assets\python38-x64.exe $PYTHON38_x64_SHA256
     Secure-Download $PYTHON39_x86_URL ${prefix}\assets\python39-x86.exe $PYTHON39_x86_SHA256
     Secure-Download $PYTHON39_x64_URL ${prefix}\assets\python39-x64.exe $PYTHON39_x64_SHA256
+    Secure-Download $PYTHON310_x86_URL ${prefix}\assets\python310-x86.exe $PYTHON310_x86_SHA256
+    Secure-Download $PYTHON310_x64_URL ${prefix}\assets\python310-x64.exe $PYTHON310_x64_SHA256
     Secure-Download $PIP_URL ${pip} $PIP_SHA256
     Secure-Download $VS_BUILD_TOOLS_URL ${prefix}\assets\vs_buildtools.exe $VS_BUILD_TOOLS_SHA256
     Secure-Download $INNO_SETUP_URL ${prefix}\assets\InnoSetup.exe $INNO_SETUP_SHA256
     Secure-Download $MINGW_BIN_URL ${prefix}\assets\mingw-get-bin.zip $MINGW_BIN_SHA256
     Secure-Download $MERCURIAL_WHEEL_URL ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME} $MERCURIAL_WHEEL_SHA256
     Secure-Download $RUSTUP_INIT_URL ${prefix}\assets\rustup-init.exe $RUSTUP_INIT_SHA256
     Secure-Download $PYOXIDIZER_URL ${prefix}\assets\PyOxidizer.msi $PYOXIDIZER_SHA256
 
     Install-Python3 "Python 3.7 32-bit" ${prefix}\assets\python37-x86.exe ${prefix}\python37-x86 ${pip}
     Install-Python3 "Python 3.7 64-bit" ${prefix}\assets\python37-x64.exe ${prefix}\python37-x64 ${pip}
     Install-Python3 "Python 3.8 32-bit" ${prefix}\assets\python38-x86.exe ${prefix}\python38-x86 ${pip}
 #    Install-Python3 "Python 3.8 64-bit" ${prefix}\assets\python38-x64.exe ${prefix}\python38-x64 ${pip}
     Install-Python3 "Python 3.9 32-bit" ${prefix}\assets\python39-x86.exe ${prefix}\python39-x86 ${pip}
     Install-Python3 "Python 3.9 64-bit" ${prefix}\assets\python39-x64.exe ${prefix}\python39-x64 ${pip}
+    Install-Python3 "Python 3.10 32-bit" ${prefix}\assets\python310-x86.exe ${prefix}\python310-x86 ${pip}
+    Install-Python3 "Python 3.10 64-bit" ${prefix}\assets\python310-x64.exe ${prefix}\python310-x64 ${pip}
 
     Write-Output "installing Visual Studio 2017 Build Tools and SDKs"
     Invoke-Process ${prefix}\assets\vs_buildtools.exe "--quiet --wait --norestart --nocache --channelUri https://aka.ms/vs/15/release/channel --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.VC.140"
 
     Write-Output "installing PyOxidizer"
     Invoke-Process msiexec.exe "/i ${prefix}\assets\PyOxidizer.msi /l* ${prefix}\assets\PyOxidizer.log /quiet"
 
     Install-Rust ${prefix}
 
     Write-Output "installing Inno Setup"
     Invoke-Process ${prefix}\assets\InnoSetup.exe "/SP- /VERYSILENT /SUPPRESSMSGBOXES"
 
     Write-Output "extracting MinGW base archive"
     Expand-Archive -Path ${prefix}\assets\mingw-get-bin.zip -DestinationPath "${prefix}\MinGW" -Force
 
     Write-Output "updating MinGW package catalogs"
     Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "update"
 
     Write-Output "installing MinGW packages"
     Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "install msys-base msys-coreutils msys-diffutils msys-unzip"
 
     # Construct a virtualenv useful for bootstrapping. It conveniently contains a
     # Mercurial install.
     Write-Output "creating bootstrap virtualenv with Mercurial"
     Invoke-Process "$prefix\python39-x64\python.exe" "-m venv ${prefix}\venv-bootstrap"
     Invoke-Process "${prefix}\venv-bootstrap\Scripts\pip.exe" "install ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME}"
 }
 
 function Clone-Mercurial-Repo($prefix, $repo_url, $dest) {
     Write-Output "cloning $repo_url to $dest"
     # TODO Figure out why CA verification isn't working in EC2 and remove
     # --insecure.
     Invoke-Process "${prefix}\venv-bootstrap\Scripts\python.exe" "${prefix}\venv-bootstrap\Scripts\hg clone --insecure $repo_url $dest"
 
     # Mark repo as non-publishing by default for convenience.
     Add-Content -Path "$dest\.hg\hgrc" -Value "`n[phases]`npublish = false"
 }
 
 $prefix = "c:\hgdev"
 Install-Dependencies $prefix
 Clone-Mercurial-Repo $prefix "https://www.mercurial-scm.org/repo/hg" $prefix\src

contrib/packaging/requirements-windows-py3.txt

@@ -1,301 +1,328 b''
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
 #    pip-compile --generate-hashes --output-file=contrib/packaging/requirements-windows-py3.txt contrib/packaging/requirements-windows.txt.in
 #
 atomicwrites==1.4.0 \
     --hash=sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197 \
-    --hash=sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a \
+    --hash=sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a
     # via pytest
 attrs==21.2.0 \
     --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \
-    --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb \
+    --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb
     # via pytest
 cached-property==1.5.2 \
     --hash=sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130 \
-    --hash=sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0 \
+    --hash=sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0
     # via pygit2
 certifi==2021.5.30 \
     --hash=sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee \
-    --hash=sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8 \
+    --hash=sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8
     # via dulwich
-cffi==1.14.4 \
-    --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \
-    --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \
-    --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \
-    --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \
-    --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \
-    --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \
-    --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \
-    --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \
-    --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \
-    --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \
-    --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \
-    --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \
-    --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \
-    --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \
-    --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \
-    --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \
-    --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \
-    --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \
-    --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \
-    --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \
-    --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \
-    --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \
-    --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \
-    --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \
-    --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \
-    --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \
-    --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \
-    --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \
-    --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \
-    --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \
-    --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \
-    --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \
-    --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \
-    --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f \
+cffi==1.15.0 \
+    --hash=sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3 \
+    --hash=sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2 \
+    --hash=sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636 \
+    --hash=sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20 \
+    --hash=sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728 \
+    --hash=sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27 \
+    --hash=sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66 \
+    --hash=sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443 \
+    --hash=sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0 \
+    --hash=sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7 \
+    --hash=sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39 \
+    --hash=sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605 \
+    --hash=sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a \
+    --hash=sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37 \
+    --hash=sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029 \
+    --hash=sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139 \
+    --hash=sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc \
+    --hash=sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df \
+    --hash=sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14 \
+    --hash=sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880 \
+    --hash=sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2 \
+    --hash=sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a \
+    --hash=sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e \
+    --hash=sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474 \
+    --hash=sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024 \
+    --hash=sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8 \
+    --hash=sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0 \
+    --hash=sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e \
+    --hash=sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a \
+    --hash=sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e \
+    --hash=sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032 \
+    --hash=sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6 \
+    --hash=sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e \
+    --hash=sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b \
+    --hash=sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e \
+    --hash=sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954 \
+    --hash=sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962 \
+    --hash=sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c \
+    --hash=sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4 \
+    --hash=sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55 \
+    --hash=sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962 \
+    --hash=sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023 \
+    --hash=sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c \
+    --hash=sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6 \
+    --hash=sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8 \
+    --hash=sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382 \
+    --hash=sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7 \
+    --hash=sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc \
+    --hash=sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997 \
+    --hash=sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796
     # via pygit2
 colorama==0.4.4 \
     --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
-    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \
+    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2
     # via pytest
 docutils==0.16 \
     --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
-    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
+    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc
     # via -r contrib/packaging/requirements-windows.txt.in
 dulwich==0.20.6 ; python_version >= "3" \
     --hash=sha256:1ccd55e38fa9f169290f93e027ab4508202f5bdd6ef534facac4edd3f6903f0d \
     --hash=sha256:2452a0379cc7bbbd7ab893ec104d18039f1ea98b0d6be6bca5646e5cf29e0ae9 \
     --hash=sha256:2f4aebc54ed2d37dcee737024421452375570a422eb682232e676aa7ebc9cb4b \
     --hash=sha256:304f52b10c49c3a6ddfbd73e2e93d8e979350225cfba9688e51110e74fa2f718 \
     --hash=sha256:49e747c72d9099e873bf6196260346d5996c3f28af788294d47a8accdc524de7 \
     --hash=sha256:4fee359928c59b53af153a582a7ed7595259a5a825df400301a29e17fd78dfd3 \
     --hash=sha256:50ef300a9fa4efd9f85009c2bd8b515266ec1529400f8834f85c04fa9f09b2c0 \
     --hash=sha256:5348310f21b2a23847342ce464461499b6652483fa42de03714d0f6421a99698 \
     --hash=sha256:7e7b5dea5178b6493fdb83adccbe81de9ddff55f79880185ed594c0e3a97209b \
     --hash=sha256:8f7a7f973be2beedfb10dd8d3eb6bdf9ec466c72ad555704897cbd6357fe5021 \
     --hash=sha256:bea6e6caffc6c73bfd1647714c5715ab96ac49deb8beb8b67511529afa25685a \
     --hash=sha256:e5871b86a079e9e290f52ab14559cea1b694a0b8ed2b9ebb898f6ced7f14a406 \
-    --hash=sha256:e593f514b8ac740b4ceeb047745b4719bfc9f334904245c6edcb3a9d002f577b \
+    --hash=sha256:e593f514b8ac740b4ceeb047745b4719bfc9f334904245c6edcb3a9d002f577b
     # via -r contrib/packaging/requirements-windows.txt.in
 fuzzywuzzy==0.18.0 \
-    --hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8 \
+    --hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8
     # via -r contrib/packaging/requirements-windows.txt.in
 idna==3.2 \
     --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \
-    --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 \
+    --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3
     # via yarl
 importlib-metadata==3.1.0 \
     --hash=sha256:590690d61efdd716ff82c39ca9a9d4209252adfe288a4b5721181050acbd4175 \
-    --hash=sha256:d9b8a46a0885337627a6430db287176970fff18ad421becec1d64cfc763c2099 \
-    # via keyring, pluggy, pytest
+    --hash=sha256:d9b8a46a0885337627a6430db287176970fff18ad421becec1d64cfc763c2099
+    # via
+    #   keyring
+    #   pluggy
+    #   pytest
 iniconfig==1.1.1 \
     --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
-    --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 \
+    --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32
     # via pytest
 keyring==21.4.0 \
     --hash=sha256:4e34ea2fdec90c1c43d6610b5a5fafa1b9097db1802948e90caf5763974b8f8d \
-    --hash=sha256:9aeadd006a852b78f4b4ef7c7556c2774d2432bbef8ee538a3e9089ac8b11466 \
+    --hash=sha256:9aeadd006a852b78f4b4ef7c7556c2774d2432bbef8ee538a3e9089ac8b11466
     # via -r contrib/packaging/requirements-windows.txt.in
 multidict==5.1.0 \
     --hash=sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a \
     --hash=sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93 \
     --hash=sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632 \
     --hash=sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656 \
     --hash=sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79 \
     --hash=sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7 \
     --hash=sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d \
     --hash=sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5 \
     --hash=sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224 \
     --hash=sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26 \
     --hash=sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea \
     --hash=sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348 \
     --hash=sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6 \
     --hash=sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76 \
     --hash=sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1 \
     --hash=sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f \
     --hash=sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952 \
     --hash=sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a \
     --hash=sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37 \
     --hash=sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9 \
     --hash=sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359 \
     --hash=sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8 \
     --hash=sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da \
     --hash=sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3 \
     --hash=sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d \
     --hash=sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf \
     --hash=sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841 \
     --hash=sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d \
     --hash=sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93 \
     --hash=sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f \
     --hash=sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647 \
     --hash=sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635 \
     --hash=sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456 \
     --hash=sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda \
     --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \
     --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \
-    --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80 \
+    --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80
     # via yarl
 packaging==21.0 \
     --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \
-    --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 \
+    --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14
     # via pytest
 pluggy==0.13.1 \
     --hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \
-    --hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d \
+    --hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d
     # via pytest
 py==1.10.0 \
     --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
-    --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a \
+    --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
     # via pytest
-pycparser==2.20 \
-    --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
-    --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 \
+pycparser==2.21 \
+    --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
+    --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
     # via cffi
-pygit2==1.4.0 ; python_version >= "3" \
-    --hash=sha256:0d298098e286eeda000e49ca7e1b41f87300e10dd8b9d06b32b008bd61f50b83 \
-    --hash=sha256:0ee135eb2cd8b07ce1374f3596cc5c3213472d6389bad6a4c5d87d8e267e93e9 \
-    --hash=sha256:32eb863d6651d4890ced318505ea8dc229bd9637deaf29c898de1ab574d727a0 \
-    --hash=sha256:37d6d7d6d7804c42a0fe23425c72e38093488525092fc5e51a05684e63503ce7 \
-    --hash=sha256:41204b6f3406d9f53147710f3cc485d77181ba67f57c34d36b7c86de1c14a18c \
-    --hash=sha256:818c91b582109d90580c5da74af783738838353f15eb12eeb734d80a974b05a3 \
-    --hash=sha256:8306a302487dac67df7af6a064bb37e8a8eb4138958f9560ff49ff162e185dab \
-    --hash=sha256:9c2f2d9ef59513007b66f6534b000792b614de3faf60313a0a68f6b8571aea85 \
-    --hash=sha256:9c8d5881eb709e2e2e13000b507a131bd5fb91a879581030088d0ddffbcd19af \
-    --hash=sha256:b422e417739def0a136a6355723dfe8a5ffc83db5098076f28a14f1d139779c1 \
-    --hash=sha256:cbeb38ab1df9b5d8896548a11e63aae8a064763ab5f1eabe4475e6b8a78ee1c8 \
-    --hash=sha256:cf00481ddf053e549a6edd0216bdc267b292d261eae02a67bb3737de920cbf88 \
-    --hash=sha256:d0d889144e9487d926fecea947c3f39ce5f477e521d7d467d2e66907e4cd657d \
-    --hash=sha256:ddb7a1f6d38063e8724abfa1cfdfb0f9b25014b8bca0546274b7a84b873a3888 \
-    --hash=sha256:e9037a7d810750fe23c9f5641ef14a0af2525ff03e14752cd4f73e1870ecfcb0 \
-    --hash=sha256:ec5c0365a9bdfcac1609d20868507b28685ec5ea7cc3a2c903c9b62ef2e0bbc0 \
-    --hash=sha256:fdd8ba30cda277290e000322f505132f590cf89bd7d31829b45a3cb57447ec32 \
+pygit2==1.7.1 ; python_version >= "3" \
+    --hash=sha256:2c9e95efb86c0b32cc07c26be3d179e851ca4a7899c47fef63c4203963144f5e \
+    --hash=sha256:3ddacbf461652d3d4900382f821d9fbd5ae2dedecd7862b5245842419ad0ccba \
+    --hash=sha256:4cb0414df6089d0072ebe93ff2f34730737172dd5f0e72289567d06a6caf09c0 \
+    --hash=sha256:56e960dc74f4582bfa3ca17a1a9d542732fc93b5cf8f82574c235d06b2d61eae \
+    --hash=sha256:6b17ab922c2a2d99b30ab9222472b07732bf7261d9f9655a4ea23b4c700049d8 \
+    --hash=sha256:73a7b471f22cb59e8729016de1f447c472b3b2c1cc2b622194e5e3b48a7f5776 \
+    --hash=sha256:761a8850e33822796c1c24d411d5cc2460c04e1a74b04ae8560efd3596bbd6bd \
+    --hash=sha256:7c467e81158f5827b3bca6362e5cc9b92857eff9de65034d338c1f18524b09be \
+    --hash=sha256:7c56e10592e62610a19bd3e2a633aafe3488c57b906c7c2fde0299937f0f0b2f \
+    --hash=sha256:7cc2a8e29cc9598310a78cf58b70d9331277cf374802be8f97d97c4a9e5d8387 \
+    --hash=sha256:812670f7994f31778e873a9eced29d2bbfa91674e8be0ab1e974c8a4bda9cbab \
+    --hash=sha256:8cdb0b1d6c3d24b44f340fed143b16e64ba23fe2a449f1a5db87aaf9339a9dbe \
+    --hash=sha256:91b77a305d8d18b649396e66e832d654cd593a3d29b5728f753f254a04533812 \
+    --hash=sha256:a75bcde32238c77eb0cf7d9698a5aa899408d7ad999a5920a29a7c4b80fdeaa7 \
+    --hash=sha256:b060240cf3038e7a0706bbfc5436dd03b8d5ac797ac1d512b613f4d04b974c80 \
+    --hash=sha256:cdfa61c0428a8182e5a6a1161c017b824cd511574f080a40b10d6413774eb0ca \
+    --hash=sha256:d7faa29558436decc2e78110f38d6677eb366b683ba5cdc2803d47195711165d \
+    --hash=sha256:d831825ad9c3b3c28e6b3ef8a2401ad2d3fd4db5455427ff27175a7e254e2592 \
+    --hash=sha256:df4c477bdfac85d32a1e3180282cd829a0980aa69be9bd0f7cbd4db1778ca72b \
+    --hash=sha256:eced3529bafcaaac015d08dfaa743b3cbad37fcd5b13ae9d280b8b7f716ec5ce \
+    --hash=sha256:fec17e2da668e6bb192d777417aad9c7ca924a166d0a0b9a81a11e00362b1bc7
     # via -r contrib/packaging/requirements-windows.txt.in
 pygments==2.7.1 \
     --hash=sha256:307543fe65c0947b126e83dd5a61bd8acbd84abec11f43caebaf5534cbc17998 \
-    --hash=sha256:926c3f319eda178d1bd90851e4317e6d8cdb5e292a3386aac9bd75eca29cf9c7 \
+    --hash=sha256:926c3f319eda178d1bd90851e4317e6d8cdb5e292a3386aac9bd75eca29cf9c7
     # via -r contrib/packaging/requirements-windows.txt.in
 pyparsing==2.4.7 \
     --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
-    --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+    --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
     # via packaging
-pytest-vcr==1.0.2 \
-    --hash=sha256:23ee51b75abbcc43d926272773aae4f39f93aceb75ed56852d0bf618f92e1896 \
-    # via -r contrib/packaging/requirements-windows.txt.in
 pytest==6.2.4 \
     --hash=sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b \
-    --hash=sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890 \
+    --hash=sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890
     # via pytest-vcr
+pytest-vcr==1.0.2 \
+    --hash=sha256:23ee51b75abbcc43d926272773aae4f39f93aceb75ed56852d0bf618f92e1896
+    # via -r contrib/packaging/requirements-windows.txt.in
 pywin32-ctypes==0.2.0 \
     --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
-    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98 \
-    # via -r contrib/packaging/requirements-windows.txt.in, keyring
+    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
+    # via
+    #   -r contrib/packaging/requirements-windows.txt.in
+    #   keyring
 pyyaml==5.4.1 \
     --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \
     --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \
     --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \
     --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \
     --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \
     --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \
     --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \
     --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \
     --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \
     --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \
     --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \
     --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \
     --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \
     --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \
     --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \
     --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \
     --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \
     --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \
     --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \
     --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \
     --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \
     --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \
     --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \
     --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \
     --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \
     --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \
     --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \
     --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \
-    --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 \
+    --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0
     # via vcrpy
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
-    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via vcrpy
 toml==0.10.2 \
     --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
-    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
     # via pytest
 typing-extensions==3.10.0.0 \
     --hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
     --hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342 \
-    --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 \
+    --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84
     # via yarl
 urllib3==1.25.11 \
     --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
-    --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
+    --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e
     # via dulwich
 vcrpy==4.1.1 \
     --hash=sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162 \
-    --hash=sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599 \
+    --hash=sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599
     # via pytest-vcr
-windows-curses==2.2.0 \
-    --hash=sha256:1452d771ec6f9b3fef037da2b169196a9a12be4e86a6c27dd579adac70c42028 \
-    --hash=sha256:267544e4f60c09af6505e50a69d7f01d7f8a281cf4bd4fc7efc3b32b9a4ef64e \
-    --hash=sha256:389228a3df556102e72450f599283094168aa82eee189f501ad9f131a0fc92e1 \
-    --hash=sha256:84336fe470fa07288daec5c684dec74c0766fec6b3511ccedb4c494804acfbb7 \
-    --hash=sha256:9aa6ff60be76f5de696dc6dbf7897e3b1e6abcf4c0f741e9a0ee22cd6ef382f8 \
-    --hash=sha256:c4a8ce00e82635f06648cc40d99f470be4e3ffeb84f9f7ae9d6a4f68ec6361e7 \
-    --hash=sha256:c5cd032bc7d0f03224ab55c925059d98e81795098d59bbd10f7d05c7ea9677ce \
-    --hash=sha256:fc0be372fe6da3c39d7093154ce029115a927bf287f34b4c615e2b3f8c23dfaa \
+windows-curses==2.3.0 \
+    --hash=sha256:170c0d941c2e0cdf864e7f0441c1bdf0709232bf4aa7ce7f54d90fc76a4c0504 \
+    --hash=sha256:4d5fb991d1b90a41c2332f02241a1f84c8a1e6bc8f6e0d26f532d0da7a9f7b51 \
+    --hash=sha256:7a35eda4cb120b9e1a5ae795f3bc06c55b92c9d391baba6be1903285a05f3551 \
+    --hash=sha256:935be95cfdb9213f6f5d3d5bcd489960e3a8fbc9b574e7b2e8a3a3cc46efff49 \
+    --hash=sha256:a3a63a0597729e10f923724c2cf972a23ea677b400d2387dee1d668cf7116177 \
+    --hash=sha256:c860f596d28377e47f322b7382be4d3573fd76d1292234996bb7f72e0bc0ed0d \
+    --hash=sha256:cc5fa913780d60f4a40824d374a4f8ca45b4e205546e83a2d85147315a57457e \
+    --hash=sha256:d5cde8ec6d582aa77af791eca54f60858339fb3f391945f9cad11b1ab71062e3 \
+    --hash=sha256:e913dc121446d92b33fe4f5bcca26d3a34e4ad19f2af160370d57c3d1e93b4e1 \
+    --hash=sha256:fbc2131cec57e422c6660e6cdb3420aff5be5169b8e45bb7c471f884b0590a2b
     # via -r contrib/packaging/requirements-windows.txt.in
 wrapt==1.12.1 \
-    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 \
+    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7
     # via vcrpy
 yarl==1.6.3 \
     --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \
     --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \
     --hash=sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366 \
     --hash=sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3 \
     --hash=sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec \
     --hash=sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959 \
     --hash=sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e \
     --hash=sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c \
     --hash=sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6 \
     --hash=sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a \
     --hash=sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6 \
     --hash=sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424 \
     --hash=sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e \
     --hash=sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f \
     --hash=sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50 \
     --hash=sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2 \
     --hash=sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc \
     --hash=sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4 \
     --hash=sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970 \
     --hash=sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10 \
     --hash=sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0 \
     --hash=sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406 \
     --hash=sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896 \
     --hash=sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643 \
     --hash=sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721 \
     --hash=sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478 \
     --hash=sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724 \
     --hash=sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e \
     --hash=sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8 \
     --hash=sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96 \
     --hash=sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25 \
     --hash=sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76 \
     --hash=sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2 \
     --hash=sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2 \
     --hash=sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c \
     --hash=sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a \
-    --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71 \
+    --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71
     # via vcrpy
 zipp==3.4.0 \
     --hash=sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108 \
-    --hash=sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb \
+    --hash=sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb
     # via importlib-metadata

contrib/packaging/requirements.txt

@@ -1,49 +1,49 b''
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
 #    pip-compile --generate-hashes --output-file=contrib/packaging/requirements.txt contrib/packaging/requirements.txt.in
 #
 docutils==0.16 \
     --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
-    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
+    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc
     # via -r contrib/packaging/requirements.txt.in
 jinja2==2.11.2 \
     --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
-    --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \
+    --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035
     # via -r contrib/packaging/requirements.txt.in
 markupsafe==1.1.1 \
     --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
     --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
     --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
     --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
     --hash=sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42 \
     --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
     --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
     --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
     --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
     --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
     --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
     --hash=sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b \
     --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
     --hash=sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15 \
     --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
     --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
     --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
     --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
     --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
     --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
     --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
     --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
     --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
     --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
     --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
     --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
     --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
     --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
     --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
     --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
     --hash=sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2 \
     --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
-    --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be \
+    --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be
     # via jinja2

contrib/phab-refresh-stack.sh

@@ -1,41 +1,47 b''
 #!/bin/bash
 set -eu
 
+if [[ "$PHABRICATOR_TOKEN" == "NO-PHAB" ]]; then
+    echo 'Skipping Phabricator Step' >&2
+    exit 0
+fi
+
 revision_in_stack=`hg log \
     --rev '.#stack and ::. and topic()' \
     -T '\nONE-REV\n' \
     | grep 'ONE-REV' | wc -l`
 revision_on_phab=`hg log \
     --rev '.#stack and ::. and topic() and desc("re:\nDifferential Revision: [^\n]+D\d+$")'\
     -T '\nONE-REV\n' \
     | grep 'ONE-REV' | wc -l`
 
 if [[ $revision_in_stack -eq 0 ]]; then
     echo "stack is empty" >&2
     exit 0
 fi
 
 if [[ $revision_on_phab -eq 0 ]]; then
     echo "no tracked diff in this stack" >&2
     exit 0
 fi
 
 if [[ $revision_on_phab -lt $revision_in_stack ]]; then
     echo "not all stack changesets (${revision_in_stack}) have matching Phabricator Diff (${revision_on_phab})" >&2
     exit 2
 fi
 
 if [[ "$PHABRICATOR_TOKEN" == "" ]]; then
     echo 'missing $PHABRICATOR_TOKEN variable' >&2
+    echo '(use PHABRICATOR_TOKEN="NO-PHAB" to disable this step)' >&2
     exit 2
 fi
 
 hg \
 --config extensions.phabricator= \
 --config phabricator.url=https://phab.mercurial-scm.org/ \
 --config phabricator.callsign=HG \
 --config auth.phabricator.schemes=https \
 --config auth.phabricator.prefix=phab.mercurial-scm.org \
 --config auth.phabricator.phabtoken=$PHABRICATOR_TOKEN \
 phabsend --rev '.#stack and ::. and topic()' \
 "$@"

contrib/simplemerge

@@ -1,102 +1,141 b''
 #!/usr/bin/env python3
 from __future__ import absolute_import
 
 import getopt
 import sys
 
 import hgdemandimport
 
 hgdemandimport.enable()
 
 from mercurial.i18n import _
 from mercurial import (
     context,
     error,
     fancyopts,
-    pycompat,
     simplemerge,
     ui as uimod,
+    util,
 )
 from mercurial.utils import procutil, stringutil
 
 options = [
     (b'L', b'label', [], _(b'labels to use on conflict markers')),
     (b'a', b'text', None, _(b'treat all files as text')),
     (b'p', b'print', None, _(b'print results instead of overwriting LOCAL')),
     (b'', b'no-minimal', None, _(b'no effect (DEPRECATED)')),
     (b'h', b'help', None, _(b'display help and exit')),
     (b'q', b'quiet', None, _(b'suppress output')),
 ]
 
 usage = _(
     b'''simplemerge [OPTS] LOCAL BASE OTHER
 
     Simple three-way file merge utility with a minimal feature set.
 
     Apply to LOCAL the changes necessary to go from BASE to OTHER.
 
     By default, LOCAL is overwritten with the results of this operation.
 '''
 )
 
 
 class ParseError(Exception):
     """Exception raised on errors in parsing the command line."""
 
 
 def showhelp():
     procutil.stdout.write(usage)
     procutil.stdout.write(b'\noptions:\n')
 
     out_opts = []
     for shortopt, longopt, default, desc in options:
         out_opts.append(
             (
                 b'%2s%s'
                 % (
                     shortopt and b'-%s' % shortopt,
                     longopt and b' --%s' % longopt,
                 ),
                 b'%s' % desc,
             )
         )
     opts_len = max([len(opt[0]) for opt in out_opts])
     for first, second in out_opts:
         procutil.stdout.write(b' %-*s  %s\n' % (opts_len, first, second))
 
 
+def _verifytext(input, ui, quiet=False, allow_binary=False):
+    """verifies that text is non-binary (unless opts[text] is passed,
+    then we just warn)"""
+    if stringutil.binary(input.text()):
+        msg = _(b"%s looks like a binary file.") % input.fctx.path()
+        if not quiet:
+            ui.warn(_(b'warning: %s\n') % msg)
+        if not allow_binary:
+            sys.exit(1)
+
+
 try:
     for fp in (sys.stdin, procutil.stdout, sys.stderr):
         procutil.setbinary(fp)
 
     opts = {}
     try:
         bargv = [a.encode('utf8') for a in sys.argv[1:]]
         args = fancyopts.fancyopts(bargv, options, opts)
     except getopt.GetoptError as e:
         raise ParseError(e)
     if opts[b'help']:
         showhelp()
         sys.exit(0)
     if len(args) != 3:
         raise ParseError(_(b'wrong number of arguments').decode('utf8'))
+    mode = b'merge'
+    if len(opts[b'label']) > 2:
+        mode = b'merge3'
     local, base, other = args
-    sys.exit(
-        simplemerge.simplemerge(
-            uimod.ui.load(),
-            context.arbitraryfilectx(local),
-            context.arbitraryfilectx(base),
-            context.arbitraryfilectx(other),
-            **pycompat.strkwargs(opts)
-        )
+    overrides = opts[b'label']
+    if len(overrides) > 3:
+        raise error.InputError(b'can only specify three labels.')
+    labels = [local, other, base]
+    labels[: len(overrides)] = overrides
+    local_input = simplemerge.MergeInput(
+        context.arbitraryfilectx(local), labels[0]
+    )
+    other_input = simplemerge.MergeInput(
+        context.arbitraryfilectx(other), labels[1]
+    )
+    base_input = simplemerge.MergeInput(
+        context.arbitraryfilectx(base), labels[2]
     )
+
+    quiet = opts.get(b'quiet')
+    allow_binary = opts.get(b'text')
+    ui = uimod.ui.load()
+    _verifytext(local_input, ui, quiet=quiet, allow_binary=allow_binary)
+    _verifytext(base_input, ui, quiet=quiet, allow_binary=allow_binary)
+    _verifytext(other_input, ui, quiet=quiet, allow_binary=allow_binary)
+
+    merged_text, conflicts = simplemerge.simplemerge(
+        local_input,
+        base_input,
+        other_input,
+        mode,
+        allow_binary=allow_binary,
+    )
+    if opts.get(b'print'):
+        ui.fout.write(merged_text)
+    else:
+        util.writefile(local, merged_text)
+    sys.exit(1 if conflicts else 0)
 except ParseError as e:
     e = stringutil.forcebytestr(e)
     procutil.stdout.write(b"%s: %s\n" % (sys.argv[0].encode('utf8'), e))
     showhelp()
     sys.exit(1)
 except error.Abort as e:
     procutil.stderr.write(b"abort: %s\n" % e)
     sys.exit(255)
 except KeyboardInterrupt:
     sys.exit(255)

hgext/blackbox.py

@@ -1,232 +1,241 b''
 # blackbox.py - log repository events to a file for post-mortem debugging
 #
 # Copyright 2010 Nicolas Dumazet
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """log repository events to a blackbox for debugging
 
 Logs event information to .hg/blackbox.log to help debug and diagnose problems.
 The events that get logged can be configured via the blackbox.track and
 blackbox.ignore config keys.
 
 Examples::
 
   [blackbox]
   track = *
   ignore = pythonhook
   # dirty is *EXPENSIVE* (slow);
   # each log entry indicates `+` if the repository is dirty, like :hg:`id`.
   dirty = True
   # record the source of log messages
   logsource = True
 
   [blackbox]
   track = command, commandfinish, commandexception, exthook, pythonhook
 
   [blackbox]
   track = incoming
 
   [blackbox]
   # limit the size of a log file
   maxsize = 1.5 MB
   # rotate up to N log files when the current one gets too big
   maxfiles = 3
 
   [blackbox]
-  # Include nanoseconds in log entries with %f (see Python function
+  # Include microseconds in log entries with %f (see Python function
   # datetime.datetime.strftime)
   date-format = %Y-%m-%d @ %H:%M:%S.%f
 
 """
 
 from __future__ import absolute_import
 
 import re
 
 from mercurial.i18n import _
 from mercurial.node import hex
 
 from mercurial import (
     encoding,
     loggingutil,
     registrar,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'blackbox',
     b'dirty',
     default=False,
 )
 configitem(
     b'blackbox',
     b'maxsize',
     default=b'1 MB',
 )
 configitem(
     b'blackbox',
     b'logsource',
     default=False,
 )
 configitem(
     b'blackbox',
     b'maxfiles',
     default=7,
 )
 configitem(
     b'blackbox',
     b'track',
     default=lambda: [b'*'],
 )
 configitem(
     b'blackbox',
     b'ignore',
     default=lambda: [b'chgserver', b'cmdserver', b'extension'],
 )
-configitem(
-    b'blackbox',
-    b'date-format',
-    default=b'%Y/%m/%d %H:%M:%S',
-)
+configitem(b'blackbox', b'date-format', default=b'')
 
 _lastlogger = loggingutil.proxylogger()
 
 
 class blackboxlogger(object):
     def __init__(self, ui, repo):
         self._repo = repo
         self._trackedevents = set(ui.configlist(b'blackbox', b'track'))
         self._ignoredevents = set(ui.configlist(b'blackbox', b'ignore'))
         self._maxfiles = ui.configint(b'blackbox', b'maxfiles')
         self._maxsize = ui.configbytes(b'blackbox', b'maxsize')
         self._inlog = False
 
     def tracked(self, event):
         return (
             b'*' in self._trackedevents and event not in self._ignoredevents
         ) or event in self._trackedevents
 
     def log(self, ui, event, msg, opts):
         # self._log() -> ctx.dirty() may create new subrepo instance, which
         # ui is derived from baseui. So the recursion guard in ui.log()
         # doesn't work as it's local to the ui instance.
         if self._inlog:
             return
         self._inlog = True
         try:
             self._log(ui, event, msg, opts)
         finally:
             self._inlog = False
 
     def _log(self, ui, event, msg, opts):
         default = ui.configdate(b'devel', b'default-date')
-        date = dateutil.datestr(default, ui.config(b'blackbox', b'date-format'))
+        dateformat = ui.config(b'blackbox', b'date-format')
+        if dateformat:
+            date = dateutil.datestr(default, dateformat)
+        else:
+            # We want to display milliseconds (more precision seems
+            # unnecessary).  Since %.3f is not supported, use %f and truncate
+            # microseconds.
+            date = dateutil.datestr(default, b'%Y-%m-%d %H:%M:%S.%f')[:-3]
         user = procutil.getuser()
         pid = b'%d' % procutil.getpid()
         changed = b''
         ctx = self._repo[None]
         parents = ctx.parents()
         rev = b'+'.join([hex(p.node()) for p in parents])
         if ui.configbool(b'blackbox', b'dirty') and ctx.dirty(
             missing=True, merge=False, branch=False
         ):
             changed = b'+'
         if ui.configbool(b'blackbox', b'logsource'):
             src = b' [%s]' % event
         else:
             src = b''
         try:
             fmt = b'%s %s @%s%s (%s)%s> %s'
             args = (date, user, rev, changed, pid, src, msg)
             with loggingutil.openlogfile(
                 ui,
                 self._repo.vfs,
                 name=b'blackbox.log',
                 maxfiles=self._maxfiles,
                 maxsize=self._maxsize,
             ) as fp:
                 fp.write(fmt % args)
         except (IOError, OSError) as err:
             # deactivate this to avoid failed logging again
             self._trackedevents.clear()
             ui.debug(
                 b'warning: cannot write to blackbox.log: %s\n'
                 % encoding.strtolocal(err.strerror)
             )
             return
         _lastlogger.logger = self
 
 
 def uipopulate(ui):
     ui.setlogger(b'blackbox', _lastlogger)
 
 
 def reposetup(ui, repo):
     # During 'hg pull' a httppeer repo is created to represent the remote repo.
     # It doesn't have a .hg directory to put a blackbox in, so we don't do
     # the blackbox setup for it.
     if not repo.local():
         return
 
     # Since blackbox.log is stored in the repo directory, the logger should be
     # instantiated per repository.
     logger = blackboxlogger(ui, repo)
     ui.setlogger(b'blackbox', logger)
 
     # Set _lastlogger even if ui.log is not called. This gives blackbox a
     # fallback place to log
     if _lastlogger.logger is None:
         _lastlogger.logger = logger
 
     repo._wlockfreeprefix.add(b'blackbox.log')
 
 
 @command(
     b'blackbox',
     [
         (b'l', b'limit', 10, _(b'the number of events to show')),
     ],
     _(b'hg blackbox [OPTION]...'),
     helpcategory=command.CATEGORY_MAINTENANCE,
     helpbasic=True,
 )
 def blackbox(ui, repo, *revs, **opts):
     """view the recent repository events"""
 
     if not repo.vfs.exists(b'blackbox.log'):
         return
 
     limit = opts.get('limit')
     fp = repo.vfs(b'blackbox.log', b'r')
     lines = fp.read().split(b'\n')
 
     count = 0
     output = []
     for line in reversed(lines):
         if count >= limit:
             break
 
-        # count the commands by matching lines like: 2013/01/23 19:13:36 root>
-        if re.match(br'^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} .*> .*', line):
+        # count the commands by matching lines like:
+        # 2013/01/23 19:13:36 root>
+        # 2013/01/23 19:13:36 root (1234)>
+        # 2013/01/23 19:13:36 root @0000000000000000000000000000000000000000 (1234)>
+        # 2013-01-23 19:13:36.000 root @0000000000000000000000000000000000000000 (1234)>
+        if re.match(
+            br'^\d{4}[-/]\d{2}[-/]\d{2} \d{2}:\d{2}:\d{2}(.\d*)? .*> .*', line
+        ):
             count += 1
         output.append(line)
 
     ui.status(b'\n'.join(reversed(output)))

hgext/commitextras.py

@@ -1,89 +1,89 b''
 # commitextras.py
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''adds a new flag extras to commit (ADVANCED)'''
 
 from __future__ import absolute_import
 
 import re
 
 from mercurial.i18n import _
 from mercurial import (
     commands,
     error,
     extensions,
     registrar,
     util,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 testedwith = b'ships-with-hg-core'
 
 usedinternally = {
     b'amend_source',
     b'branch',
     b'close',
     b'histedit_source',
     b'topic',
     b'rebase_source',
     b'intermediate-source',
     b'__touch-noise__',
     b'source',
     b'transplant_source',
 }
 
 
 def extsetup(ui):
     entry = extensions.wrapcommand(commands.table, b'commit', _commit)
     options = entry[1]
     options.append(
         (
             b'',
             b'extra',
             [],
             _(b'set a changeset\'s extra values'),
             _(b"KEY=VALUE"),
         )
     )
 
 
 def _commit(orig, ui, repo, *pats, **opts):
     if util.safehasattr(repo, 'unfiltered'):
         repo = repo.unfiltered()
 
     class repoextra(repo.__class__):
         def commit(self, *innerpats, **inneropts):
             extras = opts.get('extra')
             for raw in extras:
                 if b'=' not in raw:
                     msg = _(
                         b"unable to parse '%s', should follow "
                         b"KEY=VALUE format"
                     )
-                    raise error.Abort(msg % raw)
+                    raise error.InputError(msg % raw)
                 k, v = raw.split(b'=', 1)
                 if not k:
                     msg = _(b"unable to parse '%s', keys can't be empty")
-                    raise error.Abort(msg % raw)
+                    raise error.InputError(msg % raw)
                 if re.search(br'[^\w-]', k):
                     msg = _(
                         b"keys can only contain ascii letters, digits,"
                         b" '_' and '-'"
                     )
-                    raise error.Abort(msg)
+                    raise error.InputError(msg)
                 if k in usedinternally:
                     msg = _(
                         b"key '%s' is used internally, can't be set "
                         b"manually"
                     )
-                    raise error.Abort(msg % k)
+                    raise error.InputError(msg % k)
                 inneropts['extra'][k] = v
             return super(repoextra, self).commit(*innerpats, **inneropts)
 
     repo.__class__ = repoextra
     return orig(ui, repo, *pats, **opts)

hgext/convert/hg.py

@@ -1,732 +1,733 b''
 # hg.py - hg backend for convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Notes for hg->hg conversion:
 #
 # * Old versions of Mercurial didn't trim the whitespace from the ends
 #   of commit messages, but new versions do.  Changesets created by
 #   those older versions, then converted, may thus have different
 #   hashes for changesets that are otherwise identical.
 #
 # * Using "--config convert.hg.saverev=true" will make the source
 #   identifier to be stored in the converted revision. This will cause
 #   the converted revision to have a different identity than the
 #   source.
 from __future__ import absolute_import
 
 import os
 import re
 import time
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial.node import (
     bin,
     hex,
     sha1nodeconstants,
 )
 from mercurial import (
     bookmarks,
     context,
     error,
     exchange,
     hg,
     lock as lockmod,
     logcmdutil,
     merge as mergemod,
+    mergestate,
     phases,
     pycompat,
     util,
 )
 from mercurial.utils import dateutil
 
 stringio = util.stringio
 
 from . import common
 
 mapfile = common.mapfile
 NoRepo = common.NoRepo
 
 sha1re = re.compile(br'\b[0-9a-f]{12,40}\b')
 
 
 class mercurial_sink(common.converter_sink):
     def __init__(self, ui, repotype, path):
         common.converter_sink.__init__(self, ui, repotype, path)
         self.branchnames = ui.configbool(b'convert', b'hg.usebranchnames')
         self.clonebranches = ui.configbool(b'convert', b'hg.clonebranches')
         self.tagsbranch = ui.config(b'convert', b'hg.tagsbranch')
         self.lastbranch = None
         if os.path.isdir(path) and len(os.listdir(path)) > 0:
             try:
                 self.repo = hg.repository(self.ui, path)
                 if not self.repo.local():
                     raise NoRepo(
                         _(b'%s is not a local Mercurial repository') % path
                     )
             except error.RepoError as err:
                 ui.traceback()
                 raise NoRepo(err.args[0])
         else:
             try:
                 ui.status(_(b'initializing destination %s repository\n') % path)
                 self.repo = hg.repository(self.ui, path, create=True)
                 if not self.repo.local():
                     raise NoRepo(
                         _(b'%s is not a local Mercurial repository') % path
                     )
                 self.created.append(path)
             except error.RepoError:
                 ui.traceback()
                 raise NoRepo(
                     _(b"could not create hg repository %s as sink") % path
                 )
         self.lock = None
         self.wlock = None
         self.filemapmode = False
         self.subrevmaps = {}
 
     def before(self):
         self.ui.debug(b'run hg sink pre-conversion action\n')
         self.wlock = self.repo.wlock()
         self.lock = self.repo.lock()
 
     def after(self):
         self.ui.debug(b'run hg sink post-conversion action\n')
         if self.lock:
             self.lock.release()
         if self.wlock:
             self.wlock.release()
 
     def revmapfile(self):
         return self.repo.vfs.join(b"shamap")
 
     def authorfile(self):
         return self.repo.vfs.join(b"authormap")
 
     def setbranch(self, branch, pbranches):
         if not self.clonebranches:
             return
 
         setbranch = branch != self.lastbranch
         self.lastbranch = branch
         if not branch:
             branch = b'default'
         pbranches = [(b[0], b[1] and b[1] or b'default') for b in pbranches]
 
         branchpath = os.path.join(self.path, branch)
         if setbranch:
             self.after()
             try:
                 self.repo = hg.repository(self.ui, branchpath)
             except Exception:
                 self.repo = hg.repository(self.ui, branchpath, create=True)
             self.before()
 
         # pbranches may bring revisions from other branches (merge parents)
         # Make sure we have them, or pull them.
         missings = {}
         for b in pbranches:
             try:
                 self.repo.lookup(b[0])
             except Exception:
                 missings.setdefault(b[1], []).append(b[0])
 
         if missings:
             self.after()
             for pbranch, heads in sorted(pycompat.iteritems(missings)):
                 pbranchpath = os.path.join(self.path, pbranch)
                 prepo = hg.peer(self.ui, {}, pbranchpath)
                 self.ui.note(
                     _(b'pulling from %s into %s\n') % (pbranch, branch)
                 )
                 exchange.pull(
                     self.repo, prepo, heads=[prepo.lookup(h) for h in heads]
                 )
             self.before()
 
     def _rewritetags(self, source, revmap, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(b' ', 1)
             if len(s) != 2:
                 self.ui.warn(_(b'invalid tag entry: "%s"\n') % line)
                 fp.write(b'%s\n' % line)  # Bogus, but keep for hash stability
                 continue
             revid = revmap.get(source.lookuprev(s[0]))
             if not revid:
                 if s[0] == sha1nodeconstants.nullhex:
                     revid = s[0]
                 else:
                     # missing, but keep for hash stability
                     self.ui.warn(_(b'missing tag entry: "%s"\n') % line)
                     fp.write(b'%s\n' % line)
                     continue
             fp.write(b'%s %s\n' % (revid, s[1]))
         return fp.getvalue()
 
     def _rewritesubstate(self, source, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(b' ', 1)
             if len(s) != 2:
                 continue
 
             revid = s[0]
             subpath = s[1]
             if revid != sha1nodeconstants.nullhex:
                 revmap = self.subrevmaps.get(subpath)
                 if revmap is None:
                     revmap = mapfile(
                         self.ui, self.repo.wjoin(subpath, b'.hg/shamap')
                     )
                     self.subrevmaps[subpath] = revmap
 
                     # It is reasonable that one or more of the subrepos don't
                     # need to be converted, in which case they can be cloned
                     # into place instead of converted.  Therefore, only warn
                     # once.
                     msg = _(b'no ".hgsubstate" updates will be made for "%s"\n')
                     if len(revmap) == 0:
                         sub = self.repo.wvfs.reljoin(subpath, b'.hg')
 
                         if self.repo.wvfs.exists(sub):
                             self.ui.warn(msg % subpath)
 
                 newid = revmap.get(revid)
                 if not newid:
                     if len(revmap) > 0:
                         self.ui.warn(
                             _(b"%s is missing from %s/.hg/shamap\n")
                             % (revid, subpath)
                         )
                 else:
                     revid = newid
 
             fp.write(b'%s %s\n' % (revid, subpath))
 
         return fp.getvalue()
 
     def _calculatemergedfiles(self, source, p1ctx, p2ctx):
         """Calculates the files from p2 that we need to pull in when merging p1
         and p2, given that the merge is coming from the given source.
 
         This prevents us from losing files that only exist in the target p2 and
         that don't come from the source repo (like if you're merging multiple
         repositories together).
         """
         anc = [p1ctx.ancestor(p2ctx)]
         # Calculate what files are coming from p2
         # TODO: mresult.commitinfo might be able to get that info
         mresult = mergemod.calculateupdates(
             self.repo,
             p1ctx,
             p2ctx,
             anc,
             branchmerge=True,
             force=True,
             acceptremote=False,
             followcopies=False,
         )
 
         for file, (action, info, msg) in mresult.filemap():
             if source.targetfilebelongstosource(file):
                 # If the file belongs to the source repo, ignore the p2
                 # since it will be covered by the existing fileset.
                 continue
 
             # If the file requires actual merging, abort. We don't have enough
             # context to resolve merges correctly.
-            if action in [b'm', b'dm', b'cd', b'dc']:
+            if action in mergestate.CONVERT_MERGE_ACTIONS:
                 raise error.Abort(
                     _(
                         b"unable to convert merge commit "
                         b"since target parents do not merge cleanly (file "
                         b"%s, parents %s and %s)"
                     )
                     % (file, p1ctx, p2ctx)
                 )
-            elif action == b'k':
+            elif action == mergestate.ACTION_KEEP:
                 # 'keep' means nothing changed from p1
                 continue
             else:
                 # Any other change means we want to take the p2 version
                 yield file
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         files = dict(files)
 
         def getfilectx(repo, memctx, f):
             if p2ctx and f in p2files and f not in copies:
                 self.ui.debug(b'reusing %s from p2\n' % f)
                 try:
                     return p2ctx[f]
                 except error.ManifestLookupError:
                     # If the file doesn't exist in p2, then we're syncing a
                     # delete, so just return None.
                     return None
             try:
                 v = files[f]
             except KeyError:
                 return None
             data, mode = source.getfile(f, v)
             if data is None:
                 return None
             if f == b'.hgtags':
                 data = self._rewritetags(source, revmap, data)
             if f == b'.hgsubstate':
                 data = self._rewritesubstate(source, data)
             return context.memfilectx(
                 self.repo,
                 memctx,
                 f,
                 data,
                 b'l' in mode,
                 b'x' in mode,
                 copies.get(f),
             )
 
         pl = []
         for p in parents:
             if p not in pl:
                 pl.append(p)
         parents = pl
         nparents = len(parents)
         if self.filemapmode and nparents == 1:
             m1node = self.repo.changelog.read(bin(parents[0]))[0]
             parent = parents[0]
 
         if len(parents) < 2:
             parents.append(self.repo.nullid)
         if len(parents) < 2:
             parents.append(self.repo.nullid)
         p2 = parents.pop(0)
 
         text = commit.desc
 
         sha1s = re.findall(sha1re, text)
         for sha1 in sha1s:
             oldrev = source.lookuprev(sha1)
             newrev = revmap.get(oldrev)
             if newrev is not None:
                 text = text.replace(sha1, newrev[: len(sha1)])
 
         extra = commit.extra.copy()
 
         sourcename = self.repo.ui.config(b'convert', b'hg.sourcename')
         if sourcename:
             extra[b'convert_source'] = sourcename
 
         for label in (
             b'source',
             b'transplant_source',
             b'rebase_source',
             b'intermediate-source',
         ):
             node = extra.get(label)
 
             if node is None:
                 continue
 
             # Only transplant stores its reference in binary
             if label == b'transplant_source':
                 node = hex(node)
 
             newrev = revmap.get(node)
             if newrev is not None:
                 if label == b'transplant_source':
                     newrev = bin(newrev)
 
                 extra[label] = newrev
 
         if self.branchnames and commit.branch:
             extra[b'branch'] = commit.branch
         if commit.rev and commit.saverev:
             extra[b'convert_revision'] = commit.rev
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
             p1ctx = self.repo[p1]
             p2ctx = None
             if p2 != self.repo.nullid:
                 p2ctx = self.repo[p2]
             fileset = set(files)
             if full:
                 fileset.update(self.repo[p1])
                 fileset.update(self.repo[p2])
 
             if p2ctx:
                 p2files = set(cleanp2)
                 for file in self._calculatemergedfiles(source, p1ctx, p2ctx):
                     p2files.add(file)
                     fileset.add(file)
 
             ctx = context.memctx(
                 self.repo,
                 (p1, p2),
                 text,
                 fileset,
                 getfilectx,
                 commit.author,
                 commit.date,
                 extra,
             )
 
             # We won't know if the conversion changes the node until after the
             # commit, so copy the source's phase for now.
             self.repo.ui.setconfig(
                 b'phases',
                 b'new-commit',
                 phases.phasenames[commit.phase],
                 b'convert',
             )
 
             with self.repo.transaction(b"convert") as tr:
                 if self.repo.ui.config(b'convert', b'hg.preserve-hash'):
                     origctx = commit.ctx
                 else:
                     origctx = None
                 node = hex(self.repo.commitctx(ctx, origctx=origctx))
 
                 # If the node value has changed, but the phase is lower than
                 # draft, set it back to draft since it hasn't been exposed
                 # anywhere.
                 if commit.rev != node:
                     ctx = self.repo[node]
                     if ctx.phase() < phases.draft:
                         phases.registernew(
                             self.repo, tr, phases.draft, [ctx.rev()]
                         )
 
             text = b"(octopus merge fixup)\n"
             p2 = node
 
         if self.filemapmode and nparents == 1:
             man = self.repo.manifestlog.getstorage(b'')
             mnode = self.repo.changelog.read(bin(p2))[0]
             closed = b'close' in commit.extra
             if not closed and not man.cmp(m1node, man.revision(mnode)):
                 self.ui.status(_(b"filtering out empty revision\n"))
                 self.repo.rollback(force=True)
                 return parent
         return p2
 
     def puttags(self, tags):
         tagparent = self.repo.branchtip(self.tagsbranch, ignoremissing=True)
         tagparent = tagparent or self.repo.nullid
 
         oldlines = set()
         for branch, heads in pycompat.iteritems(self.repo.branchmap()):
             for h in heads:
                 if b'.hgtags' in self.repo[h]:
                     oldlines.update(
                         set(self.repo[h][b'.hgtags'].data().splitlines(True))
                     )
         oldlines = sorted(list(oldlines))
 
         newlines = sorted([(b"%s %s\n" % (tags[tag], tag)) for tag in tags])
         if newlines == oldlines:
             return None, None
 
         # if the old and new tags match, then there is nothing to update
         oldtags = set()
         newtags = set()
         for line in oldlines:
             s = line.strip().split(b' ', 1)
             if len(s) != 2:
                 continue
             oldtags.add(s[1])
         for line in newlines:
             s = line.strip().split(b' ', 1)
             if len(s) != 2:
                 continue
             if s[1] not in oldtags:
                 newtags.add(s[1].strip())
 
         if not newtags:
             return None, None
 
         data = b"".join(newlines)
 
         def getfilectx(repo, memctx, f):
             return context.memfilectx(repo, memctx, f, data, False, False, None)
 
         self.ui.status(_(b"updating tags\n"))
         date = b"%d 0" % int(time.mktime(time.gmtime()))
         extra = {b'branch': self.tagsbranch}
         ctx = context.memctx(
             self.repo,
             (tagparent, None),
             b"update tags",
             [b".hgtags"],
             getfilectx,
             b"convert-repo",
             date,
             extra,
         )
         node = self.repo.commitctx(ctx)
         return hex(node), hex(tagparent)
 
     def setfilemapmode(self, active):
         self.filemapmode = active
 
     def putbookmarks(self, updatedbookmark):
         if not len(updatedbookmark):
             return
         wlock = lock = tr = None
         try:
             wlock = self.repo.wlock()
             lock = self.repo.lock()
             tr = self.repo.transaction(b'bookmark')
             self.ui.status(_(b"updating bookmarks\n"))
             destmarks = self.repo._bookmarks
             changes = [
                 (bookmark, bin(updatedbookmark[bookmark]))
                 for bookmark in updatedbookmark
             ]
             destmarks.applychanges(self.repo, tr, changes)
             tr.close()
         finally:
             lockmod.release(lock, wlock, tr)
 
     def hascommitfrommap(self, rev):
         # the exact semantics of clonebranches is unclear so we can't say no
         return rev in self.repo or self.clonebranches
 
     def hascommitforsplicemap(self, rev):
         if rev not in self.repo and self.clonebranches:
             raise error.Abort(
                 _(
                     b'revision %s not found in destination '
                     b'repository (lookups with clonebranches=true '
                     b'are not implemented)'
                 )
                 % rev
             )
         return rev in self.repo
 
 
 class mercurial_source(common.converter_source):
     def __init__(self, ui, repotype, path, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         self.ignoreerrors = ui.configbool(b'convert', b'hg.ignoreerrors')
         self.ignored = set()
         self.saverev = ui.configbool(b'convert', b'hg.saverev')
         try:
             self.repo = hg.repository(self.ui, path)
             # try to provoke an exception if this isn't really a hg
             # repo, but some other bogus compatible-looking url
             if not self.repo.local():
                 raise error.RepoError
         except error.RepoError:
             ui.traceback()
             raise NoRepo(_(b"%s is not a local Mercurial repository") % path)
         self.lastrev = None
         self.lastctx = None
         self._changescache = None, None
         self.convertfp = None
         # Restrict converted revisions to startrev descendants
         startnode = ui.config(b'convert', b'hg.startrev')
         hgrevs = ui.config(b'convert', b'hg.revs')
         if hgrevs is None:
             if startnode is not None:
                 try:
                     startnode = self.repo.lookup(startnode)
                 except error.RepoError:
                     raise error.Abort(
                         _(b'%s is not a valid start revision') % startnode
                     )
                 startrev = self.repo.changelog.rev(startnode)
                 children = {startnode: 1}
                 for r in self.repo.changelog.descendants([startrev]):
                     children[self.repo.changelog.node(r)] = 1
                 self.keep = children.__contains__
             else:
                 self.keep = util.always
             if revs:
                 self._heads = [self.repo.lookup(r) for r in revs]
             else:
                 self._heads = self.repo.heads()
         else:
             if revs or startnode is not None:
                 raise error.Abort(
                     _(
                         b'hg.revs cannot be combined with '
                         b'hg.startrev or --rev'
                     )
                 )
             nodes = set()
             parents = set()
             for r in logcmdutil.revrange(self.repo, [hgrevs]):
                 ctx = self.repo[r]
                 nodes.add(ctx.node())
                 parents.update(p.node() for p in ctx.parents())
             self.keep = nodes.__contains__
             self._heads = nodes - parents
 
     def _changectx(self, rev):
         if self.lastrev != rev:
             self.lastctx = self.repo[rev]
             self.lastrev = rev
         return self.lastctx
 
     def _parents(self, ctx):
         return [p for p in ctx.parents() if p and self.keep(p.node())]
 
     def getheads(self):
         return [hex(h) for h in self._heads if self.keep(h)]
 
     def getfile(self, name, rev):
         try:
             fctx = self._changectx(rev)[name]
             return fctx.data(), fctx.flags()
         except error.LookupError:
             return None, None
 
     def _changedfiles(self, ctx1, ctx2):
         ma, r = [], []
         maappend = ma.append
         rappend = r.append
         d = ctx1.manifest().diff(ctx2.manifest())
         for f, ((node1, flag1), (node2, flag2)) in pycompat.iteritems(d):
             if node2 is None:
                 rappend(f)
             else:
                 maappend(f)
         return ma, r
 
     def getchanges(self, rev, full):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if full or not parents:
             files = copyfiles = ctx.manifest()
         if parents:
             if self._changescache[0] == rev:
                 ma, r = self._changescache[1]
             else:
                 ma, r = self._changedfiles(parents[0], ctx)
             if not full:
                 files = ma + r
             copyfiles = ma
         # _getcopies() is also run for roots and before filtering so missing
         # revlogs are detected early
         copies = self._getcopies(ctx, parents, copyfiles)
         cleanp2 = set()
         if len(parents) == 2:
             d = parents[1].manifest().diff(ctx.manifest(), clean=True)
             for f, value in pycompat.iteritems(d):
                 if value is None:
                     cleanp2.add(f)
         changes = [(f, rev) for f in files if f not in self.ignored]
         changes.sort()
         return changes, copies, cleanp2
 
     def _getcopies(self, ctx, parents, files):
         copies = {}
         for name in files:
             if name in self.ignored:
                 continue
             try:
                 copysource = ctx.filectx(name).copysource()
                 if copysource in self.ignored:
                     continue
                 # Ignore copy sources not in parent revisions
                 if not any(copysource in p for p in parents):
                     continue
                 copies[name] = copysource
             except TypeError:
                 pass
             except error.LookupError as e:
                 if not self.ignoreerrors:
                     raise
                 self.ignored.add(name)
                 self.ui.warn(_(b'ignoring: %s\n') % e)
         return copies
 
     def getcommit(self, rev):
         ctx = self._changectx(rev)
         _parents = self._parents(ctx)
         parents = [p.hex() for p in _parents]
         optparents = [p.hex() for p in ctx.parents() if p and p not in _parents]
         crev = rev
 
         return common.commit(
             author=ctx.user(),
             date=dateutil.datestr(ctx.date(), b'%Y-%m-%d %H:%M:%S %1%2'),
             desc=ctx.description(),
             rev=crev,
             parents=parents,
             optparents=optparents,
             branch=ctx.branch(),
             extra=ctx.extra(),
             sortkey=ctx.rev(),
             saverev=self.saverev,
             phase=ctx.phase(),
             ctx=ctx,
         )
 
     def numcommits(self):
         return len(self.repo)
 
     def gettags(self):
         # This will get written to .hgtags, filter non global tags out.
         tags = [
             t
             for t in self.repo.tagslist()
             if self.repo.tagtype(t[0]) == b'global'
         ]
         return {name: hex(node) for name, node in tags if self.keep(node)}
 
     def getchangedfiles(self, rev, i):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if not parents and i is None:
             i = 0
             ma, r = ctx.manifest().keys(), []
         else:
             i = i or 0
             ma, r = self._changedfiles(parents[i], ctx)
         ma, r = [[f for f in l if f not in self.ignored] for l in (ma, r)]
 
         if i == 0:
             self._changescache = (rev, (ma, r))
 
         return ma + r
 
     def converted(self, rev, destrev):
         if self.convertfp is None:
             self.convertfp = open(self.repo.vfs.join(b'shamap'), b'ab')
         self.convertfp.write(util.tonativeeol(b'%s %s\n' % (destrev, rev)))
         self.convertfp.flush()
 
     def before(self):
         self.ui.debug(b'run hg source pre-conversion action\n')
 
     def after(self):
         self.ui.debug(b'run hg source post-conversion action\n')
 
     def hasnativeorder(self):
         return True
 
     def hasnativeclose(self):
         return True
 
     def lookuprev(self, rev):
         try:
             return hex(self.repo.lookup(rev))
         except (error.RepoError, error.LookupError):
             return None
 
     def getbookmarks(self):
         return bookmarks.listbookmarks(self.repo)
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """Mercurial, revision string is a 40 byte hex"""
         self.checkhexformat(revstr, mapname)

hgext/fix.py

@@ -1,971 +1,958 b''
 # fix - rewrite file content in changesets and working copy
 #
 # Copyright 2018 Google LLC.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """rewrite file content in changesets or working copy (EXPERIMENTAL)
 
 Provides a command that runs configured tools on the contents of modified files,
 writing back any fixes to the working copy or replacing changesets.
 
 Here is an example configuration that causes :hg:`fix` to apply automatic
 formatting fixes to modified lines in C++ code::
 
   [fix]
   clang-format:command=clang-format --assume-filename={rootpath}
   clang-format:linerange=--lines={first}:{last}
   clang-format:pattern=set:**.cpp or **.hpp
 
 The :command suboption forms the first part of the shell command that will be
 used to fix a file. The content of the file is passed on standard input, and the
 fixed file content is expected on standard output. Any output on standard error
 will be displayed as a warning. If the exit status is not zero, the file will
 not be affected. A placeholder warning is displayed if there is a non-zero exit
 status but no standard error output. Some values may be substituted into the
 command::
 
   {rootpath}  The path of the file being fixed, relative to the repo root
   {basename}  The name of the file being fixed, without the directory path
 
 If the :linerange suboption is set, the tool will only be run if there are
 changed lines in a file. The value of this suboption is appended to the shell
 command once for every range of changed lines in the file. Some values may be
 substituted into the command::
 
   {first}   The 1-based line number of the first line in the modified range
   {last}    The 1-based line number of the last line in the modified range
 
 Deleted sections of a file will be ignored by :linerange, because there is no
 corresponding line range in the version being fixed.
 
 By default, tools that set :linerange will only be executed if there is at least
 one changed line range. This is meant to prevent accidents like running a code
 formatter in such a way that it unexpectedly reformats the whole file. If such a
 tool needs to operate on unchanged files, it should set the :skipclean suboption
 to false.
 
 The :pattern suboption determines which files will be passed through each
 configured tool. See :hg:`help patterns` for possible values. However, all
 patterns are relative to the repo root, even if that text says they are relative
 to the current working directory. If there are file arguments to :hg:`fix`, the
 intersection of these patterns is used.
 
 There is also a configurable limit for the maximum size of file that will be
 processed by :hg:`fix`::
 
   [fix]
   maxfilesize = 2MB
 
 Normally, execution of configured tools will continue after a failure (indicated
 by a non-zero exit status). It can also be configured to abort after the first
 such failure, so that no files will be affected if any tool fails. This abort
 will also cause :hg:`fix` to exit with a non-zero status::
 
   [fix]
   failure = abort
 
 When multiple tools are configured to affect a file, they execute in an order
 defined by the :priority suboption. The priority suboption has a default value
 of zero for each tool. Tools are executed in order of descending priority. The
 execution order of tools with equal priority is unspecified. For example, you
 could use the 'sort' and 'head' utilities to keep only the 10 smallest numbers
 in a text file by ensuring that 'sort' runs before 'head'::
 
   [fix]
   sort:command = sort -n
   head:command = head -n 10
   sort:pattern = numbers.txt
   head:pattern = numbers.txt
   sort:priority = 2
   head:priority = 1
 
 To account for changes made by each tool, the line numbers used for incremental
 formatting are recomputed before executing the next tool. So, each tool may see
 different values for the arguments added by the :linerange suboption.
 
 Each fixer tool is allowed to return some metadata in addition to the fixed file
 content. The metadata must be placed before the file content on stdout,
 separated from the file content by a zero byte. The metadata is parsed as a JSON
 value (so, it should be UTF-8 encoded and contain no zero bytes). A fixer tool
 is expected to produce this metadata encoding if and only if the :metadata
 suboption is true::
 
   [fix]
   tool:command = tool --prepend-json-metadata
   tool:metadata = true
 
 The metadata values are passed to hooks, which can be used to print summaries or
 perform other post-fixing work. The supported hooks are::
 
   "postfixfile"
     Run once for each file in each revision where any fixer tools made changes
     to the file content. Provides "$HG_REV" and "$HG_PATH" to identify the file,
     and "$HG_METADATA" with a map of fixer names to metadata values from fixer
     tools that affected the file. Fixer tools that didn't affect the file have a
     value of None. Only fixer tools that executed are present in the metadata.
 
   "postfix"
     Run once after all files and revisions have been handled. Provides
     "$HG_REPLACEMENTS" with information about what revisions were created and
     made obsolete. Provides a boolean "$HG_WDIRWRITTEN" to indicate whether any
     files in the working copy were updated. Provides a list "$HG_METADATA"
     mapping fixer tool names to lists of metadata values returned from
     executions that modified a file. This aggregates the same metadata
     previously passed to the "postfixfile" hook.
 
 Fixer tools are run in the repository's root directory. This allows them to read
 configuration files from the working copy, or even write to the working copy.
 The working copy is not updated to match the revision being fixed. In fact,
 several revisions may be fixed in parallel. Writes to the working copy are not
 amended into the revision being fixed; fixer tools should always write fixed
 file content back to stdout as documented above.
 """
 
 from __future__ import absolute_import
 
 import collections
 import itertools
 import os
 import re
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     nullrev,
     wdirrev,
 )
 
 from mercurial.utils import procutil
 
 from mercurial import (
     cmdutil,
     context,
     copies,
     error,
     logcmdutil,
     match as matchmod,
     mdiff,
     merge,
     mergestate as mergestatemod,
-    obsolete,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
     worker,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # Register the suboptions allowed for each configured fixer, and default values.
 FIXER_ATTRS = {
     b'command': None,
     b'linerange': None,
     b'pattern': None,
     b'priority': 0,
     b'metadata': False,
     b'skipclean': True,
     b'enabled': True,
 }
 
 for key, default in FIXER_ATTRS.items():
     configitem(b'fix', b'.*:%s$' % key, default=default, generic=True)
 
 # A good default size allows most source code files to be fixed, but avoids
 # letting fixer tools choke on huge inputs, which could be surprising to the
 # user.
 configitem(b'fix', b'maxfilesize', default=b'2MB')
 
 # Allow fix commands to exit non-zero if an executed fixer tool exits non-zero.
 # This helps users do shell scripts that stop when a fixer tool signals a
 # problem.
 configitem(b'fix', b'failure', default=b'continue')
 
 
 def checktoolfailureaction(ui, message, hint=None):
     """Abort with 'message' if fix.failure=abort"""
     action = ui.config(b'fix', b'failure')
     if action not in (b'continue', b'abort'):
         raise error.Abort(
             _(b'unknown fix.failure action: %s') % (action,),
             hint=_(b'use "continue" or "abort"'),
         )
     if action == b'abort':
         raise error.Abort(message, hint=hint)
 
 
 allopt = (b'', b'all', False, _(b'fix all non-public non-obsolete revisions'))
 baseopt = (
     b'',
     b'base',
     [],
     _(
         b'revisions to diff against (overrides automatic '
         b'selection, and applies to every revision being '
         b'fixed)'
     ),
     _(b'REV'),
 )
 revopt = (b'r', b'rev', [], _(b'revisions to fix (ADVANCED)'), _(b'REV'))
 sourceopt = (
     b's',
     b'source',
     [],
     _(b'fix the specified revisions and their descendants'),
     _(b'REV'),
 )
 wdiropt = (b'w', b'working-dir', False, _(b'fix the working directory'))
 wholeopt = (b'', b'whole', False, _(b'always fix every line of a file'))
 usage = _(b'[OPTION]... [FILE]...')
 
 
 @command(
     b'fix',
     [allopt, baseopt, revopt, sourceopt, wdiropt, wholeopt],
     usage,
     helpcategory=command.CATEGORY_FILE_CONTENTS,
 )
 def fix(ui, repo, *pats, **opts):
     """rewrite file content in changesets or working directory
 
     Runs any configured tools to fix the content of files. Only affects files
     with changes, unless file arguments are provided. Only affects changed lines
     of files, unless the --whole flag is used. Some tools may always affect the
     whole file regardless of --whole.
 
     If --working-dir is used, files with uncommitted changes in the working copy
     will be fixed. Note that no backup are made.
 
     If revisions are specified with --source, those revisions and their
     descendants will be checked, and they may be replaced with new revisions
     that have fixed file content. By automatically including the descendants,
     no merging, rebasing, or evolution will be required. If an ancestor of the
     working copy is included, then the working copy itself will also be fixed,
     and the working copy will be updated to the fixed parent.
 
     When determining what lines of each file to fix at each revision, the whole
     set of revisions being fixed is considered, so that fixes to earlier
     revisions are not forgotten in later ones. The --base flag can be used to
     override this default behavior, though it is not usually desirable to do so.
     """
     opts = pycompat.byteskwargs(opts)
     cmdutil.check_at_most_one_arg(opts, b'all', b'source', b'rev')
     cmdutil.check_incompatible_arguments(
         opts, b'working_dir', [b'all', b'source']
     )
 
     with repo.wlock(), repo.lock(), repo.transaction(b'fix'):
         revstofix = getrevstofix(ui, repo, opts)
         basectxs = getbasectxs(repo, opts, revstofix)
         workqueue, numitems = getworkqueue(
             ui, repo, pats, opts, revstofix, basectxs
         )
         basepaths = getbasepaths(repo, opts, workqueue, basectxs)
         fixers = getfixers(ui)
 
         # Rather than letting each worker independently fetch the files
         # (which also would add complications for shared/keepalive
         # connections), prefetch them all first.
         _prefetchfiles(repo, workqueue, basepaths)
 
         # There are no data dependencies between the workers fixing each file
         # revision, so we can use all available parallelism.
         def getfixes(items):
             for srcrev, path, dstrevs in items:
                 ctx = repo[srcrev]
                 olddata = ctx[path].data()
                 metadata, newdata = fixfile(
                     ui,
                     repo,
                     opts,
                     fixers,
                     ctx,
                     path,
                     basepaths,
                     basectxs[srcrev],
                 )
                 # We ungroup the work items now, because the code that consumes
                 # these results has to handle each dstrev separately, and in
                 # topological order. Because these are handled in topological
                 # order, it's important that we pass around references to
                 # "newdata" instead of copying it. Otherwise, we would be
                 # keeping more copies of file content in memory at a time than
                 # if we hadn't bothered to group/deduplicate the work items.
                 data = newdata if newdata != olddata else None
                 for dstrev in dstrevs:
                     yield (dstrev, path, metadata, data)
 
         results = worker.worker(
             ui, 1.0, getfixes, tuple(), workqueue, threadsafe=False
         )
 
         # We have to hold on to the data for each successor revision in memory
         # until all its parents are committed. We ensure this by committing and
         # freeing memory for the revisions in some topological order. This
         # leaves a little bit of memory efficiency on the table, but also makes
         # the tests deterministic. It might also be considered a feature since
         # it makes the results more easily reproducible.
         filedata = collections.defaultdict(dict)
         aggregatemetadata = collections.defaultdict(list)
         replacements = {}
         wdirwritten = False
         commitorder = sorted(revstofix, reverse=True)
         with ui.makeprogress(
             topic=_(b'fixing'), unit=_(b'files'), total=sum(numitems.values())
         ) as progress:
             for rev, path, filerevmetadata, newdata in results:
                 progress.increment(item=path)
                 for fixername, fixermetadata in filerevmetadata.items():
                     aggregatemetadata[fixername].append(fixermetadata)
                 if newdata is not None:
                     filedata[rev][path] = newdata
                     hookargs = {
                         b'rev': rev,
                         b'path': path,
                         b'metadata': filerevmetadata,
                     }
                     repo.hook(
                         b'postfixfile',
                         throw=False,
                         **pycompat.strkwargs(hookargs)
                     )
                 numitems[rev] -= 1
                 # Apply the fixes for this and any other revisions that are
                 # ready and sitting at the front of the queue. Using a loop here
                 # prevents the queue from being blocked by the first revision to
                 # be ready out of order.
                 while commitorder and not numitems[commitorder[-1]]:
                     rev = commitorder.pop()
                     ctx = repo[rev]
                     if rev == wdirrev:
                         writeworkingdir(repo, ctx, filedata[rev], replacements)
                         wdirwritten = bool(filedata[rev])
                     else:
                         replacerev(ui, repo, ctx, filedata[rev], replacements)
                     del filedata[rev]
 
         cleanup(repo, replacements, wdirwritten)
         hookargs = {
             b'replacements': replacements,
             b'wdirwritten': wdirwritten,
             b'metadata': aggregatemetadata,
         }
         repo.hook(b'postfix', throw=True, **pycompat.strkwargs(hookargs))
 
 
 def cleanup(repo, replacements, wdirwritten):
     """Calls scmutil.cleanupnodes() with the given replacements.
 
     "replacements" is a dict from nodeid to nodeid, with one key and one value
     for every revision that was affected by fixing. This is slightly different
     from cleanupnodes().
 
     "wdirwritten" is a bool which tells whether the working copy was affected by
     fixing, since it has no entry in "replacements".
 
     Useful as a hook point for extending "hg fix" with output summarizing the
     effects of the command, though we choose not to output anything here.
     """
     replacements = {
         prec: [succ] for prec, succ in pycompat.iteritems(replacements)
     }
     scmutil.cleanupnodes(repo, replacements, b'fix', fixphase=True)
 
 
 def getworkqueue(ui, repo, pats, opts, revstofix, basectxs):
     """Constructs a list of files to fix and which revisions each fix applies to
 
     To avoid duplicating work, there is usually only one work item for each file
     revision that might need to be fixed. There can be multiple work items per
     file revision if the same file needs to be fixed in multiple changesets with
     different baserevs. Each work item also contains a list of changesets where
     the file's data should be replaced with the fixed data. The work items for
     earlier changesets come earlier in the work queue, to improve pipelining by
     allowing the first changeset to be replaced while fixes are still being
     computed for later changesets.
 
     Also returned is a map from changesets to the count of work items that might
     affect each changeset. This is used later to count when all of a changeset's
     work items have been finished, without having to inspect the remaining work
     queue in each worker subprocess.
 
     The example work item (1, "foo/bar.txt", (1, 2, 3)) means that the data of
     bar.txt should be read from revision 1, then fixed, and written back to
     revisions 1, 2 and 3. Revision 1 is called the "srcrev" and the list of
     revisions is called the "dstrevs". In practice the srcrev is always one of
     the dstrevs, and we make that choice when constructing the work item so that
     the choice can't be made inconsistently later on. The dstrevs should all
     have the same file revision for the given path, so the choice of srcrev is
     arbitrary. The wdirrev can be a dstrev and a srcrev.
     """
     dstrevmap = collections.defaultdict(list)
     numitems = collections.defaultdict(int)
     maxfilesize = ui.configbytes(b'fix', b'maxfilesize')
     for rev in sorted(revstofix):
         fixctx = repo[rev]
         match = scmutil.match(fixctx, pats, opts)
         for path in sorted(
             pathstofix(ui, repo, pats, opts, match, basectxs[rev], fixctx)
         ):
             fctx = fixctx[path]
             if fctx.islink():
                 continue
             if fctx.size() > maxfilesize:
                 ui.warn(
                     _(b'ignoring file larger than %s: %s\n')
                     % (util.bytecount(maxfilesize), path)
                 )
                 continue
             baserevs = tuple(ctx.rev() for ctx in basectxs[rev])
             dstrevmap[(fctx.filerev(), baserevs, path)].append(rev)
             numitems[rev] += 1
     workqueue = [
         (min(dstrevs), path, dstrevs)
         for (_filerev, _baserevs, path), dstrevs in dstrevmap.items()
     ]
     # Move work items for earlier changesets to the front of the queue, so we
     # might be able to replace those changesets (in topological order) while
     # we're still processing later work items. Note the min() in the previous
     # expression, which means we don't need a custom comparator here. The path
     # is also important in the sort order to make the output order stable. There
     # are some situations where this doesn't help much, but some situations
     # where it lets us buffer O(1) files instead of O(n) files.
     workqueue.sort()
     return workqueue, numitems
 
 
 def getrevstofix(ui, repo, opts):
     """Returns the set of revision numbers that should be fixed"""
     if opts[b'all']:
         revs = repo.revs(b'(not public() and not obsolete()) or wdir()')
     elif opts[b'source']:
         source_revs = logcmdutil.revrange(repo, opts[b'source'])
         revs = set(repo.revs(b'(%ld::) - obsolete()', source_revs))
         if wdirrev in source_revs:
             # `wdir()::` is currently empty, so manually add wdir
             revs.add(wdirrev)
         if repo[b'.'].rev() in revs:
             revs.add(wdirrev)
     else:
         revs = set(logcmdutil.revrange(repo, opts[b'rev']))
         if opts.get(b'working_dir'):
             revs.add(wdirrev)
-        for rev in revs:
-            checkfixablectx(ui, repo, repo[rev])
     # Allow fixing only wdir() even if there's an unfinished operation
     if not (len(revs) == 1 and wdirrev in revs):
         cmdutil.checkunfinished(repo)
         rewriteutil.precheck(repo, revs, b'fix')
     if (
         wdirrev in revs
         and mergestatemod.mergestate.read(repo).unresolvedcount()
     ):
         raise error.Abort(b'unresolved conflicts', hint=b"use 'hg resolve'")
     if not revs:
         raise error.Abort(
             b'no changesets specified', hint=b'use --source or --working-dir'
         )
     return revs
 
 
-def checkfixablectx(ui, repo, ctx):
-    """Aborts if the revision shouldn't be replaced with a fixed one."""
-    if ctx.obsolete():
-        # It would be better to actually check if the revision has a successor.
-        if not obsolete.isenabled(repo, obsolete.allowdivergenceopt):
-            raise error.Abort(
-                b'fixing obsolete revision could cause divergence'
-            )
-
-
 def pathstofix(ui, repo, pats, opts, match, basectxs, fixctx):
     """Returns the set of files that should be fixed in a context
 
     The result depends on the base contexts; we include any file that has
     changed relative to any of the base contexts. Base contexts should be
     ancestors of the context being fixed.
     """
     files = set()
     for basectx in basectxs:
         stat = basectx.status(
             fixctx, match=match, listclean=bool(pats), listunknown=bool(pats)
         )
         files.update(
             set(
                 itertools.chain(
                     stat.added, stat.modified, stat.clean, stat.unknown
                 )
             )
         )
     return files
 
 
 def lineranges(opts, path, basepaths, basectxs, fixctx, content2):
     """Returns the set of line ranges that should be fixed in a file
 
     Of the form [(10, 20), (30, 40)].
 
     This depends on the given base contexts; we must consider lines that have
     changed versus any of the base contexts, and whether the file has been
     renamed versus any of them.
 
     Another way to understand this is that we exclude line ranges that are
     common to the file in all base contexts.
     """
     if opts.get(b'whole'):
         # Return a range containing all lines. Rely on the diff implementation's
         # idea of how many lines are in the file, instead of reimplementing it.
         return difflineranges(b'', content2)
 
     rangeslist = []
     for basectx in basectxs:
         basepath = basepaths.get((basectx.rev(), fixctx.rev(), path), path)
 
         if basepath in basectx:
             content1 = basectx[basepath].data()
         else:
             content1 = b''
         rangeslist.extend(difflineranges(content1, content2))
     return unionranges(rangeslist)
 
 
 def getbasepaths(repo, opts, workqueue, basectxs):
     if opts.get(b'whole'):
         # Base paths will never be fetched for line range determination.
         return {}
 
     basepaths = {}
     for srcrev, path, _dstrevs in workqueue:
         fixctx = repo[srcrev]
         for basectx in basectxs[srcrev]:
             basepath = copies.pathcopies(basectx, fixctx).get(path, path)
             if basepath in basectx:
                 basepaths[(basectx.rev(), fixctx.rev(), path)] = basepath
     return basepaths
 
 
 def unionranges(rangeslist):
     """Return the union of some closed intervals
 
     >>> unionranges([])
     []
     >>> unionranges([(1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (2, 100)])
     [(1, 100)]
     >>> unionranges([(1, 99), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (40, 60)])
     [(1, 100)]
     >>> unionranges([(1, 49), (50, 100)])
     [(1, 100)]
     >>> unionranges([(1, 48), (50, 100)])
     [(1, 48), (50, 100)]
     >>> unionranges([(1, 2), (3, 4), (5, 6)])
     [(1, 6)]
     """
     rangeslist = sorted(set(rangeslist))
     unioned = []
     if rangeslist:
         unioned, rangeslist = [rangeslist[0]], rangeslist[1:]
     for a, b in rangeslist:
         c, d = unioned[-1]
         if a > d + 1:
             unioned.append((a, b))
         else:
             unioned[-1] = (c, max(b, d))
     return unioned
 
 
 def difflineranges(content1, content2):
     """Return list of line number ranges in content2 that differ from content1.
 
     Line numbers are 1-based. The numbers are the first and last line contained
     in the range. Single-line ranges have the same line number for the first and
     last line. Excludes any empty ranges that result from lines that are only
     present in content1. Relies on mdiff's idea of where the line endings are in
     the string.
 
     >>> from mercurial import pycompat
     >>> lines = lambda s: b'\\n'.join([c for c in pycompat.iterbytestr(s)])
     >>> difflineranges2 = lambda a, b: difflineranges(lines(a), lines(b))
     >>> difflineranges2(b'', b'')
     []
     >>> difflineranges2(b'a', b'')
     []
     >>> difflineranges2(b'', b'A')
     [(1, 1)]
     >>> difflineranges2(b'a', b'a')
     []
     >>> difflineranges2(b'a', b'A')
     [(1, 1)]
     >>> difflineranges2(b'ab', b'')
     []
     >>> difflineranges2(b'', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abc', b'ac')
     []
     >>> difflineranges2(b'ab', b'aCb')
     [(2, 2)]
     >>> difflineranges2(b'abc', b'aBc')
     [(2, 2)]
     >>> difflineranges2(b'ab', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abcde', b'aBcDe')
     [(2, 2), (4, 4)]
     >>> difflineranges2(b'abcde', b'aBCDe')
     [(2, 4)]
     """
     ranges = []
     for lines, kind in mdiff.allblocks(content1, content2):
         firstline, lastline = lines[2:4]
         if kind == b'!' and firstline != lastline:
             ranges.append((firstline + 1, lastline))
     return ranges
 
 
 def getbasectxs(repo, opts, revstofix):
     """Returns a map of the base contexts for each revision
 
     The base contexts determine which lines are considered modified when we
     attempt to fix just the modified lines in a file. It also determines which
     files we attempt to fix, so it is important to compute this even when
     --whole is used.
     """
     # The --base flag overrides the usual logic, and we give every revision
     # exactly the set of baserevs that the user specified.
     if opts.get(b'base'):
         baserevs = set(logcmdutil.revrange(repo, opts.get(b'base')))
         if not baserevs:
             baserevs = {nullrev}
         basectxs = {repo[rev] for rev in baserevs}
         return {rev: basectxs for rev in revstofix}
 
     # Proceed in topological order so that we can easily determine each
     # revision's baserevs by looking at its parents and their baserevs.
     basectxs = collections.defaultdict(set)
     for rev in sorted(revstofix):
         ctx = repo[rev]
         for pctx in ctx.parents():
             if pctx.rev() in basectxs:
                 basectxs[rev].update(basectxs[pctx.rev()])
             else:
                 basectxs[rev].add(pctx)
     return basectxs
 
 
 def _prefetchfiles(repo, workqueue, basepaths):
     toprefetch = set()
 
     # Prefetch the files that will be fixed.
     for srcrev, path, _dstrevs in workqueue:
         if srcrev == wdirrev:
             continue
         toprefetch.add((srcrev, path))
 
     # Prefetch the base contents for lineranges().
     for (baserev, fixrev, path), basepath in basepaths.items():
         toprefetch.add((baserev, basepath))
 
     if toprefetch:
         scmutil.prefetchfiles(
             repo,
             [
                 (rev, scmutil.matchfiles(repo, [path]))
                 for rev, path in toprefetch
             ],
         )
 
 
 def fixfile(ui, repo, opts, fixers, fixctx, path, basepaths, basectxs):
     """Run any configured fixers that should affect the file in this context
 
     Returns the file content that results from applying the fixers in some order
     starting with the file's content in the fixctx. Fixers that support line
     ranges will affect lines that have changed relative to any of the basectxs
     (i.e. they will only avoid lines that are common to all basectxs).
 
     A fixer tool's stdout will become the file's new content if and only if it
     exits with code zero. The fixer tool's working directory is the repository's
     root.
     """
     metadata = {}
     newdata = fixctx[path].data()
     for fixername, fixer in pycompat.iteritems(fixers):
         if fixer.affects(opts, fixctx, path):
             ranges = lineranges(
                 opts, path, basepaths, basectxs, fixctx, newdata
             )
             command = fixer.command(ui, path, ranges)
             if command is None:
                 continue
             ui.debug(b'subprocess: %s\n' % (command,))
             proc = subprocess.Popen(
                 procutil.tonativestr(command),
                 shell=True,
                 cwd=procutil.tonativestr(repo.root),
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
             )
             stdout, stderr = proc.communicate(newdata)
             if stderr:
                 showstderr(ui, fixctx.rev(), fixername, stderr)
             newerdata = stdout
             if fixer.shouldoutputmetadata():
                 try:
                     metadatajson, newerdata = stdout.split(b'\0', 1)
                     metadata[fixername] = pycompat.json_loads(metadatajson)
                 except ValueError:
                     ui.warn(
                         _(b'ignored invalid output from fixer tool: %s\n')
                         % (fixername,)
                     )
                     continue
             else:
                 metadata[fixername] = None
             if proc.returncode == 0:
                 newdata = newerdata
             else:
                 if not stderr:
                     message = _(b'exited with status %d\n') % (proc.returncode,)
                     showstderr(ui, fixctx.rev(), fixername, message)
                 checktoolfailureaction(
                     ui,
                     _(b'no fixes will be applied'),
                     hint=_(
                         b'use --config fix.failure=continue to apply any '
                         b'successful fixes anyway'
                     ),
                 )
     return metadata, newdata
 
 
 def showstderr(ui, rev, fixername, stderr):
     """Writes the lines of the stderr string as warnings on the ui
 
     Uses the revision number and fixername to give more context to each line of
     the error message. Doesn't include file names, since those take up a lot of
     space and would tend to be included in the error message if they were
     relevant.
     """
     for line in re.split(b'[\r\n]+', stderr):
         if line:
             ui.warn(b'[')
             if rev is None:
                 ui.warn(_(b'wdir'), label=b'evolve.rev')
             else:
                 ui.warn(b'%d' % rev, label=b'evolve.rev')
             ui.warn(b'] %s: %s\n' % (fixername, line))
 
 
 def writeworkingdir(repo, ctx, filedata, replacements):
     """Write new content to the working copy and check out the new p1 if any
 
     We check out a new revision if and only if we fixed something in both the
     working directory and its parent revision. This avoids the need for a full
     update/merge, and means that the working directory simply isn't affected
     unless the --working-dir flag is given.
 
     Directly updates the dirstate for the affected files.
     """
     for path, data in pycompat.iteritems(filedata):
         fctx = ctx[path]
         fctx.write(data, fctx.flags())
 
     oldp1 = repo.dirstate.p1()
     newp1 = replacements.get(oldp1, oldp1)
     if newp1 != oldp1:
         assert repo.dirstate.p2() == nullid
         with repo.dirstate.parentchange():
             scmutil.movedirstate(repo, repo[newp1])
 
 
 def replacerev(ui, repo, ctx, filedata, replacements):
     """Commit a new revision like the given one, but with file content changes
 
     "ctx" is the original revision to be replaced by a modified one.
 
     "filedata" is a dict that maps paths to their new file content. All other
     paths will be recreated from the original revision without changes.
     "filedata" may contain paths that didn't exist in the original revision;
     they will be added.
 
     "replacements" is a dict that maps a single node to a single node, and it is
     updated to indicate the original revision is replaced by the newly created
     one. No entry is added if the replacement's node already exists.
 
     The new revision has the same parents as the old one, unless those parents
     have already been replaced, in which case those replacements are the parents
     of this new revision. Thus, if revisions are replaced in topological order,
     there is no need to rebase them into the original topology later.
     """
 
     p1rev, p2rev = repo.changelog.parentrevs(ctx.rev())
     p1ctx, p2ctx = repo[p1rev], repo[p2rev]
     newp1node = replacements.get(p1ctx.node(), p1ctx.node())
     newp2node = replacements.get(p2ctx.node(), p2ctx.node())
 
     # We don't want to create a revision that has no changes from the original,
     # but we should if the original revision's parent has been replaced.
     # Otherwise, we would produce an orphan that needs no actual human
     # intervention to evolve. We can't rely on commit() to avoid creating the
     # un-needed revision because the extra field added below produces a new hash
     # regardless of file content changes.
     if (
         not filedata
         and p1ctx.node() not in replacements
         and p2ctx.node() not in replacements
     ):
         return
 
     extra = ctx.extra().copy()
     extra[b'fix_source'] = ctx.hex()
 
     wctx = context.overlayworkingctx(repo)
     wctx.setbase(repo[newp1node])
     merge.revert_to(ctx, wc=wctx)
     copies.graftcopies(wctx, ctx, ctx.p1())
 
     for path in filedata.keys():
         fctx = ctx[path]
         copysource = fctx.copysource()
         wctx.write(path, filedata[path], flags=fctx.flags())
         if copysource:
             wctx.markcopied(path, copysource)
 
     desc = rewriteutil.update_hash_refs(
         repo,
         ctx.description(),
         {oldnode: [newnode] for oldnode, newnode in replacements.items()},
     )
 
     memctx = wctx.tomemctx(
         text=desc,
         branch=ctx.branch(),
         extra=extra,
         date=ctx.date(),
         parents=(newp1node, newp2node),
         user=ctx.user(),
     )
 
     sucnode = memctx.commit()
     prenode = ctx.node()
     if prenode == sucnode:
         ui.debug(b'node %s already existed\n' % (ctx.hex()))
     else:
         replacements[ctx.node()] = sucnode
 
 
 def getfixers(ui):
     """Returns a map of configured fixer tools indexed by their names
 
     Each value is a Fixer object with methods that implement the behavior of the
     fixer's config suboptions. Does not validate the config values.
     """
     fixers = {}
     for name in fixernames(ui):
         enabled = ui.configbool(b'fix', name + b':enabled')
         command = ui.config(b'fix', name + b':command')
         pattern = ui.config(b'fix', name + b':pattern')
         linerange = ui.config(b'fix', name + b':linerange')
         priority = ui.configint(b'fix', name + b':priority')
         metadata = ui.configbool(b'fix', name + b':metadata')
         skipclean = ui.configbool(b'fix', name + b':skipclean')
         # Don't use a fixer if it has no pattern configured. It would be
         # dangerous to let it affect all files. It would be pointless to let it
         # affect no files. There is no reasonable subset of files to use as the
         # default.
         if command is None:
             ui.warn(
                 _(b'fixer tool has no command configuration: %s\n') % (name,)
             )
         elif pattern is None:
             ui.warn(
                 _(b'fixer tool has no pattern configuration: %s\n') % (name,)
             )
         elif not enabled:
             ui.debug(b'ignoring disabled fixer tool: %s\n' % (name,))
         else:
             fixers[name] = Fixer(
                 command, pattern, linerange, priority, metadata, skipclean
             )
     return collections.OrderedDict(
         sorted(fixers.items(), key=lambda item: item[1]._priority, reverse=True)
     )
 
 
 def fixernames(ui):
     """Returns the names of [fix] config options that have suboptions"""
     names = set()
     for k, v in ui.configitems(b'fix'):
         if b':' in k:
             names.add(k.split(b':', 1)[0])
     return names
 
 
 class Fixer(object):
     """Wraps the raw config values for a fixer with methods"""
 
     def __init__(
         self, command, pattern, linerange, priority, metadata, skipclean
     ):
         self._command = command
         self._pattern = pattern
         self._linerange = linerange
         self._priority = priority
         self._metadata = metadata
         self._skipclean = skipclean
 
     def affects(self, opts, fixctx, path):
         """Should this fixer run on the file at the given path and context?"""
         repo = fixctx.repo()
         matcher = matchmod.match(
             repo.root, repo.root, [self._pattern], ctx=fixctx
         )
         return matcher(path)
 
     def shouldoutputmetadata(self):
         """Should the stdout of this fixer start with JSON and a null byte?"""
         return self._metadata
 
     def command(self, ui, path, ranges):
         """A shell command to use to invoke this fixer on the given file/lines
 
         May return None if there is no appropriate command to run for the given
         parameters.
         """
         expand = cmdutil.rendercommandtemplate
         parts = [
             expand(
                 ui,
                 self._command,
                 {b'rootpath': path, b'basename': os.path.basename(path)},
             )
         ]
         if self._linerange:
             if self._skipclean and not ranges:
                 # No line ranges to fix, so don't run the fixer.
                 return None
             for first, last in ranges:
                 parts.append(
                     expand(
                         ui, self._linerange, {b'first': first, b'last': last}
                     )
                 )
         return b' '.join(parts)

hgext/git/__init__.py

@@ -1,343 +1,344 b''
 """grant Mercurial the ability to operate on Git repositories. (EXPERIMENTAL)
 
 This is currently super experimental. It probably will consume your
 firstborn a la Rumpelstiltskin, etc.
 """
 
 from __future__ import absolute_import
 
 import os
 
 from mercurial.i18n import _
 
 from mercurial import (
     commands,
     error,
     extensions,
     localrepo,
     pycompat,
     registrar,
     scmutil,
     store,
     util,
 )
 
 from . import (
     dirstate,
     gitlog,
     gitutil,
     index,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 # git.log-index-cache-miss: internal knob for testing
 configitem(
     b"git",
     b"log-index-cache-miss",
     default=False,
 )
 
 getversion = gitutil.pygit2_version
 
 
 # TODO: extract an interface for this in core
 class gitstore(object):  # store.basicstore):
     def __init__(self, path, vfstype):
         self.vfs = vfstype(path)
+        self.opener = self.vfs
         self.path = self.vfs.base
         self.createmode = store._calcmode(self.vfs)
         # above lines should go away in favor of:
         # super(gitstore, self).__init__(path, vfstype)
 
         self.git = gitutil.get_pygit2().Repository(
             os.path.normpath(os.path.join(path, b'..', b'.git'))
         )
         self._progress_factory = lambda *args, **kwargs: None
         self._logfn = lambda x: None
 
     @util.propertycache
     def _db(self):
         # We lazy-create the database because we want to thread a
         # progress callback down to the indexing process if it's
         # required, and we don't have a ui handle in makestore().
         return index.get_index(self.git, self._logfn, self._progress_factory)
 
     def join(self, f):
         """Fake store.join method for git repositories.
 
         For the most part, store.join is used for @storecache
         decorators to invalidate caches when various files
         change. We'll map the ones we care about, and ignore the rest.
         """
         if f in (b'00changelog.i', b'00manifest.i'):
             # This is close enough: in order for the changelog cache
             # to be invalidated, HEAD will have to change.
             return os.path.join(self.path, b'HEAD')
         elif f == b'lock':
             # TODO: we probably want to map this to a git lock, I
             # suspect index.lock. We should figure out what the
             # most-alike file is in git-land. For now we're risking
             # bad concurrency errors if another git client is used.
             return os.path.join(self.path, b'hgit-bogus-lock')
         elif f in (b'obsstore', b'phaseroots', b'narrowspec', b'bookmarks'):
             return os.path.join(self.path, b'..', b'.hg', f)
         raise NotImplementedError(b'Need to pick file for %s.' % f)
 
     def changelog(self, trypending, concurrencychecker):
         # TODO we don't have a plan for trypending in hg's git support yet
         return gitlog.changelog(self.git, self._db)
 
     def manifestlog(self, repo, storenarrowmatch):
         # TODO handle storenarrowmatch and figure out if we need the repo arg
         return gitlog.manifestlog(self.git, self._db)
 
     def invalidatecaches(self):
         pass
 
     def write(self, tr=None):
         # normally this handles things like fncache writes, which we don't have
         pass
 
 
 def _makestore(orig, requirements, storebasepath, vfstype):
     if b'git' in requirements:
         if not os.path.exists(os.path.join(storebasepath, b'..', b'.git')):
             raise error.Abort(
                 _(
                     b'repository specified git format in '
                     b'.hg/requires but has no .git directory'
                 )
             )
         # Check for presence of pygit2 only here. The assumption is that we'll
         # run this code iff we'll later need pygit2.
         if gitutil.get_pygit2() is None:
             raise error.Abort(
                 _(
                     b'the git extension requires the Python '
                     b'pygit2 library to be installed'
                 )
             )
 
         return gitstore(storebasepath, vfstype)
     return orig(requirements, storebasepath, vfstype)
 
 
 class gitfilestorage(object):
     def file(self, path):
         if path[0:1] == b'/':
             path = path[1:]
         return gitlog.filelog(self.store.git, self.store._db, path)
 
 
 def _makefilestorage(orig, requirements, features, **kwargs):
     store = kwargs['store']
     if isinstance(store, gitstore):
         return gitfilestorage
     return orig(requirements, features, **kwargs)
 
 
 def _setupdothg(ui, path):
     dothg = os.path.join(path, b'.hg')
     if os.path.exists(dothg):
         ui.warn(_(b'git repo already initialized for hg\n'))
     else:
         os.mkdir(os.path.join(path, b'.hg'))
         # TODO is it ok to extend .git/info/exclude like this?
         with open(
             os.path.join(path, b'.git', b'info', b'exclude'), 'ab'
         ) as exclude:
             exclude.write(b'\n.hg\n')
     with open(os.path.join(dothg, b'requires'), 'wb') as f:
         f.write(b'git\n')
 
 
 _BMS_PREFIX = 'refs/heads/'
 
 
 class gitbmstore(object):
     def __init__(self, gitrepo):
         self.gitrepo = gitrepo
         self._aclean = True
         self._active = gitrepo.references['HEAD']  # git head, not mark
 
     def __contains__(self, name):
         return (
             _BMS_PREFIX + pycompat.fsdecode(name)
         ) in self.gitrepo.references
 
     def __iter__(self):
         for r in self.gitrepo.listall_references():
             if r.startswith(_BMS_PREFIX):
                 yield pycompat.fsencode(r[len(_BMS_PREFIX) :])
 
     def __getitem__(self, k):
         return (
             self.gitrepo.references[_BMS_PREFIX + pycompat.fsdecode(k)]
             .peel()
             .id.raw
         )
 
     def get(self, k, default=None):
         try:
             if k in self:
                 return self[k]
             return default
         except gitutil.get_pygit2().InvalidSpecError:
             return default
 
     @property
     def active(self):
         h = self.gitrepo.references['HEAD']
         if not isinstance(h.target, str) or not h.target.startswith(
             _BMS_PREFIX
         ):
             return None
         return pycompat.fsencode(h.target[len(_BMS_PREFIX) :])
 
     @active.setter
     def active(self, mark):
         githead = mark is not None and (_BMS_PREFIX + mark) or None
         if githead is not None and githead not in self.gitrepo.references:
             raise AssertionError(b'bookmark %s does not exist!' % mark)
 
         self._active = githead
         self._aclean = False
 
     def _writeactive(self):
         if self._aclean:
             return
         self.gitrepo.references.create('HEAD', self._active, True)
         self._aclean = True
 
     def names(self, node):
         r = []
         for ref in self.gitrepo.listall_references():
             if not ref.startswith(_BMS_PREFIX):
                 continue
             if self.gitrepo.references[ref].peel().id.raw != node:
                 continue
             r.append(pycompat.fsencode(ref[len(_BMS_PREFIX) :]))
         return r
 
     # Cleanup opportunity: this is *identical* to core's bookmarks store.
     def expandname(self, bname):
         if bname == b'.':
             if self.active:
                 return self.active
             raise error.RepoLookupError(_(b"no active bookmark"))
         return bname
 
     def applychanges(self, repo, tr, changes):
         """Apply a list of changes to bookmarks"""
         # TODO: this should respect transactions, but that's going to
         # require enlarging the gitbmstore to know how to do in-memory
         # temporary writes and read those back prior to transaction
         # finalization.
         for name, node in changes:
             if node is None:
                 self.gitrepo.references.delete(
                     _BMS_PREFIX + pycompat.fsdecode(name)
                 )
             else:
                 self.gitrepo.references.create(
                     _BMS_PREFIX + pycompat.fsdecode(name),
                     gitutil.togitnode(node),
                     force=True,
                 )
 
     def checkconflict(self, mark, force=False, target=None):
         githead = _BMS_PREFIX + mark
         cur = self.gitrepo.references['HEAD']
         if githead in self.gitrepo.references and not force:
             if target:
                 if self.gitrepo.references[githead] == target and target == cur:
                     # re-activating a bookmark
                     return []
                 # moving a bookmark - forward?
                 raise NotImplementedError
             raise error.Abort(
                 _(b"bookmark '%s' already exists (use -f to force)") % mark
             )
         if len(mark) > 3 and not force:
             try:
                 shadowhash = scmutil.isrevsymbol(self._repo, mark)
             except error.LookupError:  # ambiguous identifier
                 shadowhash = False
             if shadowhash:
                 self._repo.ui.warn(
                     _(
                         b"bookmark %s matches a changeset hash\n"
                         b"(did you leave a -r out of an 'hg bookmark' "
                         b"command?)\n"
                     )
                     % mark
                 )
         return []
 
 
 def init(orig, ui, dest=b'.', **opts):
     if opts.get('git', False):
         path = util.abspath(dest)
         # TODO: walk up looking for the git repo
         _setupdothg(ui, path)
         return 0
     return orig(ui, dest=dest, **opts)
 
 
 def reposetup(ui, repo):
     if repo.local() and isinstance(repo.store, gitstore):
         orig = repo.__class__
         repo.store._progress_factory = repo.ui.makeprogress
         if ui.configbool(b'git', b'log-index-cache-miss'):
             repo.store._logfn = repo.ui.warn
 
         class gitlocalrepo(orig):
             def _makedirstate(self):
                 # TODO narrow support here
                 return dirstate.gitdirstate(
                     self.ui, self.vfs.base, self.store.git
                 )
 
             def commit(self, *args, **kwargs):
                 ret = orig.commit(self, *args, **kwargs)
                 if ret is None:
                     # there was nothing to commit, so we should skip
                     # the index fixup logic we'd otherwise do.
                     return None
                 tid = self.store.git[gitutil.togitnode(ret)].tree.id
                 # DANGER! This will flush any writes staged to the
                 # index in Git, but we're sidestepping the index in a
                 # way that confuses git when we commit. Alas.
                 self.store.git.index.read_tree(tid)
                 self.store.git.index.write()
                 return ret
 
             @property
             def _bookmarks(self):
                 return gitbmstore(self.store.git)
 
         repo.__class__ = gitlocalrepo
     return repo
 
 
 def _featuresetup(ui, supported):
     # don't die on seeing a repo with the git requirement
     supported |= {b'git'}
 
 
 def extsetup(ui):
     extensions.wrapfunction(localrepo, b'makestore', _makestore)
     extensions.wrapfunction(localrepo, b'makefilestorage', _makefilestorage)
     # Inject --git flag for `hg init`
     entry = extensions.wrapcommand(commands.table, b'init', init)
     entry[1].extend(
         [(b'', b'git', None, b'setup up a git repository instead of hg')]
     )
     localrepo.featuresetupfuncs.add(_featuresetup)

hgext/git/dirstate.py

@@ -1,369 +1,369 b''
 from __future__ import absolute_import
 
 import contextlib
 import errno
 import os
 
 from mercurial.node import sha1nodeconstants
 from mercurial import (
     error,
     extensions,
     match as matchmod,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.interfaces import (
     dirstate as intdirstate,
     util as interfaceutil,
 )
 
 from . import gitutil
 
 pygit2 = gitutil.get_pygit2()
 
 
 def readpatternfile(orig, filepath, warn, sourceinfo=False):
     if not (b'info/exclude' in filepath or filepath.endswith(b'.gitignore')):
         return orig(filepath, warn, sourceinfo=False)
     result = []
     warnings = []
     with open(filepath, b'rb') as fp:
         for l in fp:
             l = l.strip()
             if not l or l.startswith(b'#'):
                 continue
             if l.startswith(b'!'):
                 warnings.append(b'unsupported ignore pattern %s' % l)
                 continue
             if l.startswith(b'/'):
                 result.append(b'rootglob:' + l[1:])
             else:
                 result.append(b'relglob:' + l)
     return result, warnings
 
 
 extensions.wrapfunction(matchmod, b'readpatternfile', readpatternfile)
 
 
 _STATUS_MAP = {}
 if pygit2:
     _STATUS_MAP = {
         pygit2.GIT_STATUS_CONFLICTED: b'm',
         pygit2.GIT_STATUS_CURRENT: b'n',
         pygit2.GIT_STATUS_IGNORED: b'?',
         pygit2.GIT_STATUS_INDEX_DELETED: b'r',
         pygit2.GIT_STATUS_INDEX_MODIFIED: b'n',
         pygit2.GIT_STATUS_INDEX_NEW: b'a',
         pygit2.GIT_STATUS_INDEX_RENAMED: b'a',
         pygit2.GIT_STATUS_INDEX_TYPECHANGE: b'n',
         pygit2.GIT_STATUS_WT_DELETED: b'r',
         pygit2.GIT_STATUS_WT_MODIFIED: b'n',
         pygit2.GIT_STATUS_WT_NEW: b'?',
         pygit2.GIT_STATUS_WT_RENAMED: b'a',
         pygit2.GIT_STATUS_WT_TYPECHANGE: b'n',
         pygit2.GIT_STATUS_WT_UNREADABLE: b'?',
         pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_WT_MODIFIED: b'm',
     }
 
 
 @interfaceutil.implementer(intdirstate.idirstate)
 class gitdirstate(object):
     def __init__(self, ui, root, gitrepo):
         self._ui = ui
         self._root = os.path.dirname(root)
         self.git = gitrepo
         self._plchangecallbacks = {}
         # TODO: context.poststatusfixup is bad and uses this attribute
         self._dirty = False
 
     def p1(self):
         try:
             return self.git.head.peel().id.raw
         except pygit2.GitError:
             # Typically happens when peeling HEAD fails, as in an
             # empty repository.
             return sha1nodeconstants.nullid
 
     def p2(self):
         # TODO: MERGE_HEAD? something like that, right?
         return sha1nodeconstants.nullid
 
     def setparents(self, p1, p2=None):
         if p2 is None:
             p2 = sha1nodeconstants.nullid
         assert p2 == sha1nodeconstants.nullid, b'TODO merging support'
         self.git.head.set_target(gitutil.togitnode(p1))
 
     @util.propertycache
     def identity(self):
         return util.filestat.frompath(
             os.path.join(self._root, b'.git', b'index')
         )
 
     def branch(self):
         return b'default'
 
     def parents(self):
         # TODO how on earth do we find p2 if a merge is in flight?
         return self.p1(), sha1nodeconstants.nullid
 
     def __iter__(self):
         return (pycompat.fsencode(f.path) for f in self.git.index)
 
     def items(self):
         for ie in self.git.index:
             yield ie.path, None  # value should be a DirstateItem
 
     # py2,3 compat forward
     iteritems = items
 
     def __getitem__(self, filename):
         try:
             gs = self.git.status_file(filename)
         except KeyError:
             return b'?'
         return _STATUS_MAP[gs]
 
     def __contains__(self, filename):
         try:
             gs = self.git.status_file(filename)
             return _STATUS_MAP[gs] != b'?'
         except KeyError:
             return False
 
     def status(self, match, subrepos, ignored, clean, unknown):
         listclean = clean
         # TODO handling of clean files - can we get that from git.status()?
         modified, added, removed, deleted, unknown, ignored, clean = (
             [],
             [],
             [],
             [],
             [],
             [],
             [],
         )
         gstatus = self.git.status()
         for path, status in gstatus.items():
             path = pycompat.fsencode(path)
             if not match(path):
                 continue
             if status == pygit2.GIT_STATUS_IGNORED:
                 if path.endswith(b'/'):
                     continue
                 ignored.append(path)
             elif status in (
                 pygit2.GIT_STATUS_WT_MODIFIED,
                 pygit2.GIT_STATUS_INDEX_MODIFIED,
                 pygit2.GIT_STATUS_WT_MODIFIED
                 | pygit2.GIT_STATUS_INDEX_MODIFIED,
             ):
                 modified.append(path)
             elif status == pygit2.GIT_STATUS_INDEX_NEW:
                 added.append(path)
             elif status == pygit2.GIT_STATUS_WT_NEW:
                 unknown.append(path)
             elif status == pygit2.GIT_STATUS_WT_DELETED:
                 deleted.append(path)
             elif status == pygit2.GIT_STATUS_INDEX_DELETED:
                 removed.append(path)
             else:
                 raise error.Abort(
                     b'unhandled case: status for %r is %r' % (path, status)
                 )
 
         if listclean:
             observed = set(
                 modified + added + removed + deleted + unknown + ignored
             )
             index = self.git.index
             index.read()
             for entry in index:
                 path = pycompat.fsencode(entry.path)
                 if not match(path):
                     continue
                 if path in observed:
                     continue  # already in some other set
                 if path[-1] == b'/':
                     continue  # directory
                 clean.append(path)
 
         # TODO are we really always sure of status here?
         return (
             False,
             scmutil.status(
                 modified, added, removed, deleted, unknown, ignored, clean
             ),
         )
 
     def flagfunc(self, buildfallback):
         # TODO we can do better
         return buildfallback()
 
     def getcwd(self):
         # TODO is this a good way to do this?
         return os.path.dirname(
             os.path.dirname(pycompat.fsencode(self.git.path))
         )
 
     def normalize(self, path):
         normed = util.normcase(path)
         assert normed == path, b"TODO handling of case folding: %s != %s" % (
             normed,
             path,
         )
         return path
 
     @property
     def _checklink(self):
         return util.checklink(os.path.dirname(pycompat.fsencode(self.git.path)))
 
     def copies(self):
         # TODO support copies?
         return {}
 
     # # TODO what the heck is this
     _filecache = set()
 
     def pendingparentchange(self):
         # TODO: we need to implement the context manager bits and
         # correctly stage/revert index edits.
         return False
 
     def write(self, tr):
         # TODO: call parent change callbacks
 
         if tr:
 
             def writeinner(category):
                 self.git.index.write()
 
             tr.addpending(b'gitdirstate', writeinner)
         else:
             self.git.index.write()
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         # TODO core dirstate does something about slashes here
         assert isinstance(f, bytes)
         r = util.pathto(self._root, cwd, f)
         return r
 
     def matches(self, match):
         for x in self.git.index:
             p = pycompat.fsencode(x.path)
             if match(p):
                 yield p
 
-    def set_clean(self, f, parentfiledata=None):
+    def set_clean(self, f, parentfiledata):
         """Mark a file normal and clean."""
         # TODO: for now we just let libgit2 re-stat the file. We can
         # clearly do better.
 
     def set_possibly_dirty(self, f):
         """Mark a file normal, but possibly dirty."""
         # TODO: for now we just let libgit2 re-stat the file. We can
         # clearly do better.
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         # TODO: we need to use .status() and not iterate the index,
         # because the index doesn't force a re-walk and so `hg add` of
         # a new file without an intervening call to status will
         # silently do nothing.
         r = {}
         cwd = self.getcwd()
         for path, status in self.git.status().items():
             if path.startswith('.hg/'):
                 continue
             path = pycompat.fsencode(path)
             if not match(path):
                 continue
             # TODO construct the stat info from the status object?
             try:
                 s = os.stat(os.path.join(cwd, path))
             except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
                 continue
             r[path] = s
         return r
 
     def savebackup(self, tr, backupname):
         # TODO: figure out a strategy for saving index backups.
         pass
 
     def restorebackup(self, tr, backupname):
         # TODO: figure out a strategy for saving index backups.
         pass
 
     def set_tracked(self, f):
         uf = pycompat.fsdecode(f)
         if uf in self.git.index:
             return False
         index = self.git.index
         index.read()
         index.add(uf)
         index.write()
         return True
 
     def add(self, f):
         index = self.git.index
         index.read()
         index.add(pycompat.fsdecode(f))
         index.write()
 
     def drop(self, f):
         index = self.git.index
         index.read()
         fs = pycompat.fsdecode(f)
         if fs in index:
             index.remove(fs)
             index.write()
 
     def set_untracked(self, f):
         index = self.git.index
         index.read()
         fs = pycompat.fsdecode(f)
         if fs in index:
             index.remove(fs)
             index.write()
             return True
         return False
 
     def remove(self, f):
         index = self.git.index
         index.read()
         index.remove(pycompat.fsdecode(f))
         index.write()
 
     def copied(self, path):
         # TODO: track copies?
         return None
 
     def prefetch_parents(self):
         # TODO
         pass
 
     def update_file(self, *args, **kwargs):
         # TODO
         pass
 
     @contextlib.contextmanager
     def parentchange(self):
         # TODO: track this maybe?
         yield
 
     def addparentchangecallback(self, category, callback):
         # TODO: should this be added to the dirstate interface?
         self._plchangecallbacks[category] = callback
 
     def clearbackup(self, tr, backupname):
         # TODO
         pass
 
     def setbranch(self, branch):
         raise error.Abort(
             b'git repos do not support branches. try using bookmarks'
         )

hgext/histedit.py

@@ -1,2678 +1,2690 b''
 # histedit.py - interactive history editing for mercurial
 #
 # Copyright 2009 Augie Fackler <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """interactive history editing
 
 With this extension installed, Mercurial gains one new command: histedit. Usage
 is as follows, assuming the following history::
 
  @  3[tip]   7c2fd3b9020c   2009-04-27 18:04 -0500   durin42
  |    Add delta
  |
  o  2   030b686bedc4   2009-04-27 18:04 -0500   durin42
  |    Add gamma
  |
  o  1   c561b4e977df   2009-04-27 18:04 -0500   durin42
  |    Add beta
  |
  o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
       Add alpha
 
 If you were to run ``hg histedit c561b4e977df``, you would see the following
 file open in your editor::
 
  pick c561b4e977df Add beta
  pick 030b686bedc4 Add gamma
  pick 7c2fd3b9020c Add delta
 
  # Edit history between c561b4e977df and 7c2fd3b9020c
  #
  # Commits are listed from least to most recent
  #
  # Commands:
  #  p, pick = use commit
  #  e, edit = use commit, but allow edits before making new commit
  #  f, fold = use commit, but combine it with the one above
  #  r, roll = like fold, but discard this commit's description and date
  #  d, drop = remove commit from history
  #  m, mess = edit commit message without changing commit content
  #  b, base = checkout changeset and apply further changesets from there
  #
 
 In this file, lines beginning with ``#`` are ignored. You must specify a rule
 for each revision in your history. For example, if you had meant to add gamma
 before beta, and then wanted to add delta in the same revision as beta, you
 would reorganize the file to look like this::
 
  pick 030b686bedc4 Add gamma
  pick c561b4e977df Add beta
  fold 7c2fd3b9020c Add delta
 
  # Edit history between c561b4e977df and 7c2fd3b9020c
  #
  # Commits are listed from least to most recent
  #
  # Commands:
  #  p, pick = use commit
  #  e, edit = use commit, but allow edits before making new commit
  #  f, fold = use commit, but combine it with the one above
  #  r, roll = like fold, but discard this commit's description and date
  #  d, drop = remove commit from history
  #  m, mess = edit commit message without changing commit content
  #  b, base = checkout changeset and apply further changesets from there
  #
 
 At which point you close the editor and ``histedit`` starts working. When you
 specify a ``fold`` operation, ``histedit`` will open an editor when it folds
 those revisions together, offering you a chance to clean up the commit message::
 
  Add beta
  ***
  Add delta
 
 Edit the commit message to your liking, then close the editor. The date used
 for the commit will be the later of the two commits' dates. For this example,
 let's assume that the commit message was changed to ``Add beta and delta.``
 After histedit has run and had a chance to remove any old or temporary
 revisions it needed, the history looks like this::
 
  @  2[tip]   989b4d060121   2009-04-27 18:04 -0500   durin42
  |    Add beta and delta.
  |
  o  1   081603921c3f   2009-04-27 18:04 -0500   durin42
  |    Add gamma
  |
  o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
       Add alpha
 
 Note that ``histedit`` does *not* remove any revisions (even its own temporary
 ones) until after it has completed all the editing operations, so it will
 probably perform several strip operations when it's done. For the above example,
 it had to run strip twice. Strip can be slow depending on a variety of factors,
 so you might need to be a little patient. You can choose to keep the original
 revisions by passing the ``--keep`` flag.
 
 The ``edit`` operation will drop you back to a command prompt,
 allowing you to edit files freely, or even use ``hg record`` to commit
 some changes as a separate commit. When you're done, any remaining
 uncommitted changes will be committed as well. When done, run ``hg
 histedit --continue`` to finish this step. If there are uncommitted
 changes, you'll be prompted for a new commit message, but the default
 commit message will be the original message for the ``edit`` ed
 revision, and the date of the original commit will be preserved.
 
 The ``message`` operation will give you a chance to revise a commit
 message without changing the contents. It's a shortcut for doing
 ``edit`` immediately followed by `hg histedit --continue``.
 
 If ``histedit`` encounters a conflict when moving a revision (while
 handling ``pick`` or ``fold``), it'll stop in a similar manner to
 ``edit`` with the difference that it won't prompt you for a commit
 message when done. If you decide at this point that you don't like how
 much work it will be to rearrange history, or that you made a mistake,
 you can use ``hg histedit --abort`` to abandon the new changes you
 have made and return to the state before you attempted to edit your
 history.
 
 If we clone the histedit-ed example repository above and add four more
 changes, such that we have the following history::
 
    @  6[tip]   038383181893   2009-04-27 18:04 -0500   stefan
    |    Add theta
    |
    o  5   140988835471   2009-04-27 18:04 -0500   stefan
    |    Add eta
    |
    o  4   122930637314   2009-04-27 18:04 -0500   stefan
    |    Add zeta
    |
    o  3   836302820282   2009-04-27 18:04 -0500   stefan
    |    Add epsilon
    |
    o  2   989b4d060121   2009-04-27 18:04 -0500   durin42
    |    Add beta and delta.
    |
    o  1   081603921c3f   2009-04-27 18:04 -0500   durin42
    |    Add gamma
    |
    o  0   d8d2fcd0e319   2009-04-27 18:04 -0500   durin42
         Add alpha
 
 If you run ``hg histedit --outgoing`` on the clone then it is the same
 as running ``hg histedit 836302820282``. If you need plan to push to a
 repository that Mercurial does not detect to be related to the source
 repo, you can add a ``--force`` option.
 
 Config
 ------
 
 Histedit rule lines are truncated to 80 characters by default. You
 can customize this behavior by setting a different length in your
 configuration file::
 
   [histedit]
   linelen = 120      # truncate rule lines at 120 characters
 
 The summary of a change can be customized as well::
 
   [histedit]
   summary-template = '{rev} {bookmarks} {desc|firstline}'
 
 The customized summary should be kept short enough that rule lines
 will fit in the configured line length. See above if that requires
 customization.
 
 ``hg histedit`` attempts to automatically choose an appropriate base
 revision to use. To change which base revision is used, define a
 revset in your configuration file::
 
   [histedit]
   defaultrev = only(.) & draft()
 
 By default each edited revision needs to be present in histedit commands.
 To remove revision you need to use ``drop`` operation. You can configure
 the drop to be implicit for missing commits by adding::
 
   [histedit]
   dropmissing = True
 
 By default, histedit will close the transaction after each action. For
 performance purposes, you can configure histedit to use a single transaction
 across the entire histedit. WARNING: This setting introduces a significant risk
 of losing the work you've done in a histedit if the histedit aborts
 unexpectedly::
 
   [histedit]
   singletransaction = True
 
 """
 
 from __future__ import absolute_import
 
 # chistedit dependencies that are not available everywhere
 try:
     import fcntl
     import termios
 except ImportError:
     fcntl = None
     termios = None
 
 import functools
 import os
 import struct
 
 from mercurial.i18n import _
 from mercurial.pycompat import (
     getattr,
     open,
 )
 from mercurial.node import (
     bin,
     hex,
     short,
 )
 from mercurial import (
     bundle2,
     cmdutil,
     context,
     copies,
     destutil,
     discovery,
     encoding,
     error,
     exchange,
     extensions,
     hg,
     logcmdutil,
     merge as mergemod,
     mergestate as mergestatemod,
     mergeutil,
     obsolete,
     pycompat,
     registrar,
     repair,
     rewriteutil,
     scmutil,
     state as statemod,
     util,
 )
 from mercurial.utils import (
     dateutil,
     stringutil,
     urlutil,
 )
 
 pickle = util.pickle
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 configitem(
     b'experimental',
     b'histedit.autoverb',
     default=False,
 )
 configitem(
     b'histedit',
     b'defaultrev',
     default=None,
 )
 configitem(
     b'histedit',
     b'dropmissing',
     default=False,
 )
 configitem(
     b'histedit',
     b'linelen',
     default=80,
 )
 configitem(
     b'histedit',
     b'singletransaction',
     default=False,
 )
 configitem(
     b'ui',
     b'interface.histedit',
     default=None,
 )
 configitem(b'histedit', b'summary-template', default=b'{rev} {desc|firstline}')
 # TODO: Teach the text-based histedit interface to respect this config option
 # before we make it non-experimental.
 configitem(
     b'histedit', b'later-commits-first', default=False, experimental=True
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 actiontable = {}
 primaryactions = set()
 secondaryactions = set()
 tertiaryactions = set()
 internalactions = set()
 
 
 def geteditcomment(ui, first, last):
     """construct the editor comment
     The comment includes::
      - an intro
      - sorted primary commands
      - sorted short commands
      - sorted long commands
      - additional hints
 
     Commands are only included once.
     """
     intro = _(
         b"""Edit history between %s and %s
 
 Commits are listed from least to most recent
 
 You can reorder changesets by reordering the lines
 
 Commands:
 """
     )
     actions = []
 
     def addverb(v):
         a = actiontable[v]
         lines = a.message.split(b"\n")
         if len(a.verbs):
             v = b', '.join(sorted(a.verbs, key=lambda v: len(v)))
         actions.append(b" %s = %s" % (v, lines[0]))
         actions.extend([b'  %s'] * (len(lines) - 1))
 
     for v in (
         sorted(primaryactions)
         + sorted(secondaryactions)
         + sorted(tertiaryactions)
     ):
         addverb(v)
     actions.append(b'')
 
     hints = []
     if ui.configbool(b'histedit', b'dropmissing'):
         hints.append(
             b"Deleting a changeset from the list "
             b"will DISCARD it from the edited history!"
         )
 
     lines = (intro % (first, last)).split(b'\n') + actions + hints
 
     return b''.join([b'# %s\n' % l if l else b'#\n' for l in lines])
 
 
 class histeditstate(object):
     def __init__(self, repo):
         self.repo = repo
         self.actions = None
         self.keep = None
         self.topmost = None
         self.parentctxnode = None
         self.lock = None
         self.wlock = None
         self.backupfile = None
         self.stateobj = statemod.cmdstate(repo, b'histedit-state')
         self.replacements = []
 
     def read(self):
         """Load histedit state from disk and set fields appropriately."""
         if not self.stateobj.exists():
             cmdutil.wrongtooltocontinue(self.repo, _(b'histedit'))
 
         data = self._read()
 
         self.parentctxnode = data[b'parentctxnode']
         actions = parserules(data[b'rules'], self)
         self.actions = actions
         self.keep = data[b'keep']
         self.topmost = data[b'topmost']
         self.replacements = data[b'replacements']
         self.backupfile = data[b'backupfile']
 
     def _read(self):
         fp = self.repo.vfs.read(b'histedit-state')
         if fp.startswith(b'v1\n'):
             data = self._load()
             parentctxnode, rules, keep, topmost, replacements, backupfile = data
         else:
             data = pickle.loads(fp)
             parentctxnode, rules, keep, topmost, replacements = data
             backupfile = None
         rules = b"\n".join([b"%s %s" % (verb, rest) for [verb, rest] in rules])
 
         return {
             b'parentctxnode': parentctxnode,
             b"rules": rules,
             b"keep": keep,
             b"topmost": topmost,
             b"replacements": replacements,
             b"backupfile": backupfile,
         }
 
     def write(self, tr=None):
         if tr:
             tr.addfilegenerator(
                 b'histedit-state',
                 (b'histedit-state',),
                 self._write,
                 location=b'plain',
             )
         else:
             with self.repo.vfs(b"histedit-state", b"w") as f:
                 self._write(f)
 
     def _write(self, fp):
         fp.write(b'v1\n')
         fp.write(b'%s\n' % hex(self.parentctxnode))
         fp.write(b'%s\n' % hex(self.topmost))
         fp.write(b'%s\n' % (b'True' if self.keep else b'False'))
         fp.write(b'%d\n' % len(self.actions))
         for action in self.actions:
             fp.write(b'%s\n' % action.tostate())
         fp.write(b'%d\n' % len(self.replacements))
         for replacement in self.replacements:
             fp.write(
                 b'%s%s\n'
                 % (
                     hex(replacement[0]),
                     b''.join(hex(r) for r in replacement[1]),
                 )
             )
         backupfile = self.backupfile
         if not backupfile:
             backupfile = b''
         fp.write(b'%s\n' % backupfile)
 
     def _load(self):
         fp = self.repo.vfs(b'histedit-state', b'r')
         lines = [l[:-1] for l in fp.readlines()]
 
         index = 0
         lines[index]  # version number
         index += 1
 
         parentctxnode = bin(lines[index])
         index += 1
 
         topmost = bin(lines[index])
         index += 1
 
         keep = lines[index] == b'True'
         index += 1
 
         # Rules
         rules = []
         rulelen = int(lines[index])
         index += 1
         for i in pycompat.xrange(rulelen):
             ruleaction = lines[index]
             index += 1
             rule = lines[index]
             index += 1
             rules.append((ruleaction, rule))
 
         # Replacements
         replacements = []
         replacementlen = int(lines[index])
         index += 1
         for i in pycompat.xrange(replacementlen):
             replacement = lines[index]
             original = bin(replacement[:40])
             succ = [
                 bin(replacement[i : i + 40])
                 for i in range(40, len(replacement), 40)
             ]
             replacements.append((original, succ))
             index += 1
 
         backupfile = lines[index]
         index += 1
 
         fp.close()
 
         return parentctxnode, rules, keep, topmost, replacements, backupfile
 
     def clear(self):
         if self.inprogress():
             self.repo.vfs.unlink(b'histedit-state')
 
     def inprogress(self):
         return self.repo.vfs.exists(b'histedit-state')
 
 
 class histeditaction(object):
     def __init__(self, state, node):
         self.state = state
         self.repo = state.repo
         self.node = node
 
     @classmethod
     def fromrule(cls, state, rule):
         """Parses the given rule, returning an instance of the histeditaction."""
         ruleid = rule.strip().split(b' ', 1)[0]
         # ruleid can be anything from rev numbers, hashes, "bookmarks" etc
         # Check for validation of rule ids and get the rulehash
         try:
             rev = bin(ruleid)
         except TypeError:
             try:
                 _ctx = scmutil.revsingle(state.repo, ruleid)
                 rulehash = _ctx.hex()
                 rev = bin(rulehash)
             except error.RepoLookupError:
                 raise error.ParseError(_(b"invalid changeset %s") % ruleid)
         return cls(state, rev)
 
     def verify(self, prev, expected, seen):
         """Verifies semantic correctness of the rule"""
         repo = self.repo
         ha = hex(self.node)
         self.node = scmutil.resolvehexnodeidprefix(repo, ha)
         if self.node is None:
             raise error.ParseError(_(b'unknown changeset %s listed') % ha[:12])
         self._verifynodeconstraints(prev, expected, seen)
 
     def _verifynodeconstraints(self, prev, expected, seen):
         # by default command need a node in the edited list
         if self.node not in expected:
             raise error.ParseError(
                 _(b'%s "%s" changeset was not a candidate')
                 % (self.verb, short(self.node)),
                 hint=_(b'only use listed changesets'),
             )
         # and only one command per node
         if self.node in seen:
             raise error.ParseError(
                 _(b'duplicated command for changeset %s') % short(self.node)
             )
 
     def torule(self):
         """build a histedit rule line for an action
 
         by default lines are in the form:
         <hash> <rev> <summary>
         """
         ctx = self.repo[self.node]
         ui = self.repo.ui
         # We don't want color codes in the commit message template, so
         # disable the label() template function while we render it.
         with ui.configoverride(
             {(b'templatealias', b'label(l,x)'): b"x"}, b'histedit'
         ):
             summary = cmdutil.rendertemplate(
                 ctx, ui.config(b'histedit', b'summary-template')
             )
         # Handle the fact that `''.splitlines() => []`
         summary = summary.splitlines()[0] if summary else b''
         line = b'%s %s %s' % (self.verb, ctx, summary)
         # trim to 75 columns by default so it's not stupidly wide in my editor
         # (the 5 more are left for verb)
         maxlen = self.repo.ui.configint(b'histedit', b'linelen')
         maxlen = max(maxlen, 22)  # avoid truncating hash
         return stringutil.ellipsis(line, maxlen)
 
     def tostate(self):
         """Print an action in format used by histedit state files
         (the first line is a verb, the remainder is the second)
         """
         return b"%s\n%s" % (self.verb, hex(self.node))
 
     def run(self):
         """Runs the action. The default behavior is simply apply the action's
         rulectx onto the current parentctx."""
         self.applychange()
         self.continuedirty()
         return self.continueclean()
 
     def applychange(self):
         """Applies the changes from this action's rulectx onto the current
         parentctx, but does not commit them."""
         repo = self.repo
         rulectx = repo[self.node]
         with repo.ui.silent():
             hg.update(repo, self.state.parentctxnode, quietempty=True)
         stats = applychanges(repo.ui, repo, rulectx, {})
         repo.dirstate.setbranch(rulectx.branch())
         if stats.unresolvedcount:
             raise error.InterventionRequired(
                 _(b'Fix up the change (%s %s)') % (self.verb, short(self.node)),
                 hint=_(b'hg histedit --continue to resume'),
             )
 
     def continuedirty(self):
         """Continues the action when changes have been applied to the working
         copy. The default behavior is to commit the dirty changes."""
         repo = self.repo
         rulectx = repo[self.node]
 
         editor = self.commiteditor()
         commit = commitfuncfor(repo, rulectx)
         if repo.ui.configbool(b'rewrite', b'update-timestamp'):
             date = dateutil.makedate()
         else:
             date = rulectx.date()
         commit(
             text=rulectx.description(),
             user=rulectx.user(),
             date=date,
             extra=rulectx.extra(),
             editor=editor,
         )
 
     def commiteditor(self):
         """The editor to be used to edit the commit message."""
         return False
 
     def continueclean(self):
         """Continues the action when the working copy is clean. The default
         behavior is to accept the current commit as the new version of the
         rulectx."""
         ctx = self.repo[b'.']
         if ctx.node() == self.state.parentctxnode:
             self.repo.ui.warn(
                 _(b'%s: skipping changeset (no changes)\n') % short(self.node)
             )
             return ctx, [(self.node, tuple())]
         if ctx.node() == self.node:
             # Nothing changed
             return ctx, []
         return ctx, [(self.node, (ctx.node(),))]
 
 
 def commitfuncfor(repo, src):
     """Build a commit function for the replacement of <src>
 
     This function ensure we apply the same treatment to all changesets.
 
     - Add a 'histedit_source' entry in extra.
 
     Note that fold has its own separated logic because its handling is a bit
     different and not easily factored out of the fold method.
     """
     phasemin = src.phase()
 
     def commitfunc(**kwargs):
         overrides = {(b'phases', b'new-commit'): phasemin}
         with repo.ui.configoverride(overrides, b'histedit'):
             extra = kwargs.get('extra', {}).copy()
             extra[b'histedit_source'] = src.hex()
             kwargs['extra'] = extra
             return repo.commit(**kwargs)
 
     return commitfunc
 
 
 def applychanges(ui, repo, ctx, opts):
     """Merge changeset from ctx (only) in the current working directory"""
     if ctx.p1().node() == repo.dirstate.p1():
         # edits are "in place" we do not need to make any merge,
         # just applies changes on parent for editing
         with ui.silent():
             cmdutil.revert(ui, repo, ctx, all=True)
             stats = mergemod.updateresult(0, 0, 0, 0)
     else:
         try:
             # ui.forcemerge is an internal variable, do not document
             repo.ui.setconfig(
                 b'ui', b'forcemerge', opts.get(b'tool', b''), b'histedit'
             )
-            stats = mergemod.graft(repo, ctx, labels=[b'local', b'histedit'])
+            stats = mergemod.graft(
+                repo,
+                ctx,
+                labels=[
+                    b'already edited',
+                    b'current change',
+                    b'parent of current change',
+                ],
+            )
         finally:
             repo.ui.setconfig(b'ui', b'forcemerge', b'', b'histedit')
     return stats
 
 
 def collapse(repo, firstctx, lastctx, commitopts, skipprompt=False):
     """collapse the set of revisions from first to last as new one.
 
     Expected commit options are:
         - message
         - date
         - username
     Commit message is edited in all cases.
 
     This function works in memory."""
     ctxs = list(repo.set(b'%d::%d', firstctx.rev(), lastctx.rev()))
     if not ctxs:
         return None
     for c in ctxs:
         if not c.mutable():
             raise error.ParseError(
                 _(b"cannot fold into public change %s") % short(c.node())
             )
     base = firstctx.p1()
 
     # commit a new version of the old changeset, including the update
     # collect all files which might be affected
     files = set()
     for ctx in ctxs:
         files.update(ctx.files())
 
     # Recompute copies (avoid recording a -> b -> a)
     copied = copies.pathcopies(base, lastctx)
 
     # prune files which were reverted by the updates
     files = [f for f in files if not cmdutil.samefile(f, lastctx, base)]
     # commit version of these files as defined by head
     headmf = lastctx.manifest()
 
     def filectxfn(repo, ctx, path):
         if path in headmf:
             fctx = lastctx[path]
             flags = fctx.flags()
             mctx = context.memfilectx(
                 repo,
                 ctx,
                 fctx.path(),
                 fctx.data(),
                 islink=b'l' in flags,
                 isexec=b'x' in flags,
                 copysource=copied.get(path),
             )
             return mctx
         return None
 
     if commitopts.get(b'message'):
         message = commitopts[b'message']
     else:
         message = firstctx.description()
     user = commitopts.get(b'user')
     date = commitopts.get(b'date')
     extra = commitopts.get(b'extra')
 
     parents = (firstctx.p1().node(), firstctx.p2().node())
     editor = None
     if not skipprompt:
         editor = cmdutil.getcommiteditor(edit=True, editform=b'histedit.fold')
     new = context.memctx(
         repo,
         parents=parents,
         text=message,
         files=files,
         filectxfn=filectxfn,
         user=user,
         date=date,
         extra=extra,
         editor=editor,
     )
     return repo.commitctx(new)
 
 
 def _isdirtywc(repo):
     return repo[None].dirty(missing=True)
 
 
 def abortdirty():
     raise error.StateError(
         _(b'working copy has pending changes'),
         hint=_(
             b'amend, commit, or revert them and run histedit '
             b'--continue, or abort with histedit --abort'
         ),
     )
 
 
 def action(verbs, message, priority=False, internal=False):
     def wrap(cls):
         assert not priority or not internal
         verb = verbs[0]
         if priority:
             primaryactions.add(verb)
         elif internal:
             internalactions.add(verb)
         elif len(verbs) > 1:
             secondaryactions.add(verb)
         else:
             tertiaryactions.add(verb)
 
         cls.verb = verb
         cls.verbs = verbs
         cls.message = message
         for verb in verbs:
             actiontable[verb] = cls
         return cls
 
     return wrap
 
 
 @action([b'pick', b'p'], _(b'use commit'), priority=True)
 class pick(histeditaction):
     def run(self):
         rulectx = self.repo[self.node]
         if rulectx.p1().node() == self.state.parentctxnode:
             self.repo.ui.debug(b'node %s unchanged\n' % short(self.node))
             return rulectx, []
 
         return super(pick, self).run()
 
 
 @action(
     [b'edit', b'e'],
     _(b'use commit, but allow edits before making new commit'),
     priority=True,
 )
 class edit(histeditaction):
     def run(self):
         repo = self.repo
         rulectx = repo[self.node]
         hg.update(repo, self.state.parentctxnode, quietempty=True)
         applychanges(repo.ui, repo, rulectx, {})
         hint = _(b'to edit %s, `hg histedit --continue` after making changes')
         raise error.InterventionRequired(
             _(b'Editing (%s), commit as needed now to split the change')
             % short(self.node),
             hint=hint % short(self.node),
         )
 
     def commiteditor(self):
         return cmdutil.getcommiteditor(edit=True, editform=b'histedit.edit')
 
 
 @action([b'fold', b'f'], _(b'use commit, but combine it with the one above'))
 class fold(histeditaction):
     def verify(self, prev, expected, seen):
         """Verifies semantic correctness of the fold rule"""
         super(fold, self).verify(prev, expected, seen)
         repo = self.repo
         if not prev:
             c = repo[self.node].p1()
         elif not prev.verb in (b'pick', b'base'):
             return
         else:
             c = repo[prev.node]
         if not c.mutable():
             raise error.ParseError(
                 _(b"cannot fold into public change %s") % short(c.node())
             )
 
     def continuedirty(self):
         repo = self.repo
         rulectx = repo[self.node]
 
         commit = commitfuncfor(repo, rulectx)
         commit(
             text=b'fold-temp-revision %s' % short(self.node),
             user=rulectx.user(),
             date=rulectx.date(),
             extra=rulectx.extra(),
         )
 
     def continueclean(self):
         repo = self.repo
         ctx = repo[b'.']
         rulectx = repo[self.node]
         parentctxnode = self.state.parentctxnode
         if ctx.node() == parentctxnode:
             repo.ui.warn(_(b'%s: empty changeset\n') % short(self.node))
             return ctx, [(self.node, (parentctxnode,))]
 
         parentctx = repo[parentctxnode]
         newcommits = {
             c.node()
             for c in repo.set(b'(%d::. - %d)', parentctx.rev(), parentctx.rev())
         }
         if not newcommits:
             repo.ui.warn(
                 _(
                     b'%s: cannot fold - working copy is not a '
                     b'descendant of previous commit %s\n'
                 )
                 % (short(self.node), short(parentctxnode))
             )
             return ctx, [(self.node, (ctx.node(),))]
 
         middlecommits = newcommits.copy()
         middlecommits.discard(ctx.node())
 
         return self.finishfold(
             repo.ui, repo, parentctx, rulectx, ctx.node(), middlecommits
         )
 
     def skipprompt(self):
         """Returns true if the rule should skip the message editor.
 
         For example, 'fold' wants to show an editor, but 'rollup'
         doesn't want to.
         """
         return False
 
     def mergedescs(self):
         """Returns true if the rule should merge messages of multiple changes.
 
         This exists mainly so that 'rollup' rules can be a subclass of
         'fold'.
         """
         return True
 
     def firstdate(self):
         """Returns true if the rule should preserve the date of the first
         change.
 
         This exists mainly so that 'rollup' rules can be a subclass of
         'fold'.
         """
         return False
 
     def finishfold(self, ui, repo, ctx, oldctx, newnode, internalchanges):
         mergemod.update(ctx.p1())
         ### prepare new commit data
         commitopts = {}
         commitopts[b'user'] = ctx.user()
         # commit message
         if not self.mergedescs():
             newmessage = ctx.description()
         else:
             newmessage = (
                 b'\n***\n'.join(
                     [ctx.description()]
                     + [repo[r].description() for r in internalchanges]
                     + [oldctx.description()]
                 )
                 + b'\n'
             )
         commitopts[b'message'] = newmessage
         # date
         if self.firstdate():
             commitopts[b'date'] = ctx.date()
         else:
             commitopts[b'date'] = max(ctx.date(), oldctx.date())
         # if date is to be updated to current
         if ui.configbool(b'rewrite', b'update-timestamp'):
             commitopts[b'date'] = dateutil.makedate()
 
         extra = ctx.extra().copy()
         # histedit_source
         # note: ctx is likely a temporary commit but that the best we can do
         #       here. This is sufficient to solve issue3681 anyway.
         extra[b'histedit_source'] = b'%s,%s' % (ctx.hex(), oldctx.hex())
         commitopts[b'extra'] = extra
         phasemin = max(ctx.phase(), oldctx.phase())
         overrides = {(b'phases', b'new-commit'): phasemin}
         with repo.ui.configoverride(overrides, b'histedit'):
             n = collapse(
                 repo,
                 ctx,
                 repo[newnode],
                 commitopts,
                 skipprompt=self.skipprompt(),
             )
         if n is None:
             return ctx, []
         mergemod.update(repo[n])
         replacements = [
             (oldctx.node(), (newnode,)),
             (ctx.node(), (n,)),
             (newnode, (n,)),
         ]
         for ich in internalchanges:
             replacements.append((ich, (n,)))
         return repo[n], replacements
 
 
 @action(
     [b'base', b'b'],
     _(b'checkout changeset and apply further changesets from there'),
 )
 class base(histeditaction):
     def run(self):
         if self.repo[b'.'].node() != self.node:
             mergemod.clean_update(self.repo[self.node])
         return self.continueclean()
 
     def continuedirty(self):
         abortdirty()
 
     def continueclean(self):
         basectx = self.repo[b'.']
         return basectx, []
 
     def _verifynodeconstraints(self, prev, expected, seen):
         # base can only be use with a node not in the edited set
         if self.node in expected:
             msg = _(b'%s "%s" changeset was an edited list candidate')
             raise error.ParseError(
                 msg % (self.verb, short(self.node)),
                 hint=_(b'base must only use unlisted changesets'),
             )
 
 
 @action(
     [b'_multifold'],
     _(
         """fold subclass used for when multiple folds happen in a row
 
     We only want to fire the editor for the folded message once when
     (say) four changes are folded down into a single change. This is
     similar to rollup, but we should preserve both messages so that
     when the last fold operation runs we can show the user all the
     commit messages in their editor.
     """
     ),
     internal=True,
 )
 class _multifold(fold):
     def skipprompt(self):
         return True
 
 
 @action(
     [b"roll", b"r"],
     _(b"like fold, but discard this commit's description and date"),
 )
 class rollup(fold):
     def mergedescs(self):
         return False
 
     def skipprompt(self):
         return True
 
     def firstdate(self):
         return True
 
 
 @action([b"drop", b"d"], _(b'remove commit from history'))
 class drop(histeditaction):
     def run(self):
         parentctx = self.repo[self.state.parentctxnode]
         return parentctx, [(self.node, tuple())]
 
 
 @action(
     [b"mess", b"m"],
     _(b'edit commit message without changing commit content'),
     priority=True,
 )
 class message(histeditaction):
     def commiteditor(self):
         return cmdutil.getcommiteditor(edit=True, editform=b'histedit.mess')
 
 
 def findoutgoing(ui, repo, remote=None, force=False, opts=None):
     """utility function to find the first outgoing changeset
 
     Used by initialization code"""
     if opts is None:
         opts = {}
     path = urlutil.get_unique_push_path(b'histedit', repo, ui, remote)
     dest = path.pushloc or path.loc
 
     ui.status(_(b'comparing with %s\n') % urlutil.hidepassword(dest))
 
     revs, checkout = hg.addbranchrevs(repo, repo, (path.branch, []), None)
     other = hg.peer(repo, opts, dest)
 
     if revs:
         revs = [repo.lookup(rev) for rev in revs]
 
     outgoing = discovery.findcommonoutgoing(repo, other, revs, force=force)
     if not outgoing.missing:
         raise error.StateError(_(b'no outgoing ancestors'))
     roots = list(repo.revs(b"roots(%ln)", outgoing.missing))
     if len(roots) > 1:
         msg = _(b'there are ambiguous outgoing revisions')
         hint = _(b"see 'hg help histedit' for more detail")
         raise error.StateError(msg, hint=hint)
     return repo[roots[0]].node()
 
 
 # Curses Support
 try:
     import curses
 except ImportError:
     curses = None
 
 KEY_LIST = [b'pick', b'edit', b'fold', b'drop', b'mess', b'roll']
 ACTION_LABELS = {
     b'fold': b'^fold',
     b'roll': b'^roll',
 }
 
 COLOR_HELP, COLOR_SELECTED, COLOR_OK, COLOR_WARN, COLOR_CURRENT = 1, 2, 3, 4, 5
 COLOR_DIFF_ADD_LINE, COLOR_DIFF_DEL_LINE, COLOR_DIFF_OFFSET = 6, 7, 8
 COLOR_ROLL, COLOR_ROLL_CURRENT, COLOR_ROLL_SELECTED = 9, 10, 11
 
 E_QUIT, E_HISTEDIT = 1, 2
 E_PAGEDOWN, E_PAGEUP, E_LINEUP, E_LINEDOWN, E_RESIZE = 3, 4, 5, 6, 7
 MODE_INIT, MODE_PATCH, MODE_RULES, MODE_HELP = 0, 1, 2, 3
 
 KEYTABLE = {
     b'global': {
         b'h': b'next-action',
         b'KEY_RIGHT': b'next-action',
         b'l': b'prev-action',
         b'KEY_LEFT': b'prev-action',
         b'q': b'quit',
         b'c': b'histedit',
         b'C': b'histedit',
         b'v': b'showpatch',
         b'?': b'help',
     },
     MODE_RULES: {
         b'd': b'action-drop',
         b'e': b'action-edit',
         b'f': b'action-fold',
         b'm': b'action-mess',
         b'p': b'action-pick',
         b'r': b'action-roll',
         b' ': b'select',
         b'j': b'down',
         b'k': b'up',
         b'KEY_DOWN': b'down',
         b'KEY_UP': b'up',
         b'J': b'move-down',
         b'K': b'move-up',
         b'KEY_NPAGE': b'move-down',
         b'KEY_PPAGE': b'move-up',
         b'0': b'goto',  # Used for 0..9
     },
     MODE_PATCH: {
         b' ': b'page-down',
         b'KEY_NPAGE': b'page-down',
         b'KEY_PPAGE': b'page-up',
         b'j': b'line-down',
         b'k': b'line-up',
         b'KEY_DOWN': b'line-down',
         b'KEY_UP': b'line-up',
         b'J': b'down',
         b'K': b'up',
     },
     MODE_HELP: {},
 }
 
 
 def screen_size():
     return struct.unpack(b'hh', fcntl.ioctl(1, termios.TIOCGWINSZ, b'    '))
 
 
 class histeditrule(object):
     def __init__(self, ui, ctx, pos, action=b'pick'):
         self.ui = ui
         self.ctx = ctx
         self.action = action
         self.origpos = pos
         self.pos = pos
         self.conflicts = []
 
     def __bytes__(self):
         # Example display of several histeditrules:
         #
         #  #10 pick   316392:06a16c25c053   add option to skip tests
         #  #11 ^roll  316393:71313c964cc5   <RED>oops a fixup commit</RED>
         #  #12 pick   316394:ab31f3973b0d   include mfbt for mozilla-config.h
         #  #13 ^fold  316395:14ce5803f4c3   fix warnings
         #
         # The carets point to the changeset being folded into ("roll this
         # changeset into the changeset above").
         return b'%s%s' % (self.prefix, self.desc)
 
     __str__ = encoding.strmethod(__bytes__)
 
     @property
     def prefix(self):
         # Some actions ('fold' and 'roll') combine a patch with a
         # previous one. Add a marker showing which patch they apply
         # to.
         action = ACTION_LABELS.get(self.action, self.action)
 
         h = self.ctx.hex()[0:12]
         r = self.ctx.rev()
 
         return b"#%s %s %d:%s   " % (
             (b'%d' % self.origpos).ljust(2),
             action.ljust(6),
             r,
             h,
         )
 
     @util.propertycache
     def desc(self):
         summary = cmdutil.rendertemplate(
             self.ctx, self.ui.config(b'histedit', b'summary-template')
         )
         if summary:
             return summary
         # This is split off from the prefix property so that we can
         # separately make the description for 'roll' red (since it
         # will get discarded).
         return self.ctx.description().splitlines()[0].strip()
 
     def checkconflicts(self, other):
         if other.pos > self.pos and other.origpos <= self.origpos:
             if set(other.ctx.files()) & set(self.ctx.files()) != set():
                 self.conflicts.append(other)
                 return self.conflicts
 
         if other in self.conflicts:
             self.conflicts.remove(other)
         return self.conflicts
 
 
 def makecommands(rules):
     """Returns a list of commands consumable by histedit --commands based on
     our list of rules"""
     commands = []
     for rules in rules:
         commands.append(b'%s %s\n' % (rules.action, rules.ctx))
     return commands
 
 
 def addln(win, y, x, line, color=None):
     """Add a line to the given window left padding but 100% filled with
     whitespace characters, so that the color appears on the whole line"""
     maxy, maxx = win.getmaxyx()
     length = maxx - 1 - x
     line = bytes(line).ljust(length)[:length]
     if y < 0:
         y = maxy + y
     if x < 0:
         x = maxx + x
     if color:
         win.addstr(y, x, line, color)
     else:
         win.addstr(y, x, line)
 
 
 def _trunc_head(line, n):
     if len(line) <= n:
         return line
     return b'> ' + line[-(n - 2) :]
 
 
 def _trunc_tail(line, n):
     if len(line) <= n:
         return line
     return line[: n - 2] + b' >'
 
 
 class _chistedit_state(object):
     def __init__(
         self,
         repo,
         rules,
         stdscr,
     ):
         self.repo = repo
         self.rules = rules
         self.stdscr = stdscr
         self.later_on_top = repo.ui.configbool(
             b'histedit', b'later-commits-first'
         )
         # The current item in display order, initialized to point to the top
         # of the screen.
         self.pos = 0
         self.selected = None
         self.mode = (MODE_INIT, MODE_INIT)
         self.page_height = None
         self.modes = {
             MODE_RULES: {
                 b'line_offset': 0,
             },
             MODE_PATCH: {
                 b'line_offset': 0,
             },
         }
 
     def render_commit(self, win):
         """Renders the commit window that shows the log of the current selected
         commit"""
         rule = self.rules[self.display_pos_to_rule_pos(self.pos)]
 
         ctx = rule.ctx
         win.box()
 
         maxy, maxx = win.getmaxyx()
         length = maxx - 3
 
         line = b"changeset: %d:%s" % (ctx.rev(), ctx.hex()[:12])
         win.addstr(1, 1, line[:length])
 
         line = b"user:      %s" % ctx.user()
         win.addstr(2, 1, line[:length])
 
         bms = self.repo.nodebookmarks(ctx.node())
         line = b"bookmark:  %s" % b' '.join(bms)
         win.addstr(3, 1, line[:length])
 
         line = b"summary:   %s" % (ctx.description().splitlines()[0])
         win.addstr(4, 1, line[:length])
 
         line = b"files:     "
         win.addstr(5, 1, line)
         fnx = 1 + len(line)
         fnmaxx = length - fnx + 1
         y = 5
         fnmaxn = maxy - (1 + y) - 1
         files = ctx.files()
         for i, line1 in enumerate(files):
             if len(files) > fnmaxn and i == fnmaxn - 1:
                 win.addstr(y, fnx, _trunc_tail(b','.join(files[i:]), fnmaxx))
                 y = y + 1
                 break
             win.addstr(y, fnx, _trunc_head(line1, fnmaxx))
             y = y + 1
 
         conflicts = rule.conflicts
         if len(conflicts) > 0:
             conflictstr = b','.join(map(lambda r: r.ctx.hex()[:12], conflicts))
             conflictstr = b"changed files overlap with %s" % conflictstr
         else:
             conflictstr = b'no overlap'
 
         win.addstr(y, 1, conflictstr[:length])
         win.noutrefresh()
 
     def helplines(self):
         if self.mode[0] == MODE_PATCH:
             help = b"""\
 ?: help, k/up: line up, j/down: line down, v: stop viewing patch
 pgup: prev page, space/pgdn: next page, c: commit, q: abort
 """
         else:
             help = b"""\
 ?: help, k/up: move up, j/down: move down, space: select, v: view patch
 d: drop, e: edit, f: fold, m: mess, p: pick, r: roll
 pgup/K: move patch up, pgdn/J: move patch down, c: commit, q: abort
 """
+            if self.later_on_top:
+                help += b"Newer commits are shown above older commits.\n"
+            else:
+                help += b"Older commits are shown above newer commits.\n"
         return help.splitlines()
 
     def render_help(self, win):
         maxy, maxx = win.getmaxyx()
         for y, line in enumerate(self.helplines()):
             if y >= maxy:
                 break
             addln(win, y, 0, line, curses.color_pair(COLOR_HELP))
         win.noutrefresh()
 
     def layout(self):
         maxy, maxx = self.stdscr.getmaxyx()
         helplen = len(self.helplines())
         mainlen = maxy - helplen - 12
         if mainlen < 1:
             raise error.Abort(
                 _(b"terminal dimensions %d by %d too small for curses histedit")
                 % (maxy, maxx),
                 hint=_(
                     b"enlarge your terminal or use --config ui.interface=text"
                 ),
             )
         return {
             b'commit': (12, maxx),
             b'help': (helplen, maxx),
             b'main': (mainlen, maxx),
         }
 
     def display_pos_to_rule_pos(self, display_pos):
         """Converts a position in display order to rule order.
 
         The `display_pos` is the order from the top in display order, not
         considering which items are currently visible on the screen. Thus,
         `display_pos=0` is the item at the top (possibly after scrolling to
         the top)
         """
         if self.later_on_top:
             return len(self.rules) - 1 - display_pos
         else:
             return display_pos
 
     def render_rules(self, rulesscr):
         start = self.modes[MODE_RULES][b'line_offset']
 
         conflicts = [r.ctx for r in self.rules if r.conflicts]
         if len(conflicts) > 0:
             line = b"potential conflict in %s" % b','.join(
                 map(pycompat.bytestr, conflicts)
             )
             addln(rulesscr, -1, 0, line, curses.color_pair(COLOR_WARN))
 
         for display_pos in range(start, len(self.rules)):
             y = display_pos - start
             if y < 0 or y >= self.page_height:
                 continue
             rule_pos = self.display_pos_to_rule_pos(display_pos)
             rule = self.rules[rule_pos]
             if len(rule.conflicts) > 0:
                 rulesscr.addstr(y, 0, b" ", curses.color_pair(COLOR_WARN))
             else:
                 rulesscr.addstr(y, 0, b" ", curses.COLOR_BLACK)
 
             if display_pos == self.selected:
                 rollcolor = COLOR_ROLL_SELECTED
                 addln(rulesscr, y, 2, rule, curses.color_pair(COLOR_SELECTED))
             elif display_pos == self.pos:
                 rollcolor = COLOR_ROLL_CURRENT
                 addln(
                     rulesscr,
                     y,
                     2,
                     rule,
                     curses.color_pair(COLOR_CURRENT) | curses.A_BOLD,
                 )
             else:
                 rollcolor = COLOR_ROLL
                 addln(rulesscr, y, 2, rule)
 
             if rule.action == b'roll':
                 rulesscr.addstr(
                     y,
                     2 + len(rule.prefix),
                     rule.desc,
                     curses.color_pair(rollcolor),
                 )
 
         rulesscr.noutrefresh()
 
     def render_string(self, win, output, diffcolors=False):
         maxy, maxx = win.getmaxyx()
         length = min(maxy - 1, len(output))
         for y in range(0, length):
             line = output[y]
             if diffcolors:
                 if line and line[0] == b'+':
                     win.addstr(
                         y, 0, line, curses.color_pair(COLOR_DIFF_ADD_LINE)
                     )
                 elif line and line[0] == b'-':
                     win.addstr(
                         y, 0, line, curses.color_pair(COLOR_DIFF_DEL_LINE)
                     )
                 elif line.startswith(b'@@ '):
                     win.addstr(y, 0, line, curses.color_pair(COLOR_DIFF_OFFSET))
                 else:
                     win.addstr(y, 0, line)
             else:
                 win.addstr(y, 0, line)
         win.noutrefresh()
 
     def render_patch(self, win):
         start = self.modes[MODE_PATCH][b'line_offset']
         content = self.modes[MODE_PATCH][b'patchcontents']
         self.render_string(win, content[start:], diffcolors=True)
 
     def event(self, ch):
         """Change state based on the current character input
 
         This takes the current state and based on the current character input from
         the user we change the state.
         """
         oldpos = self.pos
 
         if ch in (curses.KEY_RESIZE, b"KEY_RESIZE"):
             return E_RESIZE
 
         lookup_ch = ch
         if ch is not None and b'0' <= ch <= b'9':
             lookup_ch = b'0'
 
         curmode, prevmode = self.mode
         action = KEYTABLE[curmode].get(
             lookup_ch, KEYTABLE[b'global'].get(lookup_ch)
         )
         if action is None:
             return
         if action in (b'down', b'move-down'):
             newpos = min(oldpos + 1, len(self.rules) - 1)
             self.move_cursor(oldpos, newpos)
             if self.selected is not None or action == b'move-down':
                 self.swap(oldpos, newpos)
         elif action in (b'up', b'move-up'):
             newpos = max(0, oldpos - 1)
             self.move_cursor(oldpos, newpos)
             if self.selected is not None or action == b'move-up':
                 self.swap(oldpos, newpos)
         elif action == b'next-action':
             self.cycle_action(oldpos, next=True)
         elif action == b'prev-action':
             self.cycle_action(oldpos, next=False)
         elif action == b'select':
             self.selected = oldpos if self.selected is None else None
             self.make_selection(self.selected)
         elif action == b'goto' and int(ch) < len(self.rules) <= 10:
             newrule = next((r for r in self.rules if r.origpos == int(ch)))
             self.move_cursor(oldpos, newrule.pos)
             if self.selected is not None:
                 self.swap(oldpos, newrule.pos)
         elif action.startswith(b'action-'):
             self.change_action(oldpos, action[7:])
         elif action == b'showpatch':
             self.change_mode(MODE_PATCH if curmode != MODE_PATCH else prevmode)
         elif action == b'help':
             self.change_mode(MODE_HELP if curmode != MODE_HELP else prevmode)
         elif action == b'quit':
             return E_QUIT
         elif action == b'histedit':
             return E_HISTEDIT
         elif action == b'page-down':
             return E_PAGEDOWN
         elif action == b'page-up':
             return E_PAGEUP
         elif action == b'line-down':
             return E_LINEDOWN
         elif action == b'line-up':
             return E_LINEUP
 
     def patch_contents(self):
         repo = self.repo
         rule = self.rules[self.display_pos_to_rule_pos(self.pos)]
         displayer = logcmdutil.changesetdisplayer(
             repo.ui,
             repo,
             {b"patch": True, b"template": b"status"},
             buffered=True,
         )
         overrides = {(b'ui', b'verbose'): True}
         with repo.ui.configoverride(overrides, source=b'histedit'):
             displayer.show(rule.ctx)
             displayer.close()
         return displayer.hunk[rule.ctx.rev()].splitlines()
 
     def move_cursor(self, oldpos, newpos):
         """Change the rule/changeset that the cursor is pointing to, regardless of
         current mode (you can switch between patches from the view patch window)."""
         self.pos = newpos
 
         mode, _ = self.mode
         if mode == MODE_RULES:
             # Scroll through the list by updating the view for MODE_RULES, so that
             # even if we are not currently viewing the rules, switching back will
             # result in the cursor's rule being visible.
             modestate = self.modes[MODE_RULES]
             if newpos < modestate[b'line_offset']:
                 modestate[b'line_offset'] = newpos
             elif newpos > modestate[b'line_offset'] + self.page_height - 1:
                 modestate[b'line_offset'] = newpos - self.page_height + 1
 
         # Reset the patch view region to the top of the new patch.
         self.modes[MODE_PATCH][b'line_offset'] = 0
 
     def change_mode(self, mode):
         curmode, _ = self.mode
         self.mode = (mode, curmode)
         if mode == MODE_PATCH:
             self.modes[MODE_PATCH][b'patchcontents'] = self.patch_contents()
 
     def make_selection(self, pos):
         self.selected = pos
 
     def swap(self, oldpos, newpos):
         """Swap two positions and calculate necessary conflicts in
         O(|newpos-oldpos|) time"""
         old_rule_pos = self.display_pos_to_rule_pos(oldpos)
         new_rule_pos = self.display_pos_to_rule_pos(newpos)
 
         rules = self.rules
         assert 0 <= old_rule_pos < len(rules) and 0 <= new_rule_pos < len(rules)
 
         rules[old_rule_pos], rules[new_rule_pos] = (
             rules[new_rule_pos],
             rules[old_rule_pos],
         )
 
         # TODO: swap should not know about histeditrule's internals
         rules[new_rule_pos].pos = new_rule_pos
         rules[old_rule_pos].pos = old_rule_pos
 
         start = min(old_rule_pos, new_rule_pos)
         end = max(old_rule_pos, new_rule_pos)
         for r in pycompat.xrange(start, end + 1):
             rules[new_rule_pos].checkconflicts(rules[r])
             rules[old_rule_pos].checkconflicts(rules[r])
 
         if self.selected:
             self.make_selection(newpos)
 
     def change_action(self, pos, action):
         """Change the action state on the given position to the new action"""
         assert 0 <= pos < len(self.rules)
         self.rules[pos].action = action
 
     def cycle_action(self, pos, next=False):
         """Changes the action state the next or the previous action from
         the action list"""
         assert 0 <= pos < len(self.rules)
         current = self.rules[pos].action
 
         assert current in KEY_LIST
 
         index = KEY_LIST.index(current)
         if next:
             index += 1
         else:
             index -= 1
         self.change_action(pos, KEY_LIST[index % len(KEY_LIST)])
 
     def change_view(self, delta, unit):
         """Change the region of whatever is being viewed (a patch or the list of
         changesets). 'delta' is an amount (+/- 1) and 'unit' is 'page' or 'line'."""
         mode, _ = self.mode
         if mode != MODE_PATCH:
             return
         mode_state = self.modes[mode]
         num_lines = len(mode_state[b'patchcontents'])
         page_height = self.page_height
         unit = page_height if unit == b'page' else 1
         num_pages = 1 + (num_lines - 1) // page_height
         max_offset = (num_pages - 1) * page_height
         newline = mode_state[b'line_offset'] + delta * unit
         mode_state[b'line_offset'] = max(0, min(max_offset, newline))
 
 
 def _chisteditmain(repo, rules, stdscr):
     try:
         curses.use_default_colors()
     except curses.error:
         pass
 
     # initialize color pattern
     curses.init_pair(COLOR_HELP, curses.COLOR_WHITE, curses.COLOR_BLUE)
     curses.init_pair(COLOR_SELECTED, curses.COLOR_BLACK, curses.COLOR_WHITE)
     curses.init_pair(COLOR_WARN, curses.COLOR_BLACK, curses.COLOR_YELLOW)
     curses.init_pair(COLOR_OK, curses.COLOR_BLACK, curses.COLOR_GREEN)
     curses.init_pair(COLOR_CURRENT, curses.COLOR_WHITE, curses.COLOR_MAGENTA)
     curses.init_pair(COLOR_DIFF_ADD_LINE, curses.COLOR_GREEN, -1)
     curses.init_pair(COLOR_DIFF_DEL_LINE, curses.COLOR_RED, -1)
     curses.init_pair(COLOR_DIFF_OFFSET, curses.COLOR_MAGENTA, -1)
     curses.init_pair(COLOR_ROLL, curses.COLOR_RED, -1)
     curses.init_pair(
         COLOR_ROLL_CURRENT, curses.COLOR_BLACK, curses.COLOR_MAGENTA
     )
     curses.init_pair(COLOR_ROLL_SELECTED, curses.COLOR_RED, curses.COLOR_WHITE)
 
     # don't display the cursor
     try:
         curses.curs_set(0)
     except curses.error:
         pass
 
     def drawvertwin(size, y, x):
         win = curses.newwin(size[0], size[1], y, x)
         y += size[0]
         return win, y, x
 
     state = _chistedit_state(repo, rules, stdscr)
 
     # eventloop
     ch = None
     stdscr.clear()
     stdscr.refresh()
     while True:
         oldmode, unused = state.mode
         if oldmode == MODE_INIT:
             state.change_mode(MODE_RULES)
         e = state.event(ch)
 
         if e == E_QUIT:
             return False
         if e == E_HISTEDIT:
             return state.rules
         else:
             if e == E_RESIZE:
                 size = screen_size()
                 if size != stdscr.getmaxyx():
                     curses.resizeterm(*size)
 
             sizes = state.layout()
             curmode, unused = state.mode
             if curmode != oldmode:
                 state.page_height = sizes[b'main'][0]
                 # Adjust the view to fit the current screen size.
                 state.move_cursor(state.pos, state.pos)
 
             # Pack the windows against the top, each pane spread across the
             # full width of the screen.
             y, x = (0, 0)
             helpwin, y, x = drawvertwin(sizes[b'help'], y, x)
             mainwin, y, x = drawvertwin(sizes[b'main'], y, x)
             commitwin, y, x = drawvertwin(sizes[b'commit'], y, x)
 
             if e in (E_PAGEDOWN, E_PAGEUP, E_LINEDOWN, E_LINEUP):
                 if e == E_PAGEDOWN:
                     state.change_view(+1, b'page')
                 elif e == E_PAGEUP:
                     state.change_view(-1, b'page')
                 elif e == E_LINEDOWN:
                     state.change_view(+1, b'line')
                 elif e == E_LINEUP:
                     state.change_view(-1, b'line')
 
             # start rendering
             commitwin.erase()
             helpwin.erase()
             mainwin.erase()
             if curmode == MODE_PATCH:
                 state.render_patch(mainwin)
             elif curmode == MODE_HELP:
                 state.render_string(mainwin, __doc__.strip().splitlines())
             else:
                 state.render_rules(mainwin)
                 state.render_commit(commitwin)
             state.render_help(helpwin)
             curses.doupdate()
             # done rendering
             ch = encoding.strtolocal(stdscr.getkey())
 
 
 def _chistedit(ui, repo, freeargs, opts):
     """interactively edit changeset history via a curses interface
 
     Provides a ncurses interface to histedit. Press ? in chistedit mode
     to see an extensive help. Requires python-curses to be installed."""
 
     if curses is None:
         raise error.Abort(_(b"Python curses library required"))
 
     # disable color
     ui._colormode = None
 
     try:
         keep = opts.get(b'keep')
         revs = opts.get(b'rev', [])[:]
         cmdutil.checkunfinished(repo)
         cmdutil.bailifchanged(repo)
 
         revs.extend(freeargs)
         if not revs:
             defaultrev = destutil.desthistedit(ui, repo)
             if defaultrev is not None:
                 revs.append(defaultrev)
         if len(revs) != 1:
             raise error.InputError(
                 _(b'histedit requires exactly one ancestor revision')
             )
 
         rr = list(repo.set(b'roots(%ld)', logcmdutil.revrange(repo, revs)))
         if len(rr) != 1:
             raise error.InputError(
                 _(
                     b'The specified revisions must have '
                     b'exactly one common root'
                 )
             )
         root = rr[0].node()
 
         topmost = repo.dirstate.p1()
         revs = between(repo, root, topmost, keep)
         if not revs:
             raise error.InputError(
                 _(b'%s is not an ancestor of working directory') % short(root)
             )
 
         rules = []
         for i, r in enumerate(revs):
             rules.append(histeditrule(ui, repo[r], i))
         with util.with_lc_ctype():
             rc = curses.wrapper(functools.partial(_chisteditmain, repo, rules))
         curses.echo()
         curses.endwin()
         if rc is False:
             ui.write(_(b"histedit aborted\n"))
             return 0
         if type(rc) is list:
             ui.status(_(b"performing changes\n"))
             rules = makecommands(rc)
             with repo.vfs(b'chistedit', b'w+') as fp:
                 for r in rules:
                     fp.write(r)
                 opts[b'commands'] = fp.name
             return _texthistedit(ui, repo, freeargs, opts)
     except KeyboardInterrupt:
         pass
     return -1
 
 
 @command(
     b'histedit',
     [
         (
             b'',
             b'commands',
             b'',
             _(b'read history edits from the specified file'),
             _(b'FILE'),
         ),
         (b'c', b'continue', False, _(b'continue an edit already in progress')),
         (b'', b'edit-plan', False, _(b'edit remaining actions list')),
         (
             b'k',
             b'keep',
             False,
             _(b"don't strip old nodes after edit is complete"),
         ),
         (b'', b'abort', False, _(b'abort an edit in progress')),
         (b'o', b'outgoing', False, _(b'changesets not found in destination')),
         (
             b'f',
             b'force',
             False,
             _(b'force outgoing even for unrelated repositories'),
         ),
         (b'r', b'rev', [], _(b'first revision to be edited'), _(b'REV')),
     ]
     + cmdutil.formatteropts,
     _(b"[OPTIONS] ([ANCESTOR] | --outgoing [URL])"),
     helpcategory=command.CATEGORY_CHANGE_MANAGEMENT,
 )
 def histedit(ui, repo, *freeargs, **opts):
     """interactively edit changeset history
 
     This command lets you edit a linear series of changesets (up to
     and including the working directory, which should be clean).
     You can:
 
     - `pick` to [re]order a changeset
 
     - `drop` to omit changeset
 
     - `mess` to reword the changeset commit message
 
     - `fold` to combine it with the preceding changeset (using the later date)
 
     - `roll` like fold, but discarding this commit's description and date
 
     - `edit` to edit this changeset (preserving date)
 
     - `base` to checkout changeset and apply further changesets from there
 
     There are a number of ways to select the root changeset:
 
     - Specify ANCESTOR directly
 
     - Use --outgoing -- it will be the first linear changeset not
       included in destination. (See :hg:`help config.paths.default-push`)
 
     - Otherwise, the value from the "histedit.defaultrev" config option
       is used as a revset to select the base revision when ANCESTOR is not
       specified. The first revision returned by the revset is used. By
       default, this selects the editable history that is unique to the
       ancestry of the working directory.
 
     .. container:: verbose
 
        If you use --outgoing, this command will abort if there are ambiguous
        outgoing revisions. For example, if there are multiple branches
        containing outgoing revisions.
 
        Use "min(outgoing() and ::.)" or similar revset specification
        instead of --outgoing to specify edit target revision exactly in
        such ambiguous situation. See :hg:`help revsets` for detail about
        selecting revisions.
 
     .. container:: verbose
 
        Examples:
 
          - A number of changes have been made.
            Revision 3 is no longer needed.
 
            Start history editing from revision 3::
 
              hg histedit -r 3
 
            An editor opens, containing the list of revisions,
            with specific actions specified::
 
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 0a9639fcda9d 5 Morgify the cromulancy
 
            Additional information about the possible actions
            to take appears below the list of revisions.
 
            To remove revision 3 from the history,
            its action (at the beginning of the relevant line)
            is changed to 'drop'::
 
              drop 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 0a9639fcda9d 5 Morgify the cromulancy
 
          - A number of changes have been made.
            Revision 2 and 4 need to be swapped.
 
            Start history editing from revision 2::
 
              hg histedit -r 2
 
            An editor opens, containing the list of revisions,
            with specific actions specified::
 
              pick 252a1af424ad 2 Blorb a morgwazzle
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
 
            To swap revision 2 and 4, its lines are swapped
            in the editor::
 
              pick 8ef592ce7cc4 4 Bedazzle the zerlog
              pick 5339bf82f0ca 3 Zworgle the foobar
              pick 252a1af424ad 2 Blorb a morgwazzle
 
     Returns 0 on success, 1 if user intervention is required (not only
     for intentional "edit" command, but also for resolving unexpected
     conflicts).
     """
     opts = pycompat.byteskwargs(opts)
 
     # kludge: _chistedit only works for starting an edit, not aborting
     # or continuing, so fall back to regular _texthistedit for those
     # operations.
     if ui.interface(b'histedit') == b'curses' and _getgoal(opts) == goalnew:
         return _chistedit(ui, repo, freeargs, opts)
     return _texthistedit(ui, repo, freeargs, opts)
 
 
 def _texthistedit(ui, repo, freeargs, opts):
     state = histeditstate(repo)
     with repo.wlock() as wlock, repo.lock() as lock:
         state.wlock = wlock
         state.lock = lock
         _histedit(ui, repo, state, freeargs, opts)
 
 
 goalcontinue = b'continue'
 goalabort = b'abort'
 goaleditplan = b'edit-plan'
 goalnew = b'new'
 
 
 def _getgoal(opts):
     if opts.get(b'continue'):
         return goalcontinue
     if opts.get(b'abort'):
         return goalabort
     if opts.get(b'edit_plan'):
         return goaleditplan
     return goalnew
 
 
 def _readfile(ui, path):
     if path == b'-':
         with ui.timeblockedsection(b'histedit'):
             return ui.fin.read()
     else:
         with open(path, b'rb') as f:
             return f.read()
 
 
 def _validateargs(ui, repo, freeargs, opts, goal, rules, revs):
     # TODO only abort if we try to histedit mq patches, not just
     # blanket if mq patches are applied somewhere
     mq = getattr(repo, 'mq', None)
     if mq and mq.applied:
         raise error.StateError(_(b'source has mq patches applied'))
 
     # basic argument incompatibility processing
     outg = opts.get(b'outgoing')
     editplan = opts.get(b'edit_plan')
     abort = opts.get(b'abort')
     force = opts.get(b'force')
     if force and not outg:
         raise error.InputError(_(b'--force only allowed with --outgoing'))
     if goal == b'continue':
         if any((outg, abort, revs, freeargs, rules, editplan)):
             raise error.InputError(_(b'no arguments allowed with --continue'))
     elif goal == b'abort':
         if any((outg, revs, freeargs, rules, editplan)):
             raise error.InputError(_(b'no arguments allowed with --abort'))
     elif goal == b'edit-plan':
         if any((outg, revs, freeargs)):
             raise error.InputError(
                 _(b'only --commands argument allowed with --edit-plan')
             )
     else:
         if outg:
             if revs:
                 raise error.InputError(
                     _(b'no revisions allowed with --outgoing')
                 )
             if len(freeargs) > 1:
                 raise error.InputError(
                     _(b'only one repo argument allowed with --outgoing')
                 )
         else:
             revs.extend(freeargs)
             if len(revs) == 0:
                 defaultrev = destutil.desthistedit(ui, repo)
                 if defaultrev is not None:
                     revs.append(defaultrev)
 
             if len(revs) != 1:
                 raise error.InputError(
                     _(b'histedit requires exactly one ancestor revision')
                 )
 
 
 def _histedit(ui, repo, state, freeargs, opts):
     fm = ui.formatter(b'histedit', opts)
     fm.startitem()
     goal = _getgoal(opts)
     revs = opts.get(b'rev', [])
     nobackup = not ui.configbool(b'rewrite', b'backup-bundle')
     rules = opts.get(b'commands', b'')
     state.keep = opts.get(b'keep', False)
 
     _validateargs(ui, repo, freeargs, opts, goal, rules, revs)
 
     hastags = False
     if revs:
         revs = logcmdutil.revrange(repo, revs)
         ctxs = [repo[rev] for rev in revs]
         for ctx in ctxs:
             tags = [tag for tag in ctx.tags() if tag != b'tip']
             if not hastags:
                 hastags = len(tags)
     if hastags:
         if ui.promptchoice(
             _(
                 b'warning: tags associated with the given'
                 b' changeset will be lost after histedit.\n'
                 b'do you want to continue (yN)? $$ &Yes $$ &No'
             ),
             default=1,
         ):
             raise error.CanceledError(_(b'histedit cancelled\n'))
     # rebuild state
     if goal == goalcontinue:
         state.read()
         state = bootstrapcontinue(ui, state, opts)
     elif goal == goaleditplan:
         _edithisteditplan(ui, repo, state, rules)
         return
     elif goal == goalabort:
         _aborthistedit(ui, repo, state, nobackup=nobackup)
         return
     else:
         # goal == goalnew
         _newhistedit(ui, repo, state, revs, freeargs, opts)
 
     _continuehistedit(ui, repo, state)
     _finishhistedit(ui, repo, state, fm)
     fm.end()
 
 
 def _continuehistedit(ui, repo, state):
     """This function runs after either:
     - bootstrapcontinue (if the goal is 'continue')
     - _newhistedit (if the goal is 'new')
     """
     # preprocess rules so that we can hide inner folds from the user
     # and only show one editor
     actions = state.actions[:]
     for idx, (action, nextact) in enumerate(zip(actions, actions[1:] + [None])):
         if action.verb == b'fold' and nextact and nextact.verb == b'fold':
             state.actions[idx].__class__ = _multifold
 
     # Force an initial state file write, so the user can run --abort/continue
     # even if there's an exception before the first transaction serialize.
     state.write()
 
     tr = None
     # Don't use singletransaction by default since it rolls the entire
     # transaction back if an unexpected exception happens (like a
     # pretxncommit hook throws, or the user aborts the commit msg editor).
     if ui.configbool(b"histedit", b"singletransaction"):
         # Don't use a 'with' for the transaction, since actions may close
         # and reopen a transaction. For example, if the action executes an
         # external process it may choose to commit the transaction first.
         tr = repo.transaction(b'histedit')
     progress = ui.makeprogress(
         _(b"editing"), unit=_(b'changes'), total=len(state.actions)
     )
     with progress, util.acceptintervention(tr):
         while state.actions:
             state.write(tr=tr)
             actobj = state.actions[0]
             progress.increment(item=actobj.torule())
             ui.debug(
                 b'histedit: processing %s %s\n' % (actobj.verb, actobj.torule())
             )
             parentctx, replacement_ = actobj.run()
             state.parentctxnode = parentctx.node()
             state.replacements.extend(replacement_)
             state.actions.pop(0)
 
     state.write()
 
 
 def _finishhistedit(ui, repo, state, fm):
     """This action runs when histedit is finishing its session"""
     mergemod.update(repo[state.parentctxnode])
 
     mapping, tmpnodes, created, ntm = processreplacement(state)
     if mapping:
         for prec, succs in pycompat.iteritems(mapping):
             if not succs:
                 ui.debug(b'histedit: %s is dropped\n' % short(prec))
             else:
                 ui.debug(
                     b'histedit: %s is replaced by %s\n'
                     % (short(prec), short(succs[0]))
                 )
                 if len(succs) > 1:
                     m = b'histedit:                            %s'
                     for n in succs[1:]:
                         ui.debug(m % short(n))
 
     if not state.keep:
         if mapping:
             movetopmostbookmarks(repo, state.topmost, ntm)
             # TODO update mq state
     else:
         mapping = {}
 
     for n in tmpnodes:
         if n in repo:
             mapping[n] = ()
 
     # remove entries about unknown nodes
     has_node = repo.unfiltered().changelog.index.has_node
     mapping = {
         k: v
         for k, v in mapping.items()
         if has_node(k) and all(has_node(n) for n in v)
     }
     scmutil.cleanupnodes(repo, mapping, b'histedit')
     hf = fm.hexfunc
     fl = fm.formatlist
     fd = fm.formatdict
     nodechanges = fd(
         {
             hf(oldn): fl([hf(n) for n in newn], name=b'node')
             for oldn, newn in pycompat.iteritems(mapping)
         },
         key=b"oldnode",
         value=b"newnodes",
     )
     fm.data(nodechanges=nodechanges)
 
     state.clear()
     if os.path.exists(repo.sjoin(b'undo')):
         os.unlink(repo.sjoin(b'undo'))
     if repo.vfs.exists(b'histedit-last-edit.txt'):
         repo.vfs.unlink(b'histedit-last-edit.txt')
 
 
 def _aborthistedit(ui, repo, state, nobackup=False):
     try:
         state.read()
         __, leafs, tmpnodes, __ = processreplacement(state)
         ui.debug(b'restore wc to old parent %s\n' % short(state.topmost))
 
         # Recover our old commits if necessary
         if not state.topmost in repo and state.backupfile:
             backupfile = repo.vfs.join(state.backupfile)
             f = hg.openpath(ui, backupfile)
             gen = exchange.readbundle(ui, f, backupfile)
             with repo.transaction(b'histedit.abort') as tr:
                 bundle2.applybundle(
                     repo,
                     gen,
                     tr,
                     source=b'histedit',
                     url=b'bundle:' + backupfile,
                 )
 
             os.remove(backupfile)
 
         # check whether we should update away
         if repo.unfiltered().revs(
             b'parents() and (%n  or %ln::)',
             state.parentctxnode,
             leafs | tmpnodes,
         ):
             hg.clean(repo, state.topmost, show_stats=True, quietempty=True)
         cleanupnode(ui, repo, tmpnodes, nobackup=nobackup)
         cleanupnode(ui, repo, leafs, nobackup=nobackup)
     except Exception:
         if state.inprogress():
             ui.warn(
                 _(
                     b'warning: encountered an exception during histedit '
                     b'--abort; the repository may not have been completely '
                     b'cleaned up\n'
                 )
             )
         raise
     finally:
         state.clear()
 
 
 def hgaborthistedit(ui, repo):
     state = histeditstate(repo)
     nobackup = not ui.configbool(b'rewrite', b'backup-bundle')
     with repo.wlock() as wlock, repo.lock() as lock:
         state.wlock = wlock
         state.lock = lock
         _aborthistedit(ui, repo, state, nobackup=nobackup)
 
 
 def _edithisteditplan(ui, repo, state, rules):
     state.read()
     if not rules:
         comment = geteditcomment(
             ui, short(state.parentctxnode), short(state.topmost)
         )
         rules = ruleeditor(repo, ui, state.actions, comment)
     else:
         rules = _readfile(ui, rules)
     actions = parserules(rules, state)
     ctxs = [repo[act.node] for act in state.actions if act.node]
     warnverifyactions(ui, repo, actions, state, ctxs)
     state.actions = actions
     state.write()
 
 
 def _newhistedit(ui, repo, state, revs, freeargs, opts):
     outg = opts.get(b'outgoing')
     rules = opts.get(b'commands', b'')
     force = opts.get(b'force')
 
     cmdutil.checkunfinished(repo)
     cmdutil.bailifchanged(repo)
 
     topmost = repo.dirstate.p1()
     if outg:
         if freeargs:
             remote = freeargs[0]
         else:
             remote = None
         root = findoutgoing(ui, repo, remote, force, opts)
     else:
         rr = list(repo.set(b'roots(%ld)', logcmdutil.revrange(repo, revs)))
         if len(rr) != 1:
             raise error.InputError(
                 _(
                     b'The specified revisions must have '
                     b'exactly one common root'
                 )
             )
         root = rr[0].node()
 
     revs = between(repo, root, topmost, state.keep)
     if not revs:
         raise error.InputError(
             _(b'%s is not an ancestor of working directory') % short(root)
         )
 
     ctxs = [repo[r] for r in revs]
 
     wctx = repo[None]
     # Please don't ask me why `ancestors` is this value. I figured it
     # out with print-debugging, not by actually understanding what the
     # merge code is doing. :(
     ancs = [repo[b'.']]
     # Sniff-test to make sure we won't collide with untracked files in
     # the working directory. If we don't do this, we can get a
     # collision after we've started histedit and backing out gets ugly
     # for everyone, especially the user.
     for c in [ctxs[0].p1()] + ctxs:
         try:
             mergemod.calculateupdates(
                 repo,
                 wctx,
                 c,
                 ancs,
                 # These parameters were determined by print-debugging
                 # what happens later on inside histedit.
                 branchmerge=False,
                 force=False,
                 acceptremote=False,
                 followcopies=False,
             )
         except error.Abort:
             raise error.StateError(
                 _(
                     b"untracked files in working directory conflict with files in %s"
                 )
                 % c
             )
 
     if not rules:
         comment = geteditcomment(ui, short(root), short(topmost))
         actions = [pick(state, r) for r in revs]
         rules = ruleeditor(repo, ui, actions, comment)
     else:
         rules = _readfile(ui, rules)
     actions = parserules(rules, state)
     warnverifyactions(ui, repo, actions, state, ctxs)
 
     parentctxnode = repo[root].p1().node()
 
     state.parentctxnode = parentctxnode
     state.actions = actions
     state.topmost = topmost
     state.replacements = []
 
     ui.log(
         b"histedit",
         b"%d actions to histedit\n",
         len(actions),
         histedit_num_actions=len(actions),
     )
 
     # Create a backup so we can always abort completely.
     backupfile = None
     if not obsolete.isenabled(repo, obsolete.createmarkersopt):
         backupfile = repair.backupbundle(
             repo, [parentctxnode], [topmost], root, b'histedit'
         )
     state.backupfile = backupfile
 
 
 def _getsummary(ctx):
     # a common pattern is to extract the summary but default to the empty
     # string
     summary = ctx.description() or b''
     if summary:
         summary = summary.splitlines()[0]
     return summary
 
 
 def bootstrapcontinue(ui, state, opts):
     repo = state.repo
 
     ms = mergestatemod.mergestate.read(repo)
     mergeutil.checkunresolved(ms)
 
     if state.actions:
         actobj = state.actions.pop(0)
 
         if _isdirtywc(repo):
             actobj.continuedirty()
             if _isdirtywc(repo):
                 abortdirty()
 
         parentctx, replacements = actobj.continueclean()
 
         state.parentctxnode = parentctx.node()
         state.replacements.extend(replacements)
 
     return state
 
 
 def between(repo, old, new, keep):
     """select and validate the set of revision to edit
 
     When keep is false, the specified set can't have children."""
     revs = repo.revs(b'%n::%n', old, new)
     if revs and not keep:
         rewriteutil.precheck(repo, revs, b'edit')
         if repo.revs(b'(%ld) and merge()', revs):
             raise error.StateError(
                 _(b'cannot edit history that contains merges')
             )
     return pycompat.maplist(repo.changelog.node, revs)
 
 
 def ruleeditor(repo, ui, actions, editcomment=b""):
     """open an editor to edit rules
 
     rules are in the format [ [act, ctx], ...] like in state.rules
     """
     if repo.ui.configbool(b"experimental", b"histedit.autoverb"):
         newact = util.sortdict()
         for act in actions:
             ctx = repo[act.node]
             summary = _getsummary(ctx)
             fword = summary.split(b' ', 1)[0].lower()
             added = False
 
             # if it doesn't end with the special character '!' just skip this
             if fword.endswith(b'!'):
                 fword = fword[:-1]
                 if fword in primaryactions | secondaryactions | tertiaryactions:
                     act.verb = fword
                     # get the target summary
                     tsum = summary[len(fword) + 1 :].lstrip()
                     # safe but slow: reverse iterate over the actions so we
                     # don't clash on two commits having the same summary
                     for na, l in reversed(list(pycompat.iteritems(newact))):
                         actx = repo[na.node]
                         asum = _getsummary(actx)
                         if asum == tsum:
                             added = True
                             l.append(act)
                             break
 
             if not added:
                 newact[act] = []
 
         # copy over and flatten the new list
         actions = []
         for na, l in pycompat.iteritems(newact):
             actions.append(na)
             actions += l
 
     rules = b'\n'.join([act.torule() for act in actions])
     rules += b'\n\n'
     rules += editcomment
     rules = ui.edit(
         rules,
         ui.username(),
         {b'prefix': b'histedit'},
         repopath=repo.path,
         action=b'histedit',
     )
 
     # Save edit rules in .hg/histedit-last-edit.txt in case
     # the user needs to ask for help after something
     # surprising happens.
     with repo.vfs(b'histedit-last-edit.txt', b'wb') as f:
         f.write(rules)
 
     return rules
 
 
 def parserules(rules, state):
     """Read the histedit rules string and return list of action objects"""
     rules = [
         l
         for l in (r.strip() for r in rules.splitlines())
         if l and not l.startswith(b'#')
     ]
     actions = []
     for r in rules:
         if b' ' not in r:
             raise error.ParseError(_(b'malformed line "%s"') % r)
         verb, rest = r.split(b' ', 1)
 
         if verb not in actiontable:
             raise error.ParseError(_(b'unknown action "%s"') % verb)
 
         action = actiontable[verb].fromrule(state, rest)
         actions.append(action)
     return actions
 
 
 def warnverifyactions(ui, repo, actions, state, ctxs):
     try:
         verifyactions(actions, state, ctxs)
     except error.ParseError:
         if repo.vfs.exists(b'histedit-last-edit.txt'):
             ui.warn(
                 _(
                     b'warning: histedit rules saved '
                     b'to: .hg/histedit-last-edit.txt\n'
                 )
             )
         raise
 
 
 def verifyactions(actions, state, ctxs):
     """Verify that there exists exactly one action per given changeset and
     other constraints.
 
     Will abort if there are to many or too few rules, a malformed rule,
     or a rule on a changeset outside of the user-given range.
     """
     expected = {c.node() for c in ctxs}
     seen = set()
     prev = None
 
     if actions and actions[0].verb in [b'roll', b'fold']:
         raise error.ParseError(
             _(b'first changeset cannot use verb "%s"') % actions[0].verb
         )
 
     for action in actions:
         action.verify(prev, expected, seen)
         prev = action
         if action.node is not None:
             seen.add(action.node)
     missing = sorted(expected - seen)  # sort to stabilize output
 
     if state.repo.ui.configbool(b'histedit', b'dropmissing'):
         if len(actions) == 0:
             raise error.ParseError(
                 _(b'no rules provided'),
                 hint=_(b'use strip extension to remove commits'),
             )
 
         drops = [drop(state, n) for n in missing]
         # put the in the beginning so they execute immediately and
         # don't show in the edit-plan in the future
         actions[:0] = drops
     elif missing:
         raise error.ParseError(
             _(b'missing rules for changeset %s') % short(missing[0]),
             hint=_(
                 b'use "drop %s" to discard, see also: '
                 b"'hg help -e histedit.config'"
             )
             % short(missing[0]),
         )
 
 
 def adjustreplacementsfrommarkers(repo, oldreplacements):
     """Adjust replacements from obsolescence markers
 
     Replacements structure is originally generated based on
     histedit's state and does not account for changes that are
     not recorded there. This function fixes that by adding
     data read from obsolescence markers"""
     if not obsolete.isenabled(repo, obsolete.createmarkersopt):
         return oldreplacements
 
     unfi = repo.unfiltered()
     get_rev = unfi.changelog.index.get_rev
     obsstore = repo.obsstore
     newreplacements = list(oldreplacements)
     oldsuccs = [r[1] for r in oldreplacements]
     # successors that have already been added to succstocheck once
     seensuccs = set().union(
         *oldsuccs
     )  # create a set from an iterable of tuples
     succstocheck = list(seensuccs)
     while succstocheck:
         n = succstocheck.pop()
         missing = get_rev(n) is None
         markers = obsstore.successors.get(n, ())
         if missing and not markers:
             # dead end, mark it as such
             newreplacements.append((n, ()))
         for marker in markers:
             nsuccs = marker[1]
             newreplacements.append((n, nsuccs))
             for nsucc in nsuccs:
                 if nsucc not in seensuccs:
                     seensuccs.add(nsucc)
                     succstocheck.append(nsucc)
 
     return newreplacements
 
 
 def processreplacement(state):
     """process the list of replacements to return
 
     1) the final mapping between original and created nodes
     2) the list of temporary node created by histedit
     3) the list of new commit created by histedit"""
     replacements = adjustreplacementsfrommarkers(state.repo, state.replacements)
     allsuccs = set()
     replaced = set()
     fullmapping = {}
     # initialize basic set
     # fullmapping records all operations recorded in replacement
     for rep in replacements:
         allsuccs.update(rep[1])
         replaced.add(rep[0])
         fullmapping.setdefault(rep[0], set()).update(rep[1])
     new = allsuccs - replaced
     tmpnodes = allsuccs & replaced
     # Reduce content fullmapping into direct relation between original nodes
     # and final node created during history edition
     # Dropped changeset are replaced by an empty list
     toproceed = set(fullmapping)
     final = {}
     while toproceed:
         for x in list(toproceed):
             succs = fullmapping[x]
             for s in list(succs):
                 if s in toproceed:
                     # non final node with unknown closure
                     # We can't process this now
                     break
                 elif s in final:
                     # non final node, replace with closure
                     succs.remove(s)
                     succs.update(final[s])
             else:
                 final[x] = succs
                 toproceed.remove(x)
     # remove tmpnodes from final mapping
     for n in tmpnodes:
         del final[n]
     # we expect all changes involved in final to exist in the repo
     # turn `final` into list (topologically sorted)
     get_rev = state.repo.changelog.index.get_rev
     for prec, succs in final.items():
         final[prec] = sorted(succs, key=get_rev)
 
     # computed topmost element (necessary for bookmark)
     if new:
         newtopmost = sorted(new, key=state.repo.changelog.rev)[-1]
     elif not final:
         # Nothing rewritten at all. we won't need `newtopmost`
         # It is the same as `oldtopmost` and `processreplacement` know it
         newtopmost = None
     else:
         # every body died. The newtopmost is the parent of the root.
         r = state.repo.changelog.rev
         newtopmost = state.repo[sorted(final, key=r)[0]].p1().node()
 
     return final, tmpnodes, new, newtopmost
 
 
 def movetopmostbookmarks(repo, oldtopmost, newtopmost):
     """Move bookmark from oldtopmost to newly created topmost
 
     This is arguably a feature and we may only want that for the active
     bookmark. But the behavior is kept compatible with the old version for now.
     """
     if not oldtopmost or not newtopmost:
         return
     oldbmarks = repo.nodebookmarks(oldtopmost)
     if oldbmarks:
         with repo.lock(), repo.transaction(b'histedit') as tr:
             marks = repo._bookmarks
             changes = []
             for name in oldbmarks:
                 changes.append((name, newtopmost))
             marks.applychanges(repo, tr, changes)
 
 
 def cleanupnode(ui, repo, nodes, nobackup=False):
     """strip a group of nodes from the repository
 
     The set of node to strip may contains unknown nodes."""
     with repo.lock():
         # do not let filtering get in the way of the cleanse
         # we should probably get rid of obsolescence marker created during the
         # histedit, but we currently do not have such information.
         repo = repo.unfiltered()
         # Find all nodes that need to be stripped
         # (we use %lr instead of %ln to silently ignore unknown items)
         has_node = repo.changelog.index.has_node
         nodes = sorted(n for n in nodes if has_node(n))
         roots = [c.node() for c in repo.set(b"roots(%ln)", nodes)]
         if roots:
             backup = not nobackup
             repair.strip(ui, repo, roots, backup=backup)
 
 
 def stripwrapper(orig, ui, repo, nodelist, *args, **kwargs):
     if isinstance(nodelist, bytes):
         nodelist = [nodelist]
     state = histeditstate(repo)
     if state.inprogress():
         state.read()
         histedit_nodes = {
             action.node for action in state.actions if action.node
         }
         common_nodes = histedit_nodes & set(nodelist)
         if common_nodes:
             raise error.Abort(
                 _(b"histedit in progress, can't strip %s")
                 % b', '.join(short(x) for x in common_nodes)
             )
     return orig(ui, repo, nodelist, *args, **kwargs)
 
 
 extensions.wrapfunction(repair, b'strip', stripwrapper)
 
 
 def summaryhook(ui, repo):
     state = histeditstate(repo)
     if not state.inprogress():
         return
     state.read()
     if state.actions:
         # i18n: column positioning for "hg summary"
         ui.write(
             _(b'hist:   %s (histedit --continue)\n')
             % (
                 ui.label(_(b'%d remaining'), b'histedit.remaining')
                 % len(state.actions)
             )
         )
 
 
 def extsetup(ui):
     cmdutil.summaryhooks.add(b'histedit', summaryhook)
     statemod.addunfinished(
         b'histedit',
         fname=b'histedit-state',
         allowcommit=True,
         continueflag=True,
         abortfunc=hgaborthistedit,
     )

hgext/keyword.py

@@ -1,889 +1,896 b''
 # keyword.py - $Keyword$ expansion for Mercurial
 #
 # Copyright 2007-2015 Christian Ebert <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # $Id$
 #
 # Keyword expansion hack against the grain of a Distributed SCM
 #
 # There are many good reasons why this is not needed in a distributed
 # SCM, still it may be useful in very small projects based on single
 # files (like LaTeX packages), that are mostly addressed to an
 # audience not running a version control system.
 #
 # For in-depth discussion refer to
 # <https://mercurial-scm.org/wiki/KeywordPlan>.
 #
 # Keyword expansion is based on Mercurial's changeset template mappings.
 #
 # Binary files are not touched.
 #
 # Files to act upon/ignore are specified in the [keyword] section.
 # Customized keyword template mappings in the [keywordmaps] section.
 #
 # Run 'hg help keyword' and 'hg kwdemo' to get info on configuration.
 
 '''expand keywords in tracked files
 
 This extension expands RCS/CVS-like or self-customized $Keywords$ in
 tracked text files selected by your configuration.
 
 Keywords are only expanded in local repositories and not stored in the
 change history. The mechanism can be regarded as a convenience for the
 current user or for archive distribution.
 
 Keywords expand to the changeset data pertaining to the latest change
 relative to the working directory parent of each file.
 
 Configuration is done in the [keyword], [keywordset] and [keywordmaps]
 sections of hgrc files.
 
 Example::
 
     [keyword]
     # expand keywords in every python file except those matching "x*"
     **.py =
     x*    = ignore
 
     [keywordset]
     # prefer svn- over cvs-like default keywordmaps
     svn = True
 
 .. note::
 
    The more specific you are in your filename patterns the less you
    lose speed in huge repositories.
 
 For [keywordmaps] template mapping and expansion demonstration and
 control run :hg:`kwdemo`. See :hg:`help templates` for a list of
 available templates and filters.
 
 Three additional date template filters are provided:
 
 :``utcdate``:    "2006/09/18 15:13:13"
 :``svnutcdate``: "2006-09-18 15:13:13Z"
 :``svnisodate``: "2006-09-18 08:13:13 -700 (Mon, 18 Sep 2006)"
 
 The default template mappings (view with :hg:`kwdemo -d`) can be
 replaced with customized keywords and templates. Again, run
 :hg:`kwdemo` to control the results of your configuration changes.
 
 Before changing/disabling active keywords, you must run :hg:`kwshrink`
 to avoid storing expanded keywords in the change history.
 
 To force expansion after enabling it, or a configuration change, run
 :hg:`kwexpand`.
 
 Expansions spanning more than one line and incremental expansions,
 like CVS' $Log$, are not supported. A keyword template map "Log =
 {desc}" expands to the first line of the changeset description.
 '''
 
 
 from __future__ import absolute_import
 
 import os
 import re
 import weakref
 
 from mercurial.i18n import _
 from mercurial.pycompat import getattr
 from mercurial.hgweb import webcommands
 
 from mercurial import (
     cmdutil,
     context,
     dispatch,
     error,
     extensions,
     filelog,
     localrepo,
     logcmdutil,
     match,
     patch,
     pathutil,
     pycompat,
     registrar,
     scmutil,
     templatefilters,
     templateutil,
     util,
 )
 from mercurial.utils import (
     dateutil,
     stringutil,
 )
+from mercurial.dirstateutils import timestamp
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 # hg commands that do not act on keywords
 nokwcommands = (
     b'add addremove annotate bundle export grep incoming init log'
     b' outgoing push tip verify convert email glog'
 )
 
 # webcommands that do not act on keywords
 nokwwebcommands = b'annotate changeset rev filediff diff comparison'
 
 # hg commands that trigger expansion only when writing to working dir,
 # not when reading filelog, and unexpand when reading from working dir
 restricted = (
     b'merge kwexpand kwshrink record qrecord resolve transplant'
     b' unshelve rebase graft backout histedit fetch'
 )
 
 # names of extensions using dorecord
 recordextensions = b'record'
 
 colortable = {
     b'kwfiles.enabled': b'green bold',
     b'kwfiles.deleted': b'cyan bold underline',
     b'kwfiles.enabledunknown': b'green',
     b'kwfiles.ignored': b'bold',
     b'kwfiles.ignoredunknown': b'none',
 }
 
 templatefilter = registrar.templatefilter()
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'keywordset',
     b'svn',
     default=False,
 )
 # date like in cvs' $Date
 @templatefilter(b'utcdate', intype=templateutil.date)
 def utcdate(date):
     """Date. Returns a UTC-date in this format: "2009/08/18 11:00:13"."""
     dateformat = b'%Y/%m/%d %H:%M:%S'
     return dateutil.datestr((date[0], 0), dateformat)
 
 
 # date like in svn's $Date
 @templatefilter(b'svnisodate', intype=templateutil.date)
 def svnisodate(date):
     """Date. Returns a date in this format: "2009-08-18 13:00:13
     +0200 (Tue, 18 Aug 2009)".
     """
     return dateutil.datestr(date, b'%Y-%m-%d %H:%M:%S %1%2 (%a, %d %b %Y)')
 
 
 # date like in svn's $Id
 @templatefilter(b'svnutcdate', intype=templateutil.date)
 def svnutcdate(date):
     """Date. Returns a UTC-date in this format: "2009-08-18
     11:00:13Z".
     """
     dateformat = b'%Y-%m-%d %H:%M:%SZ'
     return dateutil.datestr((date[0], 0), dateformat)
 
 
 # make keyword tools accessible
 kwtools = {b'hgcmd': b''}
 
 
 def _defaultkwmaps(ui):
     '''Returns default keywordmaps according to keywordset configuration.'''
     templates = {
         b'Revision': b'{node|short}',
         b'Author': b'{author|user}',
     }
     kwsets = (
         {
             b'Date': b'{date|utcdate}',
             b'RCSfile': b'{file|basename},v',
             b'RCSFile': b'{file|basename},v',  # kept for backwards compatibility
             # with hg-keyword
             b'Source': b'{root}/{file},v',
             b'Id': b'{file|basename},v {node|short} {date|utcdate} {author|user}',
             b'Header': b'{root}/{file},v {node|short} {date|utcdate} {author|user}',
         },
         {
             b'Date': b'{date|svnisodate}',
             b'Id': b'{file|basename},v {node|short} {date|svnutcdate} {author|user}',
             b'LastChangedRevision': b'{node|short}',
             b'LastChangedBy': b'{author|user}',
             b'LastChangedDate': b'{date|svnisodate}',
         },
     )
     templates.update(kwsets[ui.configbool(b'keywordset', b'svn')])
     return templates
 
 
 def _shrinktext(text, subfunc):
     """Helper for keyword expansion removal in text.
     Depending on subfunc also returns number of substitutions."""
     return subfunc(br'$\1$', text)
 
 
 def _preselect(wstatus, changed):
     """Retrieves modified and added files from a working directory state
     and returns the subset of each contained in given changed files
     retrieved from a change context."""
     modified = [f for f in wstatus.modified if f in changed]
     added = [f for f in wstatus.added if f in changed]
     return modified, added
 
 
 class kwtemplater(object):
     """
     Sets up keyword templates, corresponding keyword regex, and
     provides keyword substitution functions.
     """
 
     def __init__(self, ui, repo, inc, exc):
         self.ui = ui
         self._repo = weakref.ref(repo)
         self.match = match.match(repo.root, b'', [], inc, exc)
         self.restrict = kwtools[b'hgcmd'] in restricted.split()
         self.postcommit = False
 
         kwmaps = self.ui.configitems(b'keywordmaps')
         if kwmaps:  # override default templates
             self.templates = dict(kwmaps)
         else:
             self.templates = _defaultkwmaps(self.ui)
 
     @property
     def repo(self):
         return self._repo()
 
     @util.propertycache
     def escape(self):
         '''Returns bar-separated and escaped keywords.'''
         return b'|'.join(map(stringutil.reescape, self.templates.keys()))
 
     @util.propertycache
     def rekw(self):
         '''Returns regex for unexpanded keywords.'''
         return re.compile(br'\$(%s)\$' % self.escape)
 
     @util.propertycache
     def rekwexp(self):
         '''Returns regex for expanded keywords.'''
         return re.compile(br'\$(%s): [^$\n\r]*? \$' % self.escape)
 
     def substitute(self, data, path, ctx, subfunc):
         '''Replaces keywords in data with expanded template.'''
 
         def kwsub(mobj):
             kw = mobj.group(1)
             ct = logcmdutil.maketemplater(
                 self.ui, self.repo, self.templates[kw]
             )
             self.ui.pushbuffer()
             ct.show(ctx, root=self.repo.root, file=path)
             ekw = templatefilters.firstline(self.ui.popbuffer())
             return b'$%s: %s $' % (kw, ekw)
 
         return subfunc(kwsub, data)
 
     def linkctx(self, path, fileid):
         '''Similar to filelog.linkrev, but returns a changectx.'''
         return self.repo.filectx(path, fileid=fileid).changectx()
 
     def expand(self, path, node, data):
         '''Returns data with keywords expanded.'''
         if (
             not self.restrict
             and self.match(path)
             and not stringutil.binary(data)
         ):
             ctx = self.linkctx(path, node)
             return self.substitute(data, path, ctx, self.rekw.sub)
         return data
 
     def iskwfile(self, cand, ctx):
         """Returns subset of candidates which are configured for keyword
         expansion but are not symbolic links."""
         return [f for f in cand if self.match(f) and b'l' not in ctx.flags(f)]
 
     def overwrite(self, ctx, candidates, lookup, expand, rekw=False):
         '''Overwrites selected files expanding/shrinking keywords.'''
         if self.restrict or lookup or self.postcommit:  # exclude kw_copy
             candidates = self.iskwfile(candidates, ctx)
         if not candidates:
             return
         kwcmd = self.restrict and lookup  # kwexpand/kwshrink
         if self.restrict or expand and lookup:
             mf = ctx.manifest()
         if self.restrict or rekw:
             re_kw = self.rekw
         else:
             re_kw = self.rekwexp
         if expand:
             msg = _(b'overwriting %s expanding keywords\n')
         else:
             msg = _(b'overwriting %s shrinking keywords\n')
+        wctx = self.repo[None]
         for f in candidates:
             if self.restrict:
                 data = self.repo.file(f).read(mf[f])
             else:
                 data = self.repo.wread(f)
             if stringutil.binary(data):
                 continue
             if expand:
                 parents = ctx.parents()
                 if lookup:
                     ctx = self.linkctx(f, mf[f])
                 elif self.restrict and len(parents) > 1:
                     # merge commit
                     # in case of conflict f is in modified state during
                     # merge, even if f does not differ from f in parent
                     for p in parents:
                         if f in p and not p[f].cmp(ctx[f]):
                             ctx = p[f].changectx()
                             break
                 data, found = self.substitute(data, f, ctx, re_kw.subn)
             elif self.restrict:
                 found = re_kw.search(data)
             else:
                 data, found = _shrinktext(data, re_kw.subn)
             if found:
                 self.ui.note(msg % f)
                 fp = self.repo.wvfs(f, b"wb", atomictemp=True)
                 fp.write(data)
                 fp.close()
                 if kwcmd:
-                    self.repo.dirstate.set_clean(f)
+                    s = wctx[f].lstat()
+                    mode = s.st_mode
+                    size = s.st_size
+                    mtime = timestamp.mtime_of(s)
+                    cache_data = (mode, size, mtime)
+                    self.repo.dirstate.set_clean(f, cache_data)
                 elif self.postcommit:
                     self.repo.dirstate.update_file_p1(f, p1_tracked=True)
 
     def shrink(self, fname, text):
         '''Returns text with all keyword substitutions removed.'''
         if self.match(fname) and not stringutil.binary(text):
             return _shrinktext(text, self.rekwexp.sub)
         return text
 
     def shrinklines(self, fname, lines):
         '''Returns lines with keyword substitutions removed.'''
         if self.match(fname):
             text = b''.join(lines)
             if not stringutil.binary(text):
                 return _shrinktext(text, self.rekwexp.sub).splitlines(True)
         return lines
 
     def wread(self, fname, data):
         """If in restricted mode returns data read from wdir with
         keyword substitutions removed."""
         if self.restrict:
             return self.shrink(fname, data)
         return data
 
 
 class kwfilelog(filelog.filelog):
     """
     Subclass of filelog to hook into its read, add, cmp methods.
     Keywords are "stored" unexpanded, and processed on reading.
     """
 
     def __init__(self, opener, kwt, path):
         super(kwfilelog, self).__init__(opener, path)
         self.kwt = kwt
         self.path = path
 
     def read(self, node):
         '''Expands keywords when reading filelog.'''
         data = super(kwfilelog, self).read(node)
         if self.renamed(node):
             return data
         return self.kwt.expand(self.path, node, data)
 
     def add(self, text, meta, tr, link, p1=None, p2=None):
         '''Removes keyword substitutions when adding to filelog.'''
         text = self.kwt.shrink(self.path, text)
         return super(kwfilelog, self).add(text, meta, tr, link, p1, p2)
 
     def cmp(self, node, text):
         '''Removes keyword substitutions for comparison.'''
         text = self.kwt.shrink(self.path, text)
         return super(kwfilelog, self).cmp(node, text)
 
 
 def _status(ui, repo, wctx, kwt, *pats, **opts):
     """Bails out if [keyword] configuration is not active.
     Returns status of working directory."""
     if kwt:
         opts = pycompat.byteskwargs(opts)
         return repo.status(
             match=scmutil.match(wctx, pats, opts),
             clean=True,
             unknown=opts.get(b'unknown') or opts.get(b'all'),
         )
     if ui.configitems(b'keyword'):
         raise error.Abort(_(b'[keyword] patterns cannot match'))
     raise error.Abort(_(b'no [keyword] patterns configured'))
 
 
 def _kwfwrite(ui, repo, expand, *pats, **opts):
     '''Selects files and passes them to kwtemplater.overwrite.'''
     wctx = repo[None]
     if len(wctx.parents()) > 1:
         raise error.Abort(_(b'outstanding uncommitted merge'))
     kwt = getattr(repo, '_keywordkwt', None)
     with repo.wlock():
         status = _status(ui, repo, wctx, kwt, *pats, **opts)
         if status.modified or status.added or status.removed or status.deleted:
             raise error.Abort(_(b'outstanding uncommitted changes'))
         kwt.overwrite(wctx, status.clean, True, expand)
 
 
 @command(
     b'kwdemo',
     [
         (b'd', b'default', None, _(b'show default keyword template maps')),
         (b'f', b'rcfile', b'', _(b'read maps from rcfile'), _(b'FILE')),
     ],
     _(b'hg kwdemo [-d] [-f RCFILE] [TEMPLATEMAP]...'),
     optionalrepo=True,
 )
 def demo(ui, repo, *args, **opts):
     """print [keywordmaps] configuration and an expansion example
 
     Show current, custom, or default keyword template maps and their
     expansions.
 
     Extend the current configuration by specifying maps as arguments
     and using -f/--rcfile to source an external hgrc file.
 
     Use -d/--default to disable current configuration.
 
     See :hg:`help templates` for information on templates and filters.
     """
 
     def demoitems(section, items):
         ui.write(b'[%s]\n' % section)
         for k, v in sorted(items):
             if isinstance(v, bool):
                 v = stringutil.pprint(v)
             ui.write(b'%s = %s\n' % (k, v))
 
     fn = b'demo.txt'
     tmpdir = pycompat.mkdtemp(b'', b'kwdemo.')
     ui.note(_(b'creating temporary repository at %s\n') % tmpdir)
     if repo is None:
         baseui = ui
     else:
         baseui = repo.baseui
     repo = localrepo.instance(baseui, tmpdir, create=True)
     ui.setconfig(b'keyword', fn, b'', b'keyword')
     svn = ui.configbool(b'keywordset', b'svn')
     # explicitly set keywordset for demo output
     ui.setconfig(b'keywordset', b'svn', svn, b'keyword')
 
     uikwmaps = ui.configitems(b'keywordmaps')
     if args or opts.get('rcfile'):
         ui.status(_(b'\n\tconfiguration using custom keyword template maps\n'))
         if uikwmaps:
             ui.status(_(b'\textending current template maps\n'))
         if opts.get('default') or not uikwmaps:
             if svn:
                 ui.status(_(b'\toverriding default svn keywordset\n'))
             else:
                 ui.status(_(b'\toverriding default cvs keywordset\n'))
         if opts.get('rcfile'):
             ui.readconfig(opts.get(b'rcfile'))
         if args:
             # simulate hgrc parsing
             rcmaps = b'[keywordmaps]\n%s\n' % b'\n'.join(args)
             repo.vfs.write(b'hgrc', rcmaps)
             ui.readconfig(repo.vfs.join(b'hgrc'))
         kwmaps = dict(ui.configitems(b'keywordmaps'))
     elif opts.get('default'):
         if svn:
             ui.status(_(b'\n\tconfiguration using default svn keywordset\n'))
         else:
             ui.status(_(b'\n\tconfiguration using default cvs keywordset\n'))
         kwmaps = _defaultkwmaps(ui)
         if uikwmaps:
             ui.status(_(b'\tdisabling current template maps\n'))
             for k, v in pycompat.iteritems(kwmaps):
                 ui.setconfig(b'keywordmaps', k, v, b'keyword')
     else:
         ui.status(_(b'\n\tconfiguration using current keyword template maps\n'))
         if uikwmaps:
             kwmaps = dict(uikwmaps)
         else:
             kwmaps = _defaultkwmaps(ui)
 
     uisetup(ui)
     reposetup(ui, repo)
     ui.writenoi18n(b'[extensions]\nkeyword =\n')
     demoitems(b'keyword', ui.configitems(b'keyword'))
     demoitems(b'keywordset', ui.configitems(b'keywordset'))
     demoitems(b'keywordmaps', pycompat.iteritems(kwmaps))
     keywords = b'$' + b'$\n$'.join(sorted(kwmaps.keys())) + b'$\n'
     repo.wvfs.write(fn, keywords)
     repo[None].add([fn])
     ui.note(_(b'\nkeywords written to %s:\n') % fn)
     ui.note(keywords)
     with repo.wlock():
         repo.dirstate.setbranch(b'demobranch')
     for name, cmd in ui.configitems(b'hooks'):
         if name.split(b'.', 1)[0].find(b'commit') > -1:
             repo.ui.setconfig(b'hooks', name, b'', b'keyword')
     msg = _(b'hg keyword configuration and expansion example')
     ui.note((b"hg ci -m '%s'\n" % msg))
     repo.commit(text=msg)
     ui.status(_(b'\n\tkeywords expanded\n'))
     ui.write(repo.wread(fn))
     repo.wvfs.rmtree(repo.root)
 
 
 @command(
     b'kwexpand',
     cmdutil.walkopts,
     _(b'hg kwexpand [OPTION]... [FILE]...'),
     inferrepo=True,
 )
 def expand(ui, repo, *pats, **opts):
     """expand keywords in the working directory
 
     Run after (re)enabling keyword expansion.
 
     kwexpand refuses to run if given files contain local changes.
     """
     # 3rd argument sets expansion to True
     _kwfwrite(ui, repo, True, *pats, **opts)
 
 
 @command(
     b'kwfiles',
     [
         (b'A', b'all', None, _(b'show keyword status flags of all files')),
         (b'i', b'ignore', None, _(b'show files excluded from expansion')),
         (b'u', b'unknown', None, _(b'only show unknown (not tracked) files')),
     ]
     + cmdutil.walkopts,
     _(b'hg kwfiles [OPTION]... [FILE]...'),
     inferrepo=True,
 )
 def files(ui, repo, *pats, **opts):
     """show files configured for keyword expansion
 
     List which files in the working directory are matched by the
     [keyword] configuration patterns.
 
     Useful to prevent inadvertent keyword expansion and to speed up
     execution by including only files that are actual candidates for
     expansion.
 
     See :hg:`help keyword` on how to construct patterns both for
     inclusion and exclusion of files.
 
     With -A/--all and -v/--verbose the codes used to show the status
     of files are::
 
       K = keyword expansion candidate
       k = keyword expansion candidate (not tracked)
       I = ignored
       i = ignored (not tracked)
     """
     kwt = getattr(repo, '_keywordkwt', None)
     wctx = repo[None]
     status = _status(ui, repo, wctx, kwt, *pats, **opts)
     if pats:
         cwd = repo.getcwd()
     else:
         cwd = b''
     files = []
     opts = pycompat.byteskwargs(opts)
     if not opts.get(b'unknown') or opts.get(b'all'):
         files = sorted(status.modified + status.added + status.clean)
     kwfiles = kwt.iskwfile(files, wctx)
     kwdeleted = kwt.iskwfile(status.deleted, wctx)
     kwunknown = kwt.iskwfile(status.unknown, wctx)
     if not opts.get(b'ignore') or opts.get(b'all'):
         showfiles = kwfiles, kwdeleted, kwunknown
     else:
         showfiles = [], [], []
     if opts.get(b'all') or opts.get(b'ignore'):
         showfiles += (
             [f for f in files if f not in kwfiles],
             [f for f in status.unknown if f not in kwunknown],
         )
     kwlabels = b'enabled deleted enabledunknown ignored ignoredunknown'.split()
     kwstates = zip(kwlabels, pycompat.bytestr(b'K!kIi'), showfiles)
     fm = ui.formatter(b'kwfiles', opts)
     fmt = b'%.0s%s\n'
     if opts.get(b'all') or ui.verbose:
         fmt = b'%s %s\n'
     for kwstate, char, filenames in kwstates:
         label = b'kwfiles.' + kwstate
         for f in filenames:
             fm.startitem()
             fm.data(kwstatus=char, path=f)
             fm.plain(fmt % (char, repo.pathto(f, cwd)), label=label)
     fm.end()
 
 
 @command(
     b'kwshrink',
     cmdutil.walkopts,
     _(b'hg kwshrink [OPTION]... [FILE]...'),
     inferrepo=True,
 )
 def shrink(ui, repo, *pats, **opts):
     """revert expanded keywords in the working directory
 
     Must be run before changing/disabling active keywords.
 
     kwshrink refuses to run if given files contain local changes.
     """
     # 3rd argument sets expansion to False
     _kwfwrite(ui, repo, False, *pats, **opts)
 
 
 # monkeypatches
 
 
 def kwpatchfile_init(orig, self, ui, gp, backend, store, eolmode=None):
     """Monkeypatch/wrap patch.patchfile.__init__ to avoid
     rejects or conflicts due to expanded keywords in working dir."""
     orig(self, ui, gp, backend, store, eolmode)
     kwt = getattr(getattr(backend, 'repo', None), '_keywordkwt', None)
     if kwt:
         # shrink keywords read from working dir
         self.lines = kwt.shrinklines(self.fname, self.lines)
 
 
 def kwdiff(orig, repo, *args, **kwargs):
     '''Monkeypatch patch.diff to avoid expansion.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt:
         restrict = kwt.restrict
         kwt.restrict = True
     try:
         for chunk in orig(repo, *args, **kwargs):
             yield chunk
     finally:
         if kwt:
             kwt.restrict = restrict
 
 
 def kwweb_skip(orig, web):
     '''Wraps webcommands.x turning off keyword expansion.'''
     kwt = getattr(web.repo, '_keywordkwt', None)
     if kwt:
         origmatch = kwt.match
         kwt.match = util.never
     try:
         for chunk in orig(web):
             yield chunk
     finally:
         if kwt:
             kwt.match = origmatch
 
 
 def kw_amend(orig, ui, repo, old, extra, pats, opts):
     '''Wraps cmdutil.amend expanding keywords after amend.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt is None:
         return orig(ui, repo, old, extra, pats, opts)
     with repo.wlock(), repo.dirstate.parentchange():
         kwt.postcommit = True
         newid = orig(ui, repo, old, extra, pats, opts)
         if newid != old.node():
             ctx = repo[newid]
             kwt.restrict = True
             kwt.overwrite(ctx, ctx.files(), False, True)
             kwt.restrict = False
         return newid
 
 
 def kw_copy(orig, ui, repo, pats, opts, rename=False):
     """Wraps cmdutil.copy so that copy/rename destinations do not
     contain expanded keywords.
     Note that the source of a regular file destination may also be a
     symlink:
     hg cp sym x                -> x is symlink
     cp sym x; hg cp -A sym x   -> x is file (maybe expanded keywords)
     For the latter we have to follow the symlink to find out whether its
     target is configured for expansion and we therefore must unexpand the
     keywords in the destination."""
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt is None:
         return orig(ui, repo, pats, opts, rename)
     with repo.wlock():
         orig(ui, repo, pats, opts, rename)
         if opts.get(b'dry_run'):
             return
         wctx = repo[None]
         cwd = repo.getcwd()
 
         def haskwsource(dest):
             """Returns true if dest is a regular file and configured for
             expansion or a symlink which points to a file configured for
             expansion."""
             source = repo.dirstate.copied(dest)
             if b'l' in wctx.flags(source):
                 source = pathutil.canonpath(
                     repo.root, cwd, os.path.realpath(source)
                 )
             return kwt.match(source)
 
         candidates = [
             f
             for f in repo.dirstate.copies()
             if b'l' not in wctx.flags(f) and haskwsource(f)
         ]
         kwt.overwrite(wctx, candidates, False, False)
 
 
 def kw_dorecord(orig, ui, repo, commitfunc, *pats, **opts):
     '''Wraps record.dorecord expanding keywords after recording.'''
     kwt = getattr(repo, '_keywordkwt', None)
     if kwt is None:
         return orig(ui, repo, commitfunc, *pats, **opts)
     with repo.wlock():
         # record returns 0 even when nothing has changed
         # therefore compare nodes before and after
         kwt.postcommit = True
         ctx = repo[b'.']
         wstatus = ctx.status()
         ret = orig(ui, repo, commitfunc, *pats, **opts)
         recctx = repo[b'.']
         if ctx != recctx:
             modified, added = _preselect(wstatus, recctx.files())
             kwt.restrict = False
             with repo.dirstate.parentchange():
                 kwt.overwrite(recctx, modified, False, True)
                 kwt.overwrite(recctx, added, False, True, True)
             kwt.restrict = True
         return ret
 
 
 def kwfilectx_cmp(orig, self, fctx):
     if fctx._customcmp:
         return fctx.cmp(self)
     kwt = getattr(self._repo, '_keywordkwt', None)
     if kwt is None:
         return orig(self, fctx)
     # keyword affects data size, comparing wdir and filelog size does
     # not make sense
     if (
         fctx._filenode is None
         and (
             self._repo._encodefilterpats
             or kwt.match(fctx.path())
             and b'l' not in fctx.flags()
             or self.size() - 4 == fctx.size()
         )
         or self.size() == fctx.size()
     ):
         return self._filelog.cmp(self._filenode, fctx.data())
     return True
 
 
 def uisetup(ui):
     """Monkeypatches dispatch._parse to retrieve user command.
     Overrides file method to return kwfilelog instead of filelog
     if file matches user configuration.
     Wraps commit to overwrite configured files with updated
     keyword substitutions.
     Monkeypatches patch and webcommands."""
 
     def kwdispatch_parse(orig, ui, args):
         '''Monkeypatch dispatch._parse to obtain running hg command.'''
         cmd, func, args, options, cmdoptions = orig(ui, args)
         kwtools[b'hgcmd'] = cmd
         return cmd, func, args, options, cmdoptions
 
     extensions.wrapfunction(dispatch, b'_parse', kwdispatch_parse)
 
     extensions.wrapfunction(context.filectx, b'cmp', kwfilectx_cmp)
     extensions.wrapfunction(patch.patchfile, b'__init__', kwpatchfile_init)
     extensions.wrapfunction(patch, b'diff', kwdiff)
     extensions.wrapfunction(cmdutil, b'amend', kw_amend)
     extensions.wrapfunction(cmdutil, b'copy', kw_copy)
     extensions.wrapfunction(cmdutil, b'dorecord', kw_dorecord)
     for c in nokwwebcommands.split():
         extensions.wrapfunction(webcommands, c, kwweb_skip)
 
 
 def reposetup(ui, repo):
     '''Sets up repo as kwrepo for keyword substitution.'''
 
     try:
         if (
             not repo.local()
             or kwtools[b'hgcmd'] in nokwcommands.split()
             or b'.hg' in util.splitpath(repo.root)
             or repo._url.startswith(b'bundle:')
         ):
             return
     except AttributeError:
         pass
 
     inc, exc = [], [b'.hg*']
     for pat, opt in ui.configitems(b'keyword'):
         if opt != b'ignore':
             inc.append(pat)
         else:
             exc.append(pat)
     if not inc:
         return
 
     kwt = kwtemplater(ui, repo, inc, exc)
 
     class kwrepo(repo.__class__):
         def file(self, f):
             if f[0] == b'/':
                 f = f[1:]
             return kwfilelog(self.svfs, kwt, f)
 
         def wread(self, filename):
             data = super(kwrepo, self).wread(filename)
             return kwt.wread(filename, data)
 
         def commit(self, *args, **opts):
             # use custom commitctx for user commands
             # other extensions can still wrap repo.commitctx directly
             self.commitctx = self.kwcommitctx
             try:
                 return super(kwrepo, self).commit(*args, **opts)
             finally:
                 del self.commitctx
 
         def kwcommitctx(self, ctx, error=False, origctx=None):
             n = super(kwrepo, self).commitctx(ctx, error, origctx)
             # no lock needed, only called from repo.commit() which already locks
             if not kwt.postcommit:
                 restrict = kwt.restrict
                 kwt.restrict = True
                 kwt.overwrite(
                     self[n], sorted(ctx.added() + ctx.modified()), False, True
                 )
                 kwt.restrict = restrict
             return n
 
         def rollback(self, dryrun=False, force=False):
             with self.wlock():
                 origrestrict = kwt.restrict
                 try:
                     if not dryrun:
                         changed = self[b'.'].files()
                     ret = super(kwrepo, self).rollback(dryrun, force)
                     if not dryrun:
                         ctx = self[b'.']
                         modified, added = _preselect(ctx.status(), changed)
                         kwt.restrict = False
                         kwt.overwrite(ctx, modified, True, True)
                         kwt.overwrite(ctx, added, True, False)
                     return ret
                 finally:
                     kwt.restrict = origrestrict
 
     repo.__class__ = kwrepo
     repo._keywordkwt = kwt

hgext/largefiles/lfutil.py

@@ -1,790 +1,798 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''largefiles utility code: must not import other modules in this package.'''
 from __future__ import absolute_import
 
 import contextlib
 import copy
 import os
 import stat
 
 from mercurial.i18n import _
 from mercurial.node import hex
 from mercurial.pycompat import open
 
 from mercurial import (
     dirstate,
     encoding,
     error,
     httpconnection,
     match as matchmod,
     pycompat,
     requirements,
     scmutil,
     sparse,
     util,
     vfs as vfsmod,
 )
 from mercurial.utils import hashutil
+from mercurial.dirstateutils import timestamp
 
 shortname = b'.hglf'
 shortnameslash = shortname + b'/'
 longname = b'largefiles'
 
 # -- Private worker functions ------------------------------------------
 
 
 @contextlib.contextmanager
 def lfstatus(repo, value=True):
     oldvalue = getattr(repo, 'lfstatus', False)
     repo.lfstatus = value
     try:
         yield
     finally:
         repo.lfstatus = oldvalue
 
 
 def getminsize(ui, assumelfiles, opt, default=10):
     lfsize = opt
     if not lfsize and assumelfiles:
         lfsize = ui.config(longname, b'minsize', default=default)
     if lfsize:
         try:
             lfsize = float(lfsize)
         except ValueError:
             raise error.Abort(
                 _(b'largefiles: size must be number (not %s)\n') % lfsize
             )
     if lfsize is None:
         raise error.Abort(_(b'minimum size for largefiles must be specified'))
     return lfsize
 
 
 def link(src, dest):
     """Try to create hardlink - if that fails, efficiently make a copy."""
     util.makedirs(os.path.dirname(dest))
     try:
         util.oslink(src, dest)
     except OSError:
         # if hardlinks fail, fallback on atomic copy
         with open(src, b'rb') as srcf, util.atomictempfile(dest) as dstf:
             for chunk in util.filechunkiter(srcf):
                 dstf.write(chunk)
         os.chmod(dest, os.stat(src).st_mode)
 
 
 def usercachepath(ui, hash):
     """Return the correct location in the "global" largefiles cache for a file
     with the given hash.
     This cache is used for sharing of largefiles across repositories - both
     to preserve download bandwidth and storage space."""
     return os.path.join(_usercachedir(ui), hash)
 
 
 def _usercachedir(ui, name=longname):
     '''Return the location of the "global" largefiles cache.'''
     path = ui.configpath(name, b'usercache')
     if path:
         return path
 
     hint = None
 
     if pycompat.iswindows:
         appdata = encoding.environ.get(
             b'LOCALAPPDATA', encoding.environ.get(b'APPDATA')
         )
         if appdata:
             return os.path.join(appdata, name)
 
         hint = _(b"define %s or %s in the environment, or set %s.usercache") % (
             b"LOCALAPPDATA",
             b"APPDATA",
             name,
         )
     elif pycompat.isdarwin:
         home = encoding.environ.get(b'HOME')
         if home:
             return os.path.join(home, b'Library', b'Caches', name)
 
         hint = _(b"define %s in the environment, or set %s.usercache") % (
             b"HOME",
             name,
         )
     elif pycompat.isposix:
         path = encoding.environ.get(b'XDG_CACHE_HOME')
         if path:
             return os.path.join(path, name)
         home = encoding.environ.get(b'HOME')
         if home:
             return os.path.join(home, b'.cache', name)
 
         hint = _(b"define %s or %s in the environment, or set %s.usercache") % (
             b"XDG_CACHE_HOME",
             b"HOME",
             name,
         )
     else:
         raise error.Abort(
             _(b'unknown operating system: %s\n') % pycompat.osname
         )
 
     raise error.Abort(_(b'unknown %s usercache location') % name, hint=hint)
 
 
 def inusercache(ui, hash):
     path = usercachepath(ui, hash)
     return os.path.exists(path)
 
 
 def findfile(repo, hash):
     """Return store path of the largefile with the specified hash.
     As a side effect, the file might be linked from user cache.
     Return None if the file can't be found locally."""
     path, exists = findstorepath(repo, hash)
     if exists:
         repo.ui.note(_(b'found %s in store\n') % hash)
         return path
     elif inusercache(repo.ui, hash):
         repo.ui.note(_(b'found %s in system cache\n') % hash)
         path = storepath(repo, hash)
         link(usercachepath(repo.ui, hash), path)
         return path
     return None
 
 
 class largefilesdirstate(dirstate.dirstate):
     def __getitem__(self, key):
         return super(largefilesdirstate, self).__getitem__(unixpath(key))
 
     def set_tracked(self, f):
         return super(largefilesdirstate, self).set_tracked(unixpath(f))
 
     def set_untracked(self, f):
         return super(largefilesdirstate, self).set_untracked(unixpath(f))
 
     def normal(self, f, parentfiledata=None):
         # not sure if we should pass the `parentfiledata` down or throw it
         # away. So throwing it away to stay on the safe side.
         return super(largefilesdirstate, self).normal(unixpath(f))
 
     def remove(self, f):
         return super(largefilesdirstate, self).remove(unixpath(f))
 
     def add(self, f):
         return super(largefilesdirstate, self).add(unixpath(f))
 
     def drop(self, f):
         return super(largefilesdirstate, self).drop(unixpath(f))
 
     def forget(self, f):
         return super(largefilesdirstate, self).forget(unixpath(f))
 
     def normallookup(self, f):
         return super(largefilesdirstate, self).normallookup(unixpath(f))
 
     def _ignore(self, f):
         return False
 
     def write(self, tr):
         # (1) disable PENDING mode always
         #     (lfdirstate isn't yet managed as a part of the transaction)
         # (2) avoid develwarn 'use dirstate.write with ....'
         if tr:
             tr.addbackup(b'largefiles/dirstate', location=b'plain')
         super(largefilesdirstate, self).write(None)
 
 
 def openlfdirstate(ui, repo, create=True):
     """
     Return a dirstate object that tracks largefiles: i.e. its root is
     the repo root, but it is saved in .hg/largefiles/dirstate.
     """
     vfs = repo.vfs
     lfstoredir = longname
     opener = vfsmod.vfs(vfs.join(lfstoredir))
     use_dirstate_v2 = requirements.DIRSTATE_V2_REQUIREMENT in repo.requirements
     lfdirstate = largefilesdirstate(
         opener,
         ui,
         repo.root,
         repo.dirstate._validate,
         lambda: sparse.matcher(repo),
         repo.nodeconstants,
         use_dirstate_v2,
     )
 
     # If the largefiles dirstate does not exist, populate and create
     # it. This ensures that we create it on the first meaningful
     # largefiles operation in a new clone.
     if create and not vfs.exists(vfs.join(lfstoredir, b'dirstate')):
         matcher = getstandinmatcher(repo)
         standins = repo.dirstate.walk(
             matcher, subrepos=[], unknown=False, ignored=False
         )
 
         if len(standins) > 0:
             vfs.makedirs(lfstoredir)
 
         with lfdirstate.parentchange():
             for standin in standins:
                 lfile = splitstandin(standin)
                 lfdirstate.update_file(
                     lfile, p1_tracked=True, wc_tracked=True, possibly_dirty=True
                 )
     return lfdirstate
 
 
 def lfdirstatestatus(lfdirstate, repo):
     pctx = repo[b'.']
     match = matchmod.always()
-    unsure, s = lfdirstate.status(
+    unsure, s, mtime_boundary = lfdirstate.status(
         match, subrepos=[], ignored=False, clean=False, unknown=False
     )
     modified, clean = s.modified, s.clean
+    wctx = repo[None]
     for lfile in unsure:
         try:
             fctx = pctx[standin(lfile)]
         except LookupError:
             fctx = None
         if not fctx or readasstandin(fctx) != hashfile(repo.wjoin(lfile)):
             modified.append(lfile)
         else:
             clean.append(lfile)
-            lfdirstate.set_clean(lfile)
+            st = wctx[lfile].lstat()
+            mode = st.st_mode
+            size = st.st_size
+            mtime = timestamp.reliable_mtime_of(st, mtime_boundary)
+            if mtime is not None:
+                cache_data = (mode, size, mtime)
+                lfdirstate.set_clean(lfile, cache_data)
     return s
 
 
 def listlfiles(repo, rev=None, matcher=None):
     """return a list of largefiles in the working copy or the
     specified changeset"""
 
     if matcher is None:
         matcher = getstandinmatcher(repo)
 
     # ignore unknown files in working directory
     return [
         splitstandin(f)
         for f in repo[rev].walk(matcher)
         if rev is not None or repo.dirstate.get_entry(f).any_tracked
     ]
 
 
 def instore(repo, hash, forcelocal=False):
     '''Return true if a largefile with the given hash exists in the store'''
     return os.path.exists(storepath(repo, hash, forcelocal))
 
 
 def storepath(repo, hash, forcelocal=False):
     """Return the correct location in the repository largefiles store for a
     file with the given hash."""
     if not forcelocal and repo.shared():
         return repo.vfs.reljoin(repo.sharedpath, longname, hash)
     return repo.vfs.join(longname, hash)
 
 
 def findstorepath(repo, hash):
     """Search through the local store path(s) to find the file for the given
     hash.  If the file is not found, its path in the primary store is returned.
     The return value is a tuple of (path, exists(path)).
     """
     # For shared repos, the primary store is in the share source.  But for
     # backward compatibility, force a lookup in the local store if it wasn't
     # found in the share source.
     path = storepath(repo, hash, False)
 
     if instore(repo, hash):
         return (path, True)
     elif repo.shared() and instore(repo, hash, True):
         return storepath(repo, hash, True), True
 
     return (path, False)
 
 
 def copyfromcache(repo, hash, filename):
     """Copy the specified largefile from the repo or system cache to
     filename in the repository. Return true on success or false if the
     file was not found in either cache (which should not happened:
     this is meant to be called only after ensuring that the needed
     largefile exists in the cache)."""
     wvfs = repo.wvfs
     path = findfile(repo, hash)
     if path is None:
         return False
     wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
     # The write may fail before the file is fully written, but we
     # don't use atomic writes in the working copy.
     with open(path, b'rb') as srcfd, wvfs(filename, b'wb') as destfd:
         gothash = copyandhash(util.filechunkiter(srcfd), destfd)
     if gothash != hash:
         repo.ui.warn(
             _(b'%s: data corruption in %s with hash %s\n')
             % (filename, path, gothash)
         )
         wvfs.unlink(filename)
         return False
     return True
 
 
 def copytostore(repo, ctx, file, fstandin):
     wvfs = repo.wvfs
     hash = readasstandin(ctx[fstandin])
     if instore(repo, hash):
         return
     if wvfs.exists(file):
         copytostoreabsolute(repo, wvfs.join(file), hash)
     else:
         repo.ui.warn(
             _(b"%s: largefile %s not available from local store\n")
             % (file, hash)
         )
 
 
 def copyalltostore(repo, node):
     '''Copy all largefiles in a given revision to the store'''
 
     ctx = repo[node]
     for filename in ctx.files():
         realfile = splitstandin(filename)
         if realfile is not None and filename in ctx.manifest():
             copytostore(repo, ctx, realfile, filename)
 
 
 def copytostoreabsolute(repo, file, hash):
     if inusercache(repo.ui, hash):
         link(usercachepath(repo.ui, hash), storepath(repo, hash))
     else:
         util.makedirs(os.path.dirname(storepath(repo, hash)))
         with open(file, b'rb') as srcf:
             with util.atomictempfile(
                 storepath(repo, hash), createmode=repo.store.createmode
             ) as dstf:
                 for chunk in util.filechunkiter(srcf):
                     dstf.write(chunk)
         linktousercache(repo, hash)
 
 
 def linktousercache(repo, hash):
     """Link / copy the largefile with the specified hash from the store
     to the cache."""
     path = usercachepath(repo.ui, hash)
     link(storepath(repo, hash), path)
 
 
 def getstandinmatcher(repo, rmatcher=None):
     '''Return a match object that applies rmatcher to the standin directory'''
     wvfs = repo.wvfs
     standindir = shortname
 
     # no warnings about missing files or directories
     badfn = lambda f, msg: None
 
     if rmatcher and not rmatcher.always():
         pats = [wvfs.join(standindir, pat) for pat in rmatcher.files()]
         if not pats:
             pats = [wvfs.join(standindir)]
         match = scmutil.match(repo[None], pats, badfn=badfn)
     else:
         # no patterns: relative to repo root
         match = scmutil.match(repo[None], [wvfs.join(standindir)], badfn=badfn)
     return match
 
 
 def composestandinmatcher(repo, rmatcher):
     """Return a matcher that accepts standins corresponding to the
     files accepted by rmatcher. Pass the list of files in the matcher
     as the paths specified by the user."""
     smatcher = getstandinmatcher(repo, rmatcher)
     isstandin = smatcher.matchfn
 
     def composedmatchfn(f):
         return isstandin(f) and rmatcher.matchfn(splitstandin(f))
 
     smatcher.matchfn = composedmatchfn
 
     return smatcher
 
 
 def standin(filename):
     """Return the repo-relative path to the standin for the specified big
     file."""
     # Notes:
     # 1) Some callers want an absolute path, but for instance addlargefiles
     #    needs it repo-relative so it can be passed to repo[None].add().  So
     #    leave it up to the caller to use repo.wjoin() to get an absolute path.
     # 2) Join with '/' because that's what dirstate always uses, even on
     #    Windows. Change existing separator to '/' first in case we are
     #    passed filenames from an external source (like the command line).
     return shortnameslash + util.pconvert(filename)
 
 
 def isstandin(filename):
     """Return true if filename is a big file standin. filename must be
     in Mercurial's internal form (slash-separated)."""
     return filename.startswith(shortnameslash)
 
 
 def splitstandin(filename):
     # Split on / because that's what dirstate always uses, even on Windows.
     # Change local separator to / first just in case we are passed filenames
     # from an external source (like the command line).
     bits = util.pconvert(filename).split(b'/', 1)
     if len(bits) == 2 and bits[0] == shortname:
         return bits[1]
     else:
         return None
 
 
 def updatestandin(repo, lfile, standin):
     """Re-calculate hash value of lfile and write it into standin
 
     This assumes that "lfutil.standin(lfile) == standin", for efficiency.
     """
     file = repo.wjoin(lfile)
     if repo.wvfs.exists(lfile):
         hash = hashfile(file)
         executable = getexecutable(file)
         writestandin(repo, standin, hash, executable)
     else:
         raise error.Abort(_(b'%s: file not found!') % lfile)
 
 
 def readasstandin(fctx):
     """read hex hash from given filectx of standin file
 
     This encapsulates how "standin" data is stored into storage layer."""
     return fctx.data().strip()
 
 
 def writestandin(repo, standin, hash, executable):
     '''write hash to <repo.root>/<standin>'''
     repo.wwrite(standin, hash + b'\n', executable and b'x' or b'')
 
 
 def copyandhash(instream, outfile):
     """Read bytes from instream (iterable) and write them to outfile,
     computing the SHA-1 hash of the data along the way. Return the hash."""
     hasher = hashutil.sha1(b'')
     for data in instream:
         hasher.update(data)
         outfile.write(data)
     return hex(hasher.digest())
 
 
 def hashfile(file):
     if not os.path.exists(file):
         return b''
     with open(file, b'rb') as fd:
         return hexsha1(fd)
 
 
 def getexecutable(filename):
     mode = os.stat(filename).st_mode
     return (
         (mode & stat.S_IXUSR)
         and (mode & stat.S_IXGRP)
         and (mode & stat.S_IXOTH)
     )
 
 
 def urljoin(first, second, *arg):
     def join(left, right):
         if not left.endswith(b'/'):
             left += b'/'
         if right.startswith(b'/'):
             right = right[1:]
         return left + right
 
     url = join(first, second)
     for a in arg:
         url = join(url, a)
     return url
 
 
 def hexsha1(fileobj):
     """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
     object data"""
     h = hashutil.sha1()
     for chunk in util.filechunkiter(fileobj):
         h.update(chunk)
     return hex(h.digest())
 
 
 def httpsendfile(ui, filename):
     return httpconnection.httpsendfile(ui, filename, b'rb')
 
 
 def unixpath(path):
     '''Return a version of path normalized for use with the lfdirstate.'''
     return util.pconvert(os.path.normpath(path))
 
 
 def islfilesrepo(repo):
     '''Return true if the repo is a largefile repo.'''
     if b'largefiles' in repo.requirements and any(
         shortnameslash in f[1] for f in repo.store.datafiles()
     ):
         return True
 
     return any(openlfdirstate(repo.ui, repo, False))
 
 
 class storeprotonotcapable(Exception):
     def __init__(self, storetypes):
         self.storetypes = storetypes
 
 
 def getstandinsstate(repo):
     standins = []
     matcher = getstandinmatcher(repo)
     wctx = repo[None]
     for standin in repo.dirstate.walk(
         matcher, subrepos=[], unknown=False, ignored=False
     ):
         lfile = splitstandin(standin)
         try:
             hash = readasstandin(wctx[standin])
         except IOError:
             hash = None
         standins.append((lfile, hash))
     return standins
 
 
 def synclfdirstate(repo, lfdirstate, lfile, normallookup):
     lfstandin = standin(lfile)
     if lfstandin not in repo.dirstate:
         lfdirstate.update_file(lfile, p1_tracked=False, wc_tracked=False)
     else:
         entry = repo.dirstate.get_entry(lfstandin)
         lfdirstate.update_file(
             lfile,
             wc_tracked=entry.tracked,
             p1_tracked=entry.p1_tracked,
             p2_info=entry.p2_info,
             possibly_dirty=True,
         )
 
 
 def markcommitted(orig, ctx, node):
     repo = ctx.repo()
 
     lfdirstate = openlfdirstate(repo.ui, repo)
     with lfdirstate.parentchange():
         orig(node)
 
         # ATTENTION: "ctx.files()" may differ from "repo[node].files()"
         # because files coming from the 2nd parent are omitted in the latter.
         #
         # The former should be used to get targets of "synclfdirstate",
         # because such files:
         # - are marked as "a" by "patch.patch()" (e.g. via transplant), and
         # - have to be marked as "n" after commit, but
         # - aren't listed in "repo[node].files()"
 
         for f in ctx.files():
             lfile = splitstandin(f)
             if lfile is not None:
                 synclfdirstate(repo, lfdirstate, lfile, False)
     lfdirstate.write(repo.currenttransaction())
 
     # As part of committing, copy all of the largefiles into the cache.
     #
     # Using "node" instead of "ctx" implies additional "repo[node]"
     # lookup while copyalltostore(), but can omit redundant check for
     # files comming from the 2nd parent, which should exist in store
     # at merging.
     copyalltostore(repo, node)
 
 
 def getlfilestoupdate(oldstandins, newstandins):
     changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
     filelist = []
     for f in changedstandins:
         if f[0] not in filelist:
             filelist.append(f[0])
     return filelist
 
 
 def getlfilestoupload(repo, missing, addfunc):
     makeprogress = repo.ui.makeprogress
     with makeprogress(
         _(b'finding outgoing largefiles'),
         unit=_(b'revisions'),
         total=len(missing),
     ) as progress:
         for i, n in enumerate(missing):
             progress.update(i)
             parents = [p for p in repo[n].parents() if p != repo.nullid]
 
             with lfstatus(repo, value=False):
                 ctx = repo[n]
 
             files = set(ctx.files())
             if len(parents) == 2:
                 mc = ctx.manifest()
                 mp1 = ctx.p1().manifest()
                 mp2 = ctx.p2().manifest()
                 for f in mp1:
                     if f not in mc:
                         files.add(f)
                 for f in mp2:
                     if f not in mc:
                         files.add(f)
                 for f in mc:
                     if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
                         files.add(f)
             for fn in files:
                 if isstandin(fn) and fn in ctx:
                     addfunc(fn, readasstandin(ctx[fn]))
 
 
 def updatestandinsbymatch(repo, match):
     """Update standins in the working directory according to specified match
 
     This returns (possibly modified) ``match`` object to be used for
     subsequent commit process.
     """
 
     ui = repo.ui
 
     # Case 1: user calls commit with no specific files or
     # include/exclude patterns: refresh and commit all files that
     # are "dirty".
     if match is None or match.always():
         # Spend a bit of time here to get a list of files we know
         # are modified so we can compare only against those.
         # It can cost a lot of time (several seconds)
         # otherwise to update all standins if the largefiles are
         # large.
         lfdirstate = openlfdirstate(ui, repo)
         dirtymatch = matchmod.always()
-        unsure, s = lfdirstate.status(
+        unsure, s, mtime_boundary = lfdirstate.status(
             dirtymatch, subrepos=[], ignored=False, clean=False, unknown=False
         )
         modifiedfiles = unsure + s.modified + s.added + s.removed
         lfiles = listlfiles(repo)
         # this only loops through largefiles that exist (not
         # removed/renamed)
         for lfile in lfiles:
             if lfile in modifiedfiles:
                 fstandin = standin(lfile)
                 if repo.wvfs.exists(fstandin):
                     # this handles the case where a rebase is being
                     # performed and the working copy is not updated
                     # yet.
                     if repo.wvfs.exists(lfile):
                         updatestandin(repo, lfile, fstandin)
 
         return match
 
     lfiles = listlfiles(repo)
     match._files = repo._subdirlfs(match.files(), lfiles)
 
     # Case 2: user calls commit with specified patterns: refresh
     # any matching big files.
     smatcher = composestandinmatcher(repo, match)
     standins = repo.dirstate.walk(
         smatcher, subrepos=[], unknown=False, ignored=False
     )
 
     # No matching big files: get out of the way and pass control to
     # the usual commit() method.
     if not standins:
         return match
 
     # Refresh all matching big files.  It's possible that the
     # commit will end up failing, in which case the big files will
     # stay refreshed.  No harm done: the user modified them and
     # asked to commit them, so sooner or later we're going to
     # refresh the standins.  Might as well leave them refreshed.
     lfdirstate = openlfdirstate(ui, repo)
     for fstandin in standins:
         lfile = splitstandin(fstandin)
         if lfdirstate.get_entry(lfile).tracked:
             updatestandin(repo, lfile, fstandin)
 
     # Cook up a new matcher that only matches regular files or
     # standins corresponding to the big files requested by the
     # user.  Have to modify _files to prevent commit() from
     # complaining "not tracked" for big files.
     match = copy.copy(match)
     origmatchfn = match.matchfn
 
     # Check both the list of largefiles and the list of
     # standins because if a largefile was removed, it
     # won't be in the list of largefiles at this point
     match._files += sorted(standins)
 
     actualfiles = []
     for f in match._files:
         fstandin = standin(f)
 
         # For largefiles, only one of the normal and standin should be
         # committed (except if one of them is a remove).  In the case of a
         # standin removal, drop the normal file if it is unknown to dirstate.
         # Thus, skip plain largefile names but keep the standin.
         if f in lfiles or fstandin in standins:
             if not repo.dirstate.get_entry(fstandin).removed:
                 if not repo.dirstate.get_entry(f).removed:
                     continue
             elif not repo.dirstate.get_entry(f).any_tracked:
                 continue
 
         actualfiles.append(f)
     match._files = actualfiles
 
     def matchfn(f):
         if origmatchfn(f):
             return f not in lfiles
         else:
             return f in standins
 
     match.matchfn = matchfn
 
     return match
 
 
 class automatedcommithook(object):
     """Stateful hook to update standins at the 1st commit of resuming
 
     For efficiency, updating standins in the working directory should
     be avoided while automated committing (like rebase, transplant and
     so on), because they should be updated before committing.
 
     But the 1st commit of resuming automated committing (e.g. ``rebase
     --continue``) should update them, because largefiles may be
     modified manually.
     """
 
     def __init__(self, resuming):
         self.resuming = resuming
 
     def __call__(self, repo, match):
         if self.resuming:
             self.resuming = False  # avoids updating at subsequent commits
             return updatestandinsbymatch(repo, match)
         else:
             return match
 
 
 def getstatuswriter(ui, repo, forcibly=None):
     """Return the function to write largefiles specific status out
 
     If ``forcibly`` is ``None``, this returns the last element of
     ``repo._lfstatuswriters`` as "default" writer function.
 
     Otherwise, this returns the function to always write out (or
     ignore if ``not forcibly``) status.
     """
     if forcibly is None and util.safehasattr(repo, b'_largefilesenabled'):
         return repo._lfstatuswriters[-1]
     else:
         if forcibly:
             return ui.status  # forcibly WRITE OUT
         else:
             return lambda *msg, **opts: None  # forcibly IGNORE

hgext/largefiles/overrides.py

@@ -1,1857 +1,1866 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Overridden Mercurial commands and functions for the largefiles extension'''
 from __future__ import absolute_import
 
 import copy
 import os
 
 from mercurial.i18n import _
 
 from mercurial.pycompat import open
 
 from mercurial.hgweb import webcommands
 
 from mercurial import (
     archival,
     cmdutil,
     copies as copiesmod,
     error,
     exchange,
     extensions,
     exthelper,
     filemerge,
     hg,
     logcmdutil,
     match as matchmod,
     merge,
     mergestate as mergestatemod,
     pathutil,
     pycompat,
     scmutil,
     smartset,
     subrepo,
     url as urlmod,
     util,
 )
 
 from mercurial.upgrade_utils import (
     actions as upgrade_actions,
 )
 
 from . import (
     lfcommands,
     lfutil,
     storefactory,
 )
 
+ACTION_ADD = mergestatemod.ACTION_ADD
+ACTION_DELETED_CHANGED = mergestatemod.ACTION_DELETED_CHANGED
+ACTION_GET = mergestatemod.ACTION_GET
+ACTION_KEEP = mergestatemod.ACTION_KEEP
+ACTION_REMOVE = mergestatemod.ACTION_REMOVE
+
 eh = exthelper.exthelper()
 
 lfstatus = lfutil.lfstatus
 
-MERGE_ACTION_LARGEFILE_MARK_REMOVED = b'lfmr'
+MERGE_ACTION_LARGEFILE_MARK_REMOVED = mergestatemod.MergeAction('lfmr')
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
 
 def composelargefilematcher(match, manifest):
     """create a matcher that matches only the largefiles in the original
     matcher"""
     m = copy.copy(match)
     lfile = lambda f: lfutil.standin(f) in manifest
     m._files = [lf for lf in m._files if lfile(lf)]
     m._fileset = set(m._files)
     m.always = lambda: False
     origmatchfn = m.matchfn
     m.matchfn = lambda f: lfile(f) and origmatchfn(f)
     return m
 
 
 def composenormalfilematcher(match, manifest, exclude=None):
     excluded = set()
     if exclude is not None:
         excluded.update(exclude)
 
     m = copy.copy(match)
     notlfile = lambda f: not (
         lfutil.isstandin(f) or lfutil.standin(f) in manifest or f in excluded
     )
     m._files = [lf for lf in m._files if notlfile(lf)]
     m._fileset = set(m._files)
     m.always = lambda: False
     origmatchfn = m.matchfn
     m.matchfn = lambda f: notlfile(f) and origmatchfn(f)
     return m
 
 
 def addlargefiles(ui, repo, isaddremove, matcher, uipathfn, **opts):
     large = opts.get('large')
     lfsize = lfutil.getminsize(
         ui, lfutil.islfilesrepo(repo), opts.get('lfsize')
     )
 
     lfmatcher = None
     if lfutil.islfilesrepo(repo):
         lfpats = ui.configlist(lfutil.longname, b'patterns')
         if lfpats:
             lfmatcher = matchmod.match(repo.root, b'', list(lfpats))
 
     lfnames = []
     m = matcher
 
     wctx = repo[None]
     for f in wctx.walk(matchmod.badmatch(m, lambda x, y: None)):
         exact = m.exact(f)
         lfile = lfutil.standin(f) in wctx
         nfile = f in wctx
         exists = lfile or nfile
 
         # Don't warn the user when they attempt to add a normal tracked file.
         # The normal add code will do that for us.
         if exact and exists:
             if lfile:
                 ui.warn(_(b'%s already a largefile\n') % uipathfn(f))
             continue
 
         if (exact or not exists) and not lfutil.isstandin(f):
             # In case the file was removed previously, but not committed
             # (issue3507)
             if not repo.wvfs.exists(f):
                 continue
 
             abovemin = (
                 lfsize and repo.wvfs.lstat(f).st_size >= lfsize * 1024 * 1024
             )
             if large or abovemin or (lfmatcher and lfmatcher(f)):
                 lfnames.append(f)
                 if ui.verbose or not exact:
                     ui.status(_(b'adding %s as a largefile\n') % uipathfn(f))
 
     bad = []
 
     # Need to lock, otherwise there could be a race condition between
     # when standins are created and added to the repo.
     with repo.wlock():
         if not opts.get('dry_run'):
             standins = []
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for f in lfnames:
                 standinname = lfutil.standin(f)
                 lfutil.writestandin(
                     repo,
                     standinname,
                     hash=b'',
                     executable=lfutil.getexecutable(repo.wjoin(f)),
                 )
                 standins.append(standinname)
                 lfdirstate.set_tracked(f)
             lfdirstate.write(repo.currenttransaction())
             bad += [
                 lfutil.splitstandin(f)
                 for f in repo[None].add(standins)
                 if f in m.files()
             ]
 
         added = [f for f in lfnames if f not in bad]
     return added, bad
 
 
 def removelargefiles(ui, repo, isaddremove, matcher, uipathfn, dryrun, **opts):
     after = opts.get('after')
     m = composelargefilematcher(matcher, repo[None].manifest())
     with lfstatus(repo):
         s = repo.status(match=m, clean=not isaddremove)
     manifest = repo[None].manifest()
     modified, added, deleted, clean = [
         [f for f in list if lfutil.standin(f) in manifest]
         for list in (s.modified, s.added, s.deleted, s.clean)
     ]
 
     def warn(files, msg):
         for f in files:
             ui.warn(msg % uipathfn(f))
         return int(len(files) > 0)
 
     if after:
         remove = deleted
         result = warn(
             modified + added + clean, _(b'not removing %s: file still exists\n')
         )
     else:
         remove = deleted + clean
         result = warn(
             modified,
             _(
                 b'not removing %s: file is modified (use -f'
                 b' to force removal)\n'
             ),
         )
         result = (
             warn(
                 added,
                 _(
                     b'not removing %s: file has been marked for add'
                     b' (use forget to undo)\n'
                 ),
             )
             or result
         )
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in sorted(remove):
             if ui.verbose or not m.exact(f):
                 ui.status(_(b'removing %s\n') % uipathfn(f))
 
             if not dryrun:
                 if not after:
                     repo.wvfs.unlinkpath(f, ignoremissing=True)
 
         if dryrun:
             return result
 
         remove = [lfutil.standin(f) for f in remove]
         # If this is being called by addremove, let the original addremove
         # function handle this.
         if not isaddremove:
             for f in remove:
                 repo.wvfs.unlinkpath(f, ignoremissing=True)
         repo[None].forget(remove)
 
         for f in remove:
             lfdirstate.set_untracked(lfutil.splitstandin(f))
 
         lfdirstate.write(repo.currenttransaction())
 
     return result
 
 
 # For overriding mercurial.hgweb.webcommands so that largefiles will
 # appear at their right place in the manifests.
 @eh.wrapfunction(webcommands, b'decodepath')
 def decodepath(orig, path):
     return lfutil.splitstandin(path) or path
 
 
 # -- Wrappers: modify existing commands --------------------------------
 
 
 @eh.wrapcommand(
     b'add',
     opts=[
         (b'', b'large', None, _(b'add as largefile')),
         (b'', b'normal', None, _(b'add as normal file')),
         (
             b'',
             b'lfsize',
             b'',
             _(
                 b'add all files above this size (in megabytes) '
                 b'as largefiles (default: 10)'
             ),
         ),
     ],
 )
 def overrideadd(orig, ui, repo, *pats, **opts):
     if opts.get('normal') and opts.get('large'):
         raise error.Abort(_(b'--normal cannot be used with --large'))
     return orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapfunction(cmdutil, b'add')
 def cmdutiladd(orig, ui, repo, matcher, prefix, uipathfn, explicitonly, **opts):
     # The --normal flag short circuits this override
     if opts.get('normal'):
         return orig(ui, repo, matcher, prefix, uipathfn, explicitonly, **opts)
 
     ladded, lbad = addlargefiles(ui, repo, False, matcher, uipathfn, **opts)
     normalmatcher = composenormalfilematcher(
         matcher, repo[None].manifest(), ladded
     )
     bad = orig(ui, repo, normalmatcher, prefix, uipathfn, explicitonly, **opts)
 
     bad.extend(f for f in lbad)
     return bad
 
 
 @eh.wrapfunction(cmdutil, b'remove')
 def cmdutilremove(
     orig, ui, repo, matcher, prefix, uipathfn, after, force, subrepos, dryrun
 ):
     normalmatcher = composenormalfilematcher(matcher, repo[None].manifest())
     result = orig(
         ui,
         repo,
         normalmatcher,
         prefix,
         uipathfn,
         after,
         force,
         subrepos,
         dryrun,
     )
     return (
         removelargefiles(
             ui, repo, False, matcher, uipathfn, dryrun, after=after, force=force
         )
         or result
     )
 
 
 @eh.wrapfunction(subrepo.hgsubrepo, b'status')
 def overridestatusfn(orig, repo, rev2, **opts):
     with lfstatus(repo._repo):
         return orig(repo, rev2, **opts)
 
 
 @eh.wrapcommand(b'status')
 def overridestatus(orig, ui, repo, *pats, **opts):
     with lfstatus(repo):
         return orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapfunction(subrepo.hgsubrepo, b'dirty')
 def overridedirty(orig, repo, ignoreupdate=False, missing=False):
     with lfstatus(repo._repo):
         return orig(repo, ignoreupdate=ignoreupdate, missing=missing)
 
 
 @eh.wrapcommand(b'log')
 def overridelog(orig, ui, repo, *pats, **opts):
     def overridematchandpats(
         orig,
         ctx,
         pats=(),
         opts=None,
         globbed=False,
         default=b'relpath',
         badfn=None,
     ):
         """Matcher that merges root directory with .hglf, suitable for log.
         It is still possible to match .hglf directly.
         For any listed files run log on the standin too.
         matchfn tries both the given filename and with .hglf stripped.
         """
         if opts is None:
             opts = {}
         matchandpats = orig(ctx, pats, opts, globbed, default, badfn=badfn)
         m, p = copy.copy(matchandpats)
 
         if m.always():
             # We want to match everything anyway, so there's no benefit trying
             # to add standins.
             return matchandpats
 
         pats = set(p)
 
         def fixpats(pat, tostandin=lfutil.standin):
             if pat.startswith(b'set:'):
                 return pat
 
             kindpat = matchmod._patsplit(pat, None)
 
             if kindpat[0] is not None:
                 return kindpat[0] + b':' + tostandin(kindpat[1])
             return tostandin(kindpat[1])
 
         cwd = repo.getcwd()
         if cwd:
             hglf = lfutil.shortname
             back = util.pconvert(repo.pathto(hglf)[: -len(hglf)])
 
             def tostandin(f):
                 # The file may already be a standin, so truncate the back
                 # prefix and test before mangling it.  This avoids turning
                 # 'glob:../.hglf/foo*' into 'glob:../.hglf/../.hglf/foo*'.
                 if f.startswith(back) and lfutil.splitstandin(f[len(back) :]):
                     return f
 
                 # An absolute path is from outside the repo, so truncate the
                 # path to the root before building the standin.  Otherwise cwd
                 # is somewhere in the repo, relative to root, and needs to be
                 # prepended before building the standin.
                 if os.path.isabs(cwd):
                     f = f[len(back) :]
                 else:
                     f = cwd + b'/' + f
                 return back + lfutil.standin(f)
 
         else:
 
             def tostandin(f):
                 if lfutil.isstandin(f):
                     return f
                 return lfutil.standin(f)
 
         pats.update(fixpats(f, tostandin) for f in p)
 
         for i in range(0, len(m._files)):
             # Don't add '.hglf' to m.files, since that is already covered by '.'
             if m._files[i] == b'.':
                 continue
             standin = lfutil.standin(m._files[i])
             # If the "standin" is a directory, append instead of replace to
             # support naming a directory on the command line with only
             # largefiles.  The original directory is kept to support normal
             # files.
             if standin in ctx:
                 m._files[i] = standin
             elif m._files[i] not in ctx and repo.wvfs.isdir(standin):
                 m._files.append(standin)
 
         m._fileset = set(m._files)
         m.always = lambda: False
         origmatchfn = m.matchfn
 
         def lfmatchfn(f):
             lf = lfutil.splitstandin(f)
             if lf is not None and origmatchfn(lf):
                 return True
             r = origmatchfn(f)
             return r
 
         m.matchfn = lfmatchfn
 
         ui.debug(b'updated patterns: %s\n' % b', '.join(sorted(pats)))
         return m, pats
 
     # For hg log --patch, the match object is used in two different senses:
     # (1) to determine what revisions should be printed out, and
     # (2) to determine what files to print out diffs for.
     # The magic matchandpats override should be used for case (1) but not for
     # case (2).
     oldmatchandpats = scmutil.matchandpats
 
     def overridemakefilematcher(orig, repo, pats, opts, badfn=None):
         wctx = repo[None]
         match, pats = oldmatchandpats(wctx, pats, opts, badfn=badfn)
         return lambda ctx: match
 
     wrappedmatchandpats = extensions.wrappedfunction(
         scmutil, b'matchandpats', overridematchandpats
     )
     wrappedmakefilematcher = extensions.wrappedfunction(
         logcmdutil, b'_makenofollowfilematcher', overridemakefilematcher
     )
     with wrappedmatchandpats, wrappedmakefilematcher:
         return orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapcommand(
     b'verify',
     opts=[
         (
             b'',
             b'large',
             None,
             _(b'verify that all largefiles in current revision exists'),
         ),
         (
             b'',
             b'lfa',
             None,
             _(b'verify largefiles in all revisions, not just current'),
         ),
         (
             b'',
             b'lfc',
             None,
             _(b'verify local largefile contents, not just existence'),
         ),
     ],
 )
 def overrideverify(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     all = opts.pop('lfa', False)
     contents = opts.pop('lfc', False)
 
     result = orig(ui, repo, *pats, **opts)
     if large or all or contents:
         result = result or lfcommands.verifylfiles(ui, repo, all, contents)
     return result
 
 
 @eh.wrapcommand(
     b'debugstate',
     opts=[(b'', b'large', None, _(b'display largefiles dirstate'))],
 )
 def overridedebugstate(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     if large:
 
         class fakerepo(object):
             dirstate = lfutil.openlfdirstate(ui, repo)
 
         orig(ui, fakerepo, *pats, **opts)
     else:
         orig(ui, repo, *pats, **opts)
 
 
 # Before starting the manifest merge, merge.updates will call
 # _checkunknownfile to check if there are any files in the merged-in
 # changeset that collide with unknown files in the working copy.
 #
 # The largefiles are seen as unknown, so this prevents us from merging
 # in a file 'foo' if we already have a largefile with the same name.
 #
 # The overridden function filters the unknown files by removing any
 # largefiles. This makes the merge proceed and we can then handle this
 # case further in the overridden calculateupdates function below.
 @eh.wrapfunction(merge, b'_checkunknownfile')
 def overridecheckunknownfile(origfn, repo, wctx, mctx, f, f2=None):
     if lfutil.standin(repo.dirstate.normalize(f)) in wctx:
         return False
     return origfn(repo, wctx, mctx, f, f2)
 
 
 # The manifest merge handles conflicts on the manifest level. We want
 # to handle changes in largefile-ness of files at this level too.
 #
 # The strategy is to run the original calculateupdates and then process
 # the action list it outputs. There are two cases we need to deal with:
 #
 # 1. Normal file in p1, largefile in p2. Here the largefile is
 #    detected via its standin file, which will enter the working copy
 #    with a "get" action. It is not "merge" since the standin is all
 #    Mercurial is concerned with at this level -- the link to the
 #    existing normal file is not relevant here.
 #
 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
 #    since the largefile will be present in the working copy and
 #    different from the normal file in p2. Mercurial therefore
 #    triggers a merge action.
 #
 # In both cases, we prompt the user and emit new actions to either
 # remove the standin (if the normal file was kept) or to remove the
 # normal file and get the standin (if the largefile was kept). The
 # default prompt answer is to use the largefile version since it was
 # presumably changed on purpose.
 #
 # Finally, the merge.applyupdates function will then take care of
 # writing the files into the working copy and lfcommands.updatelfiles
 # will update the largefiles.
 @eh.wrapfunction(merge, b'calculateupdates')
 def overridecalculateupdates(
     origfn, repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs
 ):
     overwrite = force and not branchmerge
     mresult = origfn(
         repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs
     )
 
     if overwrite:
         return mresult
 
     # Convert to dictionary with filename as key and action as value.
     lfiles = set()
     for f in mresult.files():
         splitstandin = lfutil.splitstandin(f)
         if splitstandin is not None and splitstandin in p1:
             lfiles.add(splitstandin)
         elif lfutil.standin(f) in p1:
             lfiles.add(f)
 
     for lfile in sorted(lfiles):
         standin = lfutil.standin(lfile)
         (lm, largs, lmsg) = mresult.getfile(lfile, (None, None, None))
         (sm, sargs, smsg) = mresult.getfile(standin, (None, None, None))
-        if sm in (b'g', b'dc') and lm != b'r':
-            if sm == b'dc':
+
+        if sm in (ACTION_GET, ACTION_DELETED_CHANGED) and lm != ACTION_REMOVE:
+            if sm == ACTION_DELETED_CHANGED:
                 f1, f2, fa, move, anc = sargs
                 sargs = (p2[f2].flags(), False)
             # Case 1: normal file in the working copy, largefile in
             # the second parent
             usermsg = (
                 _(
                     b'remote turned local normal file %s into a largefile\n'
                     b'use (l)argefile or keep (n)ormal file?'
                     b'$$ &Largefile $$ &Normal file'
                 )
                 % lfile
             )
             if repo.ui.promptchoice(usermsg, 0) == 0:  # pick remote largefile
-                mresult.addfile(lfile, b'r', None, b'replaced by standin')
-                mresult.addfile(standin, b'g', sargs, b'replaces standin')
+                mresult.addfile(
+                    lfile, ACTION_REMOVE, None, b'replaced by standin'
+                )
+                mresult.addfile(standin, ACTION_GET, sargs, b'replaces standin')
             else:  # keep local normal file
-                mresult.addfile(lfile, b'k', None, b'replaces standin')
+                mresult.addfile(lfile, ACTION_KEEP, None, b'replaces standin')
                 if branchmerge:
                     mresult.addfile(
                         standin,
-                        b'k',
+                        ACTION_KEEP,
                         None,
                         b'replaced by non-standin',
                     )
                 else:
                     mresult.addfile(
                         standin,
-                        b'r',
+                        ACTION_REMOVE,
                         None,
                         b'replaced by non-standin',
                     )
-        elif lm in (b'g', b'dc') and sm != b'r':
-            if lm == b'dc':
+        if lm in (ACTION_GET, ACTION_DELETED_CHANGED) and sm != ACTION_REMOVE:
+            if lm == ACTION_DELETED_CHANGED:
                 f1, f2, fa, move, anc = largs
                 largs = (p2[f2].flags(), False)
             # Case 2: largefile in the working copy, normal file in
             # the second parent
             usermsg = (
                 _(
                     b'remote turned local largefile %s into a normal file\n'
                     b'keep (l)argefile or use (n)ormal file?'
                     b'$$ &Largefile $$ &Normal file'
                 )
                 % lfile
             )
             if repo.ui.promptchoice(usermsg, 0) == 0:  # keep local largefile
                 if branchmerge:
                     # largefile can be restored from standin safely
                     mresult.addfile(
                         lfile,
-                        b'k',
+                        ACTION_KEEP,
                         None,
                         b'replaced by standin',
                     )
-                    mresult.addfile(standin, b'k', None, b'replaces standin')
+                    mresult.addfile(
+                        standin, ACTION_KEEP, None, b'replaces standin'
+                    )
                 else:
                     # "lfile" should be marked as "removed" without
                     # removal of itself
                     mresult.addfile(
                         lfile,
                         MERGE_ACTION_LARGEFILE_MARK_REMOVED,
                         None,
                         b'forget non-standin largefile',
                     )
 
                     # linear-merge should treat this largefile as 're-added'
-                    mresult.addfile(standin, b'a', None, b'keep standin')
+                    mresult.addfile(standin, ACTION_ADD, None, b'keep standin')
             else:  # pick remote normal file
-                mresult.addfile(lfile, b'g', largs, b'replaces standin')
+                mresult.addfile(lfile, ACTION_GET, largs, b'replaces standin')
                 mresult.addfile(
                     standin,
-                    b'r',
+                    ACTION_REMOVE,
                     None,
                     b'replaced by non-standin',
                 )
 
     return mresult
 
 
 @eh.wrapfunction(mergestatemod, b'recordupdates')
 def mergerecordupdates(orig, repo, actions, branchmerge, getfiledata):
     if MERGE_ACTION_LARGEFILE_MARK_REMOVED in actions:
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
         with lfdirstate.parentchange():
             for lfile, args, msg in actions[
                 MERGE_ACTION_LARGEFILE_MARK_REMOVED
             ]:
                 # this should be executed before 'orig', to execute 'remove'
                 # before all other actions
                 repo.dirstate.update_file(
                     lfile, p1_tracked=True, wc_tracked=False
                 )
                 # make sure lfile doesn't get synclfdirstate'd as normal
                 lfdirstate.update_file(lfile, p1_tracked=False, wc_tracked=True)
         lfdirstate.write(repo.currenttransaction())
 
     return orig(repo, actions, branchmerge, getfiledata)
 
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits without prompting the user.
[e-mail](filemerge, b'_filemerge')
[e-mail](filemerge, b'filemerge')
 def overridefilemerge(
-    origfn, premerge, repo, wctx, mynode, orig, fcd, fco, fca, labels=None
+    origfn, repo, wctx, mynode, orig, fcd, fco, fca, labels=None
 ):
     if not lfutil.isstandin(orig) or fcd.isabsent() or fco.isabsent():
-        return origfn(
-            premerge, repo, wctx, mynode, orig, fcd, fco, fca, labels=labels
-        )
+        return origfn(repo, wctx, mynode, orig, fcd, fco, fca, labels=labels)
 
     ahash = lfutil.readasstandin(fca).lower()
     dhash = lfutil.readasstandin(fcd).lower()
     ohash = lfutil.readasstandin(fco).lower()
     if (
         ohash != ahash
         and ohash != dhash
         and (
             dhash == ahash
             or repo.ui.promptchoice(
                 _(
                     b'largefile %s has a merge conflict\nancestor was %s\n'
                     b'you can keep (l)ocal %s or take (o)ther %s.\n'
                     b'what do you want to do?'
                     b'$$ &Local $$ &Other'
                 )
                 % (lfutil.splitstandin(orig), ahash, dhash, ohash),
                 0,
             )
             == 1
         )
     ):
         repo.wwrite(fcd.path(), fco.data(), fco.flags())
-    return True, 0, False
+    return 0, False
 
 
 @eh.wrapfunction(copiesmod, b'pathcopies')
 def copiespathcopies(orig, ctx1, ctx2, match=None):
     copies = orig(ctx1, ctx2, match=match)
     updated = {}
 
     for k, v in pycompat.iteritems(copies):
         updated[lfutil.splitstandin(k) or k] = lfutil.splitstandin(v) or v
 
     return updated
 
 
 # Copy first changes the matchers to match standins instead of
 # largefiles.  Then it overrides util.copyfile in that function it
 # checks if the destination largefile already exists. It also keeps a
 # list of copied files so that the largefiles can be copied and the
 # dirstate updated.
 @eh.wrapfunction(cmdutil, b'copy')
 def overridecopy(orig, ui, repo, pats, opts, rename=False):
     # doesn't remove largefile on rename
     if len(pats) < 2:
         # this isn't legal, let the original function deal with it
         return orig(ui, repo, pats, opts, rename)
 
     # This could copy both lfiles and normal files in one command,
     # but we don't want to do that. First replace their matcher to
     # only match normal files and run it, then replace it to just
     # match largefiles and run it again.
     nonormalfiles = False
     nolfiles = False
     manifest = repo[None].manifest()
 
     def normalfilesmatchfn(
         orig,
         ctx,
         pats=(),
         opts=None,
         globbed=False,
         default=b'relpath',
         badfn=None,
     ):
         if opts is None:
             opts = {}
         match = orig(ctx, pats, opts, globbed, default, badfn=badfn)
         return composenormalfilematcher(match, manifest)
 
     with extensions.wrappedfunction(scmutil, b'match', normalfilesmatchfn):
         try:
             result = orig(ui, repo, pats, opts, rename)
         except error.Abort as e:
             if e.message != _(b'no files to copy'):
                 raise e
             else:
                 nonormalfiles = True
             result = 0
 
     # The first rename can cause our current working directory to be removed.
     # In that case there is nothing left to copy/rename so just quit.
     try:
         repo.getcwd()
     except OSError:
         return result
 
     def makestandin(relpath):
         path = pathutil.canonpath(repo.root, repo.getcwd(), relpath)
         return repo.wvfs.join(lfutil.standin(path))
 
     fullpats = scmutil.expandpats(pats)
     dest = fullpats[-1]
 
     if os.path.isdir(dest):
         if not os.path.isdir(makestandin(dest)):
             os.makedirs(makestandin(dest))
 
     try:
         # When we call orig below it creates the standins but we don't add
         # them to the dir state until later so lock during that time.
         wlock = repo.wlock()
 
         manifest = repo[None].manifest()
 
         def overridematch(
             orig,
             ctx,
             pats=(),
             opts=None,
             globbed=False,
             default=b'relpath',
             badfn=None,
         ):
             if opts is None:
                 opts = {}
             newpats = []
             # The patterns were previously mangled to add the standin
             # directory; we need to remove that now
             for pat in pats:
                 if matchmod.patkind(pat) is None and lfutil.shortname in pat:
                     newpats.append(pat.replace(lfutil.shortname, b''))
                 else:
                     newpats.append(pat)
             match = orig(ctx, newpats, opts, globbed, default, badfn=badfn)
             m = copy.copy(match)
             lfile = lambda f: lfutil.standin(f) in manifest
             m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
             m._fileset = set(m._files)
             origmatchfn = m.matchfn
 
             def matchfn(f):
                 lfile = lfutil.splitstandin(f)
                 return (
                     lfile is not None
                     and (f in manifest)
                     and origmatchfn(lfile)
                     or None
                 )
 
             m.matchfn = matchfn
             return m
 
         listpats = []
         for pat in pats:
             if matchmod.patkind(pat) is not None:
                 listpats.append(pat)
             else:
                 listpats.append(makestandin(pat))
 
         copiedfiles = []
 
         def overridecopyfile(orig, src, dest, *args, **kwargs):
             if lfutil.shortname in src and dest.startswith(
                 repo.wjoin(lfutil.shortname)
             ):
                 destlfile = dest.replace(lfutil.shortname, b'')
                 if not opts[b'force'] and os.path.exists(destlfile):
                     raise IOError(
                         b'', _(b'destination largefile already exists')
                     )
             copiedfiles.append((src, dest))
             orig(src, dest, *args, **kwargs)
 
         with extensions.wrappedfunction(util, b'copyfile', overridecopyfile):
             with extensions.wrappedfunction(scmutil, b'match', overridematch):
                 result += orig(ui, repo, listpats, opts, rename)
 
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for (src, dest) in copiedfiles:
             if lfutil.shortname in src and dest.startswith(
                 repo.wjoin(lfutil.shortname)
             ):
                 srclfile = src.replace(repo.wjoin(lfutil.standin(b'')), b'')
                 destlfile = dest.replace(repo.wjoin(lfutil.standin(b'')), b'')
                 destlfiledir = repo.wvfs.dirname(repo.wjoin(destlfile)) or b'.'
                 if not os.path.isdir(destlfiledir):
                     os.makedirs(destlfiledir)
                 if rename:
                     os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
 
                     # The file is gone, but this deletes any empty parent
                     # directories as a side-effect.
                     repo.wvfs.unlinkpath(srclfile, ignoremissing=True)
                     lfdirstate.set_untracked(srclfile)
                 else:
                     util.copyfile(repo.wjoin(srclfile), repo.wjoin(destlfile))
 
                 lfdirstate.set_tracked(destlfile)
         lfdirstate.write(repo.currenttransaction())
     except error.Abort as e:
         if e.message != _(b'no files to copy'):
             raise e
         else:
             nolfiles = True
     finally:
         wlock.release()
 
     if nolfiles and nonormalfiles:
         raise error.Abort(_(b'no files to copy'))
 
     return result
 
 
 # When the user calls revert, we have to be careful to not revert any
 # changes to other largefiles accidentally. This means we have to keep
 # track of the largefiles that are being reverted so we only pull down
 # the necessary largefiles.
 #
 # Standins are only updated (to match the hash of largefiles) before
 # commits. Update the standins then run the original revert, changing
 # the matcher to hit standins instead of largefiles. Based on the
 # resulting standins update the largefiles.
 @eh.wrapfunction(cmdutil, b'revert')
 def overriderevert(orig, ui, repo, ctx, *pats, **opts):
     # Because we put the standins in a bad state (by updating them)
     # and then return them to a correct state we need to lock to
     # prevent others from changing them in their incorrect state.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         s = lfutil.lfdirstatestatus(lfdirstate, repo)
         lfdirstate.write(repo.currenttransaction())
         for lfile in s.modified:
             lfutil.updatestandin(repo, lfile, lfutil.standin(lfile))
         for lfile in s.deleted:
             fstandin = lfutil.standin(lfile)
             if repo.wvfs.exists(fstandin):
                 repo.wvfs.unlink(fstandin)
 
         oldstandins = lfutil.getstandinsstate(repo)
 
         def overridematch(
             orig,
             mctx,
             pats=(),
             opts=None,
             globbed=False,
             default=b'relpath',
             badfn=None,
         ):
             if opts is None:
                 opts = {}
             match = orig(mctx, pats, opts, globbed, default, badfn=badfn)
             m = copy.copy(match)
 
             # revert supports recursing into subrepos, and though largefiles
             # currently doesn't work correctly in that case, this match is
             # called, so the lfdirstate above may not be the correct one for
             # this invocation of match.
             lfdirstate = lfutil.openlfdirstate(
                 mctx.repo().ui, mctx.repo(), False
             )
 
             wctx = repo[None]
             matchfiles = []
             for f in m._files:
                 standin = lfutil.standin(f)
                 if standin in ctx or standin in mctx:
                     matchfiles.append(standin)
                 elif standin in wctx or lfdirstate.get_entry(f).removed:
                     continue
                 else:
                     matchfiles.append(f)
             m._files = matchfiles
             m._fileset = set(m._files)
             origmatchfn = m.matchfn
 
             def matchfn(f):
                 lfile = lfutil.splitstandin(f)
                 if lfile is not None:
                     return origmatchfn(lfile) and (f in ctx or f in mctx)
                 return origmatchfn(f)
 
             m.matchfn = matchfn
             return m
 
         with extensions.wrappedfunction(scmutil, b'match', overridematch):
             orig(ui, repo, ctx, *pats, **opts)
 
         newstandins = lfutil.getstandinsstate(repo)
         filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
         # lfdirstate should be 'normallookup'-ed for updated files,
         # because reverting doesn't touch dirstate for 'normal' files
         # when target revision is explicitly specified: in such case,
         # 'n' and valid timestamp in dirstate doesn't ensure 'clean'
         # of target (standin) file.
         lfcommands.updatelfiles(
             ui, repo, filelist, printmessage=False, normallookup=True
         )
 
 
 # after pulling changesets, we need to take some extra care to get
 # largefiles updated remotely
 @eh.wrapcommand(
     b'pull',
     opts=[
         (
             b'',
             b'all-largefiles',
             None,
             _(b'download all pulled versions of largefiles (DEPRECATED)'),
         ),
         (
             b'',
             b'lfrev',
             [],
             _(b'download largefiles for these revisions'),
             _(b'REV'),
         ),
     ],
 )
 def overridepull(orig, ui, repo, source=None, **opts):
     revsprepull = len(repo)
     if not source:
         source = b'default'
     repo.lfpullsource = source
     result = orig(ui, repo, source, **opts)
     revspostpull = len(repo)
     lfrevs = opts.get('lfrev', [])
     if opts.get('all_largefiles'):
         lfrevs.append(b'pulled()')
     if lfrevs and revspostpull > revsprepull:
         numcached = 0
         repo.firstpulled = revsprepull  # for pulled() revset expression
         try:
             for rev in logcmdutil.revrange(repo, lfrevs):
                 ui.note(_(b'pulling largefiles for revision %d\n') % rev)
                 (cached, missing) = lfcommands.cachelfiles(ui, repo, rev)
                 numcached += len(cached)
         finally:
             del repo.firstpulled
         ui.status(_(b"%d largefiles cached\n") % numcached)
     return result
 
 
 @eh.wrapcommand(
     b'push',
     opts=[
         (
             b'',
             b'lfrev',
             [],
             _(b'upload largefiles for these revisions'),
             _(b'REV'),
         )
     ],
 )
 def overridepush(orig, ui, repo, *args, **kwargs):
     """Override push command and store --lfrev parameters in opargs"""
     lfrevs = kwargs.pop('lfrev', None)
     if lfrevs:
         opargs = kwargs.setdefault('opargs', {})
         opargs[b'lfrevs'] = logcmdutil.revrange(repo, lfrevs)
     return orig(ui, repo, *args, **kwargs)
 
 
 @eh.wrapfunction(exchange, b'pushoperation')
 def exchangepushoperation(orig, *args, **kwargs):
     """Override pushoperation constructor and store lfrevs parameter"""
     lfrevs = kwargs.pop('lfrevs', None)
     pushop = orig(*args, **kwargs)
     pushop.lfrevs = lfrevs
     return pushop
 
 
 @eh.revsetpredicate(b'pulled()')
 def pulledrevsetsymbol(repo, subset, x):
     """Changesets that just has been pulled.
 
     Only available with largefiles from pull --lfrev expressions.
 
     .. container:: verbose
 
       Some examples:
 
       - pull largefiles for all new changesets::
 
           hg pull -lfrev "pulled()"
 
       - pull largefiles for all new branch heads::
 
           hg pull -lfrev "head(pulled()) and not closed()"
 
     """
 
     try:
         firstpulled = repo.firstpulled
     except AttributeError:
         raise error.Abort(_(b"pulled() only available in --lfrev"))
     return smartset.baseset([r for r in subset if r >= firstpulled])
 
 
 @eh.wrapcommand(
     b'clone',
     opts=[
         (
             b'',
             b'all-largefiles',
             None,
             _(b'download all versions of all largefiles'),
         )
     ],
 )
 def overrideclone(orig, ui, source, dest=None, **opts):
     d = dest
     if d is None:
         d = hg.defaultdest(source)
     if opts.get('all_largefiles') and not hg.islocal(d):
         raise error.Abort(
             _(b'--all-largefiles is incompatible with non-local destination %s')
             % d
         )
 
     return orig(ui, source, dest, **opts)
 
 
 @eh.wrapfunction(hg, b'clone')
 def hgclone(orig, ui, opts, *args, **kwargs):
     result = orig(ui, opts, *args, **kwargs)
 
     if result is not None:
         sourcerepo, destrepo = result
         repo = destrepo.local()
 
         # When cloning to a remote repo (like through SSH), no repo is available
         # from the peer.   Therefore the largefiles can't be downloaded and the
         # hgrc can't be updated.
         if not repo:
             return result
 
         # Caching is implicitly limited to 'rev' option, since the dest repo was
         # truncated at that point.  The user may expect a download count with
         # this option, so attempt whether or not this is a largefile repo.
         if opts.get(b'all_largefiles'):
             success, missing = lfcommands.downloadlfiles(ui, repo)
 
             if missing != 0:
                 return None
 
     return result
 
 
 @eh.wrapcommand(b'rebase', extension=b'rebase')
 def overriderebasecmd(orig, ui, repo, **opts):
     if not util.safehasattr(repo, b'_largefilesenabled'):
         return orig(ui, repo, **opts)
 
     resuming = opts.get('continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
     repo._lfstatuswriters.append(lambda *msg, **opts: None)
     try:
         with ui.configoverride(
             {(b'rebase', b'experimental.inmemory'): False}, b"largefiles"
         ):
             return orig(ui, repo, **opts)
     finally:
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
 
 
 @eh.extsetup
 def overriderebase(ui):
     try:
         rebase = extensions.find(b'rebase')
     except KeyError:
         pass
     else:
 
         def _dorebase(orig, *args, **kwargs):
             kwargs['inmemory'] = False
             return orig(*args, **kwargs)
 
         extensions.wrapfunction(rebase, b'_dorebase', _dorebase)
 
 
 @eh.wrapcommand(b'archive')
 def overridearchivecmd(orig, ui, repo, dest, **opts):
     with lfstatus(repo.unfiltered()):
         return orig(ui, repo.unfiltered(), dest, **opts)
 
 
 @eh.wrapfunction(webcommands, b'archive')
 def hgwebarchive(orig, web):
     with lfstatus(web.repo):
         return orig(web)
 
 
 @eh.wrapfunction(archival, b'archive')
 def overridearchive(
     orig,
     repo,
     dest,
     node,
     kind,
     decode=True,
     match=None,
     prefix=b'',
     mtime=None,
     subrepos=None,
 ):
     # For some reason setting repo.lfstatus in hgwebarchive only changes the
     # unfiltered repo's attr, so check that as well.
     if not repo.lfstatus and not repo.unfiltered().lfstatus:
         return orig(
             repo, dest, node, kind, decode, match, prefix, mtime, subrepos
         )
 
     # No need to lock because we are only reading history and
     # largefile caches, neither of which are modified.
     if node is not None:
         lfcommands.cachelfiles(repo.ui, repo, node)
 
     if kind not in archival.archivers:
         raise error.Abort(_(b"unknown archive type '%s'") % kind)
 
     ctx = repo[node]
 
     if kind == b'files':
         if prefix:
             raise error.Abort(_(b'cannot give prefix when archiving to files'))
     else:
         prefix = archival.tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         if match and not match(name):
             return
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool(b"ui", b"archivemeta"):
         write(
             b'.hg_archival.txt',
             0o644,
             False,
             lambda: archival.buildmetadata(ctx),
         )
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         lfile = lfutil.splitstandin(f)
         if lfile is not None:
             if node is not None:
                 path = lfutil.findfile(repo, getdata().strip())
 
                 if path is None:
                     raise error.Abort(
                         _(
                             b'largefile %s not found in repo store or system cache'
                         )
                         % lfile
                     )
             else:
                 path = lfile
 
             f = lfile
 
             getdata = lambda: util.readfile(path)
         write(f, b'x' in ff and 0o755 or 0o644, b'l' in ff, getdata)
 
     if subrepos:
         for subpath in sorted(ctx.substate):
             sub = ctx.workingsub(subpath)
             submatch = matchmod.subdirmatcher(subpath, match)
             subprefix = prefix + subpath + b'/'
 
             # TODO: Only hgsubrepo instances have `_repo`, so figure out how to
             # infer and possibly set lfstatus in hgsubrepoarchive.  That would
             # allow only hgsubrepos to set this, instead of the current scheme
             # where the parent sets this for the child.
             with (
                 util.safehasattr(sub, '_repo')
                 and lfstatus(sub._repo)
                 or util.nullcontextmanager()
             ):
                 sub.archive(archiver, subprefix, submatch)
 
     archiver.done()
 
 
 @eh.wrapfunction(subrepo.hgsubrepo, b'archive')
 def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
     lfenabled = util.safehasattr(repo._repo, b'_largefilesenabled')
     if not lfenabled or not repo._repo.lfstatus:
         return orig(repo, archiver, prefix, match, decode)
 
     repo._get(repo._state + (b'hg',))
     rev = repo._state[1]
     ctx = repo._repo[rev]
 
     if ctx.node() is not None:
         lfcommands.cachelfiles(repo.ui, repo._repo, ctx.node())
 
     def write(name, mode, islink, getdata):
         # At this point, the standin has been replaced with the largefile name,
         # so the normal matcher works here without the lfutil variants.
         if match and not match(f):
             return
         data = getdata()
         if decode:
             data = repo._repo.wwritedata(name, data)
 
         archiver.addfile(prefix + name, mode, islink, data)
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         lfile = lfutil.splitstandin(f)
         if lfile is not None:
             if ctx.node() is not None:
                 path = lfutil.findfile(repo._repo, getdata().strip())
 
                 if path is None:
                     raise error.Abort(
                         _(
                             b'largefile %s not found in repo store or system cache'
                         )
                         % lfile
                     )
             else:
                 path = lfile
 
             f = lfile
 
             getdata = lambda: util.readfile(os.path.join(prefix, path))
 
         write(f, b'x' in ff and 0o755 or 0o644, b'l' in ff, getdata)
 
     for subpath in sorted(ctx.substate):
         sub = ctx.workingsub(subpath)
         submatch = matchmod.subdirmatcher(subpath, match)
         subprefix = prefix + subpath + b'/'
         # TODO: Only hgsubrepo instances have `_repo`, so figure out how to
         # infer and possibly set lfstatus at the top of this function.  That
         # would allow only hgsubrepos to set this, instead of the current scheme
         # where the parent sets this for the child.
         with (
             util.safehasattr(sub, '_repo')
             and lfstatus(sub._repo)
             or util.nullcontextmanager()
         ):
             sub.archive(archiver, subprefix, submatch, decode)
 
 
 # If a largefile is modified, the change is not reflected in its
 # standin until a commit. cmdutil.bailifchanged() raises an exception
 # if the repo has uncommitted changes. Wrap it to also check if
 # largefiles were changed. This is used by bisect, backout and fetch.
 @eh.wrapfunction(cmdutil, b'bailifchanged')
 def overridebailifchanged(orig, repo, *args, **kwargs):
     orig(repo, *args, **kwargs)
     with lfstatus(repo):
         s = repo.status()
     if s.modified or s.added or s.removed or s.deleted:
         raise error.Abort(_(b'uncommitted changes'))
 
 
 @eh.wrapfunction(cmdutil, b'postcommitstatus')
 def postcommitstatus(orig, repo, *args, **kwargs):
     with lfstatus(repo):
         return orig(repo, *args, **kwargs)
 
 
 @eh.wrapfunction(cmdutil, b'forget')
 def cmdutilforget(
     orig, ui, repo, match, prefix, uipathfn, explicitonly, dryrun, interactive
 ):
     normalmatcher = composenormalfilematcher(match, repo[None].manifest())
     bad, forgot = orig(
         ui,
         repo,
         normalmatcher,
         prefix,
         uipathfn,
         explicitonly,
         dryrun,
         interactive,
     )
     m = composelargefilematcher(match, repo[None].manifest())
 
     with lfstatus(repo):
         s = repo.status(match=m, clean=True)
     manifest = repo[None].manifest()
     forget = sorted(s.modified + s.added + s.deleted + s.clean)
     forget = [f for f in forget if lfutil.standin(f) in manifest]
 
     for f in forget:
         fstandin = lfutil.standin(f)
         if fstandin not in repo.dirstate and not repo.wvfs.isdir(fstandin):
             ui.warn(
                 _(b'not removing %s: file is already untracked\n') % uipathfn(f)
             )
             bad.append(f)
 
     for f in forget:
         if ui.verbose or not m.exact(f):
             ui.status(_(b'removing %s\n') % uipathfn(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in forget:
             lfdirstate.set_untracked(f)
         lfdirstate.write(repo.currenttransaction())
         standins = [lfutil.standin(f) for f in forget]
         for f in standins:
             repo.wvfs.unlinkpath(f, ignoremissing=True)
         rejected = repo[None].forget(standins)
 
     bad.extend(f for f in rejected if f in m.files())
     forgot.extend(f for f in forget if f not in rejected)
     return bad, forgot
 
 
 def _getoutgoings(repo, other, missing, addfunc):
     """get pairs of filename and largefile hash in outgoing revisions
     in 'missing'.
 
     largefiles already existing on 'other' repository are ignored.
 
     'addfunc' is invoked with each unique pairs of filename and
     largefile hash value.
     """
     knowns = set()
     lfhashes = set()
 
     def dedup(fn, lfhash):
         k = (fn, lfhash)
         if k not in knowns:
             knowns.add(k)
             lfhashes.add(lfhash)
 
     lfutil.getlfilestoupload(repo, missing, dedup)
     if lfhashes:
         lfexists = storefactory.openstore(repo, other).exists(lfhashes)
         for fn, lfhash in knowns:
             if not lfexists[lfhash]:  # lfhash doesn't exist on "other"
                 addfunc(fn, lfhash)
 
 
 def outgoinghook(ui, repo, other, opts, missing):
     if opts.pop(b'large', None):
         lfhashes = set()
         if ui.debugflag:
             toupload = {}
 
             def addfunc(fn, lfhash):
                 if fn not in toupload:
                     toupload[fn] = []
                 toupload[fn].append(lfhash)
                 lfhashes.add(lfhash)
 
             def showhashes(fn):
                 for lfhash in sorted(toupload[fn]):
                     ui.debug(b'    %s\n' % lfhash)
 
         else:
             toupload = set()
 
             def addfunc(fn, lfhash):
                 toupload.add(fn)
                 lfhashes.add(lfhash)
 
             def showhashes(fn):
                 pass
 
         _getoutgoings(repo, other, missing, addfunc)
 
         if not toupload:
             ui.status(_(b'largefiles: no files to upload\n'))
         else:
             ui.status(
                 _(b'largefiles to upload (%d entities):\n') % (len(lfhashes))
             )
             for file in sorted(toupload):
                 ui.status(lfutil.splitstandin(file) + b'\n')
                 showhashes(file)
             ui.status(b'\n')
 
 
 @eh.wrapcommand(
     b'outgoing', opts=[(b'', b'large', None, _(b'display outgoing largefiles'))]
 )
 def _outgoingcmd(orig, *args, **kwargs):
     # Nothing to do here other than add the extra help option- the hook above
     # processes it.
     return orig(*args, **kwargs)
 
 
 def summaryremotehook(ui, repo, opts, changes):
     largeopt = opts.get(b'large', False)
     if changes is None:
         if largeopt:
             return (False, True)  # only outgoing check is needed
         else:
             return (False, False)
     elif largeopt:
         url, branch, peer, outgoing = changes[1]
         if peer is None:
             # i18n: column positioning for "hg summary"
             ui.status(_(b'largefiles: (no remote repo)\n'))
             return
 
         toupload = set()
         lfhashes = set()
 
         def addfunc(fn, lfhash):
             toupload.add(fn)
             lfhashes.add(lfhash)
 
         _getoutgoings(repo, peer, outgoing.missing, addfunc)
 
         if not toupload:
             # i18n: column positioning for "hg summary"
             ui.status(_(b'largefiles: (no files to upload)\n'))
         else:
             # i18n: column positioning for "hg summary"
             ui.status(
                 _(b'largefiles: %d entities for %d files to upload\n')
                 % (len(lfhashes), len(toupload))
             )
 
 
 @eh.wrapcommand(
     b'summary', opts=[(b'', b'large', None, _(b'display outgoing largefiles'))]
 )
 def overridesummary(orig, ui, repo, *pats, **opts):
     with lfstatus(repo):
         orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapfunction(scmutil, b'addremove')
 def scmutiladdremove(orig, repo, matcher, prefix, uipathfn, opts=None):
     if opts is None:
         opts = {}
     if not lfutil.islfilesrepo(repo):
         return orig(repo, matcher, prefix, uipathfn, opts)
     # Get the list of missing largefiles so we can remove them
     lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
-    unsure, s = lfdirstate.status(
+    unsure, s, mtime_boundary = lfdirstate.status(
         matchmod.always(),
         subrepos=[],
         ignored=False,
         clean=False,
         unknown=False,
     )
 
     # Call into the normal remove code, but the removing of the standin, we want
     # to have handled by original addremove.  Monkey patching here makes sure
     # we don't remove the standin in the largefiles code, preventing a very
     # confused state later.
     if s.deleted:
         m = copy.copy(matcher)
 
         # The m._files and m._map attributes are not changed to the deleted list
         # because that affects the m.exact() test, which in turn governs whether
         # or not the file name is printed, and how.  Simply limit the original
         # matches to those in the deleted status list.
         matchfn = m.matchfn
         m.matchfn = lambda f: f in s.deleted and matchfn(f)
 
         removelargefiles(
             repo.ui,
             repo,
             True,
             m,
             uipathfn,
             opts.get(b'dry_run'),
             **pycompat.strkwargs(opts)
         )
     # Call into the normal add code, and any files that *should* be added as
     # largefiles will be
     added, bad = addlargefiles(
         repo.ui, repo, True, matcher, uipathfn, **pycompat.strkwargs(opts)
     )
     # Now that we've handled largefiles, hand off to the original addremove
     # function to take care of the rest.  Make sure it doesn't do anything with
     # largefiles by passing a matcher that will ignore them.
     matcher = composenormalfilematcher(matcher, repo[None].manifest(), added)
     return orig(repo, matcher, prefix, uipathfn, opts)
 
 
 # Calling purge with --all will cause the largefiles to be deleted.
 # Override repo.status to prevent this from happening.
 @eh.wrapcommand(b'purge')
 def overridepurge(orig, ui, repo, *dirs, **opts):
     # XXX Monkey patching a repoview will not work. The assigned attribute will
     # be set on the unfiltered repo, but we will only lookup attributes in the
     # unfiltered repo if the lookup in the repoview object itself fails. As the
     # monkey patched method exists on the repoview class the lookup will not
     # fail. As a result, the original version will shadow the monkey patched
     # one, defeating the monkey patch.
     #
     # As a work around we use an unfiltered repo here. We should do something
     # cleaner instead.
     repo = repo.unfiltered()
     oldstatus = repo.status
 
     def overridestatus(
         node1=b'.',
         node2=None,
         match=None,
         ignored=False,
         clean=False,
         unknown=False,
         listsubrepos=False,
     ):
         r = oldstatus(
             node1, node2, match, ignored, clean, unknown, listsubrepos
         )
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         unknown = [
             f for f in r.unknown if not lfdirstate.get_entry(f).any_tracked
         ]
         ignored = [
             f for f in r.ignored if not lfdirstate.get_entry(f).any_tracked
         ]
         return scmutil.status(
             r.modified, r.added, r.removed, r.deleted, unknown, ignored, r.clean
         )
 
     repo.status = overridestatus
     orig(ui, repo, *dirs, **opts)
     repo.status = oldstatus
 
 
 @eh.wrapcommand(b'rollback')
 def overriderollback(orig, ui, repo, **opts):
     with repo.wlock():
         before = repo.dirstate.parents()
         orphans = {
             f
             for f in repo.dirstate
             if lfutil.isstandin(f) and not repo.dirstate.get_entry(f).removed
         }
         result = orig(ui, repo, **opts)
         after = repo.dirstate.parents()
         if before == after:
             return result  # no need to restore standins
 
         pctx = repo[b'.']
         for f in repo.dirstate:
             if lfutil.isstandin(f):
                 orphans.discard(f)
                 if repo.dirstate.get_entry(f).removed:
                     repo.wvfs.unlinkpath(f, ignoremissing=True)
                 elif f in pctx:
                     fctx = pctx[f]
                     repo.wwrite(f, fctx.data(), fctx.flags())
                 else:
                     # content of standin is not so important in 'a',
                     # 'm' or 'n' (coming from the 2nd parent) cases
                     lfutil.writestandin(repo, f, b'', False)
         for standin in orphans:
             repo.wvfs.unlinkpath(standin, ignoremissing=True)
 
     return result
 
 
 @eh.wrapcommand(b'transplant', extension=b'transplant')
 def overridetransplant(orig, ui, repo, *revs, **opts):
     resuming = opts.get('continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
     repo._lfstatuswriters.append(lambda *msg, **opts: None)
     try:
         result = orig(ui, repo, *revs, **opts)
     finally:
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
     return result
 
 
 @eh.wrapcommand(b'cat')
 def overridecat(orig, ui, repo, file1, *pats, **opts):
     opts = pycompat.byteskwargs(opts)
     ctx = logcmdutil.revsingle(repo, opts.get(b'rev'))
     err = 1
     notbad = set()
     m = scmutil.match(ctx, (file1,) + pats, opts)
     origmatchfn = m.matchfn
 
     def lfmatchfn(f):
         if origmatchfn(f):
             return True
         lf = lfutil.splitstandin(f)
         if lf is None:
             return False
         notbad.add(lf)
         return origmatchfn(lf)
 
     m.matchfn = lfmatchfn
     origbadfn = m.bad
 
     def lfbadfn(f, msg):
         if not f in notbad:
             origbadfn(f, msg)
 
     m.bad = lfbadfn
 
     origvisitdirfn = m.visitdir
 
     def lfvisitdirfn(dir):
         if dir == lfutil.shortname:
             return True
         ret = origvisitdirfn(dir)
         if ret:
             return ret
         lf = lfutil.splitstandin(dir)
         if lf is None:
             return False
         return origvisitdirfn(lf)
 
     m.visitdir = lfvisitdirfn
 
     for f in ctx.walk(m):
         with cmdutil.makefileobj(ctx, opts.get(b'output'), pathname=f) as fp:
             lf = lfutil.splitstandin(f)
             if lf is None or origmatchfn(f):
                 # duplicating unreachable code from commands.cat
                 data = ctx[f].data()
                 if opts.get(b'decode'):
                     data = repo.wwritedata(f, data)
                 fp.write(data)
             else:
                 hash = lfutil.readasstandin(ctx[f])
                 if not lfutil.inusercache(repo.ui, hash):
                     store = storefactory.openstore(repo)
                     success, missing = store.get([(lf, hash)])
                     if len(success) != 1:
                         raise error.Abort(
                             _(
                                 b'largefile %s is not in cache and could not be '
                                 b'downloaded'
                             )
                             % lf
                         )
                 path = lfutil.usercachepath(repo.ui, hash)
                 with open(path, b"rb") as fpin:
                     for chunk in util.filechunkiter(fpin):
                         fp.write(chunk)
         err = 0
     return err
 
 
 @eh.wrapfunction(merge, b'_update')
 def mergeupdate(orig, repo, node, branchmerge, force, *args, **kwargs):
     matcher = kwargs.get('matcher', None)
     # note if this is a partial update
     partial = matcher and not matcher.always()
     with repo.wlock():
         # branch |       |         |
         #  merge | force | partial | action
         # -------+-------+---------+--------------
         #    x   |   x   |    x    | linear-merge
         #    o   |   x   |    x    | branch-merge
         #    x   |   o   |    x    | overwrite (as clean update)
         #    o   |   o   |    x    | force-branch-merge (*1)
         #    x   |   x   |    o    |   (*)
         #    o   |   x   |    o    |   (*)
         #    x   |   o   |    o    | overwrite (as revert)
         #    o   |   o   |    o    |   (*)
         #
         # (*) don't care
         # (*1) deprecated, but used internally (e.g: "rebase --collapse")
 
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
-        unsure, s = lfdirstate.status(
+        unsure, s, mtime_boundary = lfdirstate.status(
             matchmod.always(),
             subrepos=[],
             ignored=False,
             clean=True,
             unknown=False,
         )
         oldclean = set(s.clean)
         pctx = repo[b'.']
         dctx = repo[node]
         for lfile in unsure + s.modified:
             lfileabs = repo.wvfs.join(lfile)
             if not repo.wvfs.exists(lfileabs):
                 continue
             lfhash = lfutil.hashfile(lfileabs)
             standin = lfutil.standin(lfile)
             lfutil.writestandin(
                 repo, standin, lfhash, lfutil.getexecutable(lfileabs)
             )
             if standin in pctx and lfhash == lfutil.readasstandin(
                 pctx[standin]
             ):
                 oldclean.add(lfile)
         for lfile in s.added:
             fstandin = lfutil.standin(lfile)
             if fstandin not in dctx:
                 # in this case, content of standin file is meaningless
                 # (in dctx, lfile is unknown, or normal file)
                 continue
             lfutil.updatestandin(repo, lfile, fstandin)
         # mark all clean largefiles as dirty, just in case the update gets
         # interrupted before largefiles and lfdirstate are synchronized
         for lfile in oldclean:
             lfdirstate.set_possibly_dirty(lfile)
         lfdirstate.write(repo.currenttransaction())
 
         oldstandins = lfutil.getstandinsstate(repo)
         wc = kwargs.get('wc')
         if wc and wc.isinmemory():
             # largefiles is not a good candidate for in-memory merge (large
             # files, custom dirstate, matcher usage).
             raise error.ProgrammingError(
                 b'largefiles is not compatible with in-memory merge'
             )
         with lfdirstate.parentchange():
             result = orig(repo, node, branchmerge, force, *args, **kwargs)
 
             newstandins = lfutil.getstandinsstate(repo)
             filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
 
             # to avoid leaving all largefiles as dirty and thus rehash them, mark
             # all the ones that didn't change as clean
             for lfile in oldclean.difference(filelist):
                 lfdirstate.update_file(lfile, p1_tracked=True, wc_tracked=True)
             lfdirstate.write(repo.currenttransaction())
 
             if branchmerge or force or partial:
                 filelist.extend(s.deleted + s.removed)
 
             lfcommands.updatelfiles(
                 repo.ui, repo, filelist=filelist, normallookup=partial
             )
 
         return result
 
 
 @eh.wrapfunction(scmutil, b'marktouched')
 def scmutilmarktouched(orig, repo, files, *args, **kwargs):
     result = orig(repo, files, *args, **kwargs)
 
     filelist = []
     for f in files:
         lf = lfutil.splitstandin(f)
         if lf is not None:
             filelist.append(lf)
     if filelist:
         lfcommands.updatelfiles(
             repo.ui,
             repo,
             filelist=filelist,
             printmessage=False,
             normallookup=True,
         )
 
     return result
 
 
 @eh.wrapfunction(upgrade_actions, b'preservedrequirements')
 @eh.wrapfunction(upgrade_actions, b'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if b'largefiles' in repo.requirements:
         reqs.add(b'largefiles')
     return reqs
 
 
 _lfscheme = b'largefile://'
 
 
 @eh.wrapfunction(urlmod, b'open')
 def openlargefile(orig, ui, url_, data=None, **kwargs):
     if url_.startswith(_lfscheme):
         if data:
             msg = b"cannot use data on a 'largefile://' url"
             raise error.ProgrammingError(msg)
         lfid = url_[len(_lfscheme) :]
         return storefactory.getlfile(ui, lfid)
     else:
         return orig(ui, url_, data=data, **kwargs)

hgext/largefiles/reposetup.py

@@ -1,456 +1,468 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''setup for largefiles repositories: reposetup'''
 from __future__ import absolute_import
 
 import copy
 
 from mercurial.i18n import _
 
 from mercurial import (
     error,
     extensions,
     localrepo,
     match as matchmod,
     scmutil,
     util,
 )
 
+from mercurial.dirstateutils import timestamp
+
 from . import (
     lfcommands,
     lfutil,
 )
 
 
 def reposetup(ui, repo):
     # wire repositories should be given new wireproto functions
     # by "proto.wirereposetup()" via "hg.wirepeersetupfuncs"
     if not repo.local():
         return
 
     class lfilesrepo(repo.__class__):
         # the mark to examine whether "repo" object enables largefiles or not
         _largefilesenabled = True
 
         lfstatus = False
 
         # When lfstatus is set, return a context that gives the names
         # of largefiles instead of their corresponding standins and
         # identifies the largefiles as always binary, regardless of
         # their actual contents.
         def __getitem__(self, changeid):
             ctx = super(lfilesrepo, self).__getitem__(changeid)
             if self.lfstatus:
 
                 def files(orig):
                     filenames = orig()
                     return [lfutil.splitstandin(f) or f for f in filenames]
 
                 extensions.wrapfunction(ctx, 'files', files)
 
                 def manifest(orig):
                     man1 = orig()
 
                     class lfilesmanifest(man1.__class__):
                         def __contains__(self, filename):
                             orig = super(lfilesmanifest, self).__contains__
                             return orig(filename) or orig(
                                 lfutil.standin(filename)
                             )
 
                     man1.__class__ = lfilesmanifest
                     return man1
 
                 extensions.wrapfunction(ctx, 'manifest', manifest)
 
                 def filectx(orig, path, fileid=None, filelog=None):
                     try:
                         if filelog is not None:
                             result = orig(path, fileid, filelog)
                         else:
                             result = orig(path, fileid)
                     except error.LookupError:
                         # Adding a null character will cause Mercurial to
                         # identify this as a binary file.
                         if filelog is not None:
                             result = orig(lfutil.standin(path), fileid, filelog)
                         else:
                             result = orig(lfutil.standin(path), fileid)
                         olddata = result.data
                         result.data = lambda: olddata() + b'\0'
                     return result
 
                 extensions.wrapfunction(ctx, 'filectx', filectx)
 
             return ctx
 
         # Figure out the status of big files and insert them into the
         # appropriate list in the result. Also removes standin files
         # from the listing. Revert to the original status if
         # self.lfstatus is False.
         # XXX large file status is buggy when used on repo proxy.
         # XXX this needs to be investigated.
         @localrepo.unfilteredmethod
         def status(
             self,
             node1=b'.',
             node2=None,
             match=None,
             ignored=False,
             clean=False,
             unknown=False,
             listsubrepos=False,
         ):
             listignored, listclean, listunknown = ignored, clean, unknown
             orig = super(lfilesrepo, self).status
             if not self.lfstatus:
                 return orig(
                     node1,
                     node2,
                     match,
                     listignored,
                     listclean,
                     listunknown,
                     listsubrepos,
                 )
 
             # some calls in this function rely on the old version of status
             self.lfstatus = False
             ctx1 = self[node1]
             ctx2 = self[node2]
             working = ctx2.rev() is None
             parentworking = working and ctx1 == self[b'.']
 
             if match is None:
                 match = matchmod.always()
 
             try:
                 # updating the dirstate is optional
                 # so we don't wait on the lock
                 wlock = self.wlock(False)
                 gotlock = True
             except error.LockError:
                 wlock = util.nullcontextmanager()
                 gotlock = False
             with wlock:
 
                 # First check if paths or patterns were specified on the
                 # command line.  If there were, and they don't match any
                 # largefiles, we should just bail here and let super
                 # handle it -- thus gaining a big performance boost.
                 lfdirstate = lfutil.openlfdirstate(ui, self)
                 if not match.always():
                     for f in lfdirstate:
                         if match(f):
                             break
                     else:
                         return orig(
                             node1,
                             node2,
                             match,
                             listignored,
                             listclean,
                             listunknown,
                             listsubrepos,
                         )
 
                 # Create a copy of match that matches standins instead
                 # of largefiles.
                 def tostandins(files):
                     if not working:
                         return files
                     newfiles = []
                     dirstate = self.dirstate
                     for f in files:
                         sf = lfutil.standin(f)
                         if sf in dirstate:
                             newfiles.append(sf)
                         elif dirstate.hasdir(sf):
                             # Directory entries could be regular or
                             # standin, check both
                             newfiles.extend((f, sf))
                         else:
                             newfiles.append(f)
                     return newfiles
 
                 m = copy.copy(match)
                 m._files = tostandins(m._files)
 
                 result = orig(
                     node1, node2, m, ignored, clean, unknown, listsubrepos
                 )
                 if working:
 
                     def sfindirstate(f):
                         sf = lfutil.standin(f)
                         dirstate = self.dirstate
                         return sf in dirstate or dirstate.hasdir(sf)
 
                     match._files = [f for f in match._files if sfindirstate(f)]
                     # Don't waste time getting the ignored and unknown
                     # files from lfdirstate
-                    unsure, s = lfdirstate.status(
+                    unsure, s, mtime_boundary = lfdirstate.status(
                         match,
                         subrepos=[],
                         ignored=False,
                         clean=listclean,
                         unknown=False,
                     )
                     (modified, added, removed, deleted, clean) = (
                         s.modified,
                         s.added,
                         s.removed,
                         s.deleted,
                         s.clean,
                     )
                     if parentworking:
+                        wctx = repo[None]
                         for lfile in unsure:
                             standin = lfutil.standin(lfile)
                             if standin not in ctx1:
                                 # from second parent
                                 modified.append(lfile)
                             elif lfutil.readasstandin(
                                 ctx1[standin]
                             ) != lfutil.hashfile(self.wjoin(lfile)):
                                 modified.append(lfile)
                             else:
                                 if listclean:
                                     clean.append(lfile)
-                                lfdirstate.set_clean(lfile)
+                                s = wctx[lfile].lstat()
+                                mode = s.st_mode
+                                size = s.st_size
+                                mtime = timestamp.reliable_mtime_of(
+                                    s, mtime_boundary
+                                )
+                                if mtime is not None:
+                                    cache_data = (mode, size, mtime)
+                                    lfdirstate.set_clean(lfile, cache_data)
                     else:
                         tocheck = unsure + modified + added + clean
                         modified, added, clean = [], [], []
                         checkexec = self.dirstate._checkexec
 
                         for lfile in tocheck:
                             standin = lfutil.standin(lfile)
                             if standin in ctx1:
                                 abslfile = self.wjoin(lfile)
                                 if (
                                     lfutil.readasstandin(ctx1[standin])
                                     != lfutil.hashfile(abslfile)
                                 ) or (
                                     checkexec
                                     and (b'x' in ctx1.flags(standin))
                                     != bool(lfutil.getexecutable(abslfile))
                                 ):
                                     modified.append(lfile)
                                 elif listclean:
                                     clean.append(lfile)
                             else:
                                 added.append(lfile)
 
                         # at this point, 'removed' contains largefiles
                         # marked as 'R' in the working context.
                         # then, largefiles not managed also in the target
                         # context should be excluded from 'removed'.
                         removed = [
                             lfile
                             for lfile in removed
                             if lfutil.standin(lfile) in ctx1
                         ]
 
                     # Standins no longer found in lfdirstate have been deleted
                     for standin in ctx1.walk(lfutil.getstandinmatcher(self)):
                         lfile = lfutil.splitstandin(standin)
                         if not match(lfile):
                             continue
                         if lfile not in lfdirstate:
                             deleted.append(lfile)
                             # Sync "largefile has been removed" back to the
                             # standin. Removing a file as a side effect of
                             # running status is gross, but the alternatives (if
                             # any) are worse.
                             self.wvfs.unlinkpath(standin, ignoremissing=True)
 
                     # Filter result lists
                     result = list(result)
 
                     # Largefiles are not really removed when they're
                     # still in the normal dirstate. Likewise, normal
                     # files are not really removed if they are still in
                     # lfdirstate. This happens in merges where files
                     # change type.
                     removed = [f for f in removed if f not in self.dirstate]
                     result[2] = [f for f in result[2] if f not in lfdirstate]
 
                     lfiles = set(lfdirstate)
                     # Unknown files
                     result[4] = set(result[4]).difference(lfiles)
                     # Ignored files
                     result[5] = set(result[5]).difference(lfiles)
                     # combine normal files and largefiles
                     normals = [
                         [fn for fn in filelist if not lfutil.isstandin(fn)]
                         for filelist in result
                     ]
                     lfstatus = (
                         modified,
                         added,
                         removed,
                         deleted,
                         [],
                         [],
                         clean,
                     )
                     result = [
                         sorted(list1 + list2)
                         for (list1, list2) in zip(normals, lfstatus)
                     ]
                 else:  # not against working directory
                     result = [
                         [lfutil.splitstandin(f) or f for f in items]
                         for items in result
                     ]
 
                 if gotlock:
                     lfdirstate.write(self.currenttransaction())
 
             self.lfstatus = True
             return scmutil.status(*result)
 
         def commitctx(self, ctx, *args, **kwargs):
             node = super(lfilesrepo, self).commitctx(ctx, *args, **kwargs)
 
             class lfilesctx(ctx.__class__):
                 def markcommitted(self, node):
                     orig = super(lfilesctx, self).markcommitted
                     return lfutil.markcommitted(orig, self, node)
 
             ctx.__class__ = lfilesctx
             return node
 
         # Before commit, largefile standins have not had their
         # contents updated to reflect the hash of their largefile.
         # Do that here.
         def commit(
             self,
             text=b"",
             user=None,
             date=None,
             match=None,
             force=False,
             editor=False,
             extra=None,
         ):
             if extra is None:
                 extra = {}
             orig = super(lfilesrepo, self).commit
 
             with self.wlock():
                 lfcommithook = self._lfcommithooks[-1]
                 match = lfcommithook(self, match)
                 result = orig(
                     text=text,
                     user=user,
                     date=date,
                     match=match,
                     force=force,
                     editor=editor,
                     extra=extra,
                 )
                 return result
 
         # TODO: _subdirlfs should be moved into "lfutil.py", because
         # it is referred only from "lfutil.updatestandinsbymatch"
         def _subdirlfs(self, files, lfiles):
             """
             Adjust matched file list
             If we pass a directory to commit whose only committable files
             are largefiles, the core commit code aborts before finding
             the largefiles.
             So we do the following:
             For directories that only have largefiles as matches,
             we explicitly add the largefiles to the match list and remove
             the directory.
             In other cases, we leave the match list unmodified.
             """
             actualfiles = []
             dirs = []
             regulars = []
 
             for f in files:
                 if lfutil.isstandin(f + b'/'):
                     raise error.Abort(
                         _(b'file "%s" is a largefile standin') % f,
                         hint=b'commit the largefile itself instead',
                     )
                 # Scan directories
                 if self.wvfs.isdir(f):
                     dirs.append(f)
                 else:
                     regulars.append(f)
 
             for f in dirs:
                 matcheddir = False
                 d = self.dirstate.normalize(f) + b'/'
                 # Check for matched normal files
                 for mf in regulars:
                     if self.dirstate.normalize(mf).startswith(d):
                         actualfiles.append(f)
                         matcheddir = True
                         break
                 if not matcheddir:
                     # If no normal match, manually append
                     # any matching largefiles
                     for lf in lfiles:
                         if self.dirstate.normalize(lf).startswith(d):
                             actualfiles.append(lf)
                             if not matcheddir:
                                 # There may still be normal files in the dir, so
                                 # add a directory to the list, which
                                 # forces status/dirstate to walk all files and
                                 # call the match function on the matcher, even
                                 # on case sensitive filesystems.
                                 actualfiles.append(b'.')
                                 matcheddir = True
                 # Nothing in dir, so readd it
                 # and let commit reject it
                 if not matcheddir:
                     actualfiles.append(f)
 
             # Always add normal files
             actualfiles += regulars
             return actualfiles
 
     repo.__class__ = lfilesrepo
 
     # stack of hooks being executed before committing.
     # only last element ("_lfcommithooks[-1]") is used for each committing.
     repo._lfcommithooks = [lfutil.updatestandinsbymatch]
 
     # Stack of status writer functions taking "*msg, **opts" arguments
     # like "ui.status()". Only last element ("_lfstatuswriters[-1]")
     # is used to write status out.
     repo._lfstatuswriters = [ui.status]
 
     def prepushoutgoinghook(pushop):
         """Push largefiles for pushop before pushing revisions."""
         lfrevs = pushop.lfrevs
         if lfrevs is None:
             lfrevs = pushop.outgoing.missing
         if lfrevs:
             toupload = set()
             addfunc = lambda fn, lfhash: toupload.add(lfhash)
             lfutil.getlfilestoupload(pushop.repo, lfrevs, addfunc)
             lfcommands.uploadlfiles(ui, pushop.repo, pushop.remote, toupload)
 
     repo.prepushoutgoinghooks.add(b"largefiles", prepushoutgoinghook)
 
     def checkrequireslfiles(ui, repo, **kwargs):
-        if b'largefiles' not in repo.requirements and any(
-            lfutil.shortname + b'/' in f[1] for f in repo.store.datafiles()
-        ):
-            repo.requirements.add(b'largefiles')
-            scmutil.writereporequirements(repo)
+        with repo.lock():
+            if b'largefiles' not in repo.requirements and any(
+                lfutil.shortname + b'/' in f[1] for f in repo.store.datafiles()
+            ):
+                repo.requirements.add(b'largefiles')
+                scmutil.writereporequirements(repo)
 
     ui.setconfig(
         b'hooks', b'changegroup.lfiles', checkrequireslfiles, b'largefiles'
     )
     ui.setconfig(b'hooks', b'commit.lfiles', checkrequireslfiles, b'largefiles')

hgext/lfs/__init__.py

@@ -1,444 +1,447 b''
 # lfs - hash-preserving large file support using Git-LFS protocol
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """lfs - large file support (EXPERIMENTAL)
 
 This extension allows large files to be tracked outside of the normal
 repository storage and stored on a centralized server, similar to the
 ``largefiles`` extension.  The ``git-lfs`` protocol is used when
 communicating with the server, so existing git infrastructure can be
 harnessed.  Even though the files are stored outside of the repository,
 they are still integrity checked in the same manner as normal files.
 
 The files stored outside of the repository are downloaded on demand,
 which reduces the time to clone, and possibly the local disk usage.
 This changes fundamental workflows in a DVCS, so careful thought
 should be given before deploying it.  :hg:`convert` can be used to
 convert LFS repositories to normal repositories that no longer
 require this extension, and do so without changing the commit hashes.
 This allows the extension to be disabled if the centralized workflow
 becomes burdensome.  However, the pre and post convert clones will
 not be able to communicate with each other unless the extension is
 enabled on both.
 
 To start a new repository, or to add LFS files to an existing one, just
 create an ``.hglfs`` file as described below in the root directory of
 the repository.  Typically, this file should be put under version
 control, so that the settings will propagate to other repositories with
 push and pull.  During any commit, Mercurial will consult this file to
 determine if an added or modified file should be stored externally.  The
 type of storage depends on the characteristics of the file at each
 commit.  A file that is near a size threshold may switch back and forth
 between LFS and normal storage, as needed.
 
 Alternately, both normal repositories and largefile controlled
 repositories can be converted to LFS by using :hg:`convert` and the
 ``lfs.track`` config option described below.  The ``.hglfs`` file
 should then be created and added, to control subsequent LFS selection.
 The hashes are also unchanged in this case.  The LFS and non-LFS
 repositories can be distinguished because the LFS repository will
 abort any command if this extension is disabled.
 
 Committed LFS files are held locally, until the repository is pushed.
 Prior to pushing the normal repository data, the LFS files that are
 tracked by the outgoing commits are automatically uploaded to the
 configured central server.  No LFS files are transferred on
 :hg:`pull` or :hg:`clone`.  Instead, the files are downloaded on
 demand as they need to be read, if a cached copy cannot be found
 locally.  Both committing and downloading an LFS file will link the
 file to a usercache, to speed up future access.  See the `usercache`
 config setting described below.
 
 The extension reads its configuration from a versioned ``.hglfs``
 configuration file found in the root of the working directory. The
 ``.hglfs`` file uses the same syntax as all other Mercurial
 configuration files. It uses a single section, ``[track]``.
 
 The ``[track]`` section specifies which files are stored as LFS (or
 not). Each line is keyed by a file pattern, with a predicate value.
 The first file pattern match is used, so put more specific patterns
 first.  The available predicates are ``all()``, ``none()``, and
 ``size()``. See "hg help filesets.size" for the latter.
 
 Example versioned ``.hglfs`` file::
 
   [track]
   # No Makefile or python file, anywhere, will be LFS
   **Makefile = none()
   **.py = none()
 
   **.zip = all()
   **.exe = size(">1MB")
 
   # Catchall for everything not matched above
   ** = size(">10MB")
 
 Configs::
 
     [lfs]
     # Remote endpoint. Multiple protocols are supported:
     # - http(s)://user:[e-mail]/path
     #   git-lfs endpoint
     # - file:///tmp/path
     #   local filesystem, usually for testing
     # if unset, lfs will assume the remote repository also handles blob storage
     # for http(s) URLs.  Otherwise, lfs will prompt to set this when it must
     # use this value.
     # (default: unset)
     url = https://example.com/repo.git/info/lfs
 
     # Which files to track in LFS.  Path tests are "**.extname" for file
     # extensions, and "path:under/some/directory" for path prefix.  Both
     # are relative to the repository root.
     # File size can be tested with the "size()" fileset, and tests can be
     # joined with fileset operators.  (See "hg help filesets.operators".)
     #
     # Some examples:
     # - all()                       # everything
     # - none()                      # nothing
     # - size(">20MB")               # larger than 20MB
     # - !**.txt                     # anything not a *.txt file
     # - **.zip | **.tar.gz | **.7z  # some types of compressed files
     # - path:bin                    # files under "bin" in the project root
     # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz
     #     | (path:bin & !path:/bin/README) | size(">1GB")
     # (default: none())
     #
     # This is ignored if there is a tracked '.hglfs' file, and this setting
     # will eventually be deprecated and removed.
     track = size(">10M")
 
     # how many times to retry before giving up on transferring an object
     retry = 5
 
     # the local directory to store lfs files for sharing across local clones.
     # If not set, the cache is located in an OS specific cache location.
     usercache = /path/to/global/cache
 """
 
 from __future__ import absolute_import
 
 import sys
 
 from mercurial.i18n import _
 from mercurial.node import bin
 
 from mercurial import (
     bundlecaches,
     config,
     context,
     error,
     extensions,
     exthelper,
     filelog,
     filesetlang,
     localrepo,
     logcmdutil,
     minifileset,
     pycompat,
     revlog,
     scmutil,
     templateutil,
     util,
 )
 
 from mercurial.interfaces import repository
 
 from . import (
     blobstore,
     wireprotolfsserver,
     wrapper,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 eh = exthelper.exthelper()
 eh.merge(wrapper.eh)
 eh.merge(wireprotolfsserver.eh)
 
 cmdtable = eh.cmdtable
 configtable = eh.configtable
 extsetup = eh.finalextsetup
 uisetup = eh.finaluisetup
 filesetpredicate = eh.filesetpredicate
 reposetup = eh.finalreposetup
 templatekeyword = eh.templatekeyword
 
 eh.configitem(
     b'experimental',
     b'lfs.serve',
     default=True,
 )
 eh.configitem(
     b'experimental',
     b'lfs.user-agent',
     default=None,
 )
 eh.configitem(
     b'experimental',
     b'lfs.disableusercache',
     default=False,
 )
 eh.configitem(
     b'experimental',
     b'lfs.worker-enable',
     default=True,
 )
 
 eh.configitem(
     b'lfs',
     b'url',
     default=None,
 )
 eh.configitem(
     b'lfs',
     b'usercache',
     default=None,
 )
 # Deprecated
 eh.configitem(
     b'lfs',
     b'threshold',
     default=None,
 )
 eh.configitem(
     b'lfs',
     b'track',
     default=b'none()',
 )
 eh.configitem(
     b'lfs',
     b'retry',
     default=5,
 )
 
 lfsprocessor = (
     wrapper.readfromstore,
     wrapper.writetostore,
     wrapper.bypasscheckhash,
 )
 
 
 def featuresetup(ui, supported):
     # don't die on seeing a repo with the lfs requirement
     supported |= {b'lfs'}
 
 
 @eh.uisetup
 def _uisetup(ui):
     localrepo.featuresetupfuncs.add(featuresetup)
 
 
 @eh.reposetup
 def _reposetup(ui, repo):
     # Nothing to do with a remote repo
     if not repo.local():
         return
 
     repo.svfs.lfslocalblobstore = blobstore.local(repo)
     repo.svfs.lfsremoteblobstore = blobstore.remote(repo)
 
     class lfsrepo(repo.__class__):
         @localrepo.unfilteredmethod
         def commitctx(self, ctx, error=False, origctx=None):
             repo.svfs.options[b'lfstrack'] = _trackedmatcher(self)
             return super(lfsrepo, self).commitctx(ctx, error, origctx=origctx)
 
     repo.__class__ = lfsrepo
 
     if b'lfs' not in repo.requirements:
 
         def checkrequireslfs(ui, repo, **kwargs):
-            if b'lfs' in repo.requirements:
-                return 0
+            with repo.lock():
+                if b'lfs' in repo.requirements:
+                    return 0
 
-            last = kwargs.get('node_last')
-            if last:
-                s = repo.set(b'%n:%n', bin(kwargs['node']), bin(last))
-            else:
-                s = repo.set(b'%n', bin(kwargs['node']))
-            match = repo._storenarrowmatch
-            for ctx in s:
-                # TODO: is there a way to just walk the files in the commit?
-                if any(
-                    ctx[f].islfs() for f in ctx.files() if f in ctx and match(f)
-                ):
-                    repo.requirements.add(b'lfs')
-                    repo.features.add(repository.REPO_FEATURE_LFS)
-                    scmutil.writereporequirements(repo)
-                    repo.prepushoutgoinghooks.add(b'lfs', wrapper.prepush)
-                    break
+                last = kwargs.get('node_last')
+                if last:
+                    s = repo.set(b'%n:%n', bin(kwargs['node']), bin(last))
+                else:
+                    s = repo.set(b'%n', bin(kwargs['node']))
+                match = repo._storenarrowmatch
+                for ctx in s:
+                    # TODO: is there a way to just walk the files in the commit?
+                    if any(
+                        ctx[f].islfs()
+                        for f in ctx.files()
+                        if f in ctx and match(f)
+                    ):
+                        repo.requirements.add(b'lfs')
+                        repo.features.add(repository.REPO_FEATURE_LFS)
+                        scmutil.writereporequirements(repo)
+                        repo.prepushoutgoinghooks.add(b'lfs', wrapper.prepush)
+                        break
 
         ui.setconfig(b'hooks', b'commit.lfs', checkrequireslfs, b'lfs')
         ui.setconfig(
             b'hooks', b'pretxnchangegroup.lfs', checkrequireslfs, b'lfs'
         )
     else:
         repo.prepushoutgoinghooks.add(b'lfs', wrapper.prepush)
 
 
 def _trackedmatcher(repo):
     """Return a function (path, size) -> bool indicating whether or not to
     track a given file with lfs."""
     if not repo.wvfs.exists(b'.hglfs'):
         # No '.hglfs' in wdir.  Fallback to config for now.
         trackspec = repo.ui.config(b'lfs', b'track')
 
         # deprecated config: lfs.threshold
         threshold = repo.ui.configbytes(b'lfs', b'threshold')
         if threshold:
             filesetlang.parse(trackspec)  # make sure syntax errors are confined
             trackspec = b"(%s) | size('>%d')" % (trackspec, threshold)
 
         return minifileset.compile(trackspec)
 
     data = repo.wvfs.tryread(b'.hglfs')
     if not data:
         return lambda p, s: False
 
     # Parse errors here will abort with a message that points to the .hglfs file
     # and line number.
     cfg = config.config()
     cfg.parse(b'.hglfs', data)
 
     try:
         rules = [
             (minifileset.compile(pattern), minifileset.compile(rule))
             for pattern, rule in cfg.items(b'track')
         ]
     except error.ParseError as e:
         # The original exception gives no indicator that the error is in the
         # .hglfs file, so add that.
 
         # TODO: See if the line number of the file can be made available.
         raise error.Abort(_(b'parse error in .hglfs: %s') % e)
 
     def _match(path, size):
         for pat, rule in rules:
             if pat(path, size):
                 return rule(path, size)
 
         return False
 
     return _match
 
 
 # Called by remotefilelog
 def wrapfilelog(filelog):
     wrapfunction = extensions.wrapfunction
 
     wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision)
     wrapfunction(filelog, 'renamed', wrapper.filelogrenamed)
     wrapfunction(filelog, 'size', wrapper.filelogsize)
 
 
 @eh.wrapfunction(localrepo, b'resolverevlogstorevfsoptions')
 def _resolverevlogstorevfsoptions(orig, ui, requirements, features):
     opts = orig(ui, requirements, features)
     for name, module in extensions.extensions(ui):
         if module is sys.modules[__name__]:
             if revlog.REVIDX_EXTSTORED in opts[b'flagprocessors']:
                 msg = (
                     _(b"cannot register multiple processors on flag '%#x'.")
                     % revlog.REVIDX_EXTSTORED
                 )
                 raise error.Abort(msg)
 
             opts[b'flagprocessors'][revlog.REVIDX_EXTSTORED] = lfsprocessor
             break
 
     return opts
 
 
 @eh.extsetup
 def _extsetup(ui):
     wrapfilelog(filelog.filelog)
 
     context.basefilectx.islfs = wrapper.filectxislfs
 
     scmutil.fileprefetchhooks.add(b'lfs', wrapper._prefetchfiles)
 
     # Make bundle choose changegroup3 instead of changegroup2. This affects
     # "hg bundle" command. Note: it does not cover all bundle formats like
     # "packed1". Using "packed1" with lfs will likely cause trouble.
     bundlecaches._bundlespeccontentopts[b"v2"][b"cg.version"] = b"03"
 
 
 @eh.filesetpredicate(b'lfs()')
 def lfsfileset(mctx, x):
     """File that uses LFS storage."""
     # i18n: "lfs" is a keyword
     filesetlang.getargs(x, 0, 0, _(b"lfs takes no arguments"))
     ctx = mctx.ctx
 
     def lfsfilep(f):
         return wrapper.pointerfromctx(ctx, f, removed=True) is not None
 
     return mctx.predicate(lfsfilep, predrepr=b'<lfs>')
 
 
 @eh.templatekeyword(b'lfs_files', requires={b'ctx'})
 def lfsfiles(context, mapping):
     """List of strings. All files modified, added, or removed by this
     changeset."""
     ctx = context.resource(mapping, b'ctx')
 
     pointers = wrapper.pointersfromctx(ctx, removed=True)  # {path: pointer}
     files = sorted(pointers.keys())
 
     def pointer(v):
         # In the file spec, version is first and the other keys are sorted.
         sortkeyfunc = lambda x: (x[0] != b'version', x)
         items = sorted(pycompat.iteritems(pointers[v]), key=sortkeyfunc)
         return util.sortdict(items)
 
     makemap = lambda v: {
         b'file': v,
         b'lfsoid': pointers[v].oid() if pointers[v] else None,
         b'lfspointer': templateutil.hybriddict(pointer(v)),
     }
 
     # TODO: make the separator ', '?
     f = templateutil._showcompatlist(context, mapping, b'lfs_file', files)
     return templateutil.hybrid(f, files, makemap, pycompat.identity)
 
 
 @eh.command(
     b'debuglfsupload',
     [(b'r', b'rev', [], _(b'upload large files introduced by REV'))],
 )
 def debuglfsupload(ui, repo, **opts):
     """upload lfs blobs added by the working copy parent or given revisions"""
     revs = opts.get('rev', [])
     pointers = wrapper.extractpointers(repo, logcmdutil.revrange(repo, revs))
     wrapper.uploadblobs(repo, pointers)
 
 
 @eh.wrapcommand(
     b'verify',
     opts=[(b'', b'no-lfs', None, _(b'skip missing lfs blob content'))],
 )
 def verify(orig, ui, repo, **opts):
     skipflags = repo.ui.configint(b'verify', b'skipflags')
     no_lfs = opts.pop('no_lfs')
 
     if skipflags:
         # --lfs overrides the config bit, if set.
         if no_lfs is False:
             skipflags &= ~repository.REVISION_FLAG_EXTSTORED
     else:
         skipflags = 0
 
     if no_lfs is True:
         skipflags |= repository.REVISION_FLAG_EXTSTORED
 
     with ui.configoverride({(b'verify', b'skipflags'): skipflags}):
         return orig(ui, repo, **opts)

hgext/narrow/narrowdirstate.py

@@ -1,76 +1,76 b''
 # narrowdirstate.py - extensions to mercurial dirstate to support narrow clones
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import error
 
 
 def wrapdirstate(repo, dirstate):
     """Add narrow spec dirstate ignore, block changes outside narrow spec."""
 
     def _editfunc(fn):
         def _wrapper(self, *args, **kwargs):
             narrowmatch = repo.narrowmatch()
             for f in args:
                 if f is not None and not narrowmatch(f) and f not in self:
                     raise error.Abort(
                         _(
                             b"cannot track '%s' - it is outside "
                             + b"the narrow clone"
                         )
                         % f
                     )
             return fn(self, *args, **kwargs)
 
         return _wrapper
 
     class narrowdirstate(dirstate.__class__):
         # Prevent adding/editing/copying/deleting files that are outside the
         # sparse checkout
         @_editfunc
         def normal(self, *args, **kwargs):
             return super(narrowdirstate, self).normal(*args, **kwargs)
 
         @_editfunc
-        def set_tracked(self, *args):
-            return super(narrowdirstate, self).set_tracked(*args)
+        def set_tracked(self, *args, **kwargs):
+            return super(narrowdirstate, self).set_tracked(*args, **kwargs)
 
         @_editfunc
         def set_untracked(self, *args):
             return super(narrowdirstate, self).set_untracked(*args)
 
         @_editfunc
         def add(self, *args):
             return super(narrowdirstate, self).add(*args)
 
         @_editfunc
         def normallookup(self, *args):
             return super(narrowdirstate, self).normallookup(*args)
 
         @_editfunc
         def copy(self, *args):
             return super(narrowdirstate, self).copy(*args)
 
         @_editfunc
         def remove(self, *args):
             return super(narrowdirstate, self).remove(*args)
 
         @_editfunc
         def merge(self, *args):
             return super(narrowdirstate, self).merge(*args)
 
         def rebuild(self, parent, allfiles, changedfiles=None):
             if changedfiles is None:
                 # Rebuilding entire dirstate, let's filter allfiles to match the
                 # narrowspec.
                 allfiles = [f for f in allfiles if repo.narrowmatch()(f)]
             super(narrowdirstate, self).rebuild(parent, allfiles, changedfiles)
 
     dirstate.__class__ = narrowdirstate
     return dirstate

hgext/notify.py

@@ -1,656 +1,659 b''
 # notify.py - email notifications for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''hooks for sending email push notifications
 
 This extension implements hooks to send email notifications when
 changesets are sent from or received by the local repository.
 
 First, enable the extension as explained in :hg:`help extensions`, and
 register the hook you want to run. ``incoming`` and ``changegroup`` hooks
 are run when changesets are received, while ``outgoing`` hooks are for
 changesets sent to another repository::
 
   [hooks]
   # one email for each incoming changeset
   incoming.notify = python:hgext.notify.hook
   # one email for all incoming changesets
   changegroup.notify = python:hgext.notify.hook
 
   # one email for all outgoing changesets
   outgoing.notify = python:hgext.notify.hook
 
 This registers the hooks. To enable notification, subscribers must
 be assigned to repositories. The ``[usersubs]`` section maps multiple
 repositories to a given recipient. The ``[reposubs]`` section maps
 multiple recipients to a single repository::
 
   [usersubs]
   # key is subscriber email, value is a comma-separated list of repo patterns
   user@host = pattern
 
   [reposubs]
   # key is repo pattern, value is a comma-separated list of subscriber emails
   pattern = user@host
 
 A ``pattern`` is a ``glob`` matching the absolute path to a repository,
 optionally combined with a revset expression. A revset expression, if
 present, is separated from the glob by a hash. Example::
 
   [reposubs]
   */widgets#branch(release) = [e-mail]
 
 This sends to ``[e-mail]`` whenever a changeset on the ``release``
 branch triggers a notification in any repository ending in ``widgets``.
 
 In order to place them under direct user management, ``[usersubs]`` and
 ``[reposubs]`` sections may be placed in a separate ``hgrc`` file and
 incorporated by reference::
 
   [notify]
   config = /path/to/subscriptionsfile
 
 Notifications will not be sent until the ``notify.test`` value is set
 to ``False``; see below.
 
 Notifications content can be tweaked with the following configuration entries:
 
 notify.test
   If ``True``, print messages to stdout instead of sending them. Default: True.
 
 notify.sources
   Space-separated list of change sources. Notifications are activated only
   when a changeset's source is in this list. Sources may be:
 
   :``serve``: changesets received via http or ssh
   :``pull``: changesets received via ``hg pull``
   :``unbundle``: changesets received via ``hg unbundle``
   :``push``: changesets sent or received via ``hg push``
   :``bundle``: changesets sent via ``hg unbundle``
 
   Default: serve.
 
 notify.strip
   Number of leading slashes to strip from url paths. By default, notifications
   reference repositories with their absolute path. ``notify.strip`` lets you
   turn them into relative paths. For example, ``notify.strip=3`` will change
   ``/long/path/repository`` into ``repository``. Default: 0.
 
 notify.domain
   Default email domain for sender or recipients with no explicit domain.
   It is also used for the domain part of the ``Message-Id`` when using
   ``notify.messageidseed``.
 
 notify.messageidseed
   Create deterministic ``Message-Id`` headers for the mails based on the seed
   and the revision identifier of the first commit in the changeset.
 
 notify.style
   Style file to use when formatting emails.
 
 notify.template
   Template to use when formatting emails.
 
 notify.incoming
   Template to use when run as an incoming hook, overriding ``notify.template``.
 
 notify.outgoing
   Template to use when run as an outgoing hook, overriding ``notify.template``.
 
 notify.changegroup
   Template to use when running as a changegroup hook, overriding
   ``notify.template``.
 
 notify.maxdiff
   Maximum number of diff lines to include in notification email. Set to 0
   to disable the diff, or -1 to include all of it. Default: 300.
 
 notify.maxdiffstat
   Maximum number of diffstat lines to include in notification email. Set to -1
   to include all of it. Default: -1.
 
 notify.maxsubject
   Maximum number of characters in email's subject line. Default: 67.
 
 notify.diffstat
   Set to True to include a diffstat before diff content. Default: True.
 
 notify.showfunc
   If set, override ``diff.showfunc`` for the diff content. Default: None.
 
 notify.merge
   If True, send notifications for merge changesets. Default: True.
 
 notify.mbox
   If set, append mails to this mbox file instead of sending. Default: None.
 
 notify.fromauthor
   If set, use the committer of the first changeset in a changegroup for
   the "From" field of the notification mail. If not set, take the user
   from the pushing repo.  Default: False.
 
 notify.reply-to-predecessor (EXPERIMENTAL)
   If set and the changeset has a predecessor in the repository, try to thread
   the notification mail with the predecessor. This adds the "In-Reply-To" header
   to the notification mail with a reference to the predecessor with the smallest
   revision number. Mail threads can still be torn, especially when changesets
   are folded.
 
   This option must  be used in combination with ``notify.messageidseed``.
 
 If set, the following entries will also be used to customize the
 notifications:
 
 email.from
   Email ``From`` address to use if none can be found in the generated
   email content.
 
 web.baseurl
   Root repository URL to combine with repository paths when making
   references. See also ``notify.strip``.
 
 '''
 from __future__ import absolute_import
 
 import email.errors as emailerrors
 import email.utils as emailutils
 import fnmatch
 import hashlib
 import socket
 import time
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     logcmdutil,
     mail,
     obsutil,
     patch,
     pycompat,
     registrar,
     util,
 )
 from mercurial.utils import (
     dateutil,
     stringutil,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'notify',
     b'changegroup',
     default=None,
 )
 configitem(
     b'notify',
     b'config',
     default=None,
 )
 configitem(
     b'notify',
     b'diffstat',
     default=True,
 )
 configitem(
     b'notify',
     b'domain',
     default=None,
 )
 configitem(
     b'notify',
     b'messageidseed',
     default=None,
 )
 configitem(
     b'notify',
     b'fromauthor',
     default=None,
 )
 configitem(
     b'notify',
     b'incoming',
     default=None,
 )
 configitem(
     b'notify',
     b'maxdiff',
     default=300,
 )
 configitem(
     b'notify',
     b'maxdiffstat',
     default=-1,
 )
 configitem(
     b'notify',
     b'maxsubject',
     default=67,
 )
 configitem(
     b'notify',
     b'mbox',
     default=None,
 )
 configitem(
     b'notify',
     b'merge',
     default=True,
 )
 configitem(
     b'notify',
     b'outgoing',
     default=None,
 )
 configitem(
     b'notify',
     b'reply-to-predecessor',
     default=False,
 )
 configitem(
     b'notify',
     b'sources',
     default=b'serve',
 )
 configitem(
     b'notify',
     b'showfunc',
     default=None,
 )
 configitem(
     b'notify',
     b'strip',
     default=0,
 )
 configitem(
     b'notify',
     b'style',
     default=None,
 )
 configitem(
     b'notify',
     b'template',
     default=None,
 )
 configitem(
     b'notify',
     b'test',
     default=True,
 )
 
 # template for single changeset can include email headers.
 single_template = b'''
 Subject: changeset in {webroot}: {desc|firstline|strip}
 From: {author}
 
 changeset {node|short} in {root}
 details: {baseurl}{webroot}?cmd=changeset;node={node|short}
 description:
 \t{desc|tabindent|strip}
 '''.lstrip()
 
 # template for multiple changesets should not contain email headers,
 # because only first set of headers will be used and result will look
 # strange.
 multiple_template = b'''
 changeset {node|short} in {root}
 details: {baseurl}{webroot}?cmd=changeset;node={node|short}
 summary: {desc|firstline}
 '''
 
 deftemplates = {
     b'changegroup': multiple_template,
 }
 
 
 class notifier(object):
     '''email notification class.'''
 
     def __init__(self, ui, repo, hooktype):
         self.ui = ui
         cfg = self.ui.config(b'notify', b'config')
         if cfg:
             self.ui.readconfig(cfg, sections=[b'usersubs', b'reposubs'])
         self.repo = repo
         self.stripcount = int(self.ui.config(b'notify', b'strip'))
         self.root = self.strip(self.repo.root)
         self.domain = self.ui.config(b'notify', b'domain')
         self.mbox = self.ui.config(b'notify', b'mbox')
         self.test = self.ui.configbool(b'notify', b'test')
         self.charsets = mail._charsets(self.ui)
         self.subs = self.subscribers()
         self.merge = self.ui.configbool(b'notify', b'merge')
         self.showfunc = self.ui.configbool(b'notify', b'showfunc')
         self.messageidseed = self.ui.config(b'notify', b'messageidseed')
         self.reply = self.ui.configbool(b'notify', b'reply-to-predecessor')
 
         if self.reply and not self.messageidseed:
             raise error.Abort(
                 _(
                     b'notify.reply-to-predecessor used without '
                     b'notify.messageidseed'
                 )
             )
 
         if self.showfunc is None:
             self.showfunc = self.ui.configbool(b'diff', b'showfunc')
 
         mapfile = None
         template = self.ui.config(b'notify', hooktype) or self.ui.config(
             b'notify', b'template'
         )
         if not template:
             mapfile = self.ui.config(b'notify', b'style')
         if not mapfile and not template:
             template = deftemplates.get(hooktype) or single_template
         spec = logcmdutil.templatespec(template, mapfile)
         self.t = logcmdutil.changesettemplater(self.ui, self.repo, spec)
 
     def strip(self, path):
         '''strip leading slashes from local path, turn into web-safe path.'''
 
         path = util.pconvert(path)
         count = self.stripcount
         while count > 0:
             c = path.find(b'/')
             if c == -1:
                 break
             path = path[c + 1 :]
             count -= 1
         return path
 
     def fixmail(self, addr):
         '''try to clean up email addresses.'''
 
         addr = stringutil.email(addr.strip())
         if self.domain:
             a = addr.find(b'@localhost')
             if a != -1:
                 addr = addr[:a]
             if b'@' not in addr:
                 return addr + b'@' + self.domain
         return addr
 
     def subscribers(self):
         '''return list of email addresses of subscribers to this repo.'''
         subs = set()
         for user, pats in self.ui.configitems(b'usersubs'):
             for pat in pats.split(b','):
                 if b'#' in pat:
                     pat, revs = pat.split(b'#', 1)
                 else:
                     revs = None
                 if fnmatch.fnmatch(self.repo.root, pat.strip()):
                     subs.add((self.fixmail(user), revs))
         for pat, users in self.ui.configitems(b'reposubs'):
             if b'#' in pat:
                 pat, revs = pat.split(b'#', 1)
             else:
                 revs = None
             if fnmatch.fnmatch(self.repo.root, pat):
                 for user in users.split(b','):
                     subs.add((self.fixmail(user), revs))
         return [
             (mail.addressencode(self.ui, s, self.charsets, self.test), r)
             for s, r in sorted(subs)
         ]
 
     def node(self, ctx, **props):
         '''format one changeset, unless it is a suppressed merge.'''
         if not self.merge and len(ctx.parents()) > 1:
             return False
         self.t.show(
             ctx,
             changes=ctx.changeset(),
             baseurl=self.ui.config(b'web', b'baseurl'),
             root=self.repo.root,
             webroot=self.root,
             **props
         )
         return True
 
     def skipsource(self, source):
         '''true if incoming changes from this source should be skipped.'''
         ok_sources = self.ui.config(b'notify', b'sources').split()
         return source not in ok_sources
 
     def send(self, ctx, count, data):
         '''send message.'''
 
         # Select subscribers by revset
         subs = set()
         for sub, spec in self.subs:
             if spec is None:
                 subs.add(sub)
                 continue
-            revs = self.repo.revs(b'%r and %d:', spec, ctx.rev())
+            try:
+                revs = self.repo.revs(b'%r and %d:', spec, ctx.rev())
+            except error.RepoLookupError:
+                continue
             if len(revs):
                 subs.add(sub)
                 continue
         if len(subs) == 0:
             self.ui.debug(
                 b'notify: no subscribers to selected repo and revset\n'
             )
             return
 
         try:
             msg = mail.parsebytes(data)
         except emailerrors.MessageParseError as inst:
             raise error.Abort(inst)
 
         # store sender and subject
         sender = msg['From']
         subject = msg['Subject']
         if sender is not None:
             sender = mail.headdecode(sender)
         if subject is not None:
             subject = mail.headdecode(subject)
         del msg['From'], msg['Subject']
 
         if not msg.is_multipart():
             # create fresh mime message from scratch
             # (multipart templates must take care of this themselves)
             headers = msg.items()
             payload = msg.get_payload(decode=pycompat.ispy3)
             # for notification prefer readability over data precision
             msg = mail.mimeencode(self.ui, payload, self.charsets, self.test)
             # reinstate custom headers
             for k, v in headers:
                 msg[k] = v
 
         msg['Date'] = encoding.strfromlocal(
             dateutil.datestr(format=b"%a, %d %b %Y %H:%M:%S %1%2")
         )
 
         # try to make subject line exist and be useful
         if not subject:
             if count > 1:
                 subject = _(b'%s: %d new changesets') % (self.root, count)
             else:
                 s = ctx.description().lstrip().split(b'\n', 1)[0].rstrip()
                 subject = b'%s: %s' % (self.root, s)
         maxsubject = int(self.ui.config(b'notify', b'maxsubject'))
         if maxsubject:
             subject = stringutil.ellipsis(subject, maxsubject)
         msg['Subject'] = mail.headencode(
             self.ui, subject, self.charsets, self.test
         )
 
         # try to make message have proper sender
         if not sender:
             sender = self.ui.config(b'email', b'from') or self.ui.username()
         if b'@' not in sender or b'@localhost' in sender:
             sender = self.fixmail(sender)
         msg['From'] = mail.addressencode(
             self.ui, sender, self.charsets, self.test
         )
 
         msg['X-Hg-Notification'] = 'changeset %s' % ctx
         if not msg['Message-Id']:
             msg['Message-Id'] = messageid(ctx, self.domain, self.messageidseed)
         if self.reply:
             unfi = self.repo.unfiltered()
             has_node = unfi.changelog.index.has_node
             predecessors = [
                 unfi[ctx2]
                 for ctx2 in obsutil.allpredecessors(unfi.obsstore, [ctx.node()])
                 if ctx2 != ctx.node() and has_node(ctx2)
             ]
             if predecessors:
                 # There is at least one predecessor, so which to pick?
                 # Ideally, there is a unique root because changesets have
                 # been evolved/rebased one step at a time. In this case,
                 # just picking the oldest known changeset provides a stable
                 # base. It doesn't help when changesets are folded. Any
                 # better solution would require storing more information
                 # in the repository.
                 pred = min(predecessors, key=lambda ctx: ctx.rev())
                 msg['In-Reply-To'] = messageid(
                     pred, self.domain, self.messageidseed
                 )
         msg['To'] = ', '.join(sorted(subs))
 
         msgtext = msg.as_bytes() if pycompat.ispy3 else msg.as_string()
         if self.test:
             self.ui.write(msgtext)
             if not msgtext.endswith(b'\n'):
                 self.ui.write(b'\n')
         else:
             self.ui.status(
                 _(b'notify: sending %d subscribers %d changes\n')
                 % (len(subs), count)
             )
             mail.sendmail(
                 self.ui,
                 emailutils.parseaddr(msg['From'])[1],
                 subs,
                 msgtext,
                 mbox=self.mbox,
             )
 
     def diff(self, ctx, ref=None):
 
         maxdiff = int(self.ui.config(b'notify', b'maxdiff'))
         prev = ctx.p1().node()
         if ref:
             ref = ref.node()
         else:
             ref = ctx.node()
         diffopts = patch.diffallopts(self.ui)
         diffopts.showfunc = self.showfunc
         chunks = patch.diff(self.repo, prev, ref, opts=diffopts)
         difflines = b''.join(chunks).splitlines()
 
         if self.ui.configbool(b'notify', b'diffstat'):
             maxdiffstat = int(self.ui.config(b'notify', b'maxdiffstat'))
             s = patch.diffstat(difflines)
             # s may be nil, don't include the header if it is
             if s:
                 if maxdiffstat >= 0 and s.count(b"\n") > maxdiffstat + 1:
                     s = s.split(b"\n")
                     msg = _(b'\ndiffstat (truncated from %d to %d lines):\n\n')
                     self.ui.write(msg % (len(s) - 2, maxdiffstat))
                     self.ui.write(b"\n".join(s[:maxdiffstat] + s[-2:]))
                 else:
                     self.ui.write(_(b'\ndiffstat:\n\n%s') % s)
 
         if maxdiff == 0:
             return
         elif maxdiff > 0 and len(difflines) > maxdiff:
             msg = _(b'\ndiffs (truncated from %d to %d lines):\n\n')
             self.ui.write(msg % (len(difflines), maxdiff))
             difflines = difflines[:maxdiff]
         elif difflines:
             self.ui.write(_(b'\ndiffs (%d lines):\n\n') % len(difflines))
 
         self.ui.write(b"\n".join(difflines))
 
 
 def hook(ui, repo, hooktype, node=None, source=None, **kwargs):
     """send email notifications to interested subscribers.
 
     if used as changegroup hook, send one email for all changesets in
     changegroup. else send one email per changeset."""
 
     n = notifier(ui, repo, hooktype)
     ctx = repo.unfiltered()[node]
 
     if not n.subs:
         ui.debug(b'notify: no subscribers to repository %s\n' % n.root)
         return
     if n.skipsource(source):
         ui.debug(b'notify: changes have source "%s" - skipping\n' % source)
         return
 
     ui.pushbuffer()
     data = b''
     count = 0
     author = b''
     if hooktype == b'changegroup' or hooktype == b'outgoing':
         for rev in repo.changelog.revs(start=ctx.rev()):
             if n.node(repo[rev]):
                 count += 1
                 if not author:
                     author = repo[rev].user()
             else:
                 data += ui.popbuffer()
                 ui.note(
                     _(b'notify: suppressing notification for merge %d:%s\n')
                     % (rev, repo[rev].hex()[:12])
                 )
                 ui.pushbuffer()
         if count:
             n.diff(ctx, repo[b'tip'])
     elif ctx.rev() in repo:
         if not n.node(ctx):
             ui.popbuffer()
             ui.note(
                 _(b'notify: suppressing notification for merge %d:%s\n')
                 % (ctx.rev(), ctx.hex()[:12])
             )
             return
         count += 1
         n.diff(ctx)
         if not author:
             author = ctx.user()
 
     data += ui.popbuffer()
     fromauthor = ui.config(b'notify', b'fromauthor')
     if author and fromauthor:
         data = b'\n'.join([b'From: %s' % author, data])
 
     if count:
         n.send(ctx, count, data)
 
 
 def messageid(ctx, domain, messageidseed):
     if domain and messageidseed:
         host = domain
     else:
         host = encoding.strtolocal(socket.getfqdn())
     if messageidseed:
         messagehash = hashlib.sha512(ctx.hex() + messageidseed)
         messageid = b'<hg.%s@%s>' % (
             pycompat.sysbytes(messagehash.hexdigest()[:64]),
             host,
         )
     else:
         messageid = b'<hg.%s.%d.%d@%s>' % (
             ctx,
             int(time.time()),
             hash(ctx.repo().root),
             host,
         )
     return encoding.strfromlocal(messageid)