packaging: consolidate CLI functionality into packaging.py

Consolidating functionality for invoking code in the hgpackaging
package through a single CLI entry point will make things simpler
when we add more complexity to that package. For example, it will
allow us to run things out of a virtualenv with third party
packages.

This commit consolidates functionality from the Inno and WiX
build.py scripts into a new packaging.py script. That script
simply creates a virtualenv and runs the CLI functionality in
it.

The new virtualenv is populated with jinja2 because I felt
it easier to incorporate requirements file processing in this
commit and we will soon use jinja2 in an upcoming commit.

The unified CLI functionality will also make it easier to
script other packaging workflows going forward. e.g. RPM, Debian,
and macOS packaging.

Differential Revision: https://phab.mercurial-scm.org/D7156

contrib/packaging/hgpackaging/cli.py

@@ -0,0 +1,153 b''
+# cli.py - Command line interface for automation
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import argparse
+import os
+import pathlib
+
+from . import (
+    inno,
+    wix,
+)
+
+HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
+SOURCE_DIR = HERE.parent.parent.parent
+
+
+def build_inno(python=None, iscc=None, version=None):
+    if not os.path.isabs(python):
+        raise Exception("--python arg must be an absolute path")
+
+    if iscc:
+        iscc = pathlib.Path(iscc)
+    else:
+        iscc = (
+            pathlib.Path(os.environ["ProgramFiles(x86)"])
+            / "Inno Setup 5"
+            / "ISCC.exe"
+        )
+
+    build_dir = SOURCE_DIR / "build"
+
+    inno.build(
+        SOURCE_DIR, build_dir, pathlib.Path(python), iscc, version=version,
+    )
+
+
+def build_wix(
+    name=None,
+    python=None,
+    version=None,
+    sign_sn=None,
+    sign_cert=None,
+    sign_password=None,
+    sign_timestamp_url=None,
+    extra_packages_script=None,
+    extra_wxs=None,
+    extra_features=None,
+):
+    fn = wix.build_installer
+    kwargs = {
+        "source_dir": SOURCE_DIR,
+        "python_exe": pathlib.Path(python),
+        "version": version,
+    }
+
+    if not os.path.isabs(python):
+        raise Exception("--python arg must be an absolute path")
+
+    if extra_packages_script:
+        kwargs["extra_packages_script"] = extra_packages_script
+    if extra_wxs:
+        kwargs["extra_wxs"] = dict(
+            thing.split("=") for thing in extra_wxs.split(",")
+        )
+    if extra_features:
+        kwargs["extra_features"] = extra_features.split(",")
+
+    if sign_sn or sign_cert:
+        fn = wix.build_signed_installer
+        kwargs["name"] = name
+        kwargs["subject_name"] = sign_sn
+        kwargs["cert_path"] = sign_cert
+        kwargs["cert_password"] = sign_password
+        kwargs["timestamp_url"] = sign_timestamp_url
+
+    fn(**kwargs)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    sp = subparsers.add_parser("inno", help="Build Inno Setup installer")
+    sp.add_argument("--python", required=True, help="path to python.exe to use")
+    sp.add_argument("--iscc", help="path to iscc.exe to use")
+    sp.add_argument(
+        "--version",
+        help="Mercurial version string to use "
+        "(detected from __version__.py if not defined",
+    )
+    sp.set_defaults(func=build_inno)
+
+    sp = subparsers.add_parser(
+        "wix", help="Build Windows installer with WiX Toolset"
+    )
+    sp.add_argument("--name", help="Application name", default="Mercurial")
+    sp.add_argument(
+        "--python", help="Path to Python executable to use", required=True
+    )
+    sp.add_argument(
+        "--sign-sn",
+        help="Subject name (or fragment thereof) of certificate "
+        "to use for signing",
+    )
+    sp.add_argument(
+        "--sign-cert", help="Path to certificate to use for signing"
+    )
+    sp.add_argument("--sign-password", help="Password for signing certificate")
+    sp.add_argument(
+        "--sign-timestamp-url",
+        help="URL of timestamp server to use for signing",
+    )
+    sp.add_argument("--version", help="Version string to use")
+    sp.add_argument(
+        "--extra-packages-script",
+        help=(
+            "Script to execute to include extra packages in " "py2exe binary."
+        ),
+    )
+    sp.add_argument(
+        "--extra-wxs", help="CSV of path_to_wxs_file=working_dir_for_wxs_file"
+    )
+    sp.add_argument(
+        "--extra-features",
+        help=(
+            "CSV of extra feature names to include "
+            "in the installer from the extra wxs files"
+        ),
+    )
+    sp.set_defaults(func=build_wix)
+
+    return parser
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return
+
+    kwargs = dict(vars(args))
+    del kwargs["func"]
+
+    args.func(**kwargs)

contrib/packaging/packaging.py

@@ -0,0 +1,74 b''
+#!/usr/bin/env python3
+#
+# packaging.py - Mercurial packaging functionality
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+import os
+import pathlib
+import subprocess
+import sys
+import venv
+
+
+HERE = pathlib.Path(os.path.abspath(__file__)).parent
+REQUIREMENTS_TXT = HERE / "requirements.txt"
+SOURCE_DIR = HERE.parent.parent
+VENV = SOURCE_DIR / "build" / "venv-packaging"
+
+
+def bootstrap():
+    venv_created = not VENV.exists()
+
+    VENV.parent.mkdir(exist_ok=True)
+
+    venv.create(VENV, with_pip=True)
+
+    if os.name == "nt":
+        venv_bin = VENV / "Scripts"
+        pip = venv_bin / "pip.exe"
+        python = venv_bin / "python.exe"
+    else:
+        venv_bin = VENV / "bin"
+        pip = venv_bin / "pip"
+        python = venv_bin / "python"
+
+    args = [
+        str(pip),
+        "install",
+        "-r",
+        str(REQUIREMENTS_TXT),
+        "--disable-pip-version-check",
+    ]
+
+    if not venv_created:
+        args.append("-q")
+
+    subprocess.run(args, check=True)
+
+    os.environ["HGPACKAGING_BOOTSTRAPPED"] = "1"
+    os.environ["PATH"] = "%s%s%s" % (venv_bin, os.pathsep, os.environ["PATH"])
+
+    subprocess.run([str(python), __file__] + sys.argv[1:], check=True)
+
+
+def run():
+    import hgpackaging.cli as cli
+
+    # Need to strip off main Python executable.
+    cli.main()
+
+
+if __name__ == "__main__":
+    try:
+        if "HGPACKAGING_BOOTSTRAPPED" not in os.environ:
+            bootstrap()
+        else:
+            run()
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+    except KeyboardInterrupt:
+        sys.exit(1)

contrib/packaging/requirements.txt

@@ -0,0 +1,39 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=contrib/packaging/requirements.txt contrib/packaging/requirements.txt.in
+#
+jinja2==2.10.3 \
+    --hash=sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f \
+    --hash=sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de
+markupsafe==1.1.1 \
+    --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+    --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+    --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+    --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+    --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+    --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+    --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+    --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+    --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+    --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+    --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+    --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+    --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+    --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+    --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+    --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+    --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+    --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+    --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+    --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+    --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+    --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+    --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+    --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+    --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+    --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+    --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+    --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
+    # via jinja2

contrib/packaging/requirements.txt.in

@@ -0,0 +1,1 b''
+jinja2

contrib/automation/hgautomation/windows.py

@@ -1,510 +1,510 b''
 # windows.py - Automation specific to Windows
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import datetime
 import os
 import paramiko
 import pathlib
 import re
 import subprocess
 import tempfile
 
 from .pypi import upload as pypi_upload
 from .winrm import run_powershell
 
 
 # PowerShell commands to activate a Visual Studio 2008 environment.
 # This is essentially a port of vcvarsall.bat to PowerShell.
 ACTIVATE_VC9_AMD64 = r'''
 Write-Output "activating Visual Studio 2008 environment for AMD64"
 $root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
 $Env:VCINSTALLDIR = "${root}\VC\"
 $Env:WindowsSdkDir = "${root}\WinSDK\"
 $Env:PATH = "${root}\VC\Bin\amd64;${root}\WinSDK\Bin\x64;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:PATH"
 $Env:LIB = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIB"
 $Env:LIBPATH = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIBPATH"
 '''.lstrip()
 
 ACTIVATE_VC9_X86 = r'''
 Write-Output "activating Visual Studio 2008 environment for x86"
 $root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
 $Env:VCINSTALLDIR = "${root}\VC\"
 $Env:WindowsSdkDir = "${root}\WinSDK\"
 $Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE"
 $Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB"
 $Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib;$Env:LIBPATH"
 '''.lstrip()
 
 HG_PURGE = r'''
 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
 Set-Location C:\hgdev\src
 hg.exe --config extensions.purge= purge --all
 if ($LASTEXITCODE -ne 0) {
     throw "process exited non-0: $LASTEXITCODE"
 }
 Write-Output "purged Mercurial repo"
 '''
 
 HG_UPDATE_CLEAN = r'''
 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
 Set-Location C:\hgdev\src
 hg.exe --config extensions.purge= purge --all
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 hg.exe update -C {revision}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 hg.exe log -r .
 Write-Output "updated Mercurial working directory to {revision}"
 '''.lstrip()
 
 BUILD_INNO = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
-C:\hgdev\python37-x64\python.exe contrib\packaging\inno\build.py --python $python
+C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''.lstrip()
 
 BUILD_WHEEL = r'''
 Set-Location C:\hgdev\src
 C:\hgdev\python27-{arch}\Scripts\pip.exe wheel --wheel-dir dist .
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 BUILD_WIX = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
-C:\hgdev\python37-x64\python.exe contrib\packaging\wix\build.py --python $python {extra_args}
+C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 RUN_TESTS = r'''
 C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
 X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
 X86_EXE_FILENAME = 'Mercurial-{version}.exe'
 X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
 X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
 X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
 
 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
 
 X86_USER_AGENT_PATTERN = '.*Windows.*'
 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
 
 X86_EXE_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x86 Windows '
     '- does not require admin rights'
 )
 X64_EXE_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x64 Windows '
     '- does not require admin rights'
 )
 X86_MSI_DESCRIPTION = (
     'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights'
 )
 X64_MSI_DESCRIPTION = (
     'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights'
 )
 
 
 def get_vc_prefix(arch):
     if arch == 'x86':
         return ACTIVATE_VC9_X86
     elif arch == 'x64':
         return ACTIVATE_VC9_AMD64
     else:
         raise ValueError('illegal arch: %s; must be x86 or x64' % arch)
 
 
 def fix_authorized_keys_permissions(winrm_client, path):
     commands = [
         '$ErrorActionPreference = "Stop"',
         'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
         r'icacls %s /remove:g "NT Service\sshd"' % path,
     ]
 
     run_powershell(winrm_client, '\n'.join(commands))
 
 
 def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
     """Synchronize local Mercurial repo to remote EC2 instance."""
 
     winrm_client = ec2_instance.winrm_client
 
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_dir = pathlib.Path(temp_dir)
 
         ssh_dir = temp_dir / '.ssh'
         ssh_dir.mkdir()
         ssh_dir.chmod(0o0700)
 
         # Generate SSH key to use for communication.
         subprocess.run(
             [
                 'ssh-keygen',
                 '-t',
                 'rsa',
                 '-b',
                 '4096',
                 '-N',
                 '',
                 '-f',
                 str(ssh_dir / 'id_rsa'),
             ],
             check=True,
             capture_output=True,
         )
 
         # Add it to ~/.ssh/authorized_keys on remote.
         # This assumes the file doesn't already exist.
         authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
         winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
         winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
         fix_authorized_keys_permissions(winrm_client, authorized_keys)
 
         public_ip = ec2_instance.public_ip_address
 
         ssh_config = temp_dir / '.ssh' / 'config'
 
         with open(ssh_config, 'w', encoding='utf-8') as fh:
             fh.write('Host %s\n' % public_ip)
             fh.write('  User Administrator\n')
             fh.write('  StrictHostKeyChecking no\n')
             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
             fh.write('  IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
 
         if not (hg_repo / '.hg').is_dir():
             raise Exception(
                 '%s is not a Mercurial repository; '
                 'synchronization not yet supported' % hg_repo
             )
 
         env = dict(os.environ)
         env['HGPLAIN'] = '1'
         env['HGENCODING'] = 'utf-8'
 
         hg_bin = hg_repo / 'hg'
 
         res = subprocess.run(
             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
             cwd=str(hg_repo),
             env=env,
             check=True,
             capture_output=True,
         )
 
         full_revision = res.stdout.decode('ascii')
 
         args = [
             'python2.7',
             hg_bin,
             '--config',
             'ui.ssh=ssh -F %s' % ssh_config,
             '--config',
             'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
             # Also ensure .hgtags changes are present so auto version
             # calculation works.
             'push',
             '-f',
             '-r',
             full_revision,
             '-r',
             'file(.hgtags)',
             'ssh://%s/c:/hgdev/src' % public_ip,
         ]
 
         res = subprocess.run(args, cwd=str(hg_repo), env=env)
 
         # Allow 1 (no-op) to not trigger error.
         if res.returncode not in (0, 1):
             res.check_returncode()
 
         run_powershell(
             winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
         )
 
         # TODO detect dirty local working directory and synchronize accordingly.
 
 
 def purge_hg(winrm_client):
     """Purge the Mercurial source repository on an EC2 instance."""
     run_powershell(winrm_client, HG_PURGE)
 
 
 def find_latest_dist(winrm_client, pattern):
     """Find path to newest file in dist/ directory matching a pattern."""
 
     res = winrm_client.execute_ps(
         r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
         '| Sort-Object LastWriteTime -Descending '
         '| Select-Object -First 1\n'
         '$v.name' % pattern
     )
     return res[0]
 
 
 def copy_latest_dist(winrm_client, pattern, dest_path):
     """Copy latest file matching pattern in dist/ directory.
 
     Given a WinRM client and a file pattern, find the latest file on the remote
     matching that pattern and copy it to the ``dest_path`` directory on the
     local machine.
     """
     latest = find_latest_dist(winrm_client, pattern)
     source = r'C:\hgdev\src\dist\%s' % latest
     dest = dest_path / latest
     print('copying %s to %s' % (source, dest))
     winrm_client.fetch(source, str(dest))
 
 
 def build_inno_installer(
     winrm_client, arch: str, dest_path: pathlib.Path, version=None
 ):
     """Build the Inno Setup installer on a remote machine.
 
     Using a WinRM client, remote commands are executed to build
     a Mercurial Inno Setup installer.
     """
     print('building Inno Setup installer for %s' % arch)
 
     extra_args = []
     if version:
         extra_args.extend(['--version', version])
 
     ps = get_vc_prefix(arch) + BUILD_INNO.format(
         arch=arch, extra_args=' '.join(extra_args)
     )
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.exe', dest_path)
 
 
 def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path):
     """Build Python wheels on a remote machine.
 
     Using a WinRM client, remote commands are executed to build a Python wheel
     for Mercurial.
     """
     print('Building Windows wheel for %s' % arch)
     ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch)
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.whl', dest_path)
 
 
 def build_wix_installer(
     winrm_client, arch: str, dest_path: pathlib.Path, version=None
 ):
     """Build the WiX installer on a remote machine.
 
     Using a WinRM client, remote commands are executed to build a WiX installer.
     """
     print('Building WiX installer for %s' % arch)
     extra_args = []
     if version:
         extra_args.extend(['--version', version])
 
     ps = get_vc_prefix(arch) + BUILD_WIX.format(
         arch=arch, extra_args=' '.join(extra_args)
     )
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.msi', dest_path)
 
 
 def run_tests(winrm_client, python_version, arch, test_flags=''):
     """Run tests on a remote Windows machine.
 
     ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
     ``arch`` is ``x86`` or ``x64``.
     ``test_flags`` is a str representing extra arguments to pass to
     ``run-tests.py``.
     """
     if not re.match(r'\d\.\d', python_version):
         raise ValueError(
             r'python_version must be \d.\d; got %s' % python_version
         )
 
     if arch not in ('x86', 'x64'):
         raise ValueError('arch must be x86 or x64; got %s' % arch)
 
     python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
 
     ps = RUN_TESTS.format(python_path=python_path, test_flags=test_flags or '',)
 
     run_powershell(winrm_client, ps)
 
 
 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
     return (
         dist_path / X86_WHEEL_FILENAME.format(version=version),
         dist_path / X64_WHEEL_FILENAME.format(version=version),
     )
 
 
 def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
     return (
         dist_path / X86_WHEEL_FILENAME.format(version=version),
         dist_path / X64_WHEEL_FILENAME.format(version=version),
         dist_path / X86_EXE_FILENAME.format(version=version),
         dist_path / X64_EXE_FILENAME.format(version=version),
         dist_path / X86_MSI_FILENAME.format(version=version),
         dist_path / X64_MSI_FILENAME.format(version=version),
     )
 
 
 def generate_latest_dat(version: str):
     x86_exe_filename = X86_EXE_FILENAME.format(version=version)
     x64_exe_filename = X64_EXE_FILENAME.format(version=version)
     x86_msi_filename = X86_MSI_FILENAME.format(version=version)
     x64_msi_filename = X64_MSI_FILENAME.format(version=version)
 
     entries = (
         (
             '10',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
             X86_EXE_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
             X64_EXE_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
             X86_MSI_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
             X64_MSI_DESCRIPTION.format(version=version),
         ),
     )
 
     lines = ['\t'.join(e) for e in entries]
 
     return '\n'.join(lines) + '\n'
 
 
 def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
     """Publish Windows release artifacts to PyPI."""
 
     wheel_paths = resolve_wheel_artifacts(dist_path, version)
 
     for p in wheel_paths:
         if not p.exists():
             raise Exception('%s not found' % p)
 
     print('uploading wheels to PyPI (you may be prompted for credentials)')
     pypi_upload(wheel_paths)
 
 
 def publish_artifacts_mercurial_scm_org(
     dist_path: pathlib.Path, version: str, ssh_username=None
 ):
     """Publish Windows release artifacts to mercurial-scm.org."""
     all_paths = resolve_all_artifacts(dist_path, version)
 
     for p in all_paths:
         if not p.exists():
             raise Exception('%s not found' % p)
 
     client = paramiko.SSHClient()
     client.load_system_host_keys()
     # We assume the system SSH configuration knows how to connect.
     print('connecting to mercurial-scm.org via ssh...')
     try:
         client.connect('mercurial-scm.org', username=ssh_username)
     except paramiko.AuthenticationException:
         print('error authenticating; is an SSH key available in an SSH agent?')
         raise
 
     print('SSH connection established')
 
     print('opening SFTP client...')
     sftp = client.open_sftp()
     print('SFTP client obtained')
 
     for p in all_paths:
         dest_path = '/var/www/release/windows/%s' % p.name
         print('uploading %s to %s' % (p, dest_path))
 
         with p.open('rb') as fh:
             data = fh.read()
 
         with sftp.open(dest_path, 'wb') as fh:
             fh.write(data)
             fh.chmod(0o0664)
 
     latest_dat_path = '/var/www/release/windows/latest.dat'
 
     now = datetime.datetime.utcnow()
     backup_path = dist_path / (
         'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
     )
     print('backing up %s to %s' % (latest_dat_path, backup_path))
 
     with sftp.open(latest_dat_path, 'rb') as fh:
         latest_dat_old = fh.read()
 
     with backup_path.open('wb') as fh:
         fh.write(latest_dat_old)
 
     print('writing %s with content:' % latest_dat_path)
     latest_dat_content = generate_latest_dat(version)
     print(latest_dat_content)
 
     with sftp.open(latest_dat_path, 'wb') as fh:
         fh.write(latest_dat_content.encode('ascii'))
 
 
 def publish_artifacts(
     dist_path: pathlib.Path,
     version: str,
     pypi=True,
     mercurial_scm_org=True,
     ssh_username=None,
 ):
     """Publish Windows release artifacts.
 
     Files are found in `dist_path`. We will look for files with version string
     `version`.
 
     `pypi` controls whether we upload to PyPI.
     `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
     """
     if pypi:
         publish_artifacts_pypi(dist_path, version)
 
     if mercurial_scm_org:
         publish_artifacts_mercurial_scm_org(
             dist_path, version, ssh_username=ssh_username
         )

contrib/packaging/inno/readme.rst

@@ -1,61 +1,61 b''
 Requirements
 ============
 
 Building the Inno installer requires a Windows machine.
 
 The following system dependencies must be installed:
 
 * Python 2.7 (download from https://www.python.org/downloads/)
 * Microsoft Visual C++ Compiler for Python 2.7
   (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
 * Inno Setup (http://jrsoftware.org/isdl.php) version 5.4 or newer.
   Be sure to install the optional Inno Setup Preprocessor feature,
   which is required.
-* Python 3.5+ (to run the ``build.py`` script)
+* Python 3.5+ (to run the ``packaging.py`` script)
 
 Building
 ========
 
-The ``build.py`` script automates the process of producing an
+The ``packaging.py`` script automates the process of producing an
 Inno installer. It manages fetching and configuring the
 non-system dependencies (such as py2exe, gettext, and various
 Python packages).
 
 The script requires an activated ``Visual C++ 2008`` command prompt.
 A shortcut to such a prompt was installed with ``Microsoft Visual C++
 Compiler for Python 2.7``. From your Start Menu, look for
 ``Microsoft Visual C++ Compiler Package for Python 2.7`` then launch
 either ``Visual C++ 2008 32-bit Command Prompt`` or
 ``Visual C++ 2008 64-bit Command Prompt``.
 
 From the prompt, change to the Mercurial source directory. e.g.
 ``cd c:\src\hg``.
 
-Next, invoke ``build.py`` to produce an Inno installer. You will
+Next, invoke ``packaging.py`` to produce an Inno installer. You will
 need to supply the path to the Python interpreter to use.::
 
-   $ python3.exe contrib\packaging\inno\build.py \
-       --python c:\python27\python.exe
+   $ python3.exe contrib\packaging\packaging.py \
+       inno --python c:\python27\python.exe
 
 .. note::
 
    The script validates that the Visual C++ environment is
    active and that the architecture of the specified Python
    interpreter matches the Visual C++ environment and errors
    if not.
 
 If everything runs as intended, dependencies will be fetched and
 configured into the ``build`` sub-directory, Mercurial will be built,
 and an installer placed in the ``dist`` sub-directory. The final
 line of output should print the name of the generated installer.
 
-Additional options may be configured. Run ``build.py --help`` to
-see a list of program flags.
+Additional options may be configured. Run
+``packaging.py inno --help`` to see a list of program flags.
 
 MinGW
 =====
 
 It is theoretically possible to generate an installer that uses
-MinGW. This isn't well tested and ``build.py`` and may properly
+MinGW. This isn't well tested and ``packaging.py`` and may properly
 support it. See old versions of this file in version control for
 potentially useful hints as to how to achieve this.

contrib/packaging/wix/readme.rst

@@ -1,71 +1,71 b''
 WiX Installer
 =============
 
 The files in this directory are used to produce an MSI installer using
 the WiX Toolset (http://wixtoolset.org/).
 
 The MSI installers require elevated (admin) privileges due to the
 installation of MSVC CRT libraries into the Windows system store. See
 the Inno Setup installers in the ``inno`` sibling directory for installers
 that do not have this requirement.
 
 Requirements
 ============
 
 Building the WiX installers requires a Windows machine. The following
 dependencies must be installed:
 
 * Python 2.7 (download from https://www.python.org/downloads/)
 * Microsoft Visual C++ Compiler for Python 2.7
   (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
-* Python 3.5+ (to run the ``build.py`` script)
+* Python 3.5+ (to run the ``packaging.py`` script)
 
 Building
 ========
 
-The ``build.py`` script automates the process of producing an MSI
+The ``packaging.py`` script automates the process of producing an MSI
 installer. It manages fetching and configuring non-system dependencies
 (such as py2exe, gettext, and various Python packages).
 
 The script requires an activated ``Visual C++ 2008`` command prompt.
 A shortcut to such a prompt was installed with ``Microsoft Visual
 C++ Compiler for Python 2.7``. From your Start Menu, look for
 ``Microsoft Visual C++ Compiler Package for Python 2.7`` then
 launch either ``Visual C++ 2008 32-bit Command Prompt`` or
 ``Visual C++ 2008 64-bit Command Prompt``.
 
 From the prompt, change to the Mercurial source directory. e.g.
 ``cd c:\src\hg``.
 
-Next, invoke ``build.py`` to produce an MSI installer. You will need
+Next, invoke ``packaging.py`` to produce an MSI installer. You will need
 to supply the path to the Python interpreter to use.::
 
-   $ python3 contrib\packaging\wix\build.py \
-      --python c:\python27\python.exe
+   $ python3 contrib\packaging\packaging.py \
+      wix --python c:\python27\python.exe
 
 .. note::
 
    The script validates that the Visual C++ environment is active and
    that the architecture of the specified Python interpreter matches the
    Visual C++ environment. An error is raised otherwise.
 
 If everything runs as intended, dependencies will be fetched and
 configured into the ``build`` sub-directory, Mercurial will be built,
 and an installer placed in the ``dist`` sub-directory. The final line
 of output should print the name of the generated installer.
 
-Additional options may be configured. Run ``build.py --help`` to see
-a list of program flags.
+Additional options may be configured. Run ``packaging.py wix --help`` to
+see a list of program flags.
 
 Relationship to TortoiseHG
 ==========================
 
 TortoiseHG uses the WiX files in this directory.
 
 The code for building TortoiseHG installers lives at
 https://bitbucket.org/tortoisehg/thg-winbuild and is maintained by
 Steve Borho ([e-mail]).
 
 When changing behavior of the WiX installer, be sure to notify
 the TortoiseHG Project of the changes so they have ample time
 provide feedback and react to those changes.

tests/test-check-code.t

@@ -1,86 +1,85 b''
 #require test-repo
 
   $ . "$TESTDIR/helpers-testrepo.sh"
   $ check_code="$TESTDIR"/../contrib/check-code.py
   $ cd "$TESTDIR"/..
 
 New errors are not allowed. Warnings are strongly discouraged.
 (The writing "no-che?k-code" is for not skipping this file when checking.)
 
   $ testrepohg locate \
   > -X contrib/python-zstandard \
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/thirdparty \
   > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false
   Skipping contrib/automation/hgautomation/__init__.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
+  Skipping contrib/packaging/hgpackaging/cli.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/wix.py it has no-che?k-code (glob)
-  Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob)
-  Skipping contrib/packaging/wix/build.py it has no-che?k-code (glob)
   Skipping i18n/polib.py it has no-che?k-code (glob)
   Skipping mercurial/statprof.py it has no-che?k-code (glob)
   Skipping tests/badserverext.py it has no-che?k-code (glob)
 
 @commands in debugcommands.py should be in alphabetical order.
 
   >>> import re
   >>> commands = []
   >>> with open('mercurial/debugcommands.py', 'rb') as fh:
   ...     for line in fh:
   ...         m = re.match(br"^@command\('([a-z]+)", line)
   ...         if m:
   ...             commands.append(m.group(1))
   >>> scommands = list(sorted(commands))
   >>> for i, command in enumerate(scommands):
   ...     if command != commands[i]:
   ...         print('commands in debugcommands.py not sorted; first differing '
   ...               'command is %s; expected %s' % (commands[i], command))
   ...         break
 
 Prevent adding new files in the root directory accidentally.
 
   $ testrepohg files 'glob:*'
   .arcconfig
   .clang-format
   .editorconfig
   .hgignore
   .hgsigs
   .hgtags
   .jshintrc
   CONTRIBUTING
   CONTRIBUTORS
   COPYING
   Makefile
   README.rst
   black.toml
   hg
   hgeditor
   hgweb.cgi
   setup.py
 
 Prevent adding modules which could be shadowed by ancient .so/.dylib.
 
   $ testrepohg files \
   > mercurial/base85.py \
   > mercurial/bdiff.py \
   > mercurial/diffhelpers.py \
   > mercurial/mpatch.py \
   > mercurial/osutil.py \
   > mercurial/parsers.py \
   > mercurial/zstd.py
   [1]
 
 Keep python3 tests sorted:
   $ sort < contrib/python3-whitelist > $TESTTMP/py3sorted
   $ cmp contrib/python3-whitelist $TESTTMP/py3sorted || echo 'Please sort passing tests!'

tests/test-check-py3-compat.t

@@ -1,52 +1,53 b''
 #require test-repo
 
   $ . "$TESTDIR/helpers-testrepo.sh"
   $ cd "$TESTDIR"/..
 
 #if no-py3
   $ testrepohg files 'set:(**.py)' \
   > -X contrib/automation/ \
   > -X contrib/packaging/hgpackaging/ \
   > -X contrib/packaging/inno/ \
+  > -X contrib/packaging/packaging.py \
   > -X contrib/packaging/wix/ \
   > -X hgdemandimport/demandimportpy2.py \
   > -X mercurial/thirdparty/cbor \
   > | sed 's|\\|/|g' | xargs "$PYTHON" contrib/check-py3-compat.py
   contrib/python-zstandard/setup.py not using absolute_import
   contrib/python-zstandard/setup_zstd.py not using absolute_import
   contrib/python-zstandard/tests/common.py not using absolute_import
   contrib/python-zstandard/tests/test_buffer_util.py not using absolute_import
   contrib/python-zstandard/tests/test_compressor.py not using absolute_import
   contrib/python-zstandard/tests/test_compressor_fuzzing.py not using absolute_import
   contrib/python-zstandard/tests/test_data_structures.py not using absolute_import
   contrib/python-zstandard/tests/test_data_structures_fuzzing.py not using absolute_import
   contrib/python-zstandard/tests/test_decompressor.py not using absolute_import
   contrib/python-zstandard/tests/test_decompressor_fuzzing.py not using absolute_import
   contrib/python-zstandard/tests/test_estimate_sizes.py not using absolute_import
   contrib/python-zstandard/tests/test_module_attributes.py not using absolute_import
   contrib/python-zstandard/tests/test_train_dictionary.py not using absolute_import
   setup.py not using absolute_import
 #endif
 
 #if py3
   $ testrepohg files 'set:(**.py) - grep(pygments)' \
   > -X hgdemandimport/demandimportpy2.py \
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/cffi \
   > -X mercurial/thirdparty \
   > | sed 's|\\|/|g' | xargs "$PYTHON" contrib/check-py3-compat.py \
   > | sed 's/[0-9][0-9]*)$/*)/'
   hgext/convert/transport.py: error importing: <*Error> No module named 'svn.client' (error at transport.py:*) (glob) (?)
   hgext/infinitepush/sqlindexapi.py: error importing: <*Error> No module named 'mysql' (error at sqlindexapi.py:*) (glob) (?)
   mercurial/scmwindows.py: error importing: <ValueError> _type_ 'v' not supported (error at win32.py:*) (no-windows !)
   mercurial/win32.py: error importing: <ValueError> _type_ 'v' not supported (error at win32.py:*) (no-windows !)
   mercurial/windows.py: error importing: <ModuleNotFoundError> No module named 'msvcrt' (error at windows.py:*) (no-windows !)
   mercurial/posix.py: error importing: <ModuleNotFoundError> No module named 'fcntl' (error at posix.py:*) (windows !)
   mercurial/scmposix.py: error importing: <ModuleNotFoundError> No module named 'fcntl' (error at scmposix.py:*) (windows !)
 #endif
 
 #if py3 pygments
   $ testrepohg files 'set:(**.py) and grep(pygments)' | sed 's|\\|/|g' \
   > | xargs "$PYTHON" contrib/check-py3-compat.py \
   > | sed 's/[0-9][0-9]*)$/*)/'
 #endif