automation: support building Windows wheels for Python 3.7 and 3.8

The time has come to support Python 3 on Windows.

Let's teach our automation code to produce Windows wheels for
Python 3.7 and 3.8.

We could theoretically support 3.5 and 3.6. But I don't think
it is worth it. People on Windows generally use the Mercurial
installers, not wheels. And I'd prefer we limit variability
and not have to worry about supporting earlier Python versions
if it can be helped.

As part of this, we change the invocation of pip to `python.exe -m pip`,
as this is what is being recommended in Python docs these days. And
it seemed to be required to avoid a weird build error. Why, I'm not
sure. But it looks like pip was having trouble finding a Visual Studio
files when invoked as `pip.exe` but not when using `python.exe -m pip`.
Who knows.

Differential Revision: https://phab.mercurial-scm.org/D8478

contrib/automation/hgautomation/cli.py

@@ -1,487 +1,510 b''
 # cli.py - Command line interface for automation
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import concurrent.futures as futures
 import os
 import pathlib
 import time
 
 from . import (
     aws,
     HGAutomation,
     linux,
     try_server,
     windows,
 )
 
 
 SOURCE_ROOT = pathlib.Path(
     os.path.abspath(__file__)
 ).parent.parent.parent.parent
 DIST_PATH = SOURCE_ROOT / 'dist'
 
 
 def bootstrap_linux_dev(
     hga: HGAutomation, aws_region, distros=None, parallel=False
 ):
     c = hga.aws_connection(aws_region)
 
     if distros:
         distros = distros.split(',')
     else:
         distros = sorted(linux.DISTROS)
 
     # TODO There is a wonky interaction involving KeyboardInterrupt whereby
     # the context manager that is supposed to terminate the temporary EC2
     # instance doesn't run. Until we fix this, make parallel building opt-in
     # so we don't orphan instances.
     if parallel:
         fs = []
 
         with futures.ThreadPoolExecutor(len(distros)) as e:
             for distro in distros:
                 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
 
             for f in fs:
                 f.result()
     else:
         for distro in distros:
             aws.ensure_linux_dev_ami(c, distro=distro)
 
 
 def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     print('Windows development AMI available as %s' % image.id)
 
 
 def build_inno(
     hga: HGAutomation, aws_region, arch, revision, version, base_image_name
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for a in arch:
             windows.build_inno_installer(
                 instance.winrm_client, a, DIST_PATH, version=version
             )
 
 
 def build_wix(
     hga: HGAutomation, aws_region, arch, revision, version, base_image_name
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for a in arch:
             windows.build_wix_installer(
                 instance.winrm_client, a, DIST_PATH, version=version
             )
 
 
 def build_windows_wheel(
-    hga: HGAutomation, aws_region, arch, revision, base_image_name
+    hga: HGAutomation,
+    aws_region,
+    python_version,
+    arch,
+    revision,
+    base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
-        for a in arch:
-            windows.build_wheel(instance.winrm_client, a, DIST_PATH)
+        for py_version in python_version:
+            for a in arch:
+                windows.build_wheel(
+                    instance.winrm_client, py_version, a, DIST_PATH
+                )
 
 
 def build_all_windows_packages(
     hga: HGAutomation, aws_region, revision, version, base_image_name
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         winrm_client = instance.winrm_client
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
+        for py_version in ("2.7", "3.7", "3.8"):
+            for arch in ("x86", "x64"):
+                windows.purge_hg(winrm_client)
+                windows.build_wheel(
+                    winrm_client,
+                    python_version=py_version,
+                    arch=arch,
+                    dest_path=DIST_PATH,
+                )
+
         for arch in ('x86', 'x64'):
             windows.purge_hg(winrm_client)
-            windows.build_wheel(winrm_client, arch, DIST_PATH)
-            windows.purge_hg(winrm_client)
             windows.build_inno_installer(
                 winrm_client, arch, DIST_PATH, version=version
             )
             windows.purge_hg(winrm_client)
             windows.build_wix_installer(
                 winrm_client, arch, DIST_PATH, version=version
             )
 
 
 def terminate_ec2_instances(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.terminate_ec2_instances(c.ec2resource)
 
 
 def purge_ec2_resources(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.remove_resources(c)
 
 
 def run_tests_linux(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     test_flags,
     distro,
     filesystem,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_linux_dev_ami(c, distro=distro)
 
     t_start = time.time()
 
     ensure_extra_volume = filesystem not in ('default', 'tmpfs')
 
     with aws.temporary_linux_dev_instances(
         c, image, instance_type, ensure_extra_volume=ensure_extra_volume
     ) as insts:
 
         instance = insts[0]
 
         linux.prepare_exec_environment(
             instance.ssh_client, filesystem=filesystem
         )
         linux.synchronize_hg(SOURCE_ROOT, instance, '.')
         t_prepared = time.time()
         linux.run_tests(instance.ssh_client, python_version, test_flags)
         t_done = time.time()
 
     t_setup = t_prepared - t_start
     t_all = t_done - t_start
 
     print(
         'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
         % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
     )
 
 
 def run_tests_windows(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     arch,
     test_flags,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
 
     with aws.temporary_windows_dev_instances(
         c, image, instance_type, disable_antivirus=True
     ) as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, '.', instance)
         windows.run_tests(
             instance.winrm_client, python_version, arch, test_flags
         )
 
 
 def publish_windows_artifacts(
     hg: HGAutomation,
     aws_region,
     version: str,
     pypi: bool,
     mercurial_scm_org: bool,
     ssh_username: str,
 ):
     windows.publish_artifacts(
         DIST_PATH,
         version,
         pypi=pypi,
         mercurial_scm_org=mercurial_scm_org,
         ssh_username=ssh_username,
     )
 
 
 def run_try(hga: HGAutomation, aws_region: str, rev: str):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     try_server.trigger_try(c, rev=rev)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
         '--state-path',
         default='~/.hgautomation',
         help='Path for local state files',
     )
     parser.add_argument(
         '--aws-region', help='AWS region to use', default='us-west-2',
     )
 
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser(
         'bootstrap-linux-dev', help='Bootstrap Linux development environments',
     )
     sp.add_argument(
         '--distros', help='Comma delimited list of distros to bootstrap',
     )
     sp.add_argument(
         '--parallel',
         action='store_true',
         help='Generate AMIs in parallel (not CTRL-c safe)',
     )
     sp.set_defaults(func=bootstrap_linux_dev)
 
     sp = subparsers.add_parser(
         'bootstrap-windows-dev',
         help='Bootstrap the Windows development environment',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=bootstrap_windows_dev)
 
     sp = subparsers.add_parser(
         'build-all-windows-packages', help='Build all Windows packages',
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--version', help='Mercurial version string to use',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_all_windows_packages)
 
     sp = subparsers.add_parser(
         'build-inno', help='Build Inno Setup installer(s)',
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--version', help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_inno)
 
     sp = subparsers.add_parser(
         'build-windows-wheel', help='Build Windows wheel(s)',
     )
     sp.add_argument(
+        '--python-version',
+        help='Python version to build for',
+        choices={'2.7', '3.7', '3.8'},
+        nargs='*',
+        default=['3.8'],
+    )
+    sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_windows_wheel)
 
     sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--version', help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_wix)
 
     sp = subparsers.add_parser(
         'terminate-ec2-instances',
         help='Terminate all active EC2 instances managed by us',
     )
     sp.set_defaults(func=terminate_ec2_instances)
 
     sp = subparsers.add_parser(
         'purge-ec2-resources', help='Purge all EC2 resources managed by us',
     )
     sp.set_defaults(func=purge_ec2_resources)
 
     sp = subparsers.add_parser('run-tests-linux', help='Run tests on Linux',)
     sp.add_argument(
         '--distro',
         help='Linux distribution to run tests on',
         choices=linux.DISTROS,
         default='debian10',
     )
     sp.add_argument(
         '--filesystem',
         help='Filesystem type to use',
         choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
         default='default',
     )
     sp.add_argument(
         '--instance-type',
         help='EC2 instance type to use',
         default='c5.9xlarge',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
         choices={
             'system2',
             'system3',
             '2.7',
             '3.5',
             '3.6',
             '3.7',
             '3.8',
             'pypy',
             'pypy3.5',
             'pypy3.6',
         },
         default='system2',
     )
     sp.add_argument(
         'test_flags',
         help='Extra command line flags to pass to run-tests.py',
         nargs='*',
     )
     sp.set_defaults(func=run_tests_linux)
 
     sp = subparsers.add_parser(
         'run-tests-windows', help='Run tests on Windows',
     )
     sp.add_argument(
         '--instance-type', help='EC2 instance type to use', default='t3.medium',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
         choices={'2.7', '3.5', '3.6', '3.7', '3.8'},
         default='2.7',
     )
     sp.add_argument(
         '--arch',
         help='Architecture to test',
         choices={'x86', 'x64'},
         default='x64',
     )
     sp.add_argument(
         '--test-flags', help='Extra command line flags to pass to run-tests.py',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=run_tests_windows)
 
     sp = subparsers.add_parser(
         'publish-windows-artifacts',
         help='Publish built Windows artifacts (wheels, installers, etc)',
     )
     sp.add_argument(
         '--no-pypi',
         dest='pypi',
         action='store_false',
         default=True,
         help='Skip uploading to PyPI',
     )
     sp.add_argument(
         '--no-mercurial-scm-org',
         dest='mercurial_scm_org',
         action='store_false',
         default=True,
         help='Skip uploading to www.mercurial-scm.org',
     )
     sp.add_argument(
         '--ssh-username', help='SSH username for mercurial-scm.org',
     )
     sp.add_argument(
         'version', help='Mercurial version string to locate local packages',
     )
     sp.set_defaults(func=publish_windows_artifacts)
 
     sp = subparsers.add_parser(
         'try', help='Run CI automation against a custom changeset'
     )
     sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
     sp.set_defaults(func=run_try)
 
     return parser
 
 
 def main():
     parser = get_parser()
     args = parser.parse_args()
 
     local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
     automation = HGAutomation(local_state_path)
 
     if not hasattr(args, 'func'):
         parser.print_help()
         return
 
     kwargs = dict(vars(args))
     del kwargs['func']
     del kwargs['state_path']
 
     args.func(automation, **kwargs)

contrib/automation/hgautomation/windows.py

@@ -1,510 +1,533 b''
 # windows.py - Automation specific to Windows
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import datetime
 import os
 import paramiko
 import pathlib
 import re
 import subprocess
 import tempfile
 
 from .pypi import upload as pypi_upload
 from .winrm import run_powershell
 
 
 # PowerShell commands to activate a Visual Studio 2008 environment.
 # This is essentially a port of vcvarsall.bat to PowerShell.
 ACTIVATE_VC9_AMD64 = r'''
 Write-Output "activating Visual Studio 2008 environment for AMD64"
 $root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
 $Env:VCINSTALLDIR = "${root}\VC\"
 $Env:WindowsSdkDir = "${root}\WinSDK\"
 $Env:PATH = "${root}\VC\Bin\amd64;${root}\WinSDK\Bin\x64;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:PATH"
 $Env:LIB = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIB"
 $Env:LIBPATH = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIBPATH"
 '''.lstrip()
 
 ACTIVATE_VC9_X86 = r'''
 Write-Output "activating Visual Studio 2008 environment for x86"
 $root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
 $Env:VCINSTALLDIR = "${root}\VC\"
 $Env:WindowsSdkDir = "${root}\WinSDK\"
 $Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH"
 $Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE"
 $Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB"
 $Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib;$Env:LIBPATH"
 '''.lstrip()
 
 HG_PURGE = r'''
 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
 Set-Location C:\hgdev\src
 hg.exe --config extensions.purge= purge --all
 if ($LASTEXITCODE -ne 0) {
     throw "process exited non-0: $LASTEXITCODE"
 }
 Write-Output "purged Mercurial repo"
 '''
 
 HG_UPDATE_CLEAN = r'''
 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
 Set-Location C:\hgdev\src
 hg.exe --config extensions.purge= purge --all
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 hg.exe update -C {revision}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 hg.exe log -r .
 Write-Output "updated Mercurial working directory to {revision}"
 '''.lstrip()
 
 BUILD_INNO = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''.lstrip()
 
 BUILD_WHEEL = r'''
 Set-Location C:\hgdev\src
-C:\hgdev\python27-{arch}\Scripts\pip.exe wheel --wheel-dir dist .
+C:\hgdev\python{python_version}-{arch}\python.exe -m pip wheel --wheel-dir dist .
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 BUILD_WIX = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
 RUN_TESTS = r'''
 C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
 '''
 
-X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
-X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
+WHEEL_FILENAME_PYTHON27_X86 = 'mercurial-{version}-cp27-cp27m-win32.whl'
+WHEEL_FILENAME_PYTHON27_X64 = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
+WHEEL_FILENAME_PYTHON37_X86 = 'mercurial-{version}-cp37-cp37m-win32.whl'
+WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl'
+WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl'
+WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl'
+
 X86_EXE_FILENAME = 'Mercurial-{version}.exe'
 X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
 X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
 X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
 
 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
 
 X86_USER_AGENT_PATTERN = '.*Windows.*'
 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
 
 X86_EXE_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x86 Windows '
     '- does not require admin rights'
 )
 X64_EXE_DESCRIPTION = (
     'Mercurial {version} Inno Setup installer - x64 Windows '
     '- does not require admin rights'
 )
 X86_MSI_DESCRIPTION = (
     'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights'
 )
 X64_MSI_DESCRIPTION = (
     'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights'
 )
 
 
 def get_vc_prefix(arch):
     if arch == 'x86':
         return ACTIVATE_VC9_X86
     elif arch == 'x64':
         return ACTIVATE_VC9_AMD64
     else:
         raise ValueError('illegal arch: %s; must be x86 or x64' % arch)
 
 
 def fix_authorized_keys_permissions(winrm_client, path):
     commands = [
         '$ErrorActionPreference = "Stop"',
         'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
         r'icacls %s /remove:g "NT Service\sshd"' % path,
     ]
 
     run_powershell(winrm_client, '\n'.join(commands))
 
 
 def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
     """Synchronize local Mercurial repo to remote EC2 instance."""
 
     winrm_client = ec2_instance.winrm_client
 
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_dir = pathlib.Path(temp_dir)
 
         ssh_dir = temp_dir / '.ssh'
         ssh_dir.mkdir()
         ssh_dir.chmod(0o0700)
 
         # Generate SSH key to use for communication.
         subprocess.run(
             [
                 'ssh-keygen',
                 '-t',
                 'rsa',
                 '-b',
                 '4096',
                 '-N',
                 '',
                 '-f',
                 str(ssh_dir / 'id_rsa'),
             ],
             check=True,
             capture_output=True,
         )
 
         # Add it to ~/.ssh/authorized_keys on remote.
         # This assumes the file doesn't already exist.
         authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
         winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
         winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
         fix_authorized_keys_permissions(winrm_client, authorized_keys)
 
         public_ip = ec2_instance.public_ip_address
 
         ssh_config = temp_dir / '.ssh' / 'config'
 
         with open(ssh_config, 'w', encoding='utf-8') as fh:
             fh.write('Host %s\n' % public_ip)
             fh.write('  User Administrator\n')
             fh.write('  StrictHostKeyChecking no\n')
             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
             fh.write('  IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
 
         if not (hg_repo / '.hg').is_dir():
             raise Exception(
                 '%s is not a Mercurial repository; '
                 'synchronization not yet supported' % hg_repo
             )
 
         env = dict(os.environ)
         env['HGPLAIN'] = '1'
         env['HGENCODING'] = 'utf-8'
 
         hg_bin = hg_repo / 'hg'
 
         res = subprocess.run(
             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
             cwd=str(hg_repo),
             env=env,
             check=True,
             capture_output=True,
         )
 
         full_revision = res.stdout.decode('ascii')
 
         args = [
             'python2.7',
             hg_bin,
             '--config',
             'ui.ssh=ssh -F %s' % ssh_config,
             '--config',
             'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
             # Also ensure .hgtags changes are present so auto version
             # calculation works.
             'push',
             '-f',
             '-r',
             full_revision,
             '-r',
             'file(.hgtags)',
             'ssh://%s/c:/hgdev/src' % public_ip,
         ]
 
         res = subprocess.run(args, cwd=str(hg_repo), env=env)
 
         # Allow 1 (no-op) to not trigger error.
         if res.returncode not in (0, 1):
             res.check_returncode()
 
         run_powershell(
             winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
         )
 
         # TODO detect dirty local working directory and synchronize accordingly.
 
 
 def purge_hg(winrm_client):
     """Purge the Mercurial source repository on an EC2 instance."""
     run_powershell(winrm_client, HG_PURGE)
 
 
 def find_latest_dist(winrm_client, pattern):
     """Find path to newest file in dist/ directory matching a pattern."""
 
     res = winrm_client.execute_ps(
         r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
         '| Sort-Object LastWriteTime -Descending '
         '| Select-Object -First 1\n'
         '$v.name' % pattern
     )
     return res[0]
 
 
 def copy_latest_dist(winrm_client, pattern, dest_path):
     """Copy latest file matching pattern in dist/ directory.
 
     Given a WinRM client and a file pattern, find the latest file on the remote
     matching that pattern and copy it to the ``dest_path`` directory on the
     local machine.
     """
     latest = find_latest_dist(winrm_client, pattern)
     source = r'C:\hgdev\src\dist\%s' % latest
     dest = dest_path / latest
     print('copying %s to %s' % (source, dest))
     winrm_client.fetch(source, str(dest))
 
 
 def build_inno_installer(
     winrm_client, arch: str, dest_path: pathlib.Path, version=None
 ):
     """Build the Inno Setup installer on a remote machine.
 
     Using a WinRM client, remote commands are executed to build
     a Mercurial Inno Setup installer.
     """
     print('building Inno Setup installer for %s' % arch)
 
     extra_args = []
     if version:
         extra_args.extend(['--version', version])
 
     ps = get_vc_prefix(arch) + BUILD_INNO.format(
         arch=arch, extra_args=' '.join(extra_args)
     )
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.exe', dest_path)
 
 
-def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path):
+def build_wheel(
+    winrm_client, python_version: str, arch: str, dest_path: pathlib.Path
+):
     """Build Python wheels on a remote machine.
 
     Using a WinRM client, remote commands are executed to build a Python wheel
     for Mercurial.
     """
-    print('Building Windows wheel for %s' % arch)
-    ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch)
+    print('Building Windows wheel for Python %s %s' % (python_version, arch))
+
+    ps = BUILD_WHEEL.format(
+        python_version=python_version.replace(".", ""), arch=arch
+    )
+
+    # Python 2.7 requires an activated environment.
+    if python_version == "2.7":
+        ps = get_vc_prefix(arch) + ps
+
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.whl', dest_path)
 
 
 def build_wix_installer(
     winrm_client, arch: str, dest_path: pathlib.Path, version=None
 ):
     """Build the WiX installer on a remote machine.
 
     Using a WinRM client, remote commands are executed to build a WiX installer.
     """
     print('Building WiX installer for %s' % arch)
     extra_args = []
     if version:
         extra_args.extend(['--version', version])
 
     ps = get_vc_prefix(arch) + BUILD_WIX.format(
         arch=arch, extra_args=' '.join(extra_args)
     )
     run_powershell(winrm_client, ps)
     copy_latest_dist(winrm_client, '*.msi', dest_path)
 
 
 def run_tests(winrm_client, python_version, arch, test_flags=''):
     """Run tests on a remote Windows machine.
 
     ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
     ``arch`` is ``x86`` or ``x64``.
     ``test_flags`` is a str representing extra arguments to pass to
     ``run-tests.py``.
     """
     if not re.match(r'\d\.\d', python_version):
         raise ValueError(
             r'python_version must be \d.\d; got %s' % python_version
         )
 
     if arch not in ('x86', 'x64'):
         raise ValueError('arch must be x86 or x64; got %s' % arch)
 
     python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
 
     ps = RUN_TESTS.format(python_path=python_path, test_flags=test_flags or '',)
 
     run_powershell(winrm_client, ps)
 
 
 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
     return (
-        dist_path / X86_WHEEL_FILENAME.format(version=version),
-        dist_path / X64_WHEEL_FILENAME.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON27_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON27_X64.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
     )
 
 
 def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
     return (
-        dist_path / X86_WHEEL_FILENAME.format(version=version),
-        dist_path / X64_WHEEL_FILENAME.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON27_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON27_X64.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
+        dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
         dist_path / X86_EXE_FILENAME.format(version=version),
         dist_path / X64_EXE_FILENAME.format(version=version),
         dist_path / X86_MSI_FILENAME.format(version=version),
         dist_path / X64_MSI_FILENAME.format(version=version),
     )
 
 
 def generate_latest_dat(version: str):
     x86_exe_filename = X86_EXE_FILENAME.format(version=version)
     x64_exe_filename = X64_EXE_FILENAME.format(version=version)
     x86_msi_filename = X86_MSI_FILENAME.format(version=version)
     x64_msi_filename = X64_MSI_FILENAME.format(version=version)
 
     entries = (
         (
             '10',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
             X86_EXE_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
             X64_EXE_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X86_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
             X86_MSI_DESCRIPTION.format(version=version),
         ),
         (
             '10',
             version,
             X64_USER_AGENT_PATTERN,
             '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
             X64_MSI_DESCRIPTION.format(version=version),
         ),
     )
 
     lines = ['\t'.join(e) for e in entries]
 
     return '\n'.join(lines) + '\n'
 
 
 def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
     """Publish Windows release artifacts to PyPI."""
 
     wheel_paths = resolve_wheel_artifacts(dist_path, version)
 
     for p in wheel_paths:
         if not p.exists():
             raise Exception('%s not found' % p)
 
     print('uploading wheels to PyPI (you may be prompted for credentials)')
     pypi_upload(wheel_paths)
 
 
 def publish_artifacts_mercurial_scm_org(
     dist_path: pathlib.Path, version: str, ssh_username=None
 ):
     """Publish Windows release artifacts to mercurial-scm.org."""
     all_paths = resolve_all_artifacts(dist_path, version)
 
     for p in all_paths:
         if not p.exists():
             raise Exception('%s not found' % p)
 
     client = paramiko.SSHClient()
     client.load_system_host_keys()
     # We assume the system SSH configuration knows how to connect.
     print('connecting to mercurial-scm.org via ssh...')
     try:
         client.connect('mercurial-scm.org', username=ssh_username)
     except paramiko.AuthenticationException:
         print('error authenticating; is an SSH key available in an SSH agent?')
         raise
 
     print('SSH connection established')
 
     print('opening SFTP client...')
     sftp = client.open_sftp()
     print('SFTP client obtained')
 
     for p in all_paths:
         dest_path = '/var/www/release/windows/%s' % p.name
         print('uploading %s to %s' % (p, dest_path))
 
         with p.open('rb') as fh:
             data = fh.read()
 
         with sftp.open(dest_path, 'wb') as fh:
             fh.write(data)
             fh.chmod(0o0664)
 
     latest_dat_path = '/var/www/release/windows/latest.dat'
 
     now = datetime.datetime.utcnow()
     backup_path = dist_path / (
         'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
     )
     print('backing up %s to %s' % (latest_dat_path, backup_path))
 
     with sftp.open(latest_dat_path, 'rb') as fh:
         latest_dat_old = fh.read()
 
     with backup_path.open('wb') as fh:
         fh.write(latest_dat_old)
 
     print('writing %s with content:' % latest_dat_path)
     latest_dat_content = generate_latest_dat(version)
     print(latest_dat_content)
 
     with sftp.open(latest_dat_path, 'wb') as fh:
         fh.write(latest_dat_content.encode('ascii'))
 
 
 def publish_artifacts(
     dist_path: pathlib.Path,
     version: str,
     pypi=True,
     mercurial_scm_org=True,
     ssh_username=None,
 ):
     """Publish Windows release artifacts.
 
     Files are found in `dist_path`. We will look for files with version string
     `version`.
 
     `pypi` controls whether we upload to PyPI.
     `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
     """
     if pypi:
         publish_artifacts_pypi(dist_path, version)
 
     if mercurial_scm_org:
         publish_artifacts_mercurial_scm_org(
             dist_path, version, ssh_username=ssh_username
         )