automation: support running against Python 3.9

We just added CPython 3.9.0 to the automation environment. Let's
support actions targeting it.

One side-effect of this change is that we'll produce Windows wheels
for Python 3.9 by default. This is desirable.

Differential Revision: https://phab.mercurial-scm.org/D9189

contrib/automation/hgautomation/cli.py

@@ -1,548 +1,548 b''
 # cli.py - Command line interface for automation
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import concurrent.futures as futures
 import os
 import pathlib
 import time
 
 from . import (
     aws,
     HGAutomation,
     linux,
     try_server,
     windows,
 )
 
 
 SOURCE_ROOT = pathlib.Path(
     os.path.abspath(__file__)
 ).parent.parent.parent.parent
 DIST_PATH = SOURCE_ROOT / 'dist'
 
 
 def bootstrap_linux_dev(
     hga: HGAutomation, aws_region, distros=None, parallel=False
 ):
     c = hga.aws_connection(aws_region)
 
     if distros:
         distros = distros.split(',')
     else:
         distros = sorted(linux.DISTROS)
 
     # TODO There is a wonky interaction involving KeyboardInterrupt whereby
     # the context manager that is supposed to terminate the temporary EC2
     # instance doesn't run. Until we fix this, make parallel building opt-in
     # so we don't orphan instances.
     if parallel:
         fs = []
 
         with futures.ThreadPoolExecutor(len(distros)) as e:
             for distro in distros:
                 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
 
             for f in fs:
                 f.result()
     else:
         for distro in distros:
             aws.ensure_linux_dev_ami(c, distro=distro)
 
 
 def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     print('Windows development AMI available as %s' % image.id)
 
 
 def build_inno(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     version,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_inno_installer(
                     instance.winrm_client,
                     py_version,
                     a,
                     DIST_PATH,
                     version=version,
                 )
 
 
 def build_wix(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     version,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_wix_installer(
                     instance.winrm_client,
                     py_version,
                     a,
                     DIST_PATH,
                     version=version,
                 )
 
 
 def build_windows_wheel(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_wheel(
                     instance.winrm_client, py_version, a, DIST_PATH
                 )
 
 
 def build_all_windows_packages(
     hga: HGAutomation, aws_region, revision, version, base_image_name
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         winrm_client = instance.winrm_client
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
-        for py_version in ("2.7", "3.7", "3.8"):
+        for py_version in ("2.7", "3.7", "3.8", "3.9"):
             for arch in ("x86", "x64"):
                 windows.purge_hg(winrm_client)
                 windows.build_wheel(
                     winrm_client,
                     python_version=py_version,
                     arch=arch,
                     dest_path=DIST_PATH,
                 )
 
         for py_version in (2, 3):
             for arch in ('x86', 'x64'):
                 windows.purge_hg(winrm_client)
                 windows.build_inno_installer(
                     winrm_client, py_version, arch, DIST_PATH, version=version
                 )
                 windows.build_wix_installer(
                     winrm_client, py_version, arch, DIST_PATH, version=version
                 )
 
 
 def terminate_ec2_instances(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.terminate_ec2_instances(c.ec2resource)
 
 
 def purge_ec2_resources(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.remove_resources(c)
 
 
 def run_tests_linux(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     test_flags,
     distro,
     filesystem,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_linux_dev_ami(c, distro=distro)
 
     t_start = time.time()
 
     ensure_extra_volume = filesystem not in ('default', 'tmpfs')
 
     with aws.temporary_linux_dev_instances(
         c, image, instance_type, ensure_extra_volume=ensure_extra_volume
     ) as insts:
 
         instance = insts[0]
 
         linux.prepare_exec_environment(
             instance.ssh_client, filesystem=filesystem
         )
         linux.synchronize_hg(SOURCE_ROOT, instance, '.')
         t_prepared = time.time()
         linux.run_tests(instance.ssh_client, python_version, test_flags)
         t_done = time.time()
 
     t_setup = t_prepared - t_start
     t_all = t_done - t_start
 
     print(
         'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
         % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
     )
 
 
 def run_tests_windows(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     arch,
     test_flags,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
 
     with aws.temporary_windows_dev_instances(
         c, image, instance_type, disable_antivirus=True
     ) as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, '.', instance)
         windows.run_tests(
             instance.winrm_client, python_version, arch, test_flags
         )
 
 
 def publish_windows_artifacts(
     hg: HGAutomation,
     aws_region,
     version: str,
     pypi: bool,
     mercurial_scm_org: bool,
     ssh_username: str,
 ):
     windows.publish_artifacts(
         DIST_PATH,
         version,
         pypi=pypi,
         mercurial_scm_org=mercurial_scm_org,
         ssh_username=ssh_username,
     )
 
 
 def run_try(hga: HGAutomation, aws_region: str, rev: str):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     try_server.trigger_try(c, rev=rev)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
         '--state-path',
         default='~/.hgautomation',
         help='Path for local state files',
     )
     parser.add_argument(
         '--aws-region', help='AWS region to use', default='us-west-2',
     )
 
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser(
         'bootstrap-linux-dev', help='Bootstrap Linux development environments',
     )
     sp.add_argument(
         '--distros', help='Comma delimited list of distros to bootstrap',
     )
     sp.add_argument(
         '--parallel',
         action='store_true',
         help='Generate AMIs in parallel (not CTRL-c safe)',
     )
     sp.set_defaults(func=bootstrap_linux_dev)
 
     sp = subparsers.add_parser(
         'bootstrap-windows-dev',
         help='Bootstrap the Windows development environment',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=bootstrap_windows_dev)
 
     sp = subparsers.add_parser(
         'build-all-windows-packages', help='Build all Windows packages',
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--version', help='Mercurial version string to use',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_all_windows_packages)
 
     sp = subparsers.add_parser(
         'build-inno', help='Build Inno Setup installer(s)',
     )
     sp.add_argument(
         '--python-version',
         help='Which version of Python to target',
         choices={2, 3},
         type=int,
         nargs='*',
         default=[3],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--version', help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_inno)
 
     sp = subparsers.add_parser(
         'build-windows-wheel', help='Build Windows wheel(s)',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to build for',
-        choices={'2.7', '3.7', '3.8'},
+        choices={'2.7', '3.7', '3.8', '3.9'},
         nargs='*',
         default=['3.8'],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_windows_wheel)
 
     sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
     sp.add_argument(
         '--python-version',
         help='Which version of Python to target',
         choices={2, 3},
         type=int,
         nargs='*',
         default=[3],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision', help='Mercurial revision to build', default='.',
     )
     sp.add_argument(
         '--version', help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_wix)
 
     sp = subparsers.add_parser(
         'terminate-ec2-instances',
         help='Terminate all active EC2 instances managed by us',
     )
     sp.set_defaults(func=terminate_ec2_instances)
 
     sp = subparsers.add_parser(
         'purge-ec2-resources', help='Purge all EC2 resources managed by us',
     )
     sp.set_defaults(func=purge_ec2_resources)
 
     sp = subparsers.add_parser('run-tests-linux', help='Run tests on Linux',)
     sp.add_argument(
         '--distro',
         help='Linux distribution to run tests on',
         choices=linux.DISTROS,
         default='debian10',
     )
     sp.add_argument(
         '--filesystem',
         help='Filesystem type to use',
         choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
         default='default',
     )
     sp.add_argument(
         '--instance-type',
         help='EC2 instance type to use',
         default='c5.9xlarge',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
         choices={
             'system2',
             'system3',
             '2.7',
             '3.5',
             '3.6',
             '3.7',
             '3.8',
             'pypy',
             'pypy3.5',
             'pypy3.6',
         },
         default='system2',
     )
     sp.add_argument(
         'test_flags',
         help='Extra command line flags to pass to run-tests.py',
         nargs='*',
     )
     sp.set_defaults(func=run_tests_linux)
 
     sp = subparsers.add_parser(
         'run-tests-windows', help='Run tests on Windows',
     )
     sp.add_argument(
         '--instance-type', help='EC2 instance type to use', default='t3.medium',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
-        choices={'2.7', '3.5', '3.6', '3.7', '3.8'},
+        choices={'2.7', '3.5', '3.6', '3.7', '3.8', '3.9'},
         default='2.7',
     )
     sp.add_argument(
         '--arch',
         help='Architecture to test',
         choices={'x86', 'x64'},
         default='x64',
     )
     sp.add_argument(
         '--test-flags', help='Extra command line flags to pass to run-tests.py',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=run_tests_windows)
 
     sp = subparsers.add_parser(
         'publish-windows-artifacts',
         help='Publish built Windows artifacts (wheels, installers, etc)',
     )
     sp.add_argument(
         '--no-pypi',
         dest='pypi',
         action='store_false',
         default=True,
         help='Skip uploading to PyPI',
     )
     sp.add_argument(
         '--no-mercurial-scm-org',
         dest='mercurial_scm_org',
         action='store_false',
         default=True,
         help='Skip uploading to www.mercurial-scm.org',
     )
     sp.add_argument(
         '--ssh-username', help='SSH username for mercurial-scm.org',
     )
     sp.add_argument(
         'version', help='Mercurial version string to locate local packages',
     )
     sp.set_defaults(func=publish_windows_artifacts)
 
     sp = subparsers.add_parser(
         'try', help='Run CI automation against a custom changeset'
     )
     sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
     sp.set_defaults(func=run_try)
 
     return parser
 
 
 def main():
     parser = get_parser()
     args = parser.parse_args()
 
     local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
     automation = HGAutomation(local_state_path)
 
     if not hasattr(args, 'func'):
         parser.print_help()
         return
 
     kwargs = dict(vars(args))
     del kwargs['func']
     del kwargs['state_path']
 
     args.func(automation, **kwargs)