# cli.py - Command line interface for automation # # Copyright 2019 Gregory Szorc # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. # no-check-code because Python 3 native. import argparse import concurrent.futures as futures import os import pathlib import time from . import ( aws, HGAutomation, linux, try_server, windows, ) SOURCE_ROOT = pathlib.Path( os.path.abspath(__file__) ).parent.parent.parent.parent DIST_PATH = SOURCE_ROOT / 'dist' def bootstrap_linux_dev( hga: HGAutomation, aws_region, distros=None, parallel=False ): c = hga.aws_connection(aws_region) if distros: distros = distros.split(',') else: distros = sorted(linux.DISTROS) # TODO There is a wonky interaction involving KeyboardInterrupt whereby # the context manager that is supposed to terminate the temporary EC2 # instance doesn't run. Until we fix this, make parallel building opt-in # so we don't orphan instances. if parallel: fs = [] with futures.ThreadPoolExecutor(len(distros)) as e: for distro in distros: fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro)) for f in fs: f.result() else: for distro in distros: aws.ensure_linux_dev_ami(c, distro=distro) def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) print('Windows development AMI available as %s' % image.id) def build_inno( hga: HGAutomation, aws_region, python_version, arch, revision, version, base_image_name, ): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: instance = insts[0] windows.synchronize_hg(SOURCE_ROOT, revision, instance) for py_version in python_version: for a in arch: windows.build_inno_installer( instance.winrm_client, py_version, a, DIST_PATH, version=version, ) def build_wix( hga: HGAutomation, aws_region, python_version, arch, revision, version, base_image_name, ): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: instance = insts[0] windows.synchronize_hg(SOURCE_ROOT, revision, instance) for py_version in python_version: for a in arch: windows.build_wix_installer( instance.winrm_client, py_version, a, DIST_PATH, version=version, ) def build_windows_wheel( hga: HGAutomation, aws_region, python_version, arch, revision, base_image_name, ): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: instance = insts[0] windows.synchronize_hg(SOURCE_ROOT, revision, instance) for py_version in python_version: for a in arch: windows.build_wheel( instance.winrm_client, py_version, a, DIST_PATH ) def build_all_windows_packages( hga: HGAutomation, aws_region, revision, version, base_image_name ): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) with aws.temporary_windows_dev_instances(c, image, 'm6i.large') as insts: instance = insts[0] winrm_client = instance.winrm_client windows.synchronize_hg(SOURCE_ROOT, revision, instance) for py_version in ("2.7", "3.7", "3.8", "3.9"): for arch in ("x86", "x64"): windows.purge_hg(winrm_client) windows.build_wheel( winrm_client, python_version=py_version, arch=arch, dest_path=DIST_PATH, ) for py_version in (2, 3): for arch in ('x86', 'x64'): windows.purge_hg(winrm_client) windows.build_inno_installer( winrm_client, py_version, arch, DIST_PATH, version=version ) windows.build_wix_installer( winrm_client, py_version, arch, DIST_PATH, version=version ) def terminate_ec2_instances(hga: HGAutomation, aws_region): c = hga.aws_connection(aws_region, ensure_ec2_state=False) aws.terminate_ec2_instances(c.ec2resource) def purge_ec2_resources(hga: HGAutomation, aws_region): c = hga.aws_connection(aws_region, ensure_ec2_state=False) aws.remove_resources(c) def run_tests_linux( hga: HGAutomation, aws_region, instance_type, python_version, test_flags, distro, filesystem, ): c = hga.aws_connection(aws_region) image = aws.ensure_linux_dev_ami(c, distro=distro) t_start = time.time() ensure_extra_volume = filesystem not in ('default', 'tmpfs') with aws.temporary_linux_dev_instances( c, image, instance_type, ensure_extra_volume=ensure_extra_volume ) as insts: instance = insts[0] linux.prepare_exec_environment( instance.ssh_client, filesystem=filesystem ) linux.synchronize_hg(SOURCE_ROOT, instance, '.') t_prepared = time.time() linux.run_tests(instance.ssh_client, python_version, test_flags) t_done = time.time() t_setup = t_prepared - t_start t_all = t_done - t_start print( 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0) ) def run_tests_windows( hga: HGAutomation, aws_region, instance_type, python_version, arch, test_flags, base_image_name, ): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) with aws.temporary_windows_dev_instances( c, image, instance_type, disable_antivirus=True ) as insts: instance = insts[0] windows.synchronize_hg(SOURCE_ROOT, '.', instance) windows.run_tests( instance.winrm_client, python_version, arch, test_flags ) def publish_windows_artifacts( hg: HGAutomation, aws_region, version: str, pypi: bool, mercurial_scm_org: bool, ssh_username: str, ): windows.publish_artifacts( DIST_PATH, version, pypi=pypi, mercurial_scm_org=mercurial_scm_org, ssh_username=ssh_username, ) def run_try(hga: HGAutomation, aws_region: str, rev: str): c = hga.aws_connection(aws_region, ensure_ec2_state=False) try_server.trigger_try(c, rev=rev) def get_parser(): parser = argparse.ArgumentParser() parser.add_argument( '--state-path', default='~/.hgautomation', help='Path for local state files', ) parser.add_argument( '--aws-region', help='AWS region to use', default='us-west-2', ) subparsers = parser.add_subparsers() sp = subparsers.add_parser( 'bootstrap-linux-dev', help='Bootstrap Linux development environments', ) sp.add_argument( '--distros', help='Comma delimited list of distros to bootstrap', ) sp.add_argument( '--parallel', action='store_true', help='Generate AMIs in parallel (not CTRL-c safe)', ) sp.set_defaults(func=bootstrap_linux_dev) sp = subparsers.add_parser( 'bootstrap-windows-dev', help='Bootstrap the Windows development environment', ) sp.add_argument( '--base-image-name', help='AMI name of base image', default=aws.WINDOWS_BASE_IMAGE_NAME, ) sp.set_defaults(func=bootstrap_windows_dev) sp = subparsers.add_parser( 'build-all-windows-packages', help='Build all Windows packages', ) sp.add_argument( '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( '--version', help='Mercurial version string to use', ) sp.add_argument( '--base-image-name', help='AMI name of base image', default=aws.WINDOWS_BASE_IMAGE_NAME, ) sp.set_defaults(func=build_all_windows_packages) sp = subparsers.add_parser( 'build-inno', help='Build Inno Setup installer(s)', ) sp.add_argument( '--python-version', help='Which version of Python to target', choices={2, 3}, type=int, nargs='*', default=[3], ) sp.add_argument( '--arch', help='Architecture to build for', choices={'x86', 'x64'}, nargs='*', default=['x64'], ) sp.add_argument( '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( '--version', help='Mercurial version string to use in installer', ) sp.add_argument( '--base-image-name', help='AMI name of base image', default=aws.WINDOWS_BASE_IMAGE_NAME, ) sp.set_defaults(func=build_inno) sp = subparsers.add_parser( 'build-windows-wheel', help='Build Windows wheel(s)', ) sp.add_argument( '--python-version', help='Python version to build for', choices={'2.7', '3.7', '3.8', '3.9'}, nargs='*', default=['3.8'], ) sp.add_argument( '--arch', help='Architecture to build for', choices={'x86', 'x64'}, nargs='*', default=['x64'], ) sp.add_argument( '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( '--base-image-name', help='AMI name of base image', default=aws.WINDOWS_BASE_IMAGE_NAME, ) sp.set_defaults(func=build_windows_wheel) sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)') sp.add_argument( '--python-version', help='Which version of Python to target', choices={2, 3}, type=int, nargs='*', default=[3], ) sp.add_argument( '--arch', help='Architecture to build for', choices={'x86', 'x64'}, nargs='*', default=['x64'], ) sp.add_argument( '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( '--version', help='Mercurial version string to use in installer', ) sp.add_argument( '--base-image-name', help='AMI name of base image', default=aws.WINDOWS_BASE_IMAGE_NAME, ) sp.set_defaults(func=build_wix) sp = subparsers.add_parser( 'terminate-ec2-instances', help='Terminate all active EC2 instances managed by us', ) sp.set_defaults(func=terminate_ec2_instances) sp = subparsers.add_parser( 'purge-ec2-resources', help='Purge all EC2 resources managed by us', ) sp.set_defaults(func=purge_ec2_resources) sp = subparsers.add_parser( 'run-tests-linux', help='Run tests on Linux', ) sp.add_argument( '--distro', help='Linux distribution to run tests on', choices=linux.DISTROS, default='debian10', ) sp.add_argument( '--filesystem', help='Filesystem type to use', choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'}, default='default', ) sp.add_argument( '--instance-type', help='EC2 instance type to use', default='c5.9xlarge', ) sp.add_argument( '--python-version', help='Python version to use', choices={ 'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8', 'pypy', 'pypy3.5', 'pypy3.6', }, default='system2', ) sp.add_argument( 'test_flags', help='Extra command line flags to pass to run-tests.py', nargs='*', ) sp.set_defaults(func=run_tests_linux) sp = subparsers.add_parser( 'run-tests-windows', help='Run tests on Windows', ) sp.add_argument( '--instance-type', help='EC2 instance type to use', default='m6i.large', ) sp.add_argument( '--python-version', help='Python version to use', choices={'2.7', '3.5', '3.6', '3.7', '3.8', '3.9'}, default='2.7', ) sp.add_argument( '--arch', help='Architecture to test', choices={'x86', 'x64'}, default='x64', ) sp.add_argument( '--test-flags', help='Extra command line flags to pass to run-tests.py', ) sp.add_argument( '--base-image-name', help='AMI name of base image', default=aws.WINDOWS_BASE_IMAGE_NAME, ) sp.set_defaults(func=run_tests_windows) sp = subparsers.add_parser( 'publish-windows-artifacts', help='Publish built Windows artifacts (wheels, installers, etc)', ) sp.add_argument( '--no-pypi', dest='pypi', action='store_false', default=True, help='Skip uploading to PyPI', ) sp.add_argument( '--no-mercurial-scm-org', dest='mercurial_scm_org', action='store_false', default=True, help='Skip uploading to www.mercurial-scm.org', ) sp.add_argument( '--ssh-username', help='SSH username for mercurial-scm.org', ) sp.add_argument( 'version', help='Mercurial version string to locate local packages', ) sp.set_defaults(func=publish_windows_artifacts) sp = subparsers.add_parser( 'try', help='Run CI automation against a custom changeset' ) sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on') sp.set_defaults(func=run_try) return parser def main(): parser = get_parser() args = parser.parse_args() local_state_path = pathlib.Path(os.path.expanduser(args.state_path)) automation = HGAutomation(local_state_path) if not hasattr(args, 'func'): parser.print_help() return kwargs = dict(vars(args)) del kwargs['func'] del kwargs['state_path'] args.func(automation, **kwargs)