cli.py
574 lines
| 14.9 KiB
| text/x-python
|
PythonLexer
|
r42191 | # cli.py - Command line interface for automation | ||
# | ||||
# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
# no-check-code because Python 3 native. | ||||
import argparse | ||||
|
r42471 | import concurrent.futures as futures | ||
|
r42191 | import os | ||
import pathlib | ||||
|
r42471 | import time | ||
|
r42191 | |||
from . import ( | ||||
aws, | ||||
HGAutomation, | ||||
|
r42471 | linux, | ||
|
r43327 | try_server, | ||
|
r42191 | windows, | ||
) | ||||
|
r43346 | SOURCE_ROOT = pathlib.Path( | ||
os.path.abspath(__file__) | ||||
).parent.parent.parent.parent | ||||
|
r42191 | DIST_PATH = SOURCE_ROOT / 'dist' | ||
|
r43346 | def bootstrap_linux_dev( | ||
hga: HGAutomation, aws_region, distros=None, parallel=False | ||||
): | ||||
|
r42471 | c = hga.aws_connection(aws_region) | ||
if distros: | ||||
distros = distros.split(',') | ||||
else: | ||||
distros = sorted(linux.DISTROS) | ||||
# TODO There is a wonky interaction involving KeyboardInterrupt whereby | ||||
# the context manager that is supposed to terminate the temporary EC2 | ||||
# instance doesn't run. Until we fix this, make parallel building opt-in | ||||
# so we don't orphan instances. | ||||
if parallel: | ||||
fs = [] | ||||
with futures.ThreadPoolExecutor(len(distros)) as e: | ||||
for distro in distros: | ||||
fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro)) | ||||
for f in fs: | ||||
f.result() | ||||
else: | ||||
for distro in distros: | ||||
aws.ensure_linux_dev_ami(c, distro=distro) | ||||
|
r42871 | def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | print('Windows development AMI available as %s' % image.id) | ||
|
r43346 | def build_inno( | ||
|
r45278 | hga: HGAutomation, | ||
aws_region, | ||||
python_version, | ||||
arch, | ||||
revision, | ||||
version, | ||||
base_image_name, | ||||
|
r43346 | ): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
|
r45278 | for py_version in python_version: | ||
for a in arch: | ||||
windows.build_inno_installer( | ||||
instance.winrm_client, | ||||
py_version, | ||||
a, | ||||
DIST_PATH, | ||||
version=version, | ||||
) | ||||
|
r42191 | |||
|
r43346 | def build_wix( | ||
|
r45279 | hga: HGAutomation, | ||
aws_region, | ||||
python_version, | ||||
arch, | ||||
revision, | ||||
version, | ||||
base_image_name, | ||||
|
r43346 | ): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
|
r45279 | for py_version in python_version: | ||
for a in arch: | ||||
windows.build_wix_installer( | ||||
instance.winrm_client, | ||||
py_version, | ||||
a, | ||||
DIST_PATH, | ||||
version=version, | ||||
) | ||||
|
r42191 | |||
|
r43346 | def build_windows_wheel( | ||
|
r45275 | hga: HGAutomation, | ||
aws_region, | ||||
python_version, | ||||
arch, | ||||
revision, | ||||
base_image_name, | ||||
|
r43346 | ): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
|
r45275 | for py_version in python_version: | ||
for a in arch: | ||||
windows.build_wheel( | ||||
instance.winrm_client, py_version, a, DIST_PATH | ||||
) | ||||
|
r42191 | |||
|
r43346 | def build_all_windows_packages( | ||
hga: HGAutomation, aws_region, revision, version, base_image_name | ||||
): | ||||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
|
r49638 | with aws.temporary_windows_dev_instances(c, image, 'm6i.large') as insts: | ||
|
r42191 | instance = insts[0] | ||
winrm_client = instance.winrm_client | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
|
r49701 | for py_version in ("3.7", "3.8", "3.9", "3.10"): | ||
|
r45275 | for arch in ("x86", "x64"): | ||
windows.purge_hg(winrm_client) | ||||
windows.build_wheel( | ||||
winrm_client, | ||||
python_version=py_version, | ||||
arch=arch, | ||||
dest_path=DIST_PATH, | ||||
) | ||||
|
r45278 | for py_version in (2, 3): | ||
for arch in ('x86', 'x64'): | ||||
windows.purge_hg(winrm_client) | ||||
windows.build_inno_installer( | ||||
winrm_client, py_version, arch, DIST_PATH, version=version | ||||
) | ||||
|
r45279 | windows.build_wix_installer( | ||
winrm_client, py_version, arch, DIST_PATH, version=version | ||||
) | ||||
|
r42191 | |||
def terminate_ec2_instances(hga: HGAutomation, aws_region): | ||||
|
r42463 | c = hga.aws_connection(aws_region, ensure_ec2_state=False) | ||
|
r42191 | aws.terminate_ec2_instances(c.ec2resource) | ||
def purge_ec2_resources(hga: HGAutomation, aws_region): | ||||
|
r42463 | c = hga.aws_connection(aws_region, ensure_ec2_state=False) | ||
|
r42191 | aws.remove_resources(c) | ||
|
r43346 | def run_tests_linux( | ||
hga: HGAutomation, | ||||
aws_region, | ||||
instance_type, | ||||
python_version, | ||||
test_flags, | ||||
distro, | ||||
filesystem, | ||||
): | ||||
|
r42471 | c = hga.aws_connection(aws_region) | ||
image = aws.ensure_linux_dev_ami(c, distro=distro) | ||||
t_start = time.time() | ||||
ensure_extra_volume = filesystem not in ('default', 'tmpfs') | ||||
with aws.temporary_linux_dev_instances( | ||||
|
r43346 | c, image, instance_type, ensure_extra_volume=ensure_extra_volume | ||
) as insts: | ||||
|
r42471 | |||
instance = insts[0] | ||||
|
r43346 | linux.prepare_exec_environment( | ||
instance.ssh_client, filesystem=filesystem | ||||
) | ||||
|
r42471 | linux.synchronize_hg(SOURCE_ROOT, instance, '.') | ||
t_prepared = time.time() | ||||
|
r43346 | linux.run_tests(instance.ssh_client, python_version, test_flags) | ||
|
r42471 | t_done = time.time() | ||
t_setup = t_prepared - t_start | ||||
t_all = t_done - t_start | ||||
print( | ||||
'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' | ||||
|
r43346 | % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0) | ||
) | ||||
|
r42471 | |||
|
r43346 | def run_tests_windows( | ||
hga: HGAutomation, | ||||
aws_region, | ||||
instance_type, | ||||
python_version, | ||||
arch, | ||||
test_flags, | ||||
base_image_name, | ||||
): | ||||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | |||
|
r43346 | with aws.temporary_windows_dev_instances( | ||
c, image, instance_type, disable_antivirus=True | ||||
) as insts: | ||||
|
r42191 | instance = insts[0] | ||
windows.synchronize_hg(SOURCE_ROOT, '.', instance) | ||||
|
r43346 | windows.run_tests( | ||
instance.winrm_client, python_version, arch, test_flags | ||||
) | ||||
|
r42191 | |||
|
r43346 | def publish_windows_artifacts( | ||
hg: HGAutomation, | ||||
aws_region, | ||||
version: str, | ||||
pypi: bool, | ||||
mercurial_scm_org: bool, | ||||
ssh_username: str, | ||||
): | ||||
windows.publish_artifacts( | ||||
DIST_PATH, | ||||
version, | ||||
pypi=pypi, | ||||
mercurial_scm_org=mercurial_scm_org, | ||||
ssh_username=ssh_username, | ||||
) | ||||
|
r43177 | |||
|
r43327 | def run_try(hga: HGAutomation, aws_region: str, rev: str): | ||
c = hga.aws_connection(aws_region, ensure_ec2_state=False) | ||||
try_server.trigger_try(c, rev=rev) | ||||
|
r42191 | def get_parser(): | ||
parser = argparse.ArgumentParser() | ||||
parser.add_argument( | ||||
'--state-path', | ||||
default='~/.hgautomation', | ||||
help='Path for local state files', | ||||
) | ||||
parser.add_argument( | ||||
|
r46554 | '--aws-region', | ||
help='AWS region to use', | ||||
default='us-west-2', | ||||
|
r42191 | ) | ||
subparsers = parser.add_subparsers() | ||||
sp = subparsers.add_parser( | ||||
|
r46554 | 'bootstrap-linux-dev', | ||
help='Bootstrap Linux development environments', | ||||
|
r42471 | ) | ||
sp.add_argument( | ||||
|
r46554 | '--distros', | ||
help='Comma delimited list of distros to bootstrap', | ||||
|
r42471 | ) | ||
sp.add_argument( | ||||
'--parallel', | ||||
action='store_true', | ||||
|
r43346 | help='Generate AMIs in parallel (not CTRL-c safe)', | ||
|
r42471 | ) | ||
sp.set_defaults(func=bootstrap_linux_dev) | ||||
sp = subparsers.add_parser( | ||||
|
r42191 | 'bootstrap-windows-dev', | ||
help='Bootstrap the Windows development environment', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=bootstrap_windows_dev) | ||
sp = subparsers.add_parser( | ||||
|
r46554 | 'build-all-windows-packages', | ||
help='Build all Windows packages', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
|
r46554 | '--revision', | ||
help='Mercurial revision to build', | ||||
default='.', | ||||
|
r42191 | ) | ||
|
r42469 | sp.add_argument( | ||
|
r46554 | '--version', | ||
help='Mercurial version string to use', | ||||
|
r42469 | ) | ||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_all_windows_packages) | ||
sp = subparsers.add_parser( | ||||
|
r46554 | 'build-inno', | ||
help='Build Inno Setup installer(s)', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
|
r45278 | '--python-version', | ||
help='Which version of Python to target', | ||||
|
r49701 | choices={3}, | ||
|
r45278 | type=int, | ||
nargs='*', | ||||
default=[3], | ||||
) | ||||
sp.add_argument( | ||||
|
r42191 | '--arch', | ||
help='Architecture to build for', | ||||
choices={'x86', 'x64'}, | ||||
nargs='*', | ||||
default=['x64'], | ||||
) | ||||
sp.add_argument( | ||||
|
r46554 | '--revision', | ||
help='Mercurial revision to build', | ||||
default='.', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
|
r46554 | '--version', | ||
help='Mercurial version string to use in installer', | ||||
|
r42191 | ) | ||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_inno) | ||
sp = subparsers.add_parser( | ||||
|
r46554 | 'build-windows-wheel', | ||
help='Build Windows wheel(s)', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
|
r45275 | '--python-version', | ||
help='Python version to build for', | ||||
|
r49701 | choices={'3.7', '3.8', '3.9', '3.10'}, | ||
|
r45275 | nargs='*', | ||
default=['3.8'], | ||||
) | ||||
sp.add_argument( | ||||
|
r42191 | '--arch', | ||
help='Architecture to build for', | ||||
choices={'x86', 'x64'}, | ||||
nargs='*', | ||||
default=['x64'], | ||||
) | ||||
sp.add_argument( | ||||
|
r46554 | '--revision', | ||
help='Mercurial revision to build', | ||||
default='.', | ||||
|
r42191 | ) | ||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_windows_wheel) | ||
|
r43346 | sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)') | ||
|
r42191 | sp.add_argument( | ||
|
r45279 | '--python-version', | ||
help='Which version of Python to target', | ||||
|
r49701 | choices={3}, | ||
|
r45279 | type=int, | ||
nargs='*', | ||||
default=[3], | ||||
) | ||||
sp.add_argument( | ||||
|
r42191 | '--arch', | ||
help='Architecture to build for', | ||||
choices={'x86', 'x64'}, | ||||
nargs='*', | ||||
default=['x64'], | ||||
) | ||||
sp.add_argument( | ||||
|
r46554 | '--revision', | ||
help='Mercurial revision to build', | ||||
default='.', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
|
r46554 | '--version', | ||
help='Mercurial version string to use in installer', | ||||
|
r42191 | ) | ||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_wix) | ||
sp = subparsers.add_parser( | ||||
'terminate-ec2-instances', | ||||
help='Terminate all active EC2 instances managed by us', | ||||
) | ||||
sp.set_defaults(func=terminate_ec2_instances) | ||||
sp = subparsers.add_parser( | ||||
|
r46554 | 'purge-ec2-resources', | ||
help='Purge all EC2 resources managed by us', | ||||
|
r42191 | ) | ||
sp.set_defaults(func=purge_ec2_resources) | ||||
|
r46554 | sp = subparsers.add_parser( | ||
'run-tests-linux', | ||||
help='Run tests on Linux', | ||||
) | ||||
|
r42471 | sp.add_argument( | ||
'--distro', | ||||
help='Linux distribution to run tests on', | ||||
choices=linux.DISTROS, | ||||
|
r43288 | default='debian10', | ||
|
r42471 | ) | ||
sp.add_argument( | ||||
'--filesystem', | ||||
help='Filesystem type to use', | ||||
choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'}, | ||||
default='default', | ||||
) | ||||
sp.add_argument( | ||||
'--instance-type', | ||||
help='EC2 instance type to use', | ||||
default='c5.9xlarge', | ||||
) | ||||
sp.add_argument( | ||||
'--python-version', | ||||
help='Python version to use', | ||||
|
r43346 | choices={ | ||
'system3', | ||||
'3.5', | ||||
'3.6', | ||||
'3.7', | ||||
'3.8', | ||||
'pypy', | ||||
'pypy3.5', | ||||
'pypy3.6', | ||||
}, | ||||
|
r49699 | default='system3', | ||
|
r42471 | ) | ||
sp.add_argument( | ||||
'test_flags', | ||||
help='Extra command line flags to pass to run-tests.py', | ||||
nargs='*', | ||||
) | ||||
sp.set_defaults(func=run_tests_linux) | ||||
sp = subparsers.add_parser( | ||||
|
r46554 | 'run-tests-windows', | ||
help='Run tests on Windows', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
|
r46554 | '--instance-type', | ||
help='EC2 instance type to use', | ||||
|
r49638 | default='m6i.large', | ||
|
r42191 | ) | ||
sp.add_argument( | ||||
'--python-version', | ||||
help='Python version to use', | ||||
|
r49701 | choices={'3.5', '3.6', '3.7', '3.8', '3.9', '3.10'}, | ||
default='3.9', | ||||
|
r42191 | ) | ||
sp.add_argument( | ||||
'--arch', | ||||
help='Architecture to test', | ||||
choices={'x86', 'x64'}, | ||||
default='x64', | ||||
) | ||||
sp.add_argument( | ||||
|
r46554 | '--test-flags', | ||
help='Extra command line flags to pass to run-tests.py', | ||||
|
r42191 | ) | ||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=run_tests_windows) | ||
|
r43177 | sp = subparsers.add_parser( | ||
'publish-windows-artifacts', | ||||
|
r43346 | help='Publish built Windows artifacts (wheels, installers, etc)', | ||
|
r43177 | ) | ||
sp.add_argument( | ||||
'--no-pypi', | ||||
dest='pypi', | ||||
action='store_false', | ||||
default=True, | ||||
help='Skip uploading to PyPI', | ||||
) | ||||
sp.add_argument( | ||||
'--no-mercurial-scm-org', | ||||
dest='mercurial_scm_org', | ||||
action='store_false', | ||||
default=True, | ||||
help='Skip uploading to www.mercurial-scm.org', | ||||
) | ||||
sp.add_argument( | ||||
|
r46554 | '--ssh-username', | ||
help='SSH username for mercurial-scm.org', | ||||
|
r43177 | ) | ||
sp.add_argument( | ||||
|
r46554 | 'version', | ||
help='Mercurial version string to locate local packages', | ||||
|
r43177 | ) | ||
sp.set_defaults(func=publish_windows_artifacts) | ||||
|
r43327 | sp = subparsers.add_parser( | ||
|
r43346 | 'try', help='Run CI automation against a custom changeset' | ||
|
r43327 | ) | ||
|
r43346 | sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on') | ||
|
r43327 | sp.set_defaults(func=run_try) | ||
|
r42191 | return parser | ||
def main(): | ||||
parser = get_parser() | ||||
args = parser.parse_args() | ||||
local_state_path = pathlib.Path(os.path.expanduser(args.state_path)) | ||||
automation = HGAutomation(local_state_path) | ||||
if not hasattr(args, 'func'): | ||||
parser.print_help() | ||||
return | ||||
kwargs = dict(vars(args)) | ||||
del kwargs['func'] | ||||
del kwargs['state_path'] | ||||
args.func(automation, **kwargs) | ||||