cli.py
424 lines
| 12.2 KiB
| text/x-python
|
PythonLexer
|
r42191 | # cli.py - Command line interface for automation | ||
# | ||||
# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
# no-check-code because Python 3 native. | ||||
import argparse | ||||
|
r42471 | import concurrent.futures as futures | ||
|
r42191 | import os | ||
import pathlib | ||||
|
r42471 | import time | ||
|
r42191 | |||
from . import ( | ||||
aws, | ||||
HGAutomation, | ||||
|
r42471 | linux, | ||
|
r42191 | windows, | ||
) | ||||
SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent | ||||
DIST_PATH = SOURCE_ROOT / 'dist' | ||||
|
r42471 | def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None, | ||
parallel=False): | ||||
c = hga.aws_connection(aws_region) | ||||
if distros: | ||||
distros = distros.split(',') | ||||
else: | ||||
distros = sorted(linux.DISTROS) | ||||
# TODO There is a wonky interaction involving KeyboardInterrupt whereby | ||||
# the context manager that is supposed to terminate the temporary EC2 | ||||
# instance doesn't run. Until we fix this, make parallel building opt-in | ||||
# so we don't orphan instances. | ||||
if parallel: | ||||
fs = [] | ||||
with futures.ThreadPoolExecutor(len(distros)) as e: | ||||
for distro in distros: | ||||
fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro)) | ||||
for f in fs: | ||||
f.result() | ||||
else: | ||||
for distro in distros: | ||||
aws.ensure_linux_dev_ami(c, distro=distro) | ||||
|
r42871 | def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | print('Windows development AMI available as %s' % image.id) | ||
|
r42871 | def build_inno(hga: HGAutomation, aws_region, arch, revision, version, | ||
base_image_name): | ||||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
for a in arch: | ||||
windows.build_inno_installer(instance.winrm_client, a, | ||||
DIST_PATH, | ||||
version=version) | ||||
|
r42871 | def build_wix(hga: HGAutomation, aws_region, arch, revision, version, | ||
base_image_name): | ||||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
for a in arch: | ||||
windows.build_wix_installer(instance.winrm_client, a, | ||||
DIST_PATH, version=version) | ||||
|
r42871 | def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision, | ||
base_image_name): | ||||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
for a in arch: | ||||
windows.build_wheel(instance.winrm_client, a, DIST_PATH) | ||||
|
r42469 | def build_all_windows_packages(hga: HGAutomation, aws_region, revision, | ||
|
r42871 | version, base_image_name): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | DIST_PATH.mkdir(exist_ok=True) | ||
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: | ||||
instance = insts[0] | ||||
winrm_client = instance.winrm_client | ||||
windows.synchronize_hg(SOURCE_ROOT, revision, instance) | ||||
for arch in ('x86', 'x64'): | ||||
windows.purge_hg(winrm_client) | ||||
windows.build_wheel(winrm_client, arch, DIST_PATH) | ||||
windows.purge_hg(winrm_client) | ||||
|
r42469 | windows.build_inno_installer(winrm_client, arch, DIST_PATH, | ||
version=version) | ||||
|
r42191 | windows.purge_hg(winrm_client) | ||
|
r42469 | windows.build_wix_installer(winrm_client, arch, DIST_PATH, | ||
version=version) | ||||
|
r42191 | |||
def terminate_ec2_instances(hga: HGAutomation, aws_region): | ||||
|
r42463 | c = hga.aws_connection(aws_region, ensure_ec2_state=False) | ||
|
r42191 | aws.terminate_ec2_instances(c.ec2resource) | ||
def purge_ec2_resources(hga: HGAutomation, aws_region): | ||||
|
r42463 | c = hga.aws_connection(aws_region, ensure_ec2_state=False) | ||
|
r42191 | aws.remove_resources(c) | ||
|
r42471 | def run_tests_linux(hga: HGAutomation, aws_region, instance_type, | ||
python_version, test_flags, distro, filesystem): | ||||
c = hga.aws_connection(aws_region) | ||||
image = aws.ensure_linux_dev_ami(c, distro=distro) | ||||
t_start = time.time() | ||||
ensure_extra_volume = filesystem not in ('default', 'tmpfs') | ||||
with aws.temporary_linux_dev_instances( | ||||
c, image, instance_type, | ||||
ensure_extra_volume=ensure_extra_volume) as insts: | ||||
instance = insts[0] | ||||
linux.prepare_exec_environment(instance.ssh_client, | ||||
filesystem=filesystem) | ||||
linux.synchronize_hg(SOURCE_ROOT, instance, '.') | ||||
t_prepared = time.time() | ||||
linux.run_tests(instance.ssh_client, python_version, | ||||
test_flags) | ||||
t_done = time.time() | ||||
t_setup = t_prepared - t_start | ||||
t_all = t_done - t_start | ||||
print( | ||||
'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' | ||||
% (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)) | ||||
|
r42191 | def run_tests_windows(hga: HGAutomation, aws_region, instance_type, | ||
|
r42871 | python_version, arch, test_flags, base_image_name): | ||
|
r42191 | c = hga.aws_connection(aws_region) | ||
|
r42871 | image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) | ||
|
r42191 | |||
with aws.temporary_windows_dev_instances(c, image, instance_type, | ||||
disable_antivirus=True) as insts: | ||||
instance = insts[0] | ||||
windows.synchronize_hg(SOURCE_ROOT, '.', instance) | ||||
windows.run_tests(instance.winrm_client, python_version, arch, | ||||
test_flags) | ||||
def get_parser(): | ||||
parser = argparse.ArgumentParser() | ||||
parser.add_argument( | ||||
'--state-path', | ||||
default='~/.hgautomation', | ||||
help='Path for local state files', | ||||
) | ||||
parser.add_argument( | ||||
'--aws-region', | ||||
help='AWS region to use', | ||||
default='us-west-1', | ||||
) | ||||
subparsers = parser.add_subparsers() | ||||
sp = subparsers.add_parser( | ||||
|
r42471 | 'bootstrap-linux-dev', | ||
help='Bootstrap Linux development environments', | ||||
) | ||||
sp.add_argument( | ||||
'--distros', | ||||
help='Comma delimited list of distros to bootstrap', | ||||
) | ||||
sp.add_argument( | ||||
'--parallel', | ||||
action='store_true', | ||||
help='Generate AMIs in parallel (not CTRL-c safe)' | ||||
) | ||||
sp.set_defaults(func=bootstrap_linux_dev) | ||||
sp = subparsers.add_parser( | ||||
|
r42191 | 'bootstrap-windows-dev', | ||
help='Bootstrap the Windows development environment', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=bootstrap_windows_dev) | ||
sp = subparsers.add_parser( | ||||
'build-all-windows-packages', | ||||
help='Build all Windows packages', | ||||
) | ||||
sp.add_argument( | ||||
'--revision', | ||||
help='Mercurial revision to build', | ||||
default='.', | ||||
) | ||||
|
r42469 | sp.add_argument( | ||
'--version', | ||||
help='Mercurial version string to use', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_all_windows_packages) | ||
sp = subparsers.add_parser( | ||||
'build-inno', | ||||
help='Build Inno Setup installer(s)', | ||||
) | ||||
sp.add_argument( | ||||
'--arch', | ||||
help='Architecture to build for', | ||||
choices={'x86', 'x64'}, | ||||
nargs='*', | ||||
default=['x64'], | ||||
) | ||||
sp.add_argument( | ||||
'--revision', | ||||
help='Mercurial revision to build', | ||||
default='.', | ||||
) | ||||
sp.add_argument( | ||||
'--version', | ||||
help='Mercurial version string to use in installer', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_inno) | ||
sp = subparsers.add_parser( | ||||
'build-windows-wheel', | ||||
help='Build Windows wheel(s)', | ||||
) | ||||
sp.add_argument( | ||||
'--arch', | ||||
help='Architecture to build for', | ||||
choices={'x86', 'x64'}, | ||||
nargs='*', | ||||
default=['x64'], | ||||
) | ||||
sp.add_argument( | ||||
'--revision', | ||||
help='Mercurial revision to build', | ||||
default='.', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_windows_wheel) | ||
sp = subparsers.add_parser( | ||||
'build-wix', | ||||
help='Build WiX installer(s)' | ||||
) | ||||
sp.add_argument( | ||||
'--arch', | ||||
help='Architecture to build for', | ||||
choices={'x86', 'x64'}, | ||||
nargs='*', | ||||
default=['x64'], | ||||
) | ||||
sp.add_argument( | ||||
'--revision', | ||||
help='Mercurial revision to build', | ||||
default='.', | ||||
) | ||||
sp.add_argument( | ||||
'--version', | ||||
help='Mercurial version string to use in installer', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=build_wix) | ||
sp = subparsers.add_parser( | ||||
'terminate-ec2-instances', | ||||
help='Terminate all active EC2 instances managed by us', | ||||
) | ||||
sp.set_defaults(func=terminate_ec2_instances) | ||||
sp = subparsers.add_parser( | ||||
'purge-ec2-resources', | ||||
help='Purge all EC2 resources managed by us', | ||||
) | ||||
sp.set_defaults(func=purge_ec2_resources) | ||||
sp = subparsers.add_parser( | ||||
|
r42471 | 'run-tests-linux', | ||
help='Run tests on Linux', | ||||
) | ||||
sp.add_argument( | ||||
'--distro', | ||||
help='Linux distribution to run tests on', | ||||
choices=linux.DISTROS, | ||||
default='debian9', | ||||
) | ||||
sp.add_argument( | ||||
'--filesystem', | ||||
help='Filesystem type to use', | ||||
choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'}, | ||||
default='default', | ||||
) | ||||
sp.add_argument( | ||||
'--instance-type', | ||||
help='EC2 instance type to use', | ||||
default='c5.9xlarge', | ||||
) | ||||
sp.add_argument( | ||||
'--python-version', | ||||
help='Python version to use', | ||||
choices={'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8', | ||||
'pypy', 'pypy3.5', 'pypy3.6'}, | ||||
default='system2', | ||||
) | ||||
sp.add_argument( | ||||
'test_flags', | ||||
help='Extra command line flags to pass to run-tests.py', | ||||
nargs='*', | ||||
) | ||||
sp.set_defaults(func=run_tests_linux) | ||||
sp = subparsers.add_parser( | ||||
|
r42191 | 'run-tests-windows', | ||
help='Run tests on Windows', | ||||
) | ||||
sp.add_argument( | ||||
'--instance-type', | ||||
help='EC2 instance type to use', | ||||
default='t3.medium', | ||||
) | ||||
sp.add_argument( | ||||
'--python-version', | ||||
help='Python version to use', | ||||
choices={'2.7', '3.5', '3.6', '3.7', '3.8'}, | ||||
default='2.7', | ||||
) | ||||
sp.add_argument( | ||||
'--arch', | ||||
help='Architecture to test', | ||||
choices={'x86', 'x64'}, | ||||
default='x64', | ||||
) | ||||
sp.add_argument( | ||||
'--test-flags', | ||||
help='Extra command line flags to pass to run-tests.py', | ||||
) | ||||
|
r42871 | sp.add_argument( | ||
'--base-image-name', | ||||
help='AMI name of base image', | ||||
default=aws.WINDOWS_BASE_IMAGE_NAME, | ||||
) | ||||
|
r42191 | sp.set_defaults(func=run_tests_windows) | ||
return parser | ||||
def main(): | ||||
parser = get_parser() | ||||
args = parser.parse_args() | ||||
local_state_path = pathlib.Path(os.path.expanduser(args.state_path)) | ||||
automation = HGAutomation(local_state_path) | ||||
if not hasattr(args, 'func'): | ||||
parser.print_help() | ||||
return | ||||
kwargs = dict(vars(args)) | ||||
del kwargs['func'] | ||||
del kwargs['state_path'] | ||||
args.func(automation, **kwargs) | ||||