automation: remove Ubuntu 18.10

It reached end of life on July 18, 2019. I don't think we
should care about supporting it.

Differential Revision: https://phab.mercurial-scm.org/D6912

contrib/automation/hgautomation/aws.py

@@ -1,1209 +1,1202 b''
 # aws.py - Automation code for Amazon Web Services
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import contextlib
 import copy
 import hashlib
 import json
 import os
 import pathlib
 import subprocess
 import time
 
 import boto3
 import botocore.exceptions
 
 from .linux import (
     BOOTSTRAP_DEBIAN,
 )
 from .ssh import (
     exec_command as ssh_exec_command,
     wait_for_ssh,
 )
 from .winrm import (
     run_powershell,
     wait_for_winrm,
 )
 
 
 SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent
 
 INSTALL_WINDOWS_DEPENDENCIES = (SOURCE_ROOT / 'contrib' /
                                 'install-windows-dependencies.ps1')
 
 
 INSTANCE_TYPES_WITH_STORAGE = {
     'c5d',
     'd2',
     'h1',
     'i3',
     'm5ad',
     'm5d',
     'r5d',
     'r5ad',
     'x1',
     'z1d',
 }
 
 
 AMAZON_ACCOUNT_ID = '801119661308'
 DEBIAN_ACCOUNT_ID = '379101102735'
 UBUNTU_ACCOUNT_ID = '099720109477'
 
 
 WINDOWS_BASE_IMAGE_NAME = 'Windows_Server-2019-English-Full-Base-2019.07.12'
 
 
 KEY_PAIRS = {
     'automation',
 }
 
 
 SECURITY_GROUPS = {
     'linux-dev-1': {
         'description': 'Mercurial Linux instances that perform build/test automation',
         'ingress': [
             {
                 'FromPort': 22,
                 'ToPort': 22,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'SSH from entire Internet',
                     },
                 ],
             },
         ],
     },
     'windows-dev-1': {
         'description': 'Mercurial Windows instances that perform build automation',
         'ingress': [
             {
                 'FromPort': 22,
                 'ToPort': 22,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'SSH from entire Internet',
                     },
                 ],
             },
             {
                 'FromPort': 3389,
                 'ToPort': 3389,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'RDP from entire Internet',
                     },
                 ],
 
             },
             {
                 'FromPort': 5985,
                 'ToPort': 5986,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'PowerShell Remoting (Windows Remote Management)',
                     },
                 ],
             }
         ],
     },
 }
 
 
 IAM_ROLES = {
     'ephemeral-ec2-role-1': {
         'description': 'Mercurial temporary EC2 instances',
         'policy_arns': [
             'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM',
         ],
     },
 }
 
 
 ASSUME_ROLE_POLICY_DOCUMENT = '''
 {
   "Version": "2012-10-17",
   "Statement": [
     {
       "Effect": "Allow",
       "Principal": {
         "Service": "ec2.amazonaws.com"
       },
       "Action": "sts:AssumeRole"
     }
   ]
 }
 '''.strip()
 
 
 IAM_INSTANCE_PROFILES = {
     'ephemeral-ec2-1': {
         'roles': [
             'ephemeral-ec2-role-1',
         ],
     }
 }
 
 
 # User Data for Windows EC2 instance. Mainly used to set the password
 # and configure WinRM.
 # Inspired by the User Data script used by Packer
 # (from https://www.packer.io/intro/getting-started/build-image.html).
 WINDOWS_USER_DATA = r'''
 <powershell>
 
 # TODO enable this once we figure out what is failing.
 #$ErrorActionPreference = "stop"
 
 # Set administrator password
 net user Administrator "%s"
 wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE
 
 # First, make sure WinRM can't be connected to
 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block
 
 # Delete any existing WinRM listeners
 winrm delete winrm/config/listener?Address=*+Transport=HTTP  2>$Null
 winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null
 
 # Create a new WinRM listener and configure
 winrm create winrm/config/listener?Address=*+Transport=HTTP
 winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}'
 winrm set winrm/config '@{MaxTimeoutms="7200000"}'
 winrm set winrm/config/service '@{AllowUnencrypted="true"}'
 winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}'
 winrm set winrm/config/service/auth '@{Basic="true"}'
 winrm set winrm/config/client/auth '@{Basic="true"}'
 
 # Configure UAC to allow privilege elevation in remote shells
 $Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System'
 $Setting = 'LocalAccountTokenFilterPolicy'
 Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force
 
 # Configure and restart the WinRM Service; Enable the required firewall exception
 Stop-Service -Name WinRM
 Set-Service -Name WinRM -StartupType Automatic
 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any
 Start-Service -Name WinRM
 
 # Disable firewall on private network interfaces so prompts don't appear.
 Set-NetFirewallProfile -Name private -Enabled false
 </powershell>
 '''.lstrip()
 
 
 WINDOWS_BOOTSTRAP_POWERSHELL = '''
 Write-Output "installing PowerShell dependencies"
 Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
 Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
 Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0
 
 Write-Output "installing OpenSSL server"
 Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
 # Various tools will attempt to use older versions of .NET. So we enable
 # the feature that provides them so it doesn't have to be auto-enabled
 # later.
 Write-Output "enabling .NET Framework feature"
 Install-WindowsFeature -Name Net-Framework-Core
 '''
 
 
 class AWSConnection:
     """Manages the state of a connection with AWS."""
 
     def __init__(self, automation, region: str, ensure_ec2_state: bool=True):
         self.automation = automation
         self.local_state_path = automation.state_path
 
         self.prefix = 'hg-'
 
         self.session = boto3.session.Session(region_name=region)
         self.ec2client = self.session.client('ec2')
         self.ec2resource = self.session.resource('ec2')
         self.iamclient = self.session.client('iam')
         self.iamresource = self.session.resource('iam')
         self.security_groups = {}
 
         if ensure_ec2_state:
             ensure_key_pairs(automation.state_path, self.ec2resource)
             self.security_groups = ensure_security_groups(self.ec2resource)
             ensure_iam_state(self.iamclient, self.iamresource)
 
     def key_pair_path_private(self, name):
         """Path to a key pair private key file."""
         return self.local_state_path / 'keys' / ('keypair-%s' % name)
 
     def key_pair_path_public(self, name):
         return self.local_state_path / 'keys' / ('keypair-%s.pub' % name)
 
 
 def rsa_key_fingerprint(p: pathlib.Path):
     """Compute the fingerprint of an RSA private key."""
 
     # TODO use rsa package.
     res = subprocess.run(
         ['openssl', 'pkcs8', '-in', str(p), '-nocrypt', '-topk8',
          '-outform', 'DER'],
         capture_output=True,
         check=True)
 
     sha1 = hashlib.sha1(res.stdout).hexdigest()
     return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2]))
 
 
 def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'):
     remote_existing = {}
 
     for kpi in ec2resource.key_pairs.all():
         if kpi.name.startswith(prefix):
             remote_existing[kpi.name[len(prefix):]] = kpi.key_fingerprint
 
     # Validate that we have these keys locally.
     key_path = state_path / 'keys'
     key_path.mkdir(exist_ok=True, mode=0o700)
 
     def remove_remote(name):
         print('deleting key pair %s' % name)
         key = ec2resource.KeyPair(name)
         key.delete()
 
     def remove_local(name):
         pub_full = key_path / ('keypair-%s.pub' % name)
         priv_full = key_path / ('keypair-%s' % name)
 
         print('removing %s' % pub_full)
         pub_full.unlink()
         print('removing %s' % priv_full)
         priv_full.unlink()
 
     local_existing = {}
 
     for f in sorted(os.listdir(key_path)):
         if not f.startswith('keypair-') or not f.endswith('.pub'):
             continue
 
         name = f[len('keypair-'):-len('.pub')]
 
         pub_full = key_path / f
         priv_full = key_path / ('keypair-%s' % name)
 
         with open(pub_full, 'r', encoding='ascii') as fh:
             data = fh.read()
 
         if not data.startswith('ssh-rsa '):
             print('unexpected format for key pair file: %s; removing' %
                   pub_full)
             pub_full.unlink()
             priv_full.unlink()
             continue
 
         local_existing[name] = rsa_key_fingerprint(priv_full)
 
     for name in sorted(set(remote_existing) | set(local_existing)):
         if name not in local_existing:
             actual = '%s%s' % (prefix, name)
             print('remote key %s does not exist locally' % name)
             remove_remote(actual)
             del remote_existing[name]
 
         elif name not in remote_existing:
             print('local key %s does not exist remotely' % name)
             remove_local(name)
             del local_existing[name]
 
         elif remote_existing[name] != local_existing[name]:
             print('key fingerprint mismatch for %s; '
                   'removing from local and remote' % name)
             remove_local(name)
             remove_remote('%s%s' % (prefix, name))
             del local_existing[name]
             del remote_existing[name]
 
     missing = KEY_PAIRS - set(remote_existing)
 
     for name in sorted(missing):
         actual = '%s%s' % (prefix, name)
         print('creating key pair %s' % actual)
 
         priv_full = key_path / ('keypair-%s' % name)
         pub_full = key_path / ('keypair-%s.pub' % name)
 
         kp = ec2resource.create_key_pair(KeyName=actual)
 
         with priv_full.open('w', encoding='ascii') as fh:
             fh.write(kp.key_material)
             fh.write('\n')
 
         priv_full.chmod(0o0600)
 
         # SSH public key can be extracted via `ssh-keygen`.
         with pub_full.open('w', encoding='ascii') as fh:
             subprocess.run(
                 ['ssh-keygen', '-y', '-f', str(priv_full)],
                 stdout=fh,
                 check=True)
 
         pub_full.chmod(0o0600)
 
 
 def delete_instance_profile(profile):
     for role in profile.roles:
         print('removing role %s from instance profile %s' % (role.name,
                                                              profile.name))
         profile.remove_role(RoleName=role.name)
 
     print('deleting instance profile %s' % profile.name)
     profile.delete()
 
 
 def ensure_iam_state(iamclient, iamresource, prefix='hg-'):
     """Ensure IAM state is in sync with our canonical definition."""
 
     remote_profiles = {}
 
     for profile in iamresource.instance_profiles.all():
         if profile.name.startswith(prefix):
             remote_profiles[profile.name[len(prefix):]] = profile
 
     for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)):
         delete_instance_profile(remote_profiles[name])
         del remote_profiles[name]
 
     remote_roles = {}
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
             remote_roles[role.name[len(prefix):]] = role
 
     for name in sorted(set(remote_roles) - set(IAM_ROLES)):
         role = remote_roles[name]
 
         print('removing role %s' % role.name)
         role.delete()
         del remote_roles[name]
 
     # We've purged remote state that doesn't belong. Create missing
     # instance profiles and roles.
     for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)):
         actual = '%s%s' % (prefix, name)
         print('creating IAM instance profile %s' % actual)
 
         profile = iamresource.create_instance_profile(
             InstanceProfileName=actual)
         remote_profiles[name] = profile
 
         waiter = iamclient.get_waiter('instance_profile_exists')
         waiter.wait(InstanceProfileName=actual)
         print('IAM instance profile %s is available' % actual)
 
     for name in sorted(set(IAM_ROLES) - set(remote_roles)):
         entry = IAM_ROLES[name]
 
         actual = '%s%s' % (prefix, name)
         print('creating IAM role %s' % actual)
 
         role = iamresource.create_role(
             RoleName=actual,
             Description=entry['description'],
             AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
         )
 
         waiter = iamclient.get_waiter('role_exists')
         waiter.wait(RoleName=actual)
         print('IAM role %s is available' % actual)
 
         remote_roles[name] = role
 
         for arn in entry['policy_arns']:
             print('attaching policy %s to %s' % (arn, role.name))
             role.attach_policy(PolicyArn=arn)
 
     # Now reconcile state of profiles.
     for name, meta in sorted(IAM_INSTANCE_PROFILES.items()):
         profile = remote_profiles[name]
         wanted = {'%s%s' % (prefix, role) for role in meta['roles']}
         have = {role.name for role in profile.roles}
 
         for role in sorted(have - wanted):
             print('removing role %s from %s' % (role, profile.name))
             profile.remove_role(RoleName=role)
 
         for role in sorted(wanted - have):
             print('adding role %s to %s' % (role, profile.name))
             profile.add_role(RoleName=role)
 
 
 def find_image(ec2resource, owner_id, name):
     """Find an AMI by its owner ID and name."""
 
     images = ec2resource.images.filter(
         Filters=[
             {
                 'Name': 'owner-id',
                 'Values': [owner_id],
             },
             {
                 'Name': 'state',
                 'Values': ['available'],
             },
             {
                 'Name': 'image-type',
                 'Values': ['machine'],
             },
             {
                 'Name': 'name',
                 'Values': [name],
             },
         ])
 
     for image in images:
         return image
 
     raise Exception('unable to find image for %s' % name)
 
 
 def ensure_security_groups(ec2resource, prefix='hg-'):
     """Ensure all necessary Mercurial security groups are present.
 
     All security groups are prefixed with ``hg-`` by default. Any security
     groups having this prefix but aren't in our list are deleted.
     """
     existing = {}
 
     for group in ec2resource.security_groups.all():
         if group.group_name.startswith(prefix):
             existing[group.group_name[len(prefix):]] = group
 
     purge = set(existing) - set(SECURITY_GROUPS)
 
     for name in sorted(purge):
         group = existing[name]
         print('removing legacy security group: %s' % group.group_name)
         group.delete()
 
     security_groups = {}
 
     for name, group in sorted(SECURITY_GROUPS.items()):
         if name in existing:
             security_groups[name] = existing[name]
             continue
 
         actual = '%s%s' % (prefix, name)
         print('adding security group %s' % actual)
 
         group_res = ec2resource.create_security_group(
             Description=group['description'],
             GroupName=actual,
         )
 
         group_res.authorize_ingress(
             IpPermissions=group['ingress'],
         )
 
         security_groups[name] = group_res
 
     return security_groups
 
 
 def terminate_ec2_instances(ec2resource, prefix='hg-'):
     """Terminate all EC2 instances managed by us."""
     waiting = []
 
     for instance in ec2resource.instances.all():
         if instance.state['Name'] == 'terminated':
             continue
 
         for tag in instance.tags or []:
             if tag['Key'] == 'Name' and tag['Value'].startswith(prefix):
                 print('terminating %s' % instance.id)
                 instance.terminate()
                 waiting.append(instance)
 
     for instance in waiting:
         instance.wait_until_terminated()
 
 
 def remove_resources(c, prefix='hg-'):
     """Purge all of our resources in this EC2 region."""
     ec2resource = c.ec2resource
     iamresource = c.iamresource
 
     terminate_ec2_instances(ec2resource, prefix=prefix)
 
     for image in ec2resource.images.filter(Owners=['self']):
         if image.name.startswith(prefix):
             remove_ami(ec2resource, image)
 
     for group in ec2resource.security_groups.all():
         if group.group_name.startswith(prefix):
             print('removing security group %s' % group.group_name)
             group.delete()
 
     for profile in iamresource.instance_profiles.all():
         if profile.name.startswith(prefix):
             delete_instance_profile(profile)
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
             for p in role.attached_policies.all():
                 print('detaching policy %s from %s' % (p.arn, role.name))
                 role.detach_policy(PolicyArn=p.arn)
 
             print('removing role %s' % role.name)
             role.delete()
 
 
 def wait_for_ip_addresses(instances):
     """Wait for the public IP addresses of an iterable of instances."""
     for instance in instances:
         while True:
             if not instance.public_ip_address:
                 time.sleep(2)
                 instance.reload()
                 continue
 
             print('public IP address for %s: %s' % (
                 instance.id, instance.public_ip_address))
             break
 
 
 def remove_ami(ec2resource, image):
     """Remove an AMI and its underlying snapshots."""
     snapshots = []
 
     for device in image.block_device_mappings:
         if 'Ebs' in device:
             snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId']))
 
     print('deregistering %s' % image.id)
     image.deregister()
 
     for snapshot in snapshots:
         print('deleting snapshot %s' % snapshot.id)
         snapshot.delete()
 
 
 def wait_for_ssm(ssmclient, instances):
     """Wait for SSM to come online for an iterable of instance IDs."""
     while True:
         res = ssmclient.describe_instance_information(
             Filters=[
                 {
                     'Key': 'InstanceIds',
                     'Values': [i.id for i in instances],
                 },
             ],
         )
 
         available = len(res['InstanceInformationList'])
         wanted = len(instances)
 
         print('%d/%d instances available in SSM' % (available, wanted))
 
         if available == wanted:
             return
 
         time.sleep(2)
 
 
 def run_ssm_command(ssmclient, instances, document_name, parameters):
     """Run a PowerShell script on an EC2 instance."""
 
     res = ssmclient.send_command(
         InstanceIds=[i.id for i in instances],
         DocumentName=document_name,
         Parameters=parameters,
         CloudWatchOutputConfig={
             'CloudWatchOutputEnabled': True,
         },
     )
 
     command_id = res['Command']['CommandId']
 
     for instance in instances:
         while True:
             try:
                 res = ssmclient.get_command_invocation(
                     CommandId=command_id,
                     InstanceId=instance.id,
                 )
             except botocore.exceptions.ClientError as e:
                 if e.response['Error']['Code'] == 'InvocationDoesNotExist':
                     print('could not find SSM command invocation; waiting')
                     time.sleep(1)
                     continue
                 else:
                     raise
 
             if res['Status'] == 'Success':
                 break
             elif res['Status'] in ('Pending', 'InProgress', 'Delayed'):
                 time.sleep(2)
             else:
                 raise Exception('command failed on %s: %s' % (
                     instance.id, res['Status']))
 
 
 @contextlib.contextmanager
 def temporary_ec2_instances(ec2resource, config):
     """Create temporary EC2 instances.
 
     This is a proxy to ``ec2client.run_instances(**config)`` that takes care of
     managing the lifecycle of the instances.
 
     When the context manager exits, the instances are terminated.
 
     The context manager evaluates to the list of data structures
     describing each created instance. The instances may not be available
     for work immediately: it is up to the caller to wait for the instance
     to start responding.
     """
 
     ids = None
 
     try:
         res = ec2resource.create_instances(**config)
 
         ids = [i.id for i in res]
         print('started instances: %s' % ' '.join(ids))
 
         yield res
     finally:
         if ids:
             print('terminating instances: %s' % ' '.join(ids))
             for instance in res:
                 instance.terminate()
             print('terminated %d instances' % len(ids))
 
 
 @contextlib.contextmanager
 def create_temp_windows_ec2_instances(c: AWSConnection, config):
     """Create temporary Windows EC2 instances.
 
     This is a higher-level wrapper around ``create_temp_ec2_instances()`` that
     configures the Windows instance for Windows Remote Management. The emitted
     instances will have a ``winrm_client`` attribute containing a
     ``pypsrp.client.Client`` instance bound to the instance.
     """
     if 'IamInstanceProfile' in config:
         raise ValueError('IamInstanceProfile cannot be provided in config')
     if 'UserData' in config:
         raise ValueError('UserData cannot be provided in config')
 
     password = c.automation.default_password()
 
     config = copy.deepcopy(config)
     config['IamInstanceProfile'] = {
         'Name': 'hg-ephemeral-ec2-1',
     }
     config.setdefault('TagSpecifications', []).append({
         'ResourceType': 'instance',
         'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}],
     })
     config['UserData'] = WINDOWS_USER_DATA % password
 
     with temporary_ec2_instances(c.ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         print('waiting for Windows Remote Management service...')
 
         for instance in instances:
             client = wait_for_winrm(instance.public_ip_address, 'Administrator', password)
             print('established WinRM connection to %s' % instance.id)
             instance.winrm_client = client
 
         yield instances
 
 
 def resolve_fingerprint(fingerprint):
     fingerprint = json.dumps(fingerprint, sort_keys=True)
     return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
 
 
 def find_and_reconcile_image(ec2resource, name, fingerprint):
     """Attempt to find an existing EC2 AMI with a name and fingerprint.
 
     If an image with the specified fingerprint is found, it is returned.
     Otherwise None is returned.
 
     Existing images for the specified name that don't have the specified
     fingerprint or are missing required metadata or deleted.
     """
     # Find existing AMIs with this name and delete the ones that are invalid.
     # Store a reference to a good image so it can be returned one the
     # image state is reconciled.
     images = ec2resource.images.filter(
         Filters=[{'Name': 'name', 'Values': [name]}])
 
     existing_image = None
 
     for image in images:
         if image.tags is None:
             print('image %s for %s lacks required tags; removing' % (
                 image.id, image.name))
             remove_ami(ec2resource, image)
         else:
             tags = {t['Key']: t['Value'] for t in image.tags}
 
             if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
                 existing_image = image
             else:
                 print('image %s for %s has wrong fingerprint; removing' % (
                       image.id, image.name))
                 remove_ami(ec2resource, image)
 
     return existing_image
 
 
 def create_ami_from_instance(ec2client, instance, name, description,
                              fingerprint):
     """Create an AMI from a running instance.
 
     Returns the ``ec2resource.Image`` representing the created AMI.
     """
     instance.stop()
 
     ec2client.get_waiter('instance_stopped').wait(
         InstanceIds=[instance.id],
         WaiterConfig={
             'Delay': 5,
         })
     print('%s is stopped' % instance.id)
 
     image = instance.create_image(
         Name=name,
         Description=description,
     )
 
     image.create_tags(Tags=[
         {
             'Key': 'HGIMAGEFINGERPRINT',
             'Value': fingerprint,
         },
     ])
 
     print('waiting for image %s' % image.id)
 
     ec2client.get_waiter('image_available').wait(
         ImageIds=[image.id],
     )
 
     print('image %s available as %s' % (image.id, image.name))
 
     return image
 
 
 def ensure_linux_dev_ami(c: AWSConnection, distro='debian9', prefix='hg-'):
     """Ensures a Linux development AMI is available and up-to-date.
 
     Returns an ``ec2.Image`` of either an existing AMI or a newly-built one.
     """
     ec2client = c.ec2client
     ec2resource = c.ec2resource
 
     name = '%s%s-%s' % (prefix, 'linux-dev', distro)
 
     if distro == 'debian9':
         image = find_image(
             ec2resource,
             DEBIAN_ACCOUNT_ID,
             'debian-stretch-hvm-x86_64-gp2-2019-02-19-26620',
         )
         ssh_username = 'admin'
     elif distro == 'ubuntu18.04':
         image = find_image(
             ec2resource,
             UBUNTU_ACCOUNT_ID,
             'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190403',
         )
         ssh_username = 'ubuntu'
-    elif distro == 'ubuntu18.10':
-        image = find_image(
-            ec2resource,
-            UBUNTU_ACCOUNT_ID,
-            'ubuntu/images/hvm-ssd/ubuntu-cosmic-18.10-amd64-server-20190402',
-        )
-        ssh_username = 'ubuntu'
     elif distro == 'ubuntu19.04':
         image = find_image(
             ec2resource,
             UBUNTU_ACCOUNT_ID,
             'ubuntu/images/hvm-ssd/ubuntu-disco-19.04-amd64-server-20190417',
         )
         ssh_username = 'ubuntu'
     else:
         raise ValueError('unsupported Linux distro: %s' % distro)
 
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': image.block_device_mappings[0]['DeviceName'],
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 8,
                     'VolumeType': 'gp2',
                 },
             },
         ],
         'EbsOptimized': True,
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         # 8 VCPUs for compiling Python.
         'InstanceType': 't3.2xlarge',
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
     }
 
     requirements2_path = (pathlib.Path(__file__).parent.parent /
                           'linux-requirements-py2.txt')
     requirements3_path = (pathlib.Path(__file__).parent.parent /
                           'linux-requirements-py3.txt')
     with requirements2_path.open('r', encoding='utf-8') as fh:
         requirements2 = fh.read()
     with requirements3_path.open('r', encoding='utf-8') as fh:
         requirements3 = fh.read()
 
     # Compute a deterministic fingerprint to determine whether image needs to
     # be regenerated.
     fingerprint = resolve_fingerprint({
         'instance_config': config,
         'bootstrap_script': BOOTSTRAP_DEBIAN,
         'requirements_py2': requirements2,
         'requirements_py3': requirements3,
     })
 
     existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
 
     print('no suitable %s image found; creating one...' % name)
 
     with temporary_ec2_instances(ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         instance = instances[0]
 
         client = wait_for_ssh(
             instance.public_ip_address, 22,
             username=ssh_username,
             key_filename=str(c.key_pair_path_private('automation')))
 
         home = '/home/%s' % ssh_username
 
         with client:
             print('connecting to SSH server')
             sftp = client.open_sftp()
 
             print('uploading bootstrap files')
             with sftp.open('%s/bootstrap' % home, 'wb') as fh:
                 fh.write(BOOTSTRAP_DEBIAN)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py2.txt' % home, 'wb') as fh:
                 fh.write(requirements2)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
                 fh.write(requirements3)
                 fh.chmod(0o0700)
 
             print('executing bootstrap')
             chan, stdin, stdout = ssh_exec_command(client,
                                                    '%s/bootstrap' % home)
             stdin.close()
 
             for line in stdout:
                 print(line, end='')
 
             res = chan.recv_exit_status()
             if res:
                 raise Exception('non-0 exit from bootstrap: %d' % res)
 
             print('bootstrap completed; stopping %s to create %s' % (
                   instance.id, name))
 
         return create_ami_from_instance(ec2client, instance, name,
                                         'Mercurial Linux development environment',
                                         fingerprint)
 
 
 @contextlib.contextmanager
 def temporary_linux_dev_instances(c: AWSConnection, image, instance_type,
                                   prefix='hg-', ensure_extra_volume=False):
     """Create temporary Linux development EC2 instances.
 
     Context manager resolves to a list of ``ec2.Instance`` that were created
     and are running.
 
     ``ensure_extra_volume`` can be set to ``True`` to require that instances
     have a 2nd storage volume available other than the primary AMI volume.
     For instance types with instance storage, this does nothing special.
     But for instance types without instance storage, an additional EBS volume
     will be added to the instance.
 
     Instances have an ``ssh_client`` attribute containing a paramiko SSHClient
     instance bound to the instance.
 
     Instances have an ``ssh_private_key_path`` attributing containing the
     str path to the SSH private key to connect to the instance.
     """
 
     block_device_mappings = [
         {
             'DeviceName': image.block_device_mappings[0]['DeviceName'],
             'Ebs': {
                 'DeleteOnTermination': True,
                 'VolumeSize': 12,
                 'VolumeType': 'gp2',
             },
         }
     ]
 
     # This is not an exhaustive list of instance types having instance storage.
     # But
     if (ensure_extra_volume
         and not instance_type.startswith(tuple(INSTANCE_TYPES_WITH_STORAGE))):
         main_device = block_device_mappings[0]['DeviceName']
 
         if main_device == 'xvda':
             second_device = 'xvdb'
         elif main_device == '/dev/sda1':
             second_device = '/dev/sdb'
         else:
             raise ValueError('unhandled primary EBS device name: %s' %
                              main_device)
 
         block_device_mappings.append({
             'DeviceName': second_device,
             'Ebs': {
                 'DeleteOnTermination': True,
                 'VolumeSize': 8,
                 'VolumeType': 'gp2',
             }
         })
 
     config = {
         'BlockDeviceMappings': block_device_mappings,
         'EbsOptimized': True,
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'terminate',
         'InstanceType': instance_type,
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
     }
 
     with temporary_ec2_instances(c.ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         ssh_private_key_path = str(c.key_pair_path_private('automation'))
 
         for instance in instances:
             client = wait_for_ssh(
                 instance.public_ip_address, 22,
                 username='hg',
                 key_filename=ssh_private_key_path)
 
             instance.ssh_client = client
             instance.ssh_private_key_path = ssh_private_key_path
 
         try:
             yield instances
         finally:
             for instance in instances:
                 instance.ssh_client.close()
 
 
 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-',
                            base_image_name=WINDOWS_BASE_IMAGE_NAME):
     """Ensure Windows Development AMI is available and up-to-date.
 
     If necessary, a modern AMI will be built by starting a temporary EC2
     instance and bootstrapping it.
 
     Obsolete AMIs will be deleted so there is only a single AMI having the
     desired name.
 
     Returns an ``ec2.Image`` of either an existing AMI or a newly-built
     one.
     """
     ec2client = c.ec2client
     ec2resource = c.ec2resource
     ssmclient = c.session.client('ssm')
 
     name = '%s%s' % (prefix, 'windows-dev')
 
     image = find_image(ec2resource, AMAZON_ACCOUNT_ID, base_image_name)
 
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': '/dev/sda1',
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 32,
                     'VolumeType': 'gp2',
                 },
             }
         ],
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         'InstanceType': 't3.medium',
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
     }
 
     commands = [
         # Need to start the service so sshd_config is generated.
         'Start-Service sshd',
         'Write-Output "modifying sshd_config"',
         r'$content = Get-Content C:\ProgramData\ssh\sshd_config',
         '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""',
         r'$content | Set-Content C:\ProgramData\ssh\sshd_config',
         'Import-Module OpenSSHUtils',
         r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false',
         'Restart-Service sshd',
         'Write-Output "installing OpenSSL client"',
         'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0',
         'Set-Service -Name sshd -StartupType "Automatic"',
         'Write-Output "OpenSSH server running"',
     ]
 
     with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh:
         commands.extend(l.rstrip() for l in fh)
 
     # Disable Windows Defender when bootstrapping because it just slows
     # things down.
     commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true')
     commands.append('Set-MpPreference -DisableRealtimeMonitoring $false')
 
     # Compute a deterministic fingerprint to determine whether image needs
     # to be regenerated.
     fingerprint = resolve_fingerprint({
         'instance_config': config,
         'user_data': WINDOWS_USER_DATA,
         'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
         'bootstrap_commands': commands,
         'base_image_name': base_image_name,
     })
 
     existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
 
     print('no suitable Windows development image found; creating one...')
 
     with create_temp_windows_ec2_instances(c, config) as instances:
         assert len(instances) == 1
         instance = instances[0]
 
         wait_for_ssm(ssmclient, [instance])
 
         # On first boot, install various Windows updates.
         # We would ideally use PowerShell Remoting for this. However, there are
         # trust issues that make it difficult to invoke Windows Update
         # remotely. So we use SSM, which has a mechanism for running Windows
         # Update.
         print('installing Windows features...')
         run_ssm_command(
             ssmclient,
             [instance],
             'AWS-RunPowerShellScript',
             {
                 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'),
             },
         )
 
         # Reboot so all updates are fully applied.
         #
         # We don't use instance.reboot() here because it is asynchronous and
         # we don't know when exactly the instance has rebooted. It could take
         # a while to stop and we may start trying to interact with the instance
         # before it has rebooted.
         print('rebooting instance %s' % instance.id)
         instance.stop()
         ec2client.get_waiter('instance_stopped').wait(
             InstanceIds=[instance.id],
             WaiterConfig={
                 'Delay': 5,
             })
 
         instance.start()
         wait_for_ip_addresses([instance])
 
         # There is a race condition here between the User Data PS script running
         # and us connecting to WinRM. This can manifest as
         # "AuthorizationManager check failed" failures during run_powershell().
         # TODO figure out a workaround.
 
         print('waiting for Windows Remote Management to come back...')
         client = wait_for_winrm(instance.public_ip_address, 'Administrator',
                                 c.automation.default_password())
         print('established WinRM connection to %s' % instance.id)
         instance.winrm_client = client
 
         print('bootstrapping instance...')
         run_powershell(instance.winrm_client, '\n'.join(commands))
 
         print('bootstrap completed; stopping %s to create image' % instance.id)
         return create_ami_from_instance(ec2client, instance, name,
                                         'Mercurial Windows development environment',
                                         fingerprint)
 
 
 @contextlib.contextmanager
 def temporary_windows_dev_instances(c: AWSConnection, image, instance_type,
                                     prefix='hg-', disable_antivirus=False):
     """Create a temporary Windows development EC2 instance.
 
     Context manager resolves to the list of ``EC2.Instance`` that were created.
     """
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': '/dev/sda1',
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 32,
                     'VolumeType': 'gp2',
                 },
             }
         ],
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         'InstanceType': instance_type,
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
     }
 
     with create_temp_windows_ec2_instances(c, config) as instances:
         if disable_antivirus:
             for instance in instances:
                 run_powershell(
                     instance.winrm_client,
                     'Set-MpPreference -DisableRealtimeMonitoring $true')
 
         yield instances

contrib/automation/hgautomation/linux.py

@@ -1,561 +1,560 b''
 # linux.py - Linux specific automation functionality
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import os
 import pathlib
 import shlex
 import subprocess
 import tempfile
 
 from .ssh import (
     exec_command,
 )
 
 
 # Linux distributions that are supported.
 DISTROS = {
     'debian9',
     'ubuntu18.04',
-    'ubuntu18.10',
     'ubuntu19.04',
 }
 
 INSTALL_PYTHONS = r'''
 PYENV2_VERSIONS="2.7.16 pypy2.7-7.1.1"
 PYENV3_VERSIONS="3.5.7 3.6.9 3.7.4 3.8-dev pypy3.5-7.0.0 pypy3.6-7.1.1"
 
 git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
 pushd /hgdev/pyenv
 git checkout 17f44b7cd6f58ea2fa68ec0371fb9e7a826b8be2
 popd
 
 export PYENV_ROOT="/hgdev/pyenv"
 export PATH="$PYENV_ROOT/bin:$PATH"
 
 # pip 19.0.3.
 PIP_SHA256=efe99298f3fbb1f56201ce6b81d2658067d2f7d7dfc2d412e0d3cacc9a397c61
 wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/fee32c376da1ff6496a798986d7939cd51e1644f/get-pip.py
 echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
 
 VIRTUALENV_SHA256=984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39
 VIRTUALENV_TARBALL=virtualenv-16.4.3.tar.gz
 wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/${VIRTUALENV_TARBALL}
 echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
 
 for v in ${PYENV2_VERSIONS}; do
     pyenv install -v ${v}
     ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
     ${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
     ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
 done
 
 for v in ${PYENV3_VERSIONS}; do
     pyenv install -v ${v}
     ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
     ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
 done
 
 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
 '''.lstrip().replace('\r\n', '\n')
 
 
 INSTALL_RUST = r'''
 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
 echo "${RUSTUP_INIT_SHA256} rustup-init" | sha256sum --check -
 
 chmod +x rustup-init
 sudo -H -u hg -g hg ./rustup-init -y
 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.31.1 1.34.2
 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup component add clippy
 '''
 
 
 BOOTSTRAP_VIRTUALENV = r'''
 /usr/bin/virtualenv /hgdev/venv-bootstrap
 
 HG_SHA256=1bdd21bb87d1e05fb5cd395d488d0e0cc2f2f90ce0fd248e31a03595da5ccb47
 HG_TARBALL=mercurial-4.9.1.tar.gz
 
 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
 
 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
 '''.lstrip().replace('\r\n', '\n')
 
 
 BOOTSTRAP_DEBIAN = r'''
 #!/bin/bash
 
 set -ex
 
 DISTRO=`grep DISTRIB_ID /etc/lsb-release  | awk -F= '{{print $2}}'`
 DEBIAN_VERSION=`cat /etc/debian_version`
 LSB_RELEASE=`lsb_release -cs`
 
 sudo /usr/sbin/groupadd hg
 sudo /usr/sbin/groupadd docker
 sudo /usr/sbin/useradd -g hg -G sudo,docker -d /home/hg -m -s /bin/bash hg
 sudo mkdir /home/hg/.ssh
 sudo cp ~/.ssh/authorized_keys /home/hg/.ssh/authorized_keys
 sudo chown -R hg:hg /home/hg/.ssh
 sudo chmod 700 /home/hg/.ssh
 sudo chmod 600 /home/hg/.ssh/authorized_keys
 
 cat << EOF | sudo tee /etc/sudoers.d/90-hg
 hg ALL=(ALL) NOPASSWD:ALL
 EOF
 
 sudo apt-get update
 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade
 
 # Install packages necessary to set up Docker Apt repo.
 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends \
     apt-transport-https \
     gnupg
 
 cat > docker-apt-key << EOF
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
 mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth
 lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh
 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq
 L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7
 UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N
 cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht
 ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo
 vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD
 G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ
 XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj
 q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB
 tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
 BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO
 v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd
 tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk
 jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m
 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P
 XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc
 FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8
 g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm
 ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh
 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5
 G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW
 FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB
 EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF
 M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx
 Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu
 w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk
 z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8
 eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb
 VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa
 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X
 zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ
 pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7
 ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ
 BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY
 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp
 YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI
 mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES
 KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7
 JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ
 cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0
 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5
 U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z
 VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f
 irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk
 SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz
 QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W
 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw
 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe
 dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y
 Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR
 H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh
 /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ
 M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S
 xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O
 jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG
 YT90qFF93M3v01BbxP+EIY2/9tiIPbrd
 =0YYh
 -----END PGP PUBLIC KEY BLOCK-----
 EOF
 
 sudo apt-key add docker-apt-key
 
 if [ "$LSB_RELEASE" = "stretch" ]; then
 cat << EOF | sudo tee -a /etc/apt/sources.list
 # Need backports for clang-format-6.0
 deb http://deb.debian.org/debian stretch-backports main
 
 # Sources are useful if we want to compile things locally.
 deb-src http://deb.debian.org/debian stretch main
 deb-src http://security.debian.org/debian-security stretch/updates main
 deb-src http://deb.debian.org/debian stretch-updates main
 deb-src http://deb.debian.org/debian stretch-backports main
 
 deb [arch=amd64] https://download.docker.com/linux/debian stretch stable
 EOF
 
 elif [ "$DISTRO" = "Ubuntu" ]; then
 cat << EOF | sudo tee -a /etc/apt/sources.list
 deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
 EOF
 
 fi
 
 sudo apt-get update
 
 PACKAGES="\
     btrfs-progs \
     build-essential \
     bzr \
     clang-format-6.0 \
     cvs \
     darcs \
     debhelper \
     devscripts \
     docker-ce \
     dpkg-dev \
     dstat \
     emacs \
     gettext \
     git \
     htop \
     iotop \
     jfsutils \
     libbz2-dev \
     libexpat1-dev \
     libffi-dev \
     libgdbm-dev \
     liblzma-dev \
     libncurses5-dev \
     libnss3-dev \
     libreadline-dev \
     libsqlite3-dev \
     libssl-dev \
     netbase \
     ntfs-3g \
     nvme-cli \
     pyflakes \
     pyflakes3 \
     pylint \
     pylint3 \
     python-all-dev \
     python-dev \
     python-docutils \
     python-fuzzywuzzy \
     python-pygments \
     python-subversion \
     python-vcr \
     python3-dev \
     python3-docutils \
     python3-fuzzywuzzy \
     python3-pygments \
     python3-vcr \
     rsync \
     sqlite3 \
     subversion \
     tcl-dev \
     tk-dev \
     tla \
     unzip \
     uuid-dev \
     vim \
     virtualenv \
     wget \
     xfsprogs \
     zip \
     zlib1g-dev"
 
 if [ "LSB_RELEASE" = "stretch" ]; then
     PACKAGES="$PACKAGES linux-perf"
 elif [ "$DISTRO" = "Ubuntu" ]; then
     PACKAGES="$PACKAGES linux-tools-common"
 fi
 
 # Ubuntu 19.04 removes monotone.
 if [ "$LSB_RELEASE" != "disco" ]; then
     PACKAGES="$PACKAGES monotone"
 fi
 
 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
 
 # Create clang-format symlink so test harness finds it.
 sudo update-alternatives --install /usr/bin/clang-format clang-format \
     /usr/bin/clang-format-6.0 1000
 
 sudo mkdir /hgdev
 # Will be normalized to hg:hg later.
 sudo chown `whoami` /hgdev
 
 {install_rust}
 
 cp requirements-py2.txt /hgdev/requirements-py2.txt
 cp requirements-py3.txt /hgdev/requirements-py3.txt
 
 # Disable the pip version check because it uses the network and can
 # be annoying.
 cat << EOF | sudo tee -a /etc/pip.conf
 [global]
 disable-pip-version-check = True
 EOF
 
 {install_pythons}
 {bootstrap_virtualenv}
 
 /hgdev/venv-bootstrap/bin/hg clone https://www.mercurial-scm.org/repo/hg /hgdev/src
 
 # Mark the repo as non-publishing.
 cat >> /hgdev/src/.hg/hgrc << EOF
 [phases]
 publish = false
 EOF
 
 sudo chown -R hg:hg /hgdev
 '''.lstrip().format(
     install_rust=INSTALL_RUST,
     install_pythons=INSTALL_PYTHONS,
     bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV
 ).replace('\r\n', '\n')
 
 
 # Prepares /hgdev for operations.
 PREPARE_HGDEV = '''
 #!/bin/bash
 
 set -e
 
 FS=$1
 
 ensure_device() {
     if [ -z "${DEVICE}" ]; then
         echo "could not find block device to format"
         exit 1
     fi
 }
 
 # Determine device to partition for extra filesystem.
 # If only 1 volume is present, it will be the root volume and
 # should be /dev/nvme0. If multiple volumes are present, the
 # root volume could be nvme0 or nvme1. Use whichever one doesn't have
 # a partition.
 if [ -e /dev/nvme1n1 ]; then
     if [ -e /dev/nvme0n1p1 ]; then
         DEVICE=/dev/nvme1n1
     else
         DEVICE=/dev/nvme0n1
     fi
 else
     DEVICE=
 fi
 
 sudo mkdir /hgwork
 
 if [ "${FS}" != "default" -a "${FS}" != "tmpfs" ]; then
     ensure_device
     echo "creating ${FS} filesystem on ${DEVICE}"
 fi
 
 if [ "${FS}" = "default" ]; then
     :
 
 elif [ "${FS}" = "btrfs" ]; then
     sudo mkfs.btrfs ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "ext3" ]; then
     # lazy_journal_init speeds up filesystem creation at the expense of
     # integrity if things crash. We are an ephemeral instance, so we don't
     # care about integrity.
     sudo mkfs.ext3 -E lazy_journal_init=1 ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "ext4" ]; then
     sudo mkfs.ext4 -E lazy_journal_init=1 ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "jfs" ]; then
     sudo mkfs.jfs ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 elif [ "${FS}" = "tmpfs" ]; then
     echo "creating tmpfs volume in /hgwork"
     sudo mount -t tmpfs -o size=1024M tmpfs /hgwork
 
 elif [ "${FS}" = "xfs" ]; then
     sudo mkfs.xfs ${DEVICE}
     sudo mount ${DEVICE} /hgwork
 
 else
     echo "unsupported filesystem: ${FS}"
     exit 1
 fi
 
 echo "/hgwork ready"
 
 sudo chown hg:hg /hgwork
 mkdir /hgwork/tmp
 chown hg:hg /hgwork/tmp
 
 rsync -a /hgdev/src /hgwork/
 '''.lstrip().replace('\r\n', '\n')
 
 
 HG_UPDATE_CLEAN = '''
 set -ex
 
 HG=/hgdev/venv-bootstrap/bin/hg
 
 cd /hgwork/src
 ${HG} --config extensions.purge= purge --all
 ${HG} update -C $1
 ${HG} log -r .
 '''.lstrip().replace('\r\n', '\n')
 
 
 def prepare_exec_environment(ssh_client, filesystem='default'):
     """Prepare an EC2 instance to execute things.
 
     The AMI has an ``/hgdev`` bootstrapped with various Python installs
     and a clone of the Mercurial repo.
 
     In EC2, EBS volumes launched from snapshots have wonky performance behavior.
     Notably, blocks have to be copied on first access, which makes volume
     I/O extremely slow on fresh volumes.
 
     Furthermore, we may want to run operations, tests, etc on alternative
     filesystems so we examine behavior on different filesystems.
 
     This function is used to facilitate executing operations on alternate
     volumes.
     """
     sftp = ssh_client.open_sftp()
 
     with sftp.open('/hgdev/prepare-hgdev', 'wb') as fh:
         fh.write(PREPARE_HGDEV)
         fh.chmod(0o0777)
 
     command = 'sudo /hgdev/prepare-hgdev %s' % filesystem
     chan, stdin, stdout = exec_command(ssh_client, command)
     stdin.close()
 
     for line in stdout:
         print(line, end='')
 
     res = chan.recv_exit_status()
 
     if res:
         raise Exception('non-0 exit code updating working directory; %d'
                         % res)
 
 
 def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
     """Synchronize a local Mercurial source path to remote EC2 instance."""
 
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_dir = pathlib.Path(temp_dir)
 
         ssh_dir = temp_dir / '.ssh'
         ssh_dir.mkdir()
         ssh_dir.chmod(0o0700)
 
         public_ip = ec2_instance.public_ip_address
 
         ssh_config = ssh_dir / 'config'
 
         with ssh_config.open('w', encoding='utf-8') as fh:
             fh.write('Host %s\n' % public_ip)
             fh.write('  User hg\n')
             fh.write('  StrictHostKeyChecking no\n')
             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
             fh.write('  IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
 
         if not (source_path / '.hg').is_dir():
             raise Exception('%s is not a Mercurial repository; synchronization '
                             'not yet supported' % source_path)
 
         env = dict(os.environ)
         env['HGPLAIN'] = '1'
         env['HGENCODING'] = 'utf-8'
 
         hg_bin = source_path / 'hg'
 
         res = subprocess.run(
             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
             cwd=str(source_path), env=env, check=True, capture_output=True)
 
         full_revision = res.stdout.decode('ascii')
 
         args = [
             'python2.7', str(hg_bin),
             '--config', 'ui.ssh=ssh -F %s' % ssh_config,
             '--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
             # Also ensure .hgtags changes are present so auto version
             # calculation works.
             'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)',
             'ssh://%s//hgwork/src' % public_ip,
         ]
 
         res = subprocess.run(args, cwd=str(source_path), env=env)
 
         # Allow 1 (no-op) to not trigger error.
         if res.returncode not in (0, 1):
             res.check_returncode()
 
         # TODO support synchronizing dirty working directory.
 
         sftp = ec2_instance.ssh_client.open_sftp()
 
         with sftp.open('/hgdev/hgup', 'wb') as fh:
             fh.write(HG_UPDATE_CLEAN)
             fh.chmod(0o0700)
 
         chan, stdin, stdout = exec_command(
             ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision)
         stdin.close()
 
         for line in stdout:
             print(line, end='')
 
         res = chan.recv_exit_status()
 
         if res:
             raise Exception('non-0 exit code updating working directory; %d'
                             % res)
 
 
 def run_tests(ssh_client, python_version, test_flags=None):
     """Run tests on a remote Linux machine via an SSH client."""
     test_flags = test_flags or []
 
     print('running tests')
 
     if python_version == 'system2':
         python = '/usr/bin/python2'
     elif python_version == 'system3':
         python = '/usr/bin/python3'
     elif python_version.startswith('pypy'):
         python = '/hgdev/pyenv/shims/%s' % python_version
     else:
         python = '/hgdev/pyenv/shims/python%s' % python_version
 
     test_flags = ' '.join(shlex.quote(a) for a in test_flags)
 
     command = (
         '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
         'cd /hgwork/src/tests && %s run-tests.py %s"' % (
             python, test_flags))
 
     chan, stdin, stdout = exec_command(ssh_client, command)
 
     stdin.close()
 
     for line in stdout:
         print(line, end='')
 
     return chan.recv_exit_status()