branching: merge stable into default

contrib/automation/hgautomation/aws.py

@@ -1,1335 +1,1335 b''
 # aws.py - Automation code for Amazon Web Services
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import contextlib
 import copy
 import hashlib
 import json
 import os
 import pathlib
 import subprocess
 import time
 
 import boto3
 import botocore.exceptions
 
 from .linux import BOOTSTRAP_DEBIAN
 from .ssh import (
     exec_command as ssh_exec_command,
     wait_for_ssh,
 )
 from .winrm import (
     run_powershell,
     wait_for_winrm,
 )
 
 
 SOURCE_ROOT = pathlib.Path(
     os.path.abspath(__file__)
 ).parent.parent.parent.parent
 
 INSTALL_WINDOWS_DEPENDENCIES = (
     SOURCE_ROOT / 'contrib' / 'install-windows-dependencies.ps1'
 )
 
 
 INSTANCE_TYPES_WITH_STORAGE = {
     'c5d',
     'd2',
     'h1',
     'i3',
     'm5ad',
     'm5d',
     'r5d',
     'r5ad',
     'x1',
     'z1d',
 }
 
 
 AMAZON_ACCOUNT_ID = '801119661308'
 DEBIAN_ACCOUNT_ID = '379101102735'
 DEBIAN_ACCOUNT_ID_2 = '136693071363'
 UBUNTU_ACCOUNT_ID = '099720109477'
 
 
 WINDOWS_BASE_IMAGE_NAME = 'Windows_Server-2019-English-Full-Base-*'
 
 
 KEY_PAIRS = {
     'automation',
 }
 
 
 SECURITY_GROUPS = {
     'linux-dev-1': {
         'description': 'Mercurial Linux instances that perform build/test automation',
         'ingress': [
             {
                 'FromPort': 22,
                 'ToPort': 22,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'SSH from entire Internet',
                     },
                 ],
             },
         ],
     },
     'windows-dev-1': {
         'description': 'Mercurial Windows instances that perform build automation',
         'ingress': [
             {
                 'FromPort': 22,
                 'ToPort': 22,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'SSH from entire Internet',
                     },
                 ],
             },
             {
                 'FromPort': 3389,
                 'ToPort': 3389,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'RDP from entire Internet',
                     },
                 ],
             },
             {
                 'FromPort': 5985,
                 'ToPort': 5986,
                 'IpProtocol': 'tcp',
                 'IpRanges': [
                     {
                         'CidrIp': '0.0.0.0/0',
                         'Description': 'PowerShell Remoting (Windows Remote Management)',
                     },
                 ],
             },
         ],
     },
 }
 
 
 IAM_ROLES = {
     'ephemeral-ec2-role-1': {
         'description': 'Mercurial temporary EC2 instances',
         'policy_arns': [
             'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM',
         ],
     },
 }
 
 
 ASSUME_ROLE_POLICY_DOCUMENT = '''
 {
   "Version": "2012-10-17",
   "Statement": [
     {
       "Effect": "Allow",
       "Principal": {
         "Service": "ec2.amazonaws.com"
       },
       "Action": "sts:AssumeRole"
     }
   ]
 }
 '''.strip()
 
 
 IAM_INSTANCE_PROFILES = {
     'ephemeral-ec2-1': {
         'roles': [
             'ephemeral-ec2-role-1',
         ],
     }
 }
 
 
 # User Data for Windows EC2 instance. Mainly used to set the password
 # and configure WinRM.
 # Inspired by the User Data script used by Packer
 # (from https://www.packer.io/intro/getting-started/build-image.html).
 WINDOWS_USER_DATA = r'''
 <powershell>
 
 # TODO enable this once we figure out what is failing.
 #$ErrorActionPreference = "stop"
 
 # Set administrator password
 net user Administrator "%s"
 wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE
 
 # First, make sure WinRM can't be connected to
 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block
 
 # Delete any existing WinRM listeners
 winrm delete winrm/config/listener?Address=*+Transport=HTTP  2>$Null
 winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null
 
 # Create a new WinRM listener and configure
 winrm create winrm/config/listener?Address=*+Transport=HTTP
 winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}'
 winrm set winrm/config '@{MaxTimeoutms="7200000"}'
 winrm set winrm/config/service '@{AllowUnencrypted="true"}'
 winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}'
 winrm set winrm/config/service/auth '@{Basic="true"}'
 winrm set winrm/config/client/auth '@{Basic="true"}'
 
 # Configure UAC to allow privilege elevation in remote shells
 $Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System'
 $Setting = 'LocalAccountTokenFilterPolicy'
 Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force
 
 # Avoid long usernames in the temp directory path because the '~' causes extra quoting in ssh output
 [System.Environment]::SetEnvironmentVariable('TMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
 [System.Environment]::SetEnvironmentVariable('TEMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
 
 # Configure and restart the WinRM Service; Enable the required firewall exception
 Stop-Service -Name WinRM
 Set-Service -Name WinRM -StartupType Automatic
 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any
 Start-Service -Name WinRM
 
 # Disable firewall on private network interfaces so prompts don't appear.
 Set-NetFirewallProfile -Name private -Enabled false
 </powershell>
 '''.lstrip()
 
 
 WINDOWS_BOOTSTRAP_POWERSHELL = '''
 Write-Output "installing PowerShell dependencies"
 Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
 Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
 Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0
 
 Write-Output "installing OpenSSL server"
 Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
 # Various tools will attempt to use older versions of .NET. So we enable
 # the feature that provides them so it doesn't have to be auto-enabled
 # later.
 Write-Output "enabling .NET Framework feature"
 Install-WindowsFeature -Name Net-Framework-Core
 '''
 
 
 class AWSConnection:
     """Manages the state of a connection with AWS."""
 
     def __init__(self, automation, region: str, ensure_ec2_state: bool = True):
         self.automation = automation
         self.local_state_path = automation.state_path
 
         self.prefix = 'hg-'
 
         self.session = boto3.session.Session(region_name=region)
         self.ec2client = self.session.client('ec2')
         self.ec2resource = self.session.resource('ec2')
         self.iamclient = self.session.client('iam')
         self.iamresource = self.session.resource('iam')
         self.security_groups = {}
 
         if ensure_ec2_state:
             ensure_key_pairs(automation.state_path, self.ec2resource)
             self.security_groups = ensure_security_groups(self.ec2resource)
             ensure_iam_state(self.iamclient, self.iamresource)
 
     def key_pair_path_private(self, name):
         """Path to a key pair private key file."""
         return self.local_state_path / 'keys' / ('keypair-%s' % name)
 
     def key_pair_path_public(self, name):
         return self.local_state_path / 'keys' / ('keypair-%s.pub' % name)
 
 
 def rsa_key_fingerprint(p: pathlib.Path):
     """Compute the fingerprint of an RSA private key."""
 
     # TODO use rsa package.
     res = subprocess.run(
         [
             'openssl',
             'pkcs8',
             '-in',
             str(p),
             '-nocrypt',
             '-topk8',
             '-outform',
             'DER',
         ],
         capture_output=True,
         check=True,
     )
 
     sha1 = hashlib.sha1(res.stdout).hexdigest()
     return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2]))
 
 
 def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'):
     remote_existing = {}
 
     for kpi in ec2resource.key_pairs.all():
         if kpi.name.startswith(prefix):
             remote_existing[kpi.name[len(prefix) :]] = kpi.key_fingerprint
 
     # Validate that we have these keys locally.
     key_path = state_path / 'keys'
     key_path.mkdir(exist_ok=True, mode=0o700)
 
     def remove_remote(name):
         print('deleting key pair %s' % name)
         key = ec2resource.KeyPair(name)
         key.delete()
 
     def remove_local(name):
         pub_full = key_path / ('keypair-%s.pub' % name)
         priv_full = key_path / ('keypair-%s' % name)
 
         print('removing %s' % pub_full)
         pub_full.unlink()
         print('removing %s' % priv_full)
         priv_full.unlink()
 
     local_existing = {}
 
     for f in sorted(os.listdir(key_path)):
         if not f.startswith('keypair-') or not f.endswith('.pub'):
             continue
 
         name = f[len('keypair-') : -len('.pub')]
 
         pub_full = key_path / f
         priv_full = key_path / ('keypair-%s' % name)
 
         with open(pub_full, 'r', encoding='ascii') as fh:
             data = fh.read()
 
         if not data.startswith('ssh-rsa '):
             print(
                 'unexpected format for key pair file: %s; removing' % pub_full
             )
             pub_full.unlink()
             priv_full.unlink()
             continue
 
         local_existing[name] = rsa_key_fingerprint(priv_full)
 
     for name in sorted(set(remote_existing) | set(local_existing)):
         if name not in local_existing:
             actual = '%s%s' % (prefix, name)
             print('remote key %s does not exist locally' % name)
             remove_remote(actual)
             del remote_existing[name]
 
         elif name not in remote_existing:
             print('local key %s does not exist remotely' % name)
             remove_local(name)
             del local_existing[name]
 
         elif remote_existing[name] != local_existing[name]:
             print(
                 'key fingerprint mismatch for %s; '
                 'removing from local and remote' % name
             )
             remove_local(name)
             remove_remote('%s%s' % (prefix, name))
             del local_existing[name]
             del remote_existing[name]
 
     missing = KEY_PAIRS - set(remote_existing)
 
     for name in sorted(missing):
         actual = '%s%s' % (prefix, name)
         print('creating key pair %s' % actual)
 
         priv_full = key_path / ('keypair-%s' % name)
         pub_full = key_path / ('keypair-%s.pub' % name)
 
         kp = ec2resource.create_key_pair(KeyName=actual)
 
         with priv_full.open('w', encoding='ascii') as fh:
             fh.write(kp.key_material)
             fh.write('\n')
 
         priv_full.chmod(0o0600)
 
         # SSH public key can be extracted via `ssh-keygen`.
         with pub_full.open('w', encoding='ascii') as fh:
             subprocess.run(
                 ['ssh-keygen', '-y', '-f', str(priv_full)],
                 stdout=fh,
                 check=True,
             )
 
         pub_full.chmod(0o0600)
 
 
 def delete_instance_profile(profile):
     for role in profile.roles:
         print(
             'removing role %s from instance profile %s'
             % (role.name, profile.name)
         )
         profile.remove_role(RoleName=role.name)
 
     print('deleting instance profile %s' % profile.name)
     profile.delete()
 
 
 def ensure_iam_state(iamclient, iamresource, prefix='hg-'):
     """Ensure IAM state is in sync with our canonical definition."""
 
     remote_profiles = {}
 
     for profile in iamresource.instance_profiles.all():
         if profile.name.startswith(prefix):
             remote_profiles[profile.name[len(prefix) :]] = profile
 
     for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)):
         delete_instance_profile(remote_profiles[name])
         del remote_profiles[name]
 
     remote_roles = {}
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
             remote_roles[role.name[len(prefix) :]] = role
 
     for name in sorted(set(remote_roles) - set(IAM_ROLES)):
         role = remote_roles[name]
 
         print('removing role %s' % role.name)
         role.delete()
         del remote_roles[name]
 
     # We've purged remote state that doesn't belong. Create missing
     # instance profiles and roles.
     for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)):
         actual = '%s%s' % (prefix, name)
         print('creating IAM instance profile %s' % actual)
 
         profile = iamresource.create_instance_profile(
             InstanceProfileName=actual
         )
         remote_profiles[name] = profile
 
         waiter = iamclient.get_waiter('instance_profile_exists')
         waiter.wait(InstanceProfileName=actual)
         print('IAM instance profile %s is available' % actual)
 
     for name in sorted(set(IAM_ROLES) - set(remote_roles)):
         entry = IAM_ROLES[name]
 
         actual = '%s%s' % (prefix, name)
         print('creating IAM role %s' % actual)
 
         role = iamresource.create_role(
             RoleName=actual,
             Description=entry['description'],
             AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
         )
 
         waiter = iamclient.get_waiter('role_exists')
         waiter.wait(RoleName=actual)
         print('IAM role %s is available' % actual)
 
         remote_roles[name] = role
 
         for arn in entry['policy_arns']:
             print('attaching policy %s to %s' % (arn, role.name))
             role.attach_policy(PolicyArn=arn)
 
     # Now reconcile state of profiles.
     for name, meta in sorted(IAM_INSTANCE_PROFILES.items()):
         profile = remote_profiles[name]
         wanted = {'%s%s' % (prefix, role) for role in meta['roles']}
         have = {role.name for role in profile.roles}
 
         for role in sorted(have - wanted):
             print('removing role %s from %s' % (role, profile.name))
             profile.remove_role(RoleName=role)
 
         for role in sorted(wanted - have):
             print('adding role %s to %s' % (role, profile.name))
             profile.add_role(RoleName=role)
 
 
 def find_image(ec2resource, owner_id, name, reverse_sort_field=None):
     """Find an AMI by its owner ID and name."""
 
     images = ec2resource.images.filter(
         Filters=[
             {
                 'Name': 'owner-id',
                 'Values': [owner_id],
             },
             {
                 'Name': 'state',
                 'Values': ['available'],
             },
             {
                 'Name': 'image-type',
                 'Values': ['machine'],
             },
             {
                 'Name': 'name',
                 'Values': [name],
             },
         ]
     )
 
     if reverse_sort_field:
         images = sorted(
             images,
             key=lambda image: getattr(image, reverse_sort_field),
             reverse=True,
         )
 
     for image in images:
         return image
 
     raise Exception('unable to find image for %s' % name)
 
 
 def ensure_security_groups(ec2resource, prefix='hg-'):
     """Ensure all necessary Mercurial security groups are present.
 
     All security groups are prefixed with ``hg-`` by default. Any security
     groups having this prefix but aren't in our list are deleted.
     """
     existing = {}
 
     for group in ec2resource.security_groups.all():
         if group.group_name.startswith(prefix):
             existing[group.group_name[len(prefix) :]] = group
 
     purge = set(existing) - set(SECURITY_GROUPS)
 
     for name in sorted(purge):
         group = existing[name]
         print('removing legacy security group: %s' % group.group_name)
         group.delete()
 
     security_groups = {}
 
     for name, group in sorted(SECURITY_GROUPS.items()):
         if name in existing:
             security_groups[name] = existing[name]
             continue
 
         actual = '%s%s' % (prefix, name)
         print('adding security group %s' % actual)
 
         group_res = ec2resource.create_security_group(
             Description=group['description'],
             GroupName=actual,
         )
 
         group_res.authorize_ingress(
             IpPermissions=group['ingress'],
         )
 
         security_groups[name] = group_res
 
     return security_groups
 
 
 def terminate_ec2_instances(ec2resource, prefix='hg-'):
     """Terminate all EC2 instances managed by us."""
     waiting = []
 
     for instance in ec2resource.instances.all():
         if instance.state['Name'] == 'terminated':
             continue
 
         for tag in instance.tags or []:
             if tag['Key'] == 'Name' and tag['Value'].startswith(prefix):
                 print('terminating %s' % instance.id)
                 instance.terminate()
                 waiting.append(instance)
 
     for instance in waiting:
         instance.wait_until_terminated()
 
 
 def remove_resources(c, prefix='hg-'):
     """Purge all of our resources in this EC2 region."""
     ec2resource = c.ec2resource
     iamresource = c.iamresource
 
     terminate_ec2_instances(ec2resource, prefix=prefix)
 
     for image in ec2resource.images.filter(Owners=['self']):
         if image.name.startswith(prefix):
             remove_ami(ec2resource, image)
 
     for group in ec2resource.security_groups.all():
         if group.group_name.startswith(prefix):
             print('removing security group %s' % group.group_name)
             group.delete()
 
     for profile in iamresource.instance_profiles.all():
         if profile.name.startswith(prefix):
             delete_instance_profile(profile)
 
     for role in iamresource.roles.all():
         if role.name.startswith(prefix):
             for p in role.attached_policies.all():
                 print('detaching policy %s from %s' % (p.arn, role.name))
                 role.detach_policy(PolicyArn=p.arn)
 
             print('removing role %s' % role.name)
             role.delete()
 
 
 def wait_for_ip_addresses(instances):
     """Wait for the public IP addresses of an iterable of instances."""
     for instance in instances:
         while True:
             if not instance.public_ip_address:
                 time.sleep(2)
                 instance.reload()
                 continue
 
             print(
                 'public IP address for %s: %s'
                 % (instance.id, instance.public_ip_address)
             )
             break
 
 
 def remove_ami(ec2resource, image):
     """Remove an AMI and its underlying snapshots."""
     snapshots = []
 
     for device in image.block_device_mappings:
         if 'Ebs' in device:
             snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId']))
 
     print('deregistering %s' % image.id)
     image.deregister()
 
     for snapshot in snapshots:
         print('deleting snapshot %s' % snapshot.id)
         snapshot.delete()
 
 
 def wait_for_ssm(ssmclient, instances):
     """Wait for SSM to come online for an iterable of instance IDs."""
     while True:
         res = ssmclient.describe_instance_information(
             Filters=[
                 {
                     'Key': 'InstanceIds',
                     'Values': [i.id for i in instances],
                 },
             ],
         )
 
         available = len(res['InstanceInformationList'])
         wanted = len(instances)
 
         print('%d/%d instances available in SSM' % (available, wanted))
 
         if available == wanted:
             return
 
         time.sleep(2)
 
 
 def run_ssm_command(ssmclient, instances, document_name, parameters):
     """Run a PowerShell script on an EC2 instance."""
 
     res = ssmclient.send_command(
         InstanceIds=[i.id for i in instances],
         DocumentName=document_name,
         Parameters=parameters,
         CloudWatchOutputConfig={
             'CloudWatchOutputEnabled': True,
         },
     )
 
     command_id = res['Command']['CommandId']
 
     for instance in instances:
         while True:
             try:
                 res = ssmclient.get_command_invocation(
                     CommandId=command_id,
                     InstanceId=instance.id,
                 )
             except botocore.exceptions.ClientError as e:
                 if e.response['Error']['Code'] == 'InvocationDoesNotExist':
                     print('could not find SSM command invocation; waiting')
                     time.sleep(1)
                     continue
                 else:
                     raise
 
             if res['Status'] == 'Success':
                 break
             elif res['Status'] in ('Pending', 'InProgress', 'Delayed'):
                 time.sleep(2)
             else:
                 raise Exception(
                     'command failed on %s: %s' % (instance.id, res['Status'])
                 )
 
 
 @contextlib.contextmanager
 def temporary_ec2_instances(ec2resource, config):
     """Create temporary EC2 instances.
 
     This is a proxy to ``ec2client.run_instances(**config)`` that takes care of
     managing the lifecycle of the instances.
 
     When the context manager exits, the instances are terminated.
 
     The context manager evaluates to the list of data structures
     describing each created instance. The instances may not be available
     for work immediately: it is up to the caller to wait for the instance
     to start responding.
     """
 
     ids = None
 
     try:
         res = ec2resource.create_instances(**config)
 
         ids = [i.id for i in res]
         print('started instances: %s' % ' '.join(ids))
 
         yield res
     finally:
         if ids:
             print('terminating instances: %s' % ' '.join(ids))
             for instance in res:
                 instance.terminate()
             print('terminated %d instances' % len(ids))
 
 
 @contextlib.contextmanager
 def create_temp_windows_ec2_instances(
     c: AWSConnection, config, bootstrap: bool = False
 ):
     """Create temporary Windows EC2 instances.
 
     This is a higher-level wrapper around ``create_temp_ec2_instances()`` that
     configures the Windows instance for Windows Remote Management. The emitted
     instances will have a ``winrm_client`` attribute containing a
     ``pypsrp.client.Client`` instance bound to the instance.
     """
     if 'IamInstanceProfile' in config:
         raise ValueError('IamInstanceProfile cannot be provided in config')
     if 'UserData' in config:
         raise ValueError('UserData cannot be provided in config')
 
     password = c.automation.default_password()
 
     config = copy.deepcopy(config)
     config['IamInstanceProfile'] = {
         'Name': 'hg-ephemeral-ec2-1',
     }
     config.setdefault('TagSpecifications', []).append(
         {
             'ResourceType': 'instance',
             'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}],
         }
     )
 
     if bootstrap:
         config['UserData'] = WINDOWS_USER_DATA % password
 
     with temporary_ec2_instances(c.ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         print('waiting for Windows Remote Management service...')
 
         for instance in instances:
             client = wait_for_winrm(
                 instance.public_ip_address, 'Administrator', password
             )
             print('established WinRM connection to %s' % instance.id)
             instance.winrm_client = client
 
         yield instances
 
 
 def resolve_fingerprint(fingerprint):
     fingerprint = json.dumps(fingerprint, sort_keys=True)
     return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
 
 
 def find_and_reconcile_image(ec2resource, name, fingerprint):
     """Attempt to find an existing EC2 AMI with a name and fingerprint.
 
     If an image with the specified fingerprint is found, it is returned.
     Otherwise None is returned.
 
     Existing images for the specified name that don't have the specified
     fingerprint or are missing required metadata or deleted.
     """
     # Find existing AMIs with this name and delete the ones that are invalid.
     # Store a reference to a good image so it can be returned one the
     # image state is reconciled.
     images = ec2resource.images.filter(
         Filters=[{'Name': 'name', 'Values': [name]}]
     )
 
     existing_image = None
 
     for image in images:
         if image.tags is None:
             print(
                 'image %s for %s lacks required tags; removing'
                 % (image.id, image.name)
             )
             remove_ami(ec2resource, image)
         else:
             tags = {t['Key']: t['Value'] for t in image.tags}
 
             if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
                 existing_image = image
             else:
                 print(
                     'image %s for %s has wrong fingerprint; removing'
                     % (image.id, image.name)
                 )
                 remove_ami(ec2resource, image)
 
     return existing_image
 
 
 def create_ami_from_instance(
     ec2client, instance, name, description, fingerprint
 ):
     """Create an AMI from a running instance.
 
     Returns the ``ec2resource.Image`` representing the created AMI.
     """
     instance.stop()
 
     ec2client.get_waiter('instance_stopped').wait(
         InstanceIds=[instance.id],
         WaiterConfig={
             'Delay': 5,
         },
     )
     print('%s is stopped' % instance.id)
 
     image = instance.create_image(
         Name=name,
         Description=description,
     )
 
     image.create_tags(
         Tags=[
             {
                 'Key': 'HGIMAGEFINGERPRINT',
                 'Value': fingerprint,
             },
         ]
     )
 
     print('waiting for image %s' % image.id)
 
     ec2client.get_waiter('image_available').wait(
         ImageIds=[image.id],
     )
 
     print('image %s available as %s' % (image.id, image.name))
 
     return image
 
 
 def ensure_linux_dev_ami(c: AWSConnection, distro='debian10', prefix='hg-'):
     """Ensures a Linux development AMI is available and up-to-date.
 
     Returns an ``ec2.Image`` of either an existing AMI or a newly-built one.
     """
     ec2client = c.ec2client
     ec2resource = c.ec2resource
 
     name = '%s%s-%s' % (prefix, 'linux-dev', distro)
 
     if distro == 'debian9':
         image = find_image(
             ec2resource,
             DEBIAN_ACCOUNT_ID,
             'debian-stretch-hvm-x86_64-gp2-2019-09-08-17994',
         )
         ssh_username = 'admin'
     elif distro == 'debian10':
         image = find_image(
             ec2resource,
             DEBIAN_ACCOUNT_ID_2,
             'debian-10-amd64-20190909-10',
         )
         ssh_username = 'admin'
     elif distro == 'ubuntu18.04':
         image = find_image(
             ec2resource,
             UBUNTU_ACCOUNT_ID,
             'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190918',
         )
         ssh_username = 'ubuntu'
     elif distro == 'ubuntu19.04':
         image = find_image(
             ec2resource,
             UBUNTU_ACCOUNT_ID,
             'ubuntu/images/hvm-ssd/ubuntu-disco-19.04-amd64-server-20190918',
         )
         ssh_username = 'ubuntu'
     else:
         raise ValueError('unsupported Linux distro: %s' % distro)
 
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': image.block_device_mappings[0]['DeviceName'],
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 10,
-                    'VolumeType': 'gp2',
+                    'VolumeType': 'gp3',
                 },
             },
         ],
         'EbsOptimized': True,
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         # 8 VCPUs for compiling Python.
         'InstanceType': 't3.2xlarge',
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
     }
 
     requirements2_path = (
         pathlib.Path(__file__).parent.parent / 'linux-requirements-py2.txt'
     )
     requirements3_path = (
         pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt'
     )
     requirements35_path = (
         pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt'
     )
     with requirements2_path.open('r', encoding='utf-8') as fh:
         requirements2 = fh.read()
     with requirements3_path.open('r', encoding='utf-8') as fh:
         requirements3 = fh.read()
     with requirements35_path.open('r', encoding='utf-8') as fh:
         requirements35 = fh.read()
 
     # Compute a deterministic fingerprint to determine whether image needs to
     # be regenerated.
     fingerprint = resolve_fingerprint(
         {
             'instance_config': config,
             'bootstrap_script': BOOTSTRAP_DEBIAN,
             'requirements_py2': requirements2,
             'requirements_py3': requirements3,
             'requirements_py35': requirements35,
         }
     )
 
     existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
 
     print('no suitable %s image found; creating one...' % name)
 
     with temporary_ec2_instances(ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         instance = instances[0]
 
         client = wait_for_ssh(
             instance.public_ip_address,
             22,
             username=ssh_username,
             key_filename=str(c.key_pair_path_private('automation')),
         )
 
         home = '/home/%s' % ssh_username
 
         with client:
             print('connecting to SSH server')
             sftp = client.open_sftp()
 
             print('uploading bootstrap files')
             with sftp.open('%s/bootstrap' % home, 'wb') as fh:
                 fh.write(BOOTSTRAP_DEBIAN)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py2.txt' % home, 'wb') as fh:
                 fh.write(requirements2)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
                 fh.write(requirements3)
                 fh.chmod(0o0700)
 
             with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh:
                 fh.write(requirements35)
                 fh.chmod(0o0700)
 
             print('executing bootstrap')
             chan, stdin, stdout = ssh_exec_command(
                 client, '%s/bootstrap' % home
             )
             stdin.close()
 
             for line in stdout:
                 print(line, end='')
 
             res = chan.recv_exit_status()
             if res:
                 raise Exception('non-0 exit from bootstrap: %d' % res)
 
             print(
                 'bootstrap completed; stopping %s to create %s'
                 % (instance.id, name)
             )
 
         return create_ami_from_instance(
             ec2client,
             instance,
             name,
             'Mercurial Linux development environment',
             fingerprint,
         )
 
 
 @contextlib.contextmanager
 def temporary_linux_dev_instances(
     c: AWSConnection,
     image,
     instance_type,
     prefix='hg-',
     ensure_extra_volume=False,
 ):
     """Create temporary Linux development EC2 instances.
 
     Context manager resolves to a list of ``ec2.Instance`` that were created
     and are running.
 
     ``ensure_extra_volume`` can be set to ``True`` to require that instances
     have a 2nd storage volume available other than the primary AMI volume.
     For instance types with instance storage, this does nothing special.
     But for instance types without instance storage, an additional EBS volume
     will be added to the instance.
 
     Instances have an ``ssh_client`` attribute containing a paramiko SSHClient
     instance bound to the instance.
 
     Instances have an ``ssh_private_key_path`` attributing containing the
     str path to the SSH private key to connect to the instance.
     """
 
     block_device_mappings = [
         {
             'DeviceName': image.block_device_mappings[0]['DeviceName'],
             'Ebs': {
                 'DeleteOnTermination': True,
                 'VolumeSize': 12,
-                'VolumeType': 'gp2',
+                'VolumeType': 'gp3',
             },
         }
     ]
 
     # This is not an exhaustive list of instance types having instance storage.
     # But
     if ensure_extra_volume and not instance_type.startswith(
         tuple(INSTANCE_TYPES_WITH_STORAGE)
     ):
         main_device = block_device_mappings[0]['DeviceName']
 
         if main_device == 'xvda':
             second_device = 'xvdb'
         elif main_device == '/dev/sda1':
             second_device = '/dev/sdb'
         else:
             raise ValueError(
                 'unhandled primary EBS device name: %s' % main_device
             )
 
         block_device_mappings.append(
             {
                 'DeviceName': second_device,
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 8,
-                    'VolumeType': 'gp2',
+                    'VolumeType': 'gp3',
                 },
             }
         )
 
     config = {
         'BlockDeviceMappings': block_device_mappings,
         'EbsOptimized': True,
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'terminate',
         'InstanceType': instance_type,
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
     }
 
     with temporary_ec2_instances(c.ec2resource, config) as instances:
         wait_for_ip_addresses(instances)
 
         ssh_private_key_path = str(c.key_pair_path_private('automation'))
 
         for instance in instances:
             client = wait_for_ssh(
                 instance.public_ip_address,
                 22,
                 username='hg',
                 key_filename=ssh_private_key_path,
             )
 
             instance.ssh_client = client
             instance.ssh_private_key_path = ssh_private_key_path
 
         try:
             yield instances
         finally:
             for instance in instances:
                 instance.ssh_client.close()
 
 
 def ensure_windows_dev_ami(
     c: AWSConnection,
     prefix='hg-',
     base_image_name=WINDOWS_BASE_IMAGE_NAME,
 ):
     """Ensure Windows Development AMI is available and up-to-date.
 
     If necessary, a modern AMI will be built by starting a temporary EC2
     instance and bootstrapping it.
 
     Obsolete AMIs will be deleted so there is only a single AMI having the
     desired name.
 
     Returns an ``ec2.Image`` of either an existing AMI or a newly-built
     one.
     """
     ec2client = c.ec2client
     ec2resource = c.ec2resource
     ssmclient = c.session.client('ssm')
 
     name = '%s%s' % (prefix, 'windows-dev')
 
     image = find_image(
         ec2resource,
         AMAZON_ACCOUNT_ID,
         base_image_name,
         reverse_sort_field="name",
     )
 
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': '/dev/sda1',
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 32,
-                    'VolumeType': 'gp2',
+                    'VolumeType': 'gp3',
                 },
             }
         ],
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
-        'InstanceType': 't3.medium',
+        'InstanceType': 'm6i.large',
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
     }
 
     commands = [
         # Need to start the service so sshd_config is generated.
         'Start-Service sshd',
         'Write-Output "modifying sshd_config"',
         r'$content = Get-Content C:\ProgramData\ssh\sshd_config',
         '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""',
         r'$content | Set-Content C:\ProgramData\ssh\sshd_config',
         'Import-Module OpenSSHUtils',
         r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false',
         'Restart-Service sshd',
         'Write-Output "installing OpenSSL client"',
         'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0',
         'Set-Service -Name sshd -StartupType "Automatic"',
         'Write-Output "OpenSSH server running"',
     ]
 
     with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh:
         commands.extend(l.rstrip() for l in fh)
 
     # Schedule run of EC2Launch on next boot. This ensures that UserData
     # is executed.
     # We disable setComputerName because it forces a reboot.
     # We set an explicit admin password because this causes UserData to run
     # as Administrator instead of System.
     commands.extend(
         [
             r'''Set-Content -Path C:\ProgramData\Amazon\EC2-Windows\Launch\Config\LaunchConfig.json '''
             r'''-Value '{"setComputerName": false, "setWallpaper": true, "addDnsSuffixList": true, '''
             r'''"extendBootVolumeSize": true, "handleUserData": true, '''
             r'''"adminPasswordType": "Specify", "adminPassword": "%s"}' '''
             % c.automation.default_password(),
             r'C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 '
             r'–Schedule',
         ]
     )
 
     # Disable Windows Defender when bootstrapping because it just slows
     # things down.
     commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true')
     commands.append('Set-MpPreference -DisableRealtimeMonitoring $false')
 
     # Compute a deterministic fingerprint to determine whether image needs
     # to be regenerated.
     fingerprint = resolve_fingerprint(
         {
             'instance_config': config,
             'user_data': WINDOWS_USER_DATA,
             'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
             'bootstrap_commands': commands,
             'base_image_name': base_image_name,
         }
     )
 
     existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
 
     if existing_image:
         return existing_image
 
     print('no suitable Windows development image found; creating one...')
 
     with create_temp_windows_ec2_instances(
         c, config, bootstrap=True
     ) as instances:
         assert len(instances) == 1
         instance = instances[0]
 
         wait_for_ssm(ssmclient, [instance])
 
         # On first boot, install various Windows updates.
         # We would ideally use PowerShell Remoting for this. However, there are
         # trust issues that make it difficult to invoke Windows Update
         # remotely. So we use SSM, which has a mechanism for running Windows
         # Update.
         print('installing Windows features...')
         run_ssm_command(
             ssmclient,
             [instance],
             'AWS-RunPowerShellScript',
             {
                 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'),
             },
         )
 
         # Reboot so all updates are fully applied.
         #
         # We don't use instance.reboot() here because it is asynchronous and
         # we don't know when exactly the instance has rebooted. It could take
         # a while to stop and we may start trying to interact with the instance
         # before it has rebooted.
         print('rebooting instance %s' % instance.id)
         instance.stop()
         ec2client.get_waiter('instance_stopped').wait(
             InstanceIds=[instance.id],
             WaiterConfig={
                 'Delay': 5,
             },
         )
 
         instance.start()
         wait_for_ip_addresses([instance])
 
         # There is a race condition here between the User Data PS script running
         # and us connecting to WinRM. This can manifest as
         # "AuthorizationManager check failed" failures during run_powershell().
         # TODO figure out a workaround.
 
         print('waiting for Windows Remote Management to come back...')
         client = wait_for_winrm(
             instance.public_ip_address,
             'Administrator',
             c.automation.default_password(),
         )
         print('established WinRM connection to %s' % instance.id)
         instance.winrm_client = client
 
         print('bootstrapping instance...')
         run_powershell(instance.winrm_client, '\n'.join(commands))
 
         print('bootstrap completed; stopping %s to create image' % instance.id)
         return create_ami_from_instance(
             ec2client,
             instance,
             name,
             'Mercurial Windows development environment',
             fingerprint,
         )
 
 
 @contextlib.contextmanager
 def temporary_windows_dev_instances(
     c: AWSConnection,
     image,
     instance_type,
     prefix='hg-',
     disable_antivirus=False,
 ):
     """Create a temporary Windows development EC2 instance.
 
     Context manager resolves to the list of ``EC2.Instance`` that were created.
     """
     config = {
         'BlockDeviceMappings': [
             {
                 'DeviceName': '/dev/sda1',
                 'Ebs': {
                     'DeleteOnTermination': True,
                     'VolumeSize': 32,
-                    'VolumeType': 'gp2',
+                    'VolumeType': 'gp3',
                 },
             }
         ],
         'ImageId': image.id,
         'InstanceInitiatedShutdownBehavior': 'stop',
         'InstanceType': instance_type,
         'KeyName': '%sautomation' % prefix,
         'MaxCount': 1,
         'MinCount': 1,
         'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
     }
 
     with create_temp_windows_ec2_instances(c, config) as instances:
         if disable_antivirus:
             for instance in instances:
                 run_powershell(
                     instance.winrm_client,
                     'Set-MpPreference -DisableRealtimeMonitoring $true',
                 )
 
         yield instances

contrib/automation/hgautomation/cli.py

@@ -1,576 +1,576 b''
 # cli.py - Command line interface for automation
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import concurrent.futures as futures
 import os
 import pathlib
 import time
 
 from . import (
     aws,
     HGAutomation,
     linux,
     try_server,
     windows,
 )
 
 
 SOURCE_ROOT = pathlib.Path(
     os.path.abspath(__file__)
 ).parent.parent.parent.parent
 DIST_PATH = SOURCE_ROOT / 'dist'
 
 
 def bootstrap_linux_dev(
     hga: HGAutomation, aws_region, distros=None, parallel=False
 ):
     c = hga.aws_connection(aws_region)
 
     if distros:
         distros = distros.split(',')
     else:
         distros = sorted(linux.DISTROS)
 
     # TODO There is a wonky interaction involving KeyboardInterrupt whereby
     # the context manager that is supposed to terminate the temporary EC2
     # instance doesn't run. Until we fix this, make parallel building opt-in
     # so we don't orphan instances.
     if parallel:
         fs = []
 
         with futures.ThreadPoolExecutor(len(distros)) as e:
             for distro in distros:
                 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
 
             for f in fs:
                 f.result()
     else:
         for distro in distros:
             aws.ensure_linux_dev_ami(c, distro=distro)
 
 
 def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     print('Windows development AMI available as %s' % image.id)
 
 
 def build_inno(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     version,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_inno_installer(
                     instance.winrm_client,
                     py_version,
                     a,
                     DIST_PATH,
                     version=version,
                 )
 
 
 def build_wix(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     version,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_wix_installer(
                     instance.winrm_client,
                     py_version,
                     a,
                     DIST_PATH,
                     version=version,
                 )
 
 
 def build_windows_wheel(
     hga: HGAutomation,
     aws_region,
     python_version,
     arch,
     revision,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
     with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in python_version:
             for a in arch:
                 windows.build_wheel(
                     instance.winrm_client, py_version, a, DIST_PATH
                 )
 
 
 def build_all_windows_packages(
     hga: HGAutomation, aws_region, revision, version, base_image_name
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
     DIST_PATH.mkdir(exist_ok=True)
 
-    with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
+    with aws.temporary_windows_dev_instances(c, image, 'm6i.large') as insts:
         instance = insts[0]
 
         winrm_client = instance.winrm_client
 
         windows.synchronize_hg(SOURCE_ROOT, revision, instance)
 
         for py_version in ("2.7", "3.7", "3.8", "3.9", "3.10"):
             for arch in ("x86", "x64"):
                 windows.purge_hg(winrm_client)
                 windows.build_wheel(
                     winrm_client,
                     python_version=py_version,
                     arch=arch,
                     dest_path=DIST_PATH,
                 )
 
         for py_version in (2, 3):
             for arch in ('x86', 'x64'):
                 windows.purge_hg(winrm_client)
                 windows.build_inno_installer(
                     winrm_client, py_version, arch, DIST_PATH, version=version
                 )
                 windows.build_wix_installer(
                     winrm_client, py_version, arch, DIST_PATH, version=version
                 )
 
 
 def terminate_ec2_instances(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.terminate_ec2_instances(c.ec2resource)
 
 
 def purge_ec2_resources(hga: HGAutomation, aws_region):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     aws.remove_resources(c)
 
 
 def run_tests_linux(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     test_flags,
     distro,
     filesystem,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_linux_dev_ami(c, distro=distro)
 
     t_start = time.time()
 
     ensure_extra_volume = filesystem not in ('default', 'tmpfs')
 
     with aws.temporary_linux_dev_instances(
         c, image, instance_type, ensure_extra_volume=ensure_extra_volume
     ) as insts:
 
         instance = insts[0]
 
         linux.prepare_exec_environment(
             instance.ssh_client, filesystem=filesystem
         )
         linux.synchronize_hg(SOURCE_ROOT, instance, '.')
         t_prepared = time.time()
         linux.run_tests(instance.ssh_client, python_version, test_flags)
         t_done = time.time()
 
     t_setup = t_prepared - t_start
     t_all = t_done - t_start
 
     print(
         'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
         % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
     )
 
 
 def run_tests_windows(
     hga: HGAutomation,
     aws_region,
     instance_type,
     python_version,
     arch,
     test_flags,
     base_image_name,
 ):
     c = hga.aws_connection(aws_region)
     image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
 
     with aws.temporary_windows_dev_instances(
         c, image, instance_type, disable_antivirus=True
     ) as insts:
         instance = insts[0]
 
         windows.synchronize_hg(SOURCE_ROOT, '.', instance)
         windows.run_tests(
             instance.winrm_client, python_version, arch, test_flags
         )
 
 
 def publish_windows_artifacts(
     hg: HGAutomation,
     aws_region,
     version: str,
     pypi: bool,
     mercurial_scm_org: bool,
     ssh_username: str,
 ):
     windows.publish_artifacts(
         DIST_PATH,
         version,
         pypi=pypi,
         mercurial_scm_org=mercurial_scm_org,
         ssh_username=ssh_username,
     )
 
 
 def run_try(hga: HGAutomation, aws_region: str, rev: str):
     c = hga.aws_connection(aws_region, ensure_ec2_state=False)
     try_server.trigger_try(c, rev=rev)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
         '--state-path',
         default='~/.hgautomation',
         help='Path for local state files',
     )
     parser.add_argument(
         '--aws-region',
         help='AWS region to use',
         default='us-west-2',
     )
 
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser(
         'bootstrap-linux-dev',
         help='Bootstrap Linux development environments',
     )
     sp.add_argument(
         '--distros',
         help='Comma delimited list of distros to bootstrap',
     )
     sp.add_argument(
         '--parallel',
         action='store_true',
         help='Generate AMIs in parallel (not CTRL-c safe)',
     )
     sp.set_defaults(func=bootstrap_linux_dev)
 
     sp = subparsers.add_parser(
         'bootstrap-windows-dev',
         help='Bootstrap the Windows development environment',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=bootstrap_windows_dev)
 
     sp = subparsers.add_parser(
         'build-all-windows-packages',
         help='Build all Windows packages',
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--version',
         help='Mercurial version string to use',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_all_windows_packages)
 
     sp = subparsers.add_parser(
         'build-inno',
         help='Build Inno Setup installer(s)',
     )
     sp.add_argument(
         '--python-version',
         help='Which version of Python to target',
         choices={2, 3},
         type=int,
         nargs='*',
         default=[3],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--version',
         help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_inno)
 
     sp = subparsers.add_parser(
         'build-windows-wheel',
         help='Build Windows wheel(s)',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to build for',
         choices={'2.7', '3.7', '3.8', '3.9', '3.10'},
         nargs='*',
         default=['3.8'],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_windows_wheel)
 
     sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
     sp.add_argument(
         '--python-version',
         help='Which version of Python to target',
         choices={2, 3},
         type=int,
         nargs='*',
         default=[3],
     )
     sp.add_argument(
         '--arch',
         help='Architecture to build for',
         choices={'x86', 'x64'},
         nargs='*',
         default=['x64'],
     )
     sp.add_argument(
         '--revision',
         help='Mercurial revision to build',
         default='.',
     )
     sp.add_argument(
         '--version',
         help='Mercurial version string to use in installer',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=build_wix)
 
     sp = subparsers.add_parser(
         'terminate-ec2-instances',
         help='Terminate all active EC2 instances managed by us',
     )
     sp.set_defaults(func=terminate_ec2_instances)
 
     sp = subparsers.add_parser(
         'purge-ec2-resources',
         help='Purge all EC2 resources managed by us',
     )
     sp.set_defaults(func=purge_ec2_resources)
 
     sp = subparsers.add_parser(
         'run-tests-linux',
         help='Run tests on Linux',
     )
     sp.add_argument(
         '--distro',
         help='Linux distribution to run tests on',
         choices=linux.DISTROS,
         default='debian10',
     )
     sp.add_argument(
         '--filesystem',
         help='Filesystem type to use',
         choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
         default='default',
     )
     sp.add_argument(
         '--instance-type',
         help='EC2 instance type to use',
         default='c5.9xlarge',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
         choices={
             'system2',
             'system3',
             '2.7',
             '3.5',
             '3.6',
             '3.7',
             '3.8',
             'pypy',
             'pypy3.5',
             'pypy3.6',
         },
         default='system2',
     )
     sp.add_argument(
         'test_flags',
         help='Extra command line flags to pass to run-tests.py',
         nargs='*',
     )
     sp.set_defaults(func=run_tests_linux)
 
     sp = subparsers.add_parser(
         'run-tests-windows',
         help='Run tests on Windows',
     )
     sp.add_argument(
         '--instance-type',
         help='EC2 instance type to use',
-        default='t3.medium',
+        default='m6i.large',
     )
     sp.add_argument(
         '--python-version',
         help='Python version to use',
         choices={'2.7', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10'},
         default='2.7',
     )
     sp.add_argument(
         '--arch',
         help='Architecture to test',
         choices={'x86', 'x64'},
         default='x64',
     )
     sp.add_argument(
         '--test-flags',
         help='Extra command line flags to pass to run-tests.py',
     )
     sp.add_argument(
         '--base-image-name',
         help='AMI name of base image',
         default=aws.WINDOWS_BASE_IMAGE_NAME,
     )
     sp.set_defaults(func=run_tests_windows)
 
     sp = subparsers.add_parser(
         'publish-windows-artifacts',
         help='Publish built Windows artifacts (wheels, installers, etc)',
     )
     sp.add_argument(
         '--no-pypi',
         dest='pypi',
         action='store_false',
         default=True,
         help='Skip uploading to PyPI',
     )
     sp.add_argument(
         '--no-mercurial-scm-org',
         dest='mercurial_scm_org',
         action='store_false',
         default=True,
         help='Skip uploading to www.mercurial-scm.org',
     )
     sp.add_argument(
         '--ssh-username',
         help='SSH username for mercurial-scm.org',
     )
     sp.add_argument(
         'version',
         help='Mercurial version string to locate local packages',
     )
     sp.set_defaults(func=publish_windows_artifacts)
 
     sp = subparsers.add_parser(
         'try', help='Run CI automation against a custom changeset'
     )
     sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
     sp.set_defaults(func=run_try)
 
     return parser
 
 
 def main():
     parser = get_parser()
     args = parser.parse_args()
 
     local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
     automation = HGAutomation(local_state_path)
 
     if not hasattr(args, 'func'):
         parser.print_help()
         return
 
     kwargs = dict(vars(args))
     del kwargs['func']
     del kwargs['state_path']
 
     args.func(automation, **kwargs)

rust/hg-core/src/dirstate_tree/status.rs

@@ -1,827 +1,837 b''
 use crate::dirstate::entry::TruncatedTimestamp;
 use crate::dirstate::status::IgnoreFnType;
 use crate::dirstate::status::StatusPath;
 use crate::dirstate_tree::dirstate_map::BorrowedPath;
 use crate::dirstate_tree::dirstate_map::ChildNodesRef;
 use crate::dirstate_tree::dirstate_map::DirstateMap;
 use crate::dirstate_tree::dirstate_map::NodeData;
 use crate::dirstate_tree::dirstate_map::NodeRef;
 use crate::dirstate_tree::on_disk::DirstateV2ParseError;
 use crate::matchers::get_ignore_function;
 use crate::matchers::Matcher;
 use crate::utils::files::get_bytes_from_os_string;
 use crate::utils::files::get_path_from_bytes;
 use crate::utils::hg_path::HgPath;
 use crate::BadMatch;
 use crate::DirstateStatus;
 use crate::EntryState;
 use crate::HgPathBuf;
 use crate::HgPathCow;
 use crate::PatternFileWarning;
 use crate::StatusError;
 use crate::StatusOptions;
 use micro_timer::timed;
 use rayon::prelude::*;
 use sha1::{Digest, Sha1};
 use std::borrow::Cow;
 use std::io;
 use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Mutex;
 use std::time::SystemTime;
 
 /// Returns the status of the working directory compared to its parent
 /// changeset.
 ///
 /// This algorithm is based on traversing the filesystem tree (`fs` in function
 /// and variable names) and dirstate tree at the same time. The core of this
 /// traversal is the recursive `traverse_fs_directory_and_dirstate` function
 /// and its use of `itertools::merge_join_by`. When reaching a path that only
 /// exists in one of the two trees, depending on information requested by
 /// `options` we may need to traverse the remaining subtree.
 #[timed]
 pub fn status<'tree, 'on_disk: 'tree>(
     dmap: &'tree mut DirstateMap<'on_disk>,
     matcher: &(dyn Matcher + Sync),
     root_dir: PathBuf,
     ignore_files: Vec<PathBuf>,
     options: StatusOptions,
 ) -> Result<(DirstateStatus<'on_disk>, Vec<PatternFileWarning>), StatusError> {
     let (ignore_fn, warnings, patterns_changed): (IgnoreFnType, _, _) =
         if options.list_ignored || options.list_unknown {
             let mut hasher = Sha1::new();
             let (ignore_fn, warnings) = get_ignore_function(
                 ignore_files,
                 &root_dir,
                 &mut |pattern_bytes| hasher.update(pattern_bytes),
             )?;
             let new_hash = *hasher.finalize().as_ref();
             let changed = new_hash != dmap.ignore_patterns_hash;
             dmap.ignore_patterns_hash = new_hash;
             (ignore_fn, warnings, Some(changed))
         } else {
             (Box::new(|&_| true), vec![], None)
         };
 
     let filesystem_time_at_status_start =
         filesystem_now(&root_dir).ok().map(TruncatedTimestamp::from);
 
     // If the repository is under the current directory, prefer using a
     // relative path, so the kernel needs to traverse fewer directory in every
     // call to `read_dir` or `symlink_metadata`.
     // This is effective in the common case where the current directory is the
     // repository root.
 
     // TODO: Better yet would be to use libc functions like `openat` and
     // `fstatat` to remove such repeated traversals entirely, but the standard
     // library does not provide APIs based on those.
     // Maybe with a crate like https://crates.io/crates/openat instead?
     let root_dir = if let Some(relative) = std::env::current_dir()
         .ok()
         .and_then(|cwd| root_dir.strip_prefix(cwd).ok())
     {
         relative
     } else {
         &root_dir
     };
 
     let outcome = DirstateStatus {
         filesystem_time_at_status_start,
         ..Default::default()
     };
     let common = StatusCommon {
         dmap,
         options,
         matcher,
         ignore_fn,
         outcome: Mutex::new(outcome),
         ignore_patterns_have_changed: patterns_changed,
         new_cachable_directories: Default::default(),
         outated_cached_directories: Default::default(),
         filesystem_time_at_status_start,
     };
     let is_at_repo_root = true;
     let hg_path = &BorrowedPath::OnDisk(HgPath::new(""));
     let has_ignored_ancestor = false;
     let root_cached_mtime = None;
     let root_dir_metadata = None;
     // If the path we have for the repository root is a symlink, do follow it.
     // (As opposed to symlinks within the working directory which are not
     // followed, using `std::fs::symlink_metadata`.)
     common.traverse_fs_directory_and_dirstate(
         has_ignored_ancestor,
         dmap.root.as_ref(),
         hg_path,
         &root_dir,
         root_dir_metadata,
         root_cached_mtime,
         is_at_repo_root,
     )?;
     let mut outcome = common.outcome.into_inner().unwrap();
     let new_cachable = common.new_cachable_directories.into_inner().unwrap();
     let outdated = common.outated_cached_directories.into_inner().unwrap();
 
     outcome.dirty = common.ignore_patterns_have_changed == Some(true)
         || !outdated.is_empty()
         || !new_cachable.is_empty();
 
     // Remove outdated mtimes before adding new mtimes, in case a given
     // directory is both
     for path in &outdated {
         let node = dmap.get_or_insert(path)?;
         if let NodeData::CachedDirectory { .. } = &node.data {
             node.data = NodeData::None
         }
     }
     for (path, mtime) in &new_cachable {
         let node = dmap.get_or_insert(path)?;
         match &node.data {
             NodeData::Entry(_) => {} // Don’t overwrite an entry
             NodeData::CachedDirectory { .. } | NodeData::None => {
                 node.data = NodeData::CachedDirectory { mtime: *mtime }
             }
         }
     }
 
     Ok((outcome, warnings))
 }
 
 /// Bag of random things needed by various parts of the algorithm. Reduces the
 /// number of parameters passed to functions.
 struct StatusCommon<'a, 'tree, 'on_disk: 'tree> {
     dmap: &'tree DirstateMap<'on_disk>,
     options: StatusOptions,
     matcher: &'a (dyn Matcher + Sync),
     ignore_fn: IgnoreFnType<'a>,
     outcome: Mutex<DirstateStatus<'on_disk>>,
     new_cachable_directories:
         Mutex<Vec<(Cow<'on_disk, HgPath>, TruncatedTimestamp)>>,
     outated_cached_directories: Mutex<Vec<Cow<'on_disk, HgPath>>>,
 
     /// Whether ignore files like `.hgignore` have changed since the previous
     /// time a `status()` call wrote their hash to the dirstate. `None` means
     /// we don’t know as this run doesn’t list either ignored or uknown files
     /// and therefore isn’t reading `.hgignore`.
     ignore_patterns_have_changed: Option<bool>,
 
     /// The current time at the start of the `status()` algorithm, as measured
     /// and possibly truncated by the filesystem.
     filesystem_time_at_status_start: Option<TruncatedTimestamp>,
 }
 
 enum Outcome {
     Modified,
     Added,
     Removed,
     Deleted,
     Clean,
     Ignored,
     Unknown,
     Unsure,
 }
 
 impl<'a, 'tree, 'on_disk> StatusCommon<'a, 'tree, 'on_disk> {
     fn push_outcome(
         &self,
         which: Outcome,
         dirstate_node: &NodeRef<'tree, 'on_disk>,
     ) -> Result<(), DirstateV2ParseError> {
         let path = dirstate_node
             .full_path_borrowed(self.dmap.on_disk)?
             .detach_from_tree();
         let copy_source = if self.options.list_copies {
             dirstate_node
                 .copy_source_borrowed(self.dmap.on_disk)?
                 .map(|source| source.detach_from_tree())
         } else {
             None
         };
         self.push_outcome_common(which, path, copy_source);
         Ok(())
     }
 
     fn push_outcome_without_copy_source(
         &self,
         which: Outcome,
         path: &BorrowedPath<'_, 'on_disk>,
     ) {
         self.push_outcome_common(which, path.detach_from_tree(), None)
     }
 
     fn push_outcome_common(
         &self,
         which: Outcome,
         path: HgPathCow<'on_disk>,
         copy_source: Option<HgPathCow<'on_disk>>,
     ) {
         let mut outcome = self.outcome.lock().unwrap();
         let vec = match which {
             Outcome::Modified => &mut outcome.modified,
             Outcome::Added => &mut outcome.added,
             Outcome::Removed => &mut outcome.removed,
             Outcome::Deleted => &mut outcome.deleted,
             Outcome::Clean => &mut outcome.clean,
             Outcome::Ignored => &mut outcome.ignored,
             Outcome::Unknown => &mut outcome.unknown,
             Outcome::Unsure => &mut outcome.unsure,
         };
         vec.push(StatusPath { path, copy_source });
     }
 
     fn read_dir(
         &self,
         hg_path: &HgPath,
         fs_path: &Path,
         is_at_repo_root: bool,
     ) -> Result<Vec<DirEntry>, ()> {
         DirEntry::read_dir(fs_path, is_at_repo_root)
             .map_err(|error| self.io_error(error, hg_path))
     }
 
     fn io_error(&self, error: std::io::Error, hg_path: &HgPath) {
         let errno = error.raw_os_error().expect("expected real OS error");
         self.outcome
             .lock()
             .unwrap()
             .bad
             .push((hg_path.to_owned().into(), BadMatch::OsError(errno)))
     }
 
     fn check_for_outdated_directory_cache(
         &self,
         dirstate_node: &NodeRef<'tree, 'on_disk>,
     ) -> Result<(), DirstateV2ParseError> {
         if self.ignore_patterns_have_changed == Some(true)
             && dirstate_node.cached_directory_mtime()?.is_some()
         {
             self.outated_cached_directories.lock().unwrap().push(
                 dirstate_node
                     .full_path_borrowed(self.dmap.on_disk)?
                     .detach_from_tree(),
             )
         }
         Ok(())
     }
 
     /// If this returns true, we can get accurate results by only using
     /// `symlink_metadata` for child nodes that exist in the dirstate and don’t
     /// need to call `read_dir`.
     fn can_skip_fs_readdir(
         &self,
         directory_metadata: Option<&std::fs::Metadata>,
         cached_directory_mtime: Option<TruncatedTimestamp>,
     ) -> bool {
         if !self.options.list_unknown && !self.options.list_ignored {
             // All states that we care about listing have corresponding
             // dirstate entries.
             // This happens for example with `hg status -mard`.
             return true;
         }
         if !self.options.list_ignored
             && self.ignore_patterns_have_changed == Some(false)
         {
             if let Some(cached_mtime) = cached_directory_mtime {
                 // The dirstate contains a cached mtime for this directory, set
                 // by a previous run of the `status` algorithm which found this
                 // directory eligible for `read_dir` caching.
                 if let Some(meta) = directory_metadata {
                     if cached_mtime
                         .likely_equal_to_mtime_of(meta)
                         .unwrap_or(false)
                     {
                         // The mtime of that directory has not changed
                         // since then, which means that the results of
                         // `read_dir` should also be unchanged.
                         return true;
                     }
                 }
             }
         }
         false
     }
 
     /// Returns whether all child entries of the filesystem directory have a
     /// corresponding dirstate node or are ignored.
     fn traverse_fs_directory_and_dirstate(
         &self,
         has_ignored_ancestor: bool,
         dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,
         directory_hg_path: &BorrowedPath<'tree, 'on_disk>,
         directory_fs_path: &Path,
         directory_metadata: Option<&std::fs::Metadata>,
         cached_directory_mtime: Option<TruncatedTimestamp>,
         is_at_repo_root: bool,
     ) -> Result<bool, DirstateV2ParseError> {
         if self.can_skip_fs_readdir(directory_metadata, cached_directory_mtime)
         {
             dirstate_nodes
                 .par_iter()
                 .map(|dirstate_node| {
                     let fs_path = directory_fs_path.join(get_path_from_bytes(
                         dirstate_node.base_name(self.dmap.on_disk)?.as_bytes(),
                     ));
                     match std::fs::symlink_metadata(&fs_path) {
                         Ok(fs_metadata) => self.traverse_fs_and_dirstate(
                             &fs_path,
                             &fs_metadata,
                             dirstate_node,
                             has_ignored_ancestor,
                         ),
                         Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
                             self.traverse_dirstate_only(dirstate_node)
                         }
                         Err(error) => {
                             let hg_path =
                                 dirstate_node.full_path(self.dmap.on_disk)?;
                             Ok(self.io_error(error, hg_path))
                         }
                     }
                 })
                 .collect::<Result<_, _>>()?;
 
             // We don’t know, so conservatively say this isn’t the case
             let children_all_have_dirstate_node_or_are_ignored = false;
 
             return Ok(children_all_have_dirstate_node_or_are_ignored);
         }
 
         let mut fs_entries = if let Ok(entries) = self.read_dir(
             directory_hg_path,
             directory_fs_path,
             is_at_repo_root,
         ) {
             entries
         } else {
             // Treat an unreadable directory (typically because of insufficient
             // permissions) like an empty directory. `self.read_dir` has
             // already called `self.io_error` so a warning will be emitted.
             Vec::new()
         };
 
         // `merge_join_by` requires both its input iterators to be sorted:
 
         let dirstate_nodes = dirstate_nodes.sorted();
         // `sort_unstable_by_key` doesn’t allow keys borrowing from the value:
         // https://github.com/rust-lang/rust/issues/34162
         fs_entries.sort_unstable_by(|e1, e2| e1.base_name.cmp(&e2.base_name));
 
         // Propagate here any error that would happen inside the comparison
         // callback below
         for dirstate_node in &dirstate_nodes {
             dirstate_node.base_name(self.dmap.on_disk)?;
         }
         itertools::merge_join_by(
             dirstate_nodes,
             &fs_entries,
             |dirstate_node, fs_entry| {
                 // This `unwrap` never panics because we already propagated
                 // those errors above
                 dirstate_node
                     .base_name(self.dmap.on_disk)
                     .unwrap()
                     .cmp(&fs_entry.base_name)
             },
         )
         .par_bridge()
         .map(|pair| {
             use itertools::EitherOrBoth::*;
             let has_dirstate_node_or_is_ignored;
             match pair {
                 Both(dirstate_node, fs_entry) => {
                     self.traverse_fs_and_dirstate(
                         &fs_entry.full_path,
                         &fs_entry.metadata,
                         dirstate_node,
                         has_ignored_ancestor,
                     )?;
                     has_dirstate_node_or_is_ignored = true
                 }
                 Left(dirstate_node) => {
                     self.traverse_dirstate_only(dirstate_node)?;
                     has_dirstate_node_or_is_ignored = true;
                 }
                 Right(fs_entry) => {
                     has_dirstate_node_or_is_ignored = self.traverse_fs_only(
                         has_ignored_ancestor,
                         directory_hg_path,
                         fs_entry,
                     )
                 }
             }
             Ok(has_dirstate_node_or_is_ignored)
         })
         .try_reduce(|| true, |a, b| Ok(a && b))
     }
 
     fn traverse_fs_and_dirstate(
         &self,
         fs_path: &Path,
         fs_metadata: &std::fs::Metadata,
         dirstate_node: NodeRef<'tree, 'on_disk>,
         has_ignored_ancestor: bool,
     ) -> Result<(), DirstateV2ParseError> {
         self.check_for_outdated_directory_cache(&dirstate_node)?;
         let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;
         let file_type = fs_metadata.file_type();
         let file_or_symlink = file_type.is_file() || file_type.is_symlink();
         if !file_or_symlink {
             // If we previously had a file here, it was removed (with
             // `hg rm` or similar) or deleted before it could be
             // replaced by a directory or something else.
             self.mark_removed_or_deleted_if_file(&dirstate_node)?;
         }
         if file_type.is_dir() {
             if self.options.collect_traversed_dirs {
                 self.outcome
                     .lock()
                     .unwrap()
                     .traversed
                     .push(hg_path.detach_from_tree())
             }
             let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path);
             let is_at_repo_root = false;
             let children_all_have_dirstate_node_or_are_ignored = self
                 .traverse_fs_directory_and_dirstate(
                     is_ignored,
                     dirstate_node.children(self.dmap.on_disk)?,
                     hg_path,
                     fs_path,
                     Some(fs_metadata),
                     dirstate_node.cached_directory_mtime()?,
                     is_at_repo_root,
                 )?;
             self.maybe_save_directory_mtime(
                 children_all_have_dirstate_node_or_are_ignored,
                 fs_metadata,
                 dirstate_node,
             )?
         } else {
             if file_or_symlink && self.matcher.matches(hg_path) {
                 if let Some(state) = dirstate_node.state()? {
                     match state {
                         EntryState::Added => {
                             self.push_outcome(Outcome::Added, &dirstate_node)?
                         }
                         EntryState::Removed => self
                             .push_outcome(Outcome::Removed, &dirstate_node)?,
                         EntryState::Merged => self
                             .push_outcome(Outcome::Modified, &dirstate_node)?,
                         EntryState::Normal => self
                             .handle_normal_file(&dirstate_node, fs_metadata)?,
                     }
                 } else {
                     // `node.entry.is_none()` indicates a "directory"
                     // node, but the filesystem has a file
                     self.mark_unknown_or_ignored(
                         has_ignored_ancestor,
                         hg_path,
                     );
                 }
             }
 
             for child_node in dirstate_node.children(self.dmap.on_disk)?.iter()
             {
                 self.traverse_dirstate_only(child_node)?
             }
         }
         Ok(())
     }
 
     fn maybe_save_directory_mtime(
         &self,
         children_all_have_dirstate_node_or_are_ignored: bool,
         directory_metadata: &std::fs::Metadata,
         dirstate_node: NodeRef<'tree, 'on_disk>,
     ) -> Result<(), DirstateV2ParseError> {
         if !children_all_have_dirstate_node_or_are_ignored {
             return Ok(());
         }
         // All filesystem directory entries from `read_dir` have a
         // corresponding node in the dirstate, so we can reconstitute the
         // names of those entries without calling `read_dir` again.
 
         // TODO: use let-else here and below when available:
         // https://github.com/rust-lang/rust/issues/87335
         let status_start = if let Some(status_start) =
             &self.filesystem_time_at_status_start
         {
             status_start
         } else {
             return Ok(());
         };
 
         // Although the Rust standard library’s `SystemTime` type
         // has nanosecond precision, the times reported for a
         // directory’s (or file’s) modified time may have lower
         // resolution based on the filesystem (for example ext3
         // only stores integer seconds), kernel (see
         // https://stackoverflow.com/a/14393315/1162888), etc.
         let directory_mtime = if let Ok(option) =
             TruncatedTimestamp::for_reliable_mtime_of(
                 directory_metadata,
                 status_start,
             ) {
             if let Some(directory_mtime) = option {
                 directory_mtime
             } else {
                 // The directory was modified too recently,
                 // don’t cache its `read_dir` results.
                 //
                 // 1. A change to this directory (direct child was
                 //    added or removed) cause its mtime to be set
                 //    (possibly truncated) to `directory_mtime`
                 // 2. This `status` algorithm calls `read_dir`
                 // 3. An other change is made to the same directory is
                 //    made so that calling `read_dir` agin would give
                 //    different results, but soon enough after 1. that
                 //    the mtime stays the same
                 //
                 // On a system where the time resolution poor, this
                 // scenario is not unlikely if all three steps are caused
                 // by the same script.
                 return Ok(());
             }
         } else {
             // OS/libc does not support mtime?
             return Ok(());
         };
         // We’ve observed (through `status_start`) that time has
         // “progressed” since `directory_mtime`, so any further
         // change to this directory is extremely likely to cause a
         // different mtime.
         //
         // Having the same mtime again is not entirely impossible
         // since the system clock is not monotonous. It could jump
         // backward to some point before `directory_mtime`, then a
         // directory change could potentially happen during exactly
         // the wrong tick.
         //
         // We deem this scenario (unlike the previous one) to be
         // unlikely enough in practice.
 
         let is_up_to_date =
             if let Some(cached) = dirstate_node.cached_directory_mtime()? {
                 cached.likely_equal(directory_mtime)
             } else {
                 false
             };
         if !is_up_to_date {
             let hg_path = dirstate_node
                 .full_path_borrowed(self.dmap.on_disk)?
                 .detach_from_tree();
             self.new_cachable_directories
                 .lock()
                 .unwrap()
                 .push((hg_path, directory_mtime))
         }
         Ok(())
     }
 
     /// A file with `EntryState::Normal` in the dirstate was found in the
     /// filesystem
     fn handle_normal_file(
         &self,
         dirstate_node: &NodeRef<'tree, 'on_disk>,
         fs_metadata: &std::fs::Metadata,
     ) -> Result<(), DirstateV2ParseError> {
         // Keep the low 31 bits
         fn truncate_u64(value: u64) -> i32 {
             (value & 0x7FFF_FFFF) as i32
         }
 
         let entry = dirstate_node
             .entry()?
             .expect("handle_normal_file called with entry-less node");
         let mode_changed =
             || self.options.check_exec && entry.mode_changed(fs_metadata);
         let size = entry.size();
         let size_changed = size != truncate_u64(fs_metadata.len());
         if size >= 0 && size_changed && fs_metadata.file_type().is_symlink() {
             // issue6456: Size returned may be longer due to encryption
             // on EXT-4 fscrypt. TODO maybe only do it on EXT4?
             self.push_outcome(Outcome::Unsure, dirstate_node)?
         } else if dirstate_node.has_copy_source()
             || entry.is_from_other_parent()
             || (size >= 0 && (size_changed || mode_changed()))
         {
             self.push_outcome(Outcome::Modified, dirstate_node)?
         } else {
             let mtime_looks_clean;
             if let Some(dirstate_mtime) = entry.truncated_mtime() {
                 let fs_mtime = TruncatedTimestamp::for_mtime_of(fs_metadata)
                     .expect("OS/libc does not support mtime?");
                 // There might be a change in the future if for example the
                 // internal clock become off while process run, but this is a
                 // case where the issues the user would face
                 // would be a lot worse and there is nothing we
                 // can really do.
                 mtime_looks_clean = fs_mtime.likely_equal(dirstate_mtime)
             } else {
                 // No mtime in the dirstate entry
                 mtime_looks_clean = false
             };
             if !mtime_looks_clean {
                 self.push_outcome(Outcome::Unsure, dirstate_node)?
             } else if self.options.list_clean {
                 self.push_outcome(Outcome::Clean, dirstate_node)?
             }
         }
         Ok(())
     }
 
     /// A node in the dirstate tree has no corresponding filesystem entry
     fn traverse_dirstate_only(
         &self,
         dirstate_node: NodeRef<'tree, 'on_disk>,
     ) -> Result<(), DirstateV2ParseError> {
         self.check_for_outdated_directory_cache(&dirstate_node)?;
         self.mark_removed_or_deleted_if_file(&dirstate_node)?;
         dirstate_node
             .children(self.dmap.on_disk)?
             .par_iter()
             .map(|child_node| self.traverse_dirstate_only(child_node))
             .collect()
     }
 
     /// A node in the dirstate tree has no corresponding *file* on the
     /// filesystem
     ///
     /// Does nothing on a "directory" node
     fn mark_removed_or_deleted_if_file(
         &self,
         dirstate_node: &NodeRef<'tree, 'on_disk>,
     ) -> Result<(), DirstateV2ParseError> {
         if let Some(state) = dirstate_node.state()? {
             let path = dirstate_node.full_path(self.dmap.on_disk)?;
             if self.matcher.matches(path) {
                 if let EntryState::Removed = state {
                     self.push_outcome(Outcome::Removed, dirstate_node)?
                 } else {
                     self.push_outcome(Outcome::Deleted, &dirstate_node)?
                 }
             }
         }
         Ok(())
     }
 
     /// Something in the filesystem has no corresponding dirstate node
     ///
     /// Returns whether that path is ignored
     fn traverse_fs_only(
         &self,
         has_ignored_ancestor: bool,
         directory_hg_path: &HgPath,
         fs_entry: &DirEntry,
     ) -> bool {
         let hg_path = directory_hg_path.join(&fs_entry.base_name);
         let file_type = fs_entry.metadata.file_type();
         let file_or_symlink = file_type.is_file() || file_type.is_symlink();
         if file_type.is_dir() {
             let is_ignored =
                 has_ignored_ancestor || (self.ignore_fn)(&hg_path);
             let traverse_children = if is_ignored {
                 // Descendants of an ignored directory are all ignored
                 self.options.list_ignored
             } else {
                 // Descendants of an unknown directory may be either unknown or
                 // ignored
                 self.options.list_unknown || self.options.list_ignored
             };
             if traverse_children {
                 let is_at_repo_root = false;
                 if let Ok(children_fs_entries) = self.read_dir(
                     &hg_path,
                     &fs_entry.full_path,
                     is_at_repo_root,
                 ) {
                     children_fs_entries.par_iter().for_each(|child_fs_entry| {
                         self.traverse_fs_only(
                             is_ignored,
                             &hg_path,
                             child_fs_entry,
                         );
                     })
                 }
             }
             if self.options.collect_traversed_dirs {
                 self.outcome.lock().unwrap().traversed.push(hg_path.into())
             }
             is_ignored
         } else {
             if file_or_symlink {
                 if self.matcher.matches(&hg_path) {
                     self.mark_unknown_or_ignored(
                         has_ignored_ancestor,
                         &BorrowedPath::InMemory(&hg_path),
                     )
                 } else {
                     // We haven’t computed whether this path is ignored. It
                     // might not be, and a future run of status might have a
                     // different matcher that matches it. So treat it as not
                     // ignored. That is, inhibit readdir caching of the parent
                     // directory.
                     false
                 }
             } else {
                 // This is neither a directory, a plain file, or a symlink.
                 // Treat it like an ignored file.
                 true
             }
         }
     }
 
     /// Returns whether that path is ignored
     fn mark_unknown_or_ignored(
         &self,
         has_ignored_ancestor: bool,
         hg_path: &BorrowedPath<'_, 'on_disk>,
     ) -> bool {
         let is_ignored = has_ignored_ancestor || (self.ignore_fn)(&hg_path);
         if is_ignored {
             if self.options.list_ignored {
                 self.push_outcome_without_copy_source(
                     Outcome::Ignored,
                     hg_path,
                 )
             }
         } else {
             if self.options.list_unknown {
                 self.push_outcome_without_copy_source(
                     Outcome::Unknown,
                     hg_path,
                 )
             }
         }
         is_ignored
     }
 }
 
 struct DirEntry {
     base_name: HgPathBuf,
     full_path: PathBuf,
     metadata: std::fs::Metadata,
 }
 
 impl DirEntry {
     /// Returns **unsorted** entries in the given directory, with name and
     /// metadata.
     ///
     /// If a `.hg` sub-directory is encountered:
     ///
     /// * At the repository root, ignore that sub-directory
     /// * Elsewhere, we’re listing the content of a sub-repo. Return an empty
     ///   list instead.
     fn read_dir(path: &Path, is_at_repo_root: bool) -> io::Result<Vec<Self>> {
         // `read_dir` returns a "not found" error for the empty path
         let at_cwd = path == Path::new("");
         let read_dir_path = if at_cwd { Path::new(".") } else { path };
         let mut results = Vec::new();
         for entry in read_dir_path.read_dir()? {
             let entry = entry?;
-            let metadata = entry.metadata()?;
+            let metadata = match entry.metadata() {
+                Ok(v) => v,
+                Err(e) => {
+                    // race with file deletion?
+                    if e.kind() == std::io::ErrorKind::NotFound {
+                        continue;
+                    } else {
+                        return Err(e);
+                    }
+                }
+            };
             let file_name = entry.file_name();
             // FIXME don't do this when cached
             if file_name == ".hg" {
                 if is_at_repo_root {
                     // Skip the repo’s own .hg (might be a symlink)
                     continue;
                 } else if metadata.is_dir() {
                     // A .hg sub-directory at another location means a subrepo,
                     // skip it entirely.
                     return Ok(Vec::new());
                 }
             }
             let full_path = if at_cwd {
                 file_name.clone().into()
             } else {
                 entry.path()
             };
             let base_name = get_bytes_from_os_string(file_name).into();
             results.push(DirEntry {
                 base_name,
                 full_path,
                 metadata,
             })
         }
         Ok(results)
     }
 }
 
 /// Return the `mtime` of a temporary file newly-created in the `.hg` directory
 /// of the give repository.
 ///
 /// This is similar to `SystemTime::now()`, with the result truncated to the
 /// same time resolution as other files’ modification times. Using `.hg`
 /// instead of the system’s default temporary directory (such as `/tmp`) makes
 /// it more likely the temporary file is in the same disk partition as contents
 /// of the working directory, which can matter since different filesystems may
 /// store timestamps with different resolutions.
 ///
 /// This may fail, typically if we lack write permissions. In that case we
 /// should continue the `status()` algoritm anyway and consider the current
 /// date/time to be unknown.
 fn filesystem_now(repo_root: &Path) -> Result<SystemTime, io::Error> {
     tempfile::tempfile_in(repo_root.join(".hg"))?
         .metadata()?
         .modified()
 }