|
|
# windows.py - Automation specific to Windows
|
|
|
#
|
|
|
# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
|
|
|
#
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
|
|
# no-check-code because Python 3 native.
|
|
|
|
|
|
import datetime
|
|
|
import os
|
|
|
import paramiko
|
|
|
import pathlib
|
|
|
import re
|
|
|
import subprocess
|
|
|
import tempfile
|
|
|
|
|
|
from .pypi import upload as pypi_upload
|
|
|
from .winrm import run_powershell
|
|
|
|
|
|
|
|
|
# PowerShell commands to activate a Visual Studio 2008 environment.
|
|
|
# This is essentially a port of vcvarsall.bat to PowerShell.
|
|
|
ACTIVATE_VC9_AMD64 = r'''
|
|
|
Write-Output "activating Visual Studio 2008 environment for AMD64"
|
|
|
$root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
|
|
|
$Env:VCINSTALLDIR = "${root}\VC\"
|
|
|
$Env:WindowsSdkDir = "${root}\WinSDK\"
|
|
|
$Env:PATH = "${root}\VC\Bin\amd64;${root}\WinSDK\Bin\x64;${root}\WinSDK\Bin;$Env:PATH"
|
|
|
$Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:PATH"
|
|
|
$Env:LIB = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIB"
|
|
|
$Env:LIBPATH = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIBPATH"
|
|
|
'''.lstrip()
|
|
|
|
|
|
ACTIVATE_VC9_X86 = r'''
|
|
|
Write-Output "activating Visual Studio 2008 environment for x86"
|
|
|
$root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
|
|
|
$Env:VCINSTALLDIR = "${root}\VC\"
|
|
|
$Env:WindowsSdkDir = "${root}\WinSDK\"
|
|
|
$Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH"
|
|
|
$Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE"
|
|
|
$Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB"
|
|
|
$Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib;$Env:LIBPATH"
|
|
|
'''.lstrip()
|
|
|
|
|
|
HG_PURGE = r'''
|
|
|
$Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
|
|
|
Set-Location C:\hgdev\src
|
|
|
hg.exe --config extensions.purge= purge --all
|
|
|
if ($LASTEXITCODE -ne 0) {
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}
|
|
|
Write-Output "purged Mercurial repo"
|
|
|
'''
|
|
|
|
|
|
HG_UPDATE_CLEAN = r'''
|
|
|
$Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
|
|
|
Set-Location C:\hgdev\src
|
|
|
hg.exe --config extensions.purge= purge --all
|
|
|
if ($LASTEXITCODE -ne 0) {{
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}}
|
|
|
hg.exe update -C {revision}
|
|
|
if ($LASTEXITCODE -ne 0) {{
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}}
|
|
|
hg.exe log -r .
|
|
|
Write-Output "updated Mercurial working directory to {revision}"
|
|
|
'''.lstrip()
|
|
|
|
|
|
BUILD_INNO = r'''
|
|
|
Set-Location C:\hgdev\src
|
|
|
$python = "C:\hgdev\python27-{arch}\python.exe"
|
|
|
C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python
|
|
|
if ($LASTEXITCODE -ne 0) {{
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}}
|
|
|
'''.lstrip()
|
|
|
|
|
|
BUILD_WHEEL = r'''
|
|
|
Set-Location C:\hgdev\src
|
|
|
C:\hgdev\python27-{arch}\Scripts\pip.exe wheel --wheel-dir dist .
|
|
|
if ($LASTEXITCODE -ne 0) {{
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}}
|
|
|
'''
|
|
|
|
|
|
BUILD_WIX = r'''
|
|
|
Set-Location C:\hgdev\src
|
|
|
$python = "C:\hgdev\python27-{arch}\python.exe"
|
|
|
C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args}
|
|
|
if ($LASTEXITCODE -ne 0) {{
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}}
|
|
|
'''
|
|
|
|
|
|
RUN_TESTS = r'''
|
|
|
C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
|
|
|
if ($LASTEXITCODE -ne 0) {{
|
|
|
throw "process exited non-0: $LASTEXITCODE"
|
|
|
}}
|
|
|
'''
|
|
|
|
|
|
X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
|
|
|
X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
|
|
|
X86_EXE_FILENAME = 'Mercurial-{version}.exe'
|
|
|
X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
|
|
|
X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
|
|
|
X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
|
|
|
|
|
|
MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
|
|
|
|
|
|
X86_USER_AGENT_PATTERN = '.*Windows.*'
|
|
|
X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
|
|
|
|
|
|
X86_EXE_DESCRIPTION = (
|
|
|
'Mercurial {version} Inno Setup installer - x86 Windows '
|
|
|
'- does not require admin rights'
|
|
|
)
|
|
|
X64_EXE_DESCRIPTION = (
|
|
|
'Mercurial {version} Inno Setup installer - x64 Windows '
|
|
|
'- does not require admin rights'
|
|
|
)
|
|
|
X86_MSI_DESCRIPTION = (
|
|
|
'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights'
|
|
|
)
|
|
|
X64_MSI_DESCRIPTION = (
|
|
|
'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights'
|
|
|
)
|
|
|
|
|
|
|
|
|
def get_vc_prefix(arch):
|
|
|
if arch == 'x86':
|
|
|
return ACTIVATE_VC9_X86
|
|
|
elif arch == 'x64':
|
|
|
return ACTIVATE_VC9_AMD64
|
|
|
else:
|
|
|
raise ValueError('illegal arch: %s; must be x86 or x64' % arch)
|
|
|
|
|
|
|
|
|
def fix_authorized_keys_permissions(winrm_client, path):
|
|
|
commands = [
|
|
|
'$ErrorActionPreference = "Stop"',
|
|
|
'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
|
|
|
r'icacls %s /remove:g "NT Service\sshd"' % path,
|
|
|
]
|
|
|
|
|
|
run_powershell(winrm_client, '\n'.join(commands))
|
|
|
|
|
|
|
|
|
def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
|
|
|
"""Synchronize local Mercurial repo to remote EC2 instance."""
|
|
|
|
|
|
winrm_client = ec2_instance.winrm_client
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
|
temp_dir = pathlib.Path(temp_dir)
|
|
|
|
|
|
ssh_dir = temp_dir / '.ssh'
|
|
|
ssh_dir.mkdir()
|
|
|
ssh_dir.chmod(0o0700)
|
|
|
|
|
|
# Generate SSH key to use for communication.
|
|
|
subprocess.run(
|
|
|
[
|
|
|
'ssh-keygen',
|
|
|
'-t',
|
|
|
'rsa',
|
|
|
'-b',
|
|
|
'4096',
|
|
|
'-N',
|
|
|
'',
|
|
|
'-f',
|
|
|
str(ssh_dir / 'id_rsa'),
|
|
|
],
|
|
|
check=True,
|
|
|
capture_output=True,
|
|
|
)
|
|
|
|
|
|
# Add it to ~/.ssh/authorized_keys on remote.
|
|
|
# This assumes the file doesn't already exist.
|
|
|
authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
|
|
|
winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
|
|
|
winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
|
|
|
fix_authorized_keys_permissions(winrm_client, authorized_keys)
|
|
|
|
|
|
public_ip = ec2_instance.public_ip_address
|
|
|
|
|
|
ssh_config = temp_dir / '.ssh' / 'config'
|
|
|
|
|
|
with open(ssh_config, 'w', encoding='utf-8') as fh:
|
|
|
fh.write('Host %s\n' % public_ip)
|
|
|
fh.write(' User Administrator\n')
|
|
|
fh.write(' StrictHostKeyChecking no\n')
|
|
|
fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
|
|
|
fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
|
|
|
|
|
|
if not (hg_repo / '.hg').is_dir():
|
|
|
raise Exception(
|
|
|
'%s is not a Mercurial repository; '
|
|
|
'synchronization not yet supported' % hg_repo
|
|
|
)
|
|
|
|
|
|
env = dict(os.environ)
|
|
|
env['HGPLAIN'] = '1'
|
|
|
env['HGENCODING'] = 'utf-8'
|
|
|
|
|
|
hg_bin = hg_repo / 'hg'
|
|
|
|
|
|
res = subprocess.run(
|
|
|
['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
|
|
|
cwd=str(hg_repo),
|
|
|
env=env,
|
|
|
check=True,
|
|
|
capture_output=True,
|
|
|
)
|
|
|
|
|
|
full_revision = res.stdout.decode('ascii')
|
|
|
|
|
|
args = [
|
|
|
'python2.7',
|
|
|
hg_bin,
|
|
|
'--config',
|
|
|
'ui.ssh=ssh -F %s' % ssh_config,
|
|
|
'--config',
|
|
|
'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
|
|
|
# Also ensure .hgtags changes are present so auto version
|
|
|
# calculation works.
|
|
|
'push',
|
|
|
'-f',
|
|
|
'-r',
|
|
|
full_revision,
|
|
|
'-r',
|
|
|
'file(.hgtags)',
|
|
|
'ssh://%s/c:/hgdev/src' % public_ip,
|
|
|
]
|
|
|
|
|
|
res = subprocess.run(args, cwd=str(hg_repo), env=env)
|
|
|
|
|
|
# Allow 1 (no-op) to not trigger error.
|
|
|
if res.returncode not in (0, 1):
|
|
|
res.check_returncode()
|
|
|
|
|
|
run_powershell(
|
|
|
winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
|
|
|
)
|
|
|
|
|
|
# TODO detect dirty local working directory and synchronize accordingly.
|
|
|
|
|
|
|
|
|
def purge_hg(winrm_client):
|
|
|
"""Purge the Mercurial source repository on an EC2 instance."""
|
|
|
run_powershell(winrm_client, HG_PURGE)
|
|
|
|
|
|
|
|
|
def find_latest_dist(winrm_client, pattern):
|
|
|
"""Find path to newest file in dist/ directory matching a pattern."""
|
|
|
|
|
|
res = winrm_client.execute_ps(
|
|
|
r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
|
|
|
'| Sort-Object LastWriteTime -Descending '
|
|
|
'| Select-Object -First 1\n'
|
|
|
'$v.name' % pattern
|
|
|
)
|
|
|
return res[0]
|
|
|
|
|
|
|
|
|
def copy_latest_dist(winrm_client, pattern, dest_path):
|
|
|
"""Copy latest file matching pattern in dist/ directory.
|
|
|
|
|
|
Given a WinRM client and a file pattern, find the latest file on the remote
|
|
|
matching that pattern and copy it to the ``dest_path`` directory on the
|
|
|
local machine.
|
|
|
"""
|
|
|
latest = find_latest_dist(winrm_client, pattern)
|
|
|
source = r'C:\hgdev\src\dist\%s' % latest
|
|
|
dest = dest_path / latest
|
|
|
print('copying %s to %s' % (source, dest))
|
|
|
winrm_client.fetch(source, str(dest))
|
|
|
|
|
|
|
|
|
def build_inno_installer(
|
|
|
winrm_client, arch: str, dest_path: pathlib.Path, version=None
|
|
|
):
|
|
|
"""Build the Inno Setup installer on a remote machine.
|
|
|
|
|
|
Using a WinRM client, remote commands are executed to build
|
|
|
a Mercurial Inno Setup installer.
|
|
|
"""
|
|
|
print('building Inno Setup installer for %s' % arch)
|
|
|
|
|
|
extra_args = []
|
|
|
if version:
|
|
|
extra_args.extend(['--version', version])
|
|
|
|
|
|
ps = get_vc_prefix(arch) + BUILD_INNO.format(
|
|
|
arch=arch, extra_args=' '.join(extra_args)
|
|
|
)
|
|
|
run_powershell(winrm_client, ps)
|
|
|
copy_latest_dist(winrm_client, '*.exe', dest_path)
|
|
|
|
|
|
|
|
|
def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path):
|
|
|
"""Build Python wheels on a remote machine.
|
|
|
|
|
|
Using a WinRM client, remote commands are executed to build a Python wheel
|
|
|
for Mercurial.
|
|
|
"""
|
|
|
print('Building Windows wheel for %s' % arch)
|
|
|
ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch)
|
|
|
run_powershell(winrm_client, ps)
|
|
|
copy_latest_dist(winrm_client, '*.whl', dest_path)
|
|
|
|
|
|
|
|
|
def build_wix_installer(
|
|
|
winrm_client, arch: str, dest_path: pathlib.Path, version=None
|
|
|
):
|
|
|
"""Build the WiX installer on a remote machine.
|
|
|
|
|
|
Using a WinRM client, remote commands are executed to build a WiX installer.
|
|
|
"""
|
|
|
print('Building WiX installer for %s' % arch)
|
|
|
extra_args = []
|
|
|
if version:
|
|
|
extra_args.extend(['--version', version])
|
|
|
|
|
|
ps = get_vc_prefix(arch) + BUILD_WIX.format(
|
|
|
arch=arch, extra_args=' '.join(extra_args)
|
|
|
)
|
|
|
run_powershell(winrm_client, ps)
|
|
|
copy_latest_dist(winrm_client, '*.msi', dest_path)
|
|
|
|
|
|
|
|
|
def run_tests(winrm_client, python_version, arch, test_flags=''):
|
|
|
"""Run tests on a remote Windows machine.
|
|
|
|
|
|
``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
|
|
|
``arch`` is ``x86`` or ``x64``.
|
|
|
``test_flags`` is a str representing extra arguments to pass to
|
|
|
``run-tests.py``.
|
|
|
"""
|
|
|
if not re.match(r'\d\.\d', python_version):
|
|
|
raise ValueError(
|
|
|
r'python_version must be \d.\d; got %s' % python_version
|
|
|
)
|
|
|
|
|
|
if arch not in ('x86', 'x64'):
|
|
|
raise ValueError('arch must be x86 or x64; got %s' % arch)
|
|
|
|
|
|
python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
|
|
|
|
|
|
ps = RUN_TESTS.format(python_path=python_path, test_flags=test_flags or '',)
|
|
|
|
|
|
run_powershell(winrm_client, ps)
|
|
|
|
|
|
|
|
|
def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
|
|
|
return (
|
|
|
dist_path / X86_WHEEL_FILENAME.format(version=version),
|
|
|
dist_path / X64_WHEEL_FILENAME.format(version=version),
|
|
|
)
|
|
|
|
|
|
|
|
|
def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
|
|
|
return (
|
|
|
dist_path / X86_WHEEL_FILENAME.format(version=version),
|
|
|
dist_path / X64_WHEEL_FILENAME.format(version=version),
|
|
|
dist_path / X86_EXE_FILENAME.format(version=version),
|
|
|
dist_path / X64_EXE_FILENAME.format(version=version),
|
|
|
dist_path / X86_MSI_FILENAME.format(version=version),
|
|
|
dist_path / X64_MSI_FILENAME.format(version=version),
|
|
|
)
|
|
|
|
|
|
|
|
|
def generate_latest_dat(version: str):
|
|
|
x86_exe_filename = X86_EXE_FILENAME.format(version=version)
|
|
|
x64_exe_filename = X64_EXE_FILENAME.format(version=version)
|
|
|
x86_msi_filename = X86_MSI_FILENAME.format(version=version)
|
|
|
x64_msi_filename = X64_MSI_FILENAME.format(version=version)
|
|
|
|
|
|
entries = (
|
|
|
(
|
|
|
'10',
|
|
|
version,
|
|
|
X86_USER_AGENT_PATTERN,
|
|
|
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
|
|
|
X86_EXE_DESCRIPTION.format(version=version),
|
|
|
),
|
|
|
(
|
|
|
'10',
|
|
|
version,
|
|
|
X64_USER_AGENT_PATTERN,
|
|
|
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
|
|
|
X64_EXE_DESCRIPTION.format(version=version),
|
|
|
),
|
|
|
(
|
|
|
'10',
|
|
|
version,
|
|
|
X86_USER_AGENT_PATTERN,
|
|
|
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
|
|
|
X86_MSI_DESCRIPTION.format(version=version),
|
|
|
),
|
|
|
(
|
|
|
'10',
|
|
|
version,
|
|
|
X64_USER_AGENT_PATTERN,
|
|
|
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
|
|
|
X64_MSI_DESCRIPTION.format(version=version),
|
|
|
),
|
|
|
)
|
|
|
|
|
|
lines = ['\t'.join(e) for e in entries]
|
|
|
|
|
|
return '\n'.join(lines) + '\n'
|
|
|
|
|
|
|
|
|
def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
|
|
|
"""Publish Windows release artifacts to PyPI."""
|
|
|
|
|
|
wheel_paths = resolve_wheel_artifacts(dist_path, version)
|
|
|
|
|
|
for p in wheel_paths:
|
|
|
if not p.exists():
|
|
|
raise Exception('%s not found' % p)
|
|
|
|
|
|
print('uploading wheels to PyPI (you may be prompted for credentials)')
|
|
|
pypi_upload(wheel_paths)
|
|
|
|
|
|
|
|
|
def publish_artifacts_mercurial_scm_org(
|
|
|
dist_path: pathlib.Path, version: str, ssh_username=None
|
|
|
):
|
|
|
"""Publish Windows release artifacts to mercurial-scm.org."""
|
|
|
all_paths = resolve_all_artifacts(dist_path, version)
|
|
|
|
|
|
for p in all_paths:
|
|
|
if not p.exists():
|
|
|
raise Exception('%s not found' % p)
|
|
|
|
|
|
client = paramiko.SSHClient()
|
|
|
client.load_system_host_keys()
|
|
|
# We assume the system SSH configuration knows how to connect.
|
|
|
print('connecting to mercurial-scm.org via ssh...')
|
|
|
try:
|
|
|
client.connect('mercurial-scm.org', username=ssh_username)
|
|
|
except paramiko.AuthenticationException:
|
|
|
print('error authenticating; is an SSH key available in an SSH agent?')
|
|
|
raise
|
|
|
|
|
|
print('SSH connection established')
|
|
|
|
|
|
print('opening SFTP client...')
|
|
|
sftp = client.open_sftp()
|
|
|
print('SFTP client obtained')
|
|
|
|
|
|
for p in all_paths:
|
|
|
dest_path = '/var/www/release/windows/%s' % p.name
|
|
|
print('uploading %s to %s' % (p, dest_path))
|
|
|
|
|
|
with p.open('rb') as fh:
|
|
|
data = fh.read()
|
|
|
|
|
|
with sftp.open(dest_path, 'wb') as fh:
|
|
|
fh.write(data)
|
|
|
fh.chmod(0o0664)
|
|
|
|
|
|
latest_dat_path = '/var/www/release/windows/latest.dat'
|
|
|
|
|
|
now = datetime.datetime.utcnow()
|
|
|
backup_path = dist_path / (
|
|
|
'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
|
|
|
)
|
|
|
print('backing up %s to %s' % (latest_dat_path, backup_path))
|
|
|
|
|
|
with sftp.open(latest_dat_path, 'rb') as fh:
|
|
|
latest_dat_old = fh.read()
|
|
|
|
|
|
with backup_path.open('wb') as fh:
|
|
|
fh.write(latest_dat_old)
|
|
|
|
|
|
print('writing %s with content:' % latest_dat_path)
|
|
|
latest_dat_content = generate_latest_dat(version)
|
|
|
print(latest_dat_content)
|
|
|
|
|
|
with sftp.open(latest_dat_path, 'wb') as fh:
|
|
|
fh.write(latest_dat_content.encode('ascii'))
|
|
|
|
|
|
|
|
|
def publish_artifacts(
|
|
|
dist_path: pathlib.Path,
|
|
|
version: str,
|
|
|
pypi=True,
|
|
|
mercurial_scm_org=True,
|
|
|
ssh_username=None,
|
|
|
):
|
|
|
"""Publish Windows release artifacts.
|
|
|
|
|
|
Files are found in `dist_path`. We will look for files with version string
|
|
|
`version`.
|
|
|
|
|
|
`pypi` controls whether we upload to PyPI.
|
|
|
`mercurial_scm_org` controls whether we upload to mercurial-scm.org.
|
|
|
"""
|
|
|
if pypi:
|
|
|
publish_artifacts_pypi(dist_path, version)
|
|
|
|
|
|
if mercurial_scm_org:
|
|
|
publish_artifacts_mercurial_scm_org(
|
|
|
dist_path, version, ssh_username=ssh_username
|
|
|
)
|
|
|
|
