##// END OF EJS Templates
automation: implement "publish-windows-artifacts" command...
Gregory Szorc -
r43177:92593d72 default
parent child Browse files
Show More
@@ -0,0 +1,25 b''
1 # pypi.py - Automation around PyPI
2 #
3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8 # no-check-code because Python 3 native.
9
10 from twine.commands.upload import (
11 upload as twine_upload,
12 )
13 from twine.settings import (
14 Settings,
15 )
16
17
18 def upload(paths):
19 """Upload files to PyPI.
20
21 `paths` is an iterable of `pathlib.Path`.
22 """
23 settings = Settings()
24
25 twine_upload(settings, [str(p) for p in paths])
@@ -181,3 +181,25 b' Various dependencies to run the Mercuria'
181 Documenting them is beyond the scope of this document. Various tests
181 Documenting them is beyond the scope of this document. Various tests
182 also require other optional dependencies and missing dependencies will
182 also require other optional dependencies and missing dependencies will
183 be printed by the test runner when a test is skipped.
183 be printed by the test runner when a test is skipped.
184
185 Releasing Windows Artifacts
186 ===========================
187
188 The `automation.py` script can be used to automate the release of Windows
189 artifacts::
190
191 $ ./automation.py build-all-windows-packages --revision 5.1.1
192 $ ./automation.py publish-windows-artifacts 5.1.1
193
194 The first command will launch an EC2 instance to build all Windows packages
195 and copy them into the `dist` directory relative to the repository root. The
196 second command will then attempt to upload these files to PyPI (via `twine`)
197 and to `mercurial-scm.org` (via SSH).
198
199 Uploading to PyPI requires a PyPI account with write access to the `Mercurial`
200 package. You can skip PyPI uploading by passing `--no-pypi`.
201
202 Uploading to `mercurial-scm.org` requires an SSH account on that server
203 with `windows` group membership and for the SSH key for that account to be the
204 default SSH key (e.g. `~/.ssh/id_rsa`) or in a running SSH agent. You can
205 skip `mercurial-scm.org` uploading by passing `--no-mercurial-scm-org`.
@@ -185,6 +185,14 b' def run_tests_windows(hga: HGAutomation,'
185 test_flags)
185 test_flags)
186
186
187
187
188 def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str,
189 pypi: bool, mercurial_scm_org: bool,
190 ssh_username: str):
191 windows.publish_artifacts(DIST_PATH, version,
192 pypi=pypi, mercurial_scm_org=mercurial_scm_org,
193 ssh_username=ssh_username)
194
195
188 def get_parser():
196 def get_parser():
189 parser = argparse.ArgumentParser()
197 parser = argparse.ArgumentParser()
190
198
@@ -403,6 +411,34 b' def get_parser():'
403 )
411 )
404 sp.set_defaults(func=run_tests_windows)
412 sp.set_defaults(func=run_tests_windows)
405
413
414 sp = subparsers.add_parser(
415 'publish-windows-artifacts',
416 help='Publish built Windows artifacts (wheels, installers, etc)'
417 )
418 sp.add_argument(
419 '--no-pypi',
420 dest='pypi',
421 action='store_false',
422 default=True,
423 help='Skip uploading to PyPI',
424 )
425 sp.add_argument(
426 '--no-mercurial-scm-org',
427 dest='mercurial_scm_org',
428 action='store_false',
429 default=True,
430 help='Skip uploading to www.mercurial-scm.org',
431 )
432 sp.add_argument(
433 '--ssh-username',
434 help='SSH username for mercurial-scm.org',
435 )
436 sp.add_argument(
437 'version',
438 help='Mercurial version string to locate local packages',
439 )
440 sp.set_defaults(func=publish_windows_artifacts)
441
406 return parser
442 return parser
407
443
408
444
@@ -7,12 +7,17 b''
7
7
8 # no-check-code because Python 3 native.
8 # no-check-code because Python 3 native.
9
9
10 import datetime
10 import os
11 import os
12 import paramiko
11 import pathlib
13 import pathlib
12 import re
14 import re
13 import subprocess
15 import subprocess
14 import tempfile
16 import tempfile
15
17
18 from .pypi import (
19 upload as pypi_upload,
20 )
16 from .winrm import (
21 from .winrm import (
17 run_powershell,
22 run_powershell,
18 )
23 )
@@ -100,6 +105,26 b' if ($LASTEXITCODE -ne 0) {{'
100 }}
105 }}
101 '''
106 '''
102
107
108 X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
109 X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
110 X86_EXE_FILENAME = 'Mercurial-{version}.exe'
111 X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
112 X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
113 X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
114
115 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
116
117 X86_USER_AGENT_PATTERN = '.*Windows.*'
118 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
119
120 X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows '
121 '- does not require admin rights')
122 X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows '
123 '- does not require admin rights')
124 X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows '
125 '- requires admin rights')
126 X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows '
127 '- requires admin rights')
103
128
104 def get_vc_prefix(arch):
129 def get_vc_prefix(arch):
105 if arch == 'x86':
130 if arch == 'x86':
@@ -296,3 +321,152 b' def run_tests(winrm_client, python_versi'
296 )
321 )
297
322
298 run_powershell(winrm_client, ps)
323 run_powershell(winrm_client, ps)
324
325
326 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
327 return (
328 dist_path / X86_WHEEL_FILENAME.format(version=version),
329 dist_path / X64_WHEEL_FILENAME.format(version=version),
330 )
331
332
333 def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
334 return (
335 dist_path / X86_WHEEL_FILENAME.format(version=version),
336 dist_path / X64_WHEEL_FILENAME.format(version=version),
337 dist_path / X86_EXE_FILENAME.format(version=version),
338 dist_path / X64_EXE_FILENAME.format(version=version),
339 dist_path / X86_MSI_FILENAME.format(version=version),
340 dist_path / X64_MSI_FILENAME.format(version=version),
341 )
342
343
344 def generate_latest_dat(version: str):
345 x86_exe_filename = X86_EXE_FILENAME.format(version=version)
346 x64_exe_filename = X64_EXE_FILENAME.format(version=version)
347 x86_msi_filename = X86_MSI_FILENAME.format(version=version)
348 x64_msi_filename = X64_MSI_FILENAME.format(version=version)
349
350 entries = (
351 (
352 '10',
353 version,
354 X86_USER_AGENT_PATTERN,
355 '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
356 X86_EXE_DESCRIPTION.format(version=version),
357 ),
358 (
359 '10',
360 version,
361 X64_USER_AGENT_PATTERN,
362 '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
363 X64_EXE_DESCRIPTION.format(version=version),
364 ),
365 (
366 '10',
367 version,
368 X86_USER_AGENT_PATTERN,
369 '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
370 X86_MSI_DESCRIPTION.format(version=version),
371 ),
372 (
373 '10',
374 version,
375 X64_USER_AGENT_PATTERN,
376 '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
377 X64_MSI_DESCRIPTION.format(version=version)
378 )
379 )
380
381 lines = ['\t'.join(e) for e in entries]
382
383 return '\n'.join(lines) + '\n'
384
385
386 def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
387 """Publish Windows release artifacts to PyPI."""
388
389 wheel_paths = resolve_wheel_artifacts(dist_path, version)
390
391 for p in wheel_paths:
392 if not p.exists():
393 raise Exception('%s not found' % p)
394
395 print('uploading wheels to PyPI (you may be prompted for credentials)')
396 pypi_upload(wheel_paths)
397
398
399 def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
400 ssh_username=None):
401 """Publish Windows release artifacts to mercurial-scm.org."""
402 all_paths = resolve_all_artifacts(dist_path, version)
403
404 for p in all_paths:
405 if not p.exists():
406 raise Exception('%s not found' % p)
407
408 client = paramiko.SSHClient()
409 client.load_system_host_keys()
410 # We assume the system SSH configuration knows how to connect.
411 print('connecting to mercurial-scm.org via ssh...')
412 try:
413 client.connect('mercurial-scm.org', username=ssh_username)
414 except paramiko.AuthenticationException:
415 print('error authenticating; is an SSH key available in an SSH agent?')
416 raise
417
418 print('SSH connection established')
419
420 print('opening SFTP client...')
421 sftp = client.open_sftp()
422 print('SFTP client obtained')
423
424 for p in all_paths:
425 dest_path = '/var/www/release/windows/%s' % p.name
426 print('uploading %s to %s' % (p, dest_path))
427
428 with p.open('rb') as fh:
429 data = fh.read()
430
431 with sftp.open(dest_path, 'wb') as fh:
432 fh.write(data)
433 fh.chmod(0o0664)
434
435 latest_dat_path = '/var/www/release/windows/latest.dat'
436
437 now = datetime.datetime.utcnow()
438 backup_path = dist_path / (
439 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S'))
440 print('backing up %s to %s' % (latest_dat_path, backup_path))
441
442 with sftp.open(latest_dat_path, 'rb') as fh:
443 latest_dat_old = fh.read()
444
445 with backup_path.open('wb') as fh:
446 fh.write(latest_dat_old)
447
448 print('writing %s with content:' % latest_dat_path)
449 latest_dat_content = generate_latest_dat(version)
450 print(latest_dat_content)
451
452 with sftp.open(latest_dat_path, 'wb') as fh:
453 fh.write(latest_dat_content.encode('ascii'))
454
455
456 def publish_artifacts(dist_path: pathlib.Path, version: str,
457 pypi=True, mercurial_scm_org=True,
458 ssh_username=None):
459 """Publish Windows release artifacts.
460
461 Files are found in `dist_path`. We will look for files with version string
462 `version`.
463
464 `pypi` controls whether we upload to PyPI.
465 `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
466 """
467 if pypi:
468 publish_artifacts_pypi(dist_path, version)
469
470 if mercurial_scm_org:
471 publish_artifacts_mercurial_scm_org(dist_path, version,
472 ssh_username=ssh_username)
@@ -26,6 +26,10 b' bcrypt==3.1.7 \\'
26 --hash=sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7 \
26 --hash=sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7 \
27 --hash=sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc \
27 --hash=sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc \
28 # via paramiko
28 # via paramiko
29 bleach==3.1.0 \
30 --hash=sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16 \
31 --hash=sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa \
32 # via readme-renderer
29 boto3==1.9.223 \
33 boto3==1.9.223 \
30 --hash=sha256:12ceb047c3cfbd2363b35e1c24b082808a1bb9b90f4f0b7375e83d21015bf47b \
34 --hash=sha256:12ceb047c3cfbd2363b35e1c24b082808a1bb9b90f4f0b7375e83d21015bf47b \
31 --hash=sha256:6e833a9068309c24d7752e280b2925cf5968a88111bc95fcebc451a09f8b424e
35 --hash=sha256:6e833a9068309c24d7752e280b2925cf5968a88111bc95fcebc451a09f8b424e
@@ -93,7 +97,7 b' docutils==0.15.2 \\'
93 --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
97 --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
94 --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
98 --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
95 --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
99 --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
96 # via botocore
100 # via botocore, readme-renderer
97 idna==2.8 \
101 idna==2.8 \
98 --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
102 --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
99 --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
103 --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
@@ -109,9 +113,17 b' ntlm-auth==1.4.0 \\'
109 paramiko==2.6.0 \
113 paramiko==2.6.0 \
110 --hash=sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf \
114 --hash=sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf \
111 --hash=sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041
115 --hash=sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041
116 pkginfo==1.5.0.1 \
117 --hash=sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb \
118 --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \
119 # via twine
112 pycparser==2.19 \
120 pycparser==2.19 \
113 --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
121 --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
114 # via cffi
122 # via cffi
123 pygments==2.4.2 \
124 --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \
125 --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 \
126 # via readme-renderer
115 pynacl==1.3.0 \
127 pynacl==1.3.0 \
116 --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
128 --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
117 --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
129 --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
@@ -140,10 +152,18 b' python-dateutil==2.8.0 \\'
140 --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
152 --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
141 --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
153 --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
142 # via botocore
154 # via botocore
155 readme-renderer==24.0 \
156 --hash=sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f \
157 --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d \
158 # via twine
159 requests-toolbelt==0.9.1 \
160 --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \
161 --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 \
162 # via twine
143 requests==2.22.0 \
163 requests==2.22.0 \
144 --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
164 --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
145 --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \
165 --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \
146 # via pypsrp
166 # via pypsrp, requests-toolbelt, twine
147 s3transfer==0.2.1 \
167 s3transfer==0.2.1 \
148 --hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \
168 --hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \
149 --hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba \
169 --hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba \
@@ -151,8 +171,23 b' s3transfer==0.2.1 \\'
151 six==1.12.0 \
171 six==1.12.0 \
152 --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
172 --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
153 --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
173 --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
154 # via bcrypt, cryptography, pynacl, pypsrp, python-dateutil
174 # via bcrypt, bleach, cryptography, pynacl, pypsrp, python-dateutil, readme-renderer
175 tqdm==4.35.0 \
176 --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \
177 --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457 \
178 # via twine
179 twine==1.13.0 \
180 --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \
181 --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc
155 urllib3==1.25.3 \
182 urllib3==1.25.3 \
156 --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
183 --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
157 --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \
184 --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \
158 # via botocore, requests
185 # via botocore, requests
186 webencodings==0.5.1 \
187 --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
188 --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \
189 # via bleach
190
191 # WARNING: The following packages were not pinned, but pip requires them to be
192 # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
193 # setuptools==41.2.0 # via twine
@@ -1,3 +1,4 b''
1 boto3
1 boto3
2 paramiko
2 paramiko
3 pypsrp
3 pypsrp
4 twine
@@ -16,6 +16,7 b' New errors are not allowed. Warnings are'
16 Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
16 Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
17 Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
17 Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
18 Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
18 Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
19 Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob)
19 Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
20 Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
20 Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
21 Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
21 Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
22 Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
General Comments 0
You need to be logged in to leave comments. Login now