##// END OF EJS Templates
automation: delete code related to Python 2.7 support...
Gregory Szorc -
r49702:4561ec90 default
parent child Browse files
Show More
@@ -1,574 +1,551 b''
1 1 # cli.py - Command line interface for automation
2 2 #
3 3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 # no-check-code because Python 3 native.
9 9
10 10 import argparse
11 11 import concurrent.futures as futures
12 12 import os
13 13 import pathlib
14 14 import time
15 15
16 16 from . import (
17 17 aws,
18 18 HGAutomation,
19 19 linux,
20 20 try_server,
21 21 windows,
22 22 )
23 23
24 24
25 25 SOURCE_ROOT = pathlib.Path(
26 26 os.path.abspath(__file__)
27 27 ).parent.parent.parent.parent
28 28 DIST_PATH = SOURCE_ROOT / 'dist'
29 29
30 30
31 31 def bootstrap_linux_dev(
32 32 hga: HGAutomation, aws_region, distros=None, parallel=False
33 33 ):
34 34 c = hga.aws_connection(aws_region)
35 35
36 36 if distros:
37 37 distros = distros.split(',')
38 38 else:
39 39 distros = sorted(linux.DISTROS)
40 40
41 41 # TODO There is a wonky interaction involving KeyboardInterrupt whereby
42 42 # the context manager that is supposed to terminate the temporary EC2
43 43 # instance doesn't run. Until we fix this, make parallel building opt-in
44 44 # so we don't orphan instances.
45 45 if parallel:
46 46 fs = []
47 47
48 48 with futures.ThreadPoolExecutor(len(distros)) as e:
49 49 for distro in distros:
50 50 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
51 51
52 52 for f in fs:
53 53 f.result()
54 54 else:
55 55 for distro in distros:
56 56 aws.ensure_linux_dev_ami(c, distro=distro)
57 57
58 58
59 59 def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
60 60 c = hga.aws_connection(aws_region)
61 61 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
62 62 print('Windows development AMI available as %s' % image.id)
63 63
64 64
65 65 def build_inno(
66 66 hga: HGAutomation,
67 67 aws_region,
68 python_version,
69 68 arch,
70 69 revision,
71 70 version,
72 71 base_image_name,
73 72 ):
74 73 c = hga.aws_connection(aws_region)
75 74 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
76 75 DIST_PATH.mkdir(exist_ok=True)
77 76
78 77 with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
79 78 instance = insts[0]
80 79
81 80 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
82 81
83 for py_version in python_version:
84 for a in arch:
85 windows.build_inno_installer(
86 instance.winrm_client,
87 py_version,
88 a,
89 DIST_PATH,
90 version=version,
91 )
82 for a in arch:
83 windows.build_inno_installer(
84 instance.winrm_client,
85 a,
86 DIST_PATH,
87 version=version,
88 )
92 89
93 90
94 91 def build_wix(
95 92 hga: HGAutomation,
96 93 aws_region,
97 python_version,
98 94 arch,
99 95 revision,
100 96 version,
101 97 base_image_name,
102 98 ):
103 99 c = hga.aws_connection(aws_region)
104 100 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
105 101 DIST_PATH.mkdir(exist_ok=True)
106 102
107 103 with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
108 104 instance = insts[0]
109 105
110 106 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
111 107
112 for py_version in python_version:
113 for a in arch:
114 windows.build_wix_installer(
115 instance.winrm_client,
116 py_version,
117 a,
118 DIST_PATH,
119 version=version,
120 )
108 for a in arch:
109 windows.build_wix_installer(
110 instance.winrm_client,
111 a,
112 DIST_PATH,
113 version=version,
114 )
121 115
122 116
123 117 def build_windows_wheel(
124 118 hga: HGAutomation,
125 119 aws_region,
126 120 python_version,
127 121 arch,
128 122 revision,
129 123 base_image_name,
130 124 ):
131 125 c = hga.aws_connection(aws_region)
132 126 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
133 127 DIST_PATH.mkdir(exist_ok=True)
134 128
135 129 with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
136 130 instance = insts[0]
137 131
138 132 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
139 133
140 134 for py_version in python_version:
141 135 for a in arch:
142 136 windows.build_wheel(
143 137 instance.winrm_client, py_version, a, DIST_PATH
144 138 )
145 139
146 140
147 141 def build_all_windows_packages(
148 142 hga: HGAutomation, aws_region, revision, version, base_image_name
149 143 ):
150 144 c = hga.aws_connection(aws_region)
151 145 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
152 146 DIST_PATH.mkdir(exist_ok=True)
153 147
154 148 with aws.temporary_windows_dev_instances(c, image, 'm6i.large') as insts:
155 149 instance = insts[0]
156 150
157 151 winrm_client = instance.winrm_client
158 152
159 153 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
160 154
161 155 for py_version in ("3.7", "3.8", "3.9", "3.10"):
162 156 for arch in ("x86", "x64"):
163 157 windows.purge_hg(winrm_client)
164 158 windows.build_wheel(
165 159 winrm_client,
166 160 python_version=py_version,
167 161 arch=arch,
168 162 dest_path=DIST_PATH,
169 163 )
170 164
171 for py_version in (2, 3):
172 for arch in ('x86', 'x64'):
173 windows.purge_hg(winrm_client)
174 windows.build_inno_installer(
175 winrm_client, py_version, arch, DIST_PATH, version=version
176 )
177 windows.build_wix_installer(
178 winrm_client, py_version, arch, DIST_PATH, version=version
179 )
165 for arch in ('x86', 'x64'):
166 windows.purge_hg(winrm_client)
167 windows.build_inno_installer(
168 winrm_client, arch, DIST_PATH, version=version
169 )
170 windows.build_wix_installer(
171 winrm_client, arch, DIST_PATH, version=version
172 )
180 173
181 174
182 175 def terminate_ec2_instances(hga: HGAutomation, aws_region):
183 176 c = hga.aws_connection(aws_region, ensure_ec2_state=False)
184 177 aws.terminate_ec2_instances(c.ec2resource)
185 178
186 179
187 180 def purge_ec2_resources(hga: HGAutomation, aws_region):
188 181 c = hga.aws_connection(aws_region, ensure_ec2_state=False)
189 182 aws.remove_resources(c)
190 183
191 184
192 185 def run_tests_linux(
193 186 hga: HGAutomation,
194 187 aws_region,
195 188 instance_type,
196 189 python_version,
197 190 test_flags,
198 191 distro,
199 192 filesystem,
200 193 ):
201 194 c = hga.aws_connection(aws_region)
202 195 image = aws.ensure_linux_dev_ami(c, distro=distro)
203 196
204 197 t_start = time.time()
205 198
206 199 ensure_extra_volume = filesystem not in ('default', 'tmpfs')
207 200
208 201 with aws.temporary_linux_dev_instances(
209 202 c, image, instance_type, ensure_extra_volume=ensure_extra_volume
210 203 ) as insts:
211 204
212 205 instance = insts[0]
213 206
214 207 linux.prepare_exec_environment(
215 208 instance.ssh_client, filesystem=filesystem
216 209 )
217 210 linux.synchronize_hg(SOURCE_ROOT, instance, '.')
218 211 t_prepared = time.time()
219 212 linux.run_tests(instance.ssh_client, python_version, test_flags)
220 213 t_done = time.time()
221 214
222 215 t_setup = t_prepared - t_start
223 216 t_all = t_done - t_start
224 217
225 218 print(
226 219 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
227 220 % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
228 221 )
229 222
230 223
231 224 def run_tests_windows(
232 225 hga: HGAutomation,
233 226 aws_region,
234 227 instance_type,
235 228 python_version,
236 229 arch,
237 230 test_flags,
238 231 base_image_name,
239 232 ):
240 233 c = hga.aws_connection(aws_region)
241 234 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
242 235
243 236 with aws.temporary_windows_dev_instances(
244 237 c, image, instance_type, disable_antivirus=True
245 238 ) as insts:
246 239 instance = insts[0]
247 240
248 241 windows.synchronize_hg(SOURCE_ROOT, '.', instance)
249 242 windows.run_tests(
250 243 instance.winrm_client, python_version, arch, test_flags
251 244 )
252 245
253 246
254 247 def publish_windows_artifacts(
255 248 hg: HGAutomation,
256 249 aws_region,
257 250 version: str,
258 251 pypi: bool,
259 252 mercurial_scm_org: bool,
260 253 ssh_username: str,
261 254 ):
262 255 windows.publish_artifacts(
263 256 DIST_PATH,
264 257 version,
265 258 pypi=pypi,
266 259 mercurial_scm_org=mercurial_scm_org,
267 260 ssh_username=ssh_username,
268 261 )
269 262
270 263
271 264 def run_try(hga: HGAutomation, aws_region: str, rev: str):
272 265 c = hga.aws_connection(aws_region, ensure_ec2_state=False)
273 266 try_server.trigger_try(c, rev=rev)
274 267
275 268
276 269 def get_parser():
277 270 parser = argparse.ArgumentParser()
278 271
279 272 parser.add_argument(
280 273 '--state-path',
281 274 default='~/.hgautomation',
282 275 help='Path for local state files',
283 276 )
284 277 parser.add_argument(
285 278 '--aws-region',
286 279 help='AWS region to use',
287 280 default='us-west-2',
288 281 )
289 282
290 283 subparsers = parser.add_subparsers()
291 284
292 285 sp = subparsers.add_parser(
293 286 'bootstrap-linux-dev',
294 287 help='Bootstrap Linux development environments',
295 288 )
296 289 sp.add_argument(
297 290 '--distros',
298 291 help='Comma delimited list of distros to bootstrap',
299 292 )
300 293 sp.add_argument(
301 294 '--parallel',
302 295 action='store_true',
303 296 help='Generate AMIs in parallel (not CTRL-c safe)',
304 297 )
305 298 sp.set_defaults(func=bootstrap_linux_dev)
306 299
307 300 sp = subparsers.add_parser(
308 301 'bootstrap-windows-dev',
309 302 help='Bootstrap the Windows development environment',
310 303 )
311 304 sp.add_argument(
312 305 '--base-image-name',
313 306 help='AMI name of base image',
314 307 default=aws.WINDOWS_BASE_IMAGE_NAME,
315 308 )
316 309 sp.set_defaults(func=bootstrap_windows_dev)
317 310
318 311 sp = subparsers.add_parser(
319 312 'build-all-windows-packages',
320 313 help='Build all Windows packages',
321 314 )
322 315 sp.add_argument(
323 316 '--revision',
324 317 help='Mercurial revision to build',
325 318 default='.',
326 319 )
327 320 sp.add_argument(
328 321 '--version',
329 322 help='Mercurial version string to use',
330 323 )
331 324 sp.add_argument(
332 325 '--base-image-name',
333 326 help='AMI name of base image',
334 327 default=aws.WINDOWS_BASE_IMAGE_NAME,
335 328 )
336 329 sp.set_defaults(func=build_all_windows_packages)
337 330
338 331 sp = subparsers.add_parser(
339 332 'build-inno',
340 333 help='Build Inno Setup installer(s)',
341 334 )
342 335 sp.add_argument(
343 '--python-version',
344 help='Which version of Python to target',
345 choices={3},
346 type=int,
347 nargs='*',
348 default=[3],
349 )
350 sp.add_argument(
351 336 '--arch',
352 337 help='Architecture to build for',
353 338 choices={'x86', 'x64'},
354 339 nargs='*',
355 340 default=['x64'],
356 341 )
357 342 sp.add_argument(
358 343 '--revision',
359 344 help='Mercurial revision to build',
360 345 default='.',
361 346 )
362 347 sp.add_argument(
363 348 '--version',
364 349 help='Mercurial version string to use in installer',
365 350 )
366 351 sp.add_argument(
367 352 '--base-image-name',
368 353 help='AMI name of base image',
369 354 default=aws.WINDOWS_BASE_IMAGE_NAME,
370 355 )
371 356 sp.set_defaults(func=build_inno)
372 357
373 358 sp = subparsers.add_parser(
374 359 'build-windows-wheel',
375 360 help='Build Windows wheel(s)',
376 361 )
377 362 sp.add_argument(
378 363 '--python-version',
379 364 help='Python version to build for',
380 365 choices={'3.7', '3.8', '3.9', '3.10'},
381 366 nargs='*',
382 367 default=['3.8'],
383 368 )
384 369 sp.add_argument(
385 370 '--arch',
386 371 help='Architecture to build for',
387 372 choices={'x86', 'x64'},
388 373 nargs='*',
389 374 default=['x64'],
390 375 )
391 376 sp.add_argument(
392 377 '--revision',
393 378 help='Mercurial revision to build',
394 379 default='.',
395 380 )
396 381 sp.add_argument(
397 382 '--base-image-name',
398 383 help='AMI name of base image',
399 384 default=aws.WINDOWS_BASE_IMAGE_NAME,
400 385 )
401 386 sp.set_defaults(func=build_windows_wheel)
402 387
403 388 sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
404 389 sp.add_argument(
405 '--python-version',
406 help='Which version of Python to target',
407 choices={3},
408 type=int,
409 nargs='*',
410 default=[3],
411 )
412 sp.add_argument(
413 390 '--arch',
414 391 help='Architecture to build for',
415 392 choices={'x86', 'x64'},
416 393 nargs='*',
417 394 default=['x64'],
418 395 )
419 396 sp.add_argument(
420 397 '--revision',
421 398 help='Mercurial revision to build',
422 399 default='.',
423 400 )
424 401 sp.add_argument(
425 402 '--version',
426 403 help='Mercurial version string to use in installer',
427 404 )
428 405 sp.add_argument(
429 406 '--base-image-name',
430 407 help='AMI name of base image',
431 408 default=aws.WINDOWS_BASE_IMAGE_NAME,
432 409 )
433 410 sp.set_defaults(func=build_wix)
434 411
435 412 sp = subparsers.add_parser(
436 413 'terminate-ec2-instances',
437 414 help='Terminate all active EC2 instances managed by us',
438 415 )
439 416 sp.set_defaults(func=terminate_ec2_instances)
440 417
441 418 sp = subparsers.add_parser(
442 419 'purge-ec2-resources',
443 420 help='Purge all EC2 resources managed by us',
444 421 )
445 422 sp.set_defaults(func=purge_ec2_resources)
446 423
447 424 sp = subparsers.add_parser(
448 425 'run-tests-linux',
449 426 help='Run tests on Linux',
450 427 )
451 428 sp.add_argument(
452 429 '--distro',
453 430 help='Linux distribution to run tests on',
454 431 choices=linux.DISTROS,
455 432 default='debian10',
456 433 )
457 434 sp.add_argument(
458 435 '--filesystem',
459 436 help='Filesystem type to use',
460 437 choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
461 438 default='default',
462 439 )
463 440 sp.add_argument(
464 441 '--instance-type',
465 442 help='EC2 instance type to use',
466 443 default='c5.9xlarge',
467 444 )
468 445 sp.add_argument(
469 446 '--python-version',
470 447 help='Python version to use',
471 448 choices={
472 449 'system3',
473 450 '3.5',
474 451 '3.6',
475 452 '3.7',
476 453 '3.8',
477 454 'pypy',
478 455 'pypy3.5',
479 456 'pypy3.6',
480 457 },
481 458 default='system3',
482 459 )
483 460 sp.add_argument(
484 461 'test_flags',
485 462 help='Extra command line flags to pass to run-tests.py',
486 463 nargs='*',
487 464 )
488 465 sp.set_defaults(func=run_tests_linux)
489 466
490 467 sp = subparsers.add_parser(
491 468 'run-tests-windows',
492 469 help='Run tests on Windows',
493 470 )
494 471 sp.add_argument(
495 472 '--instance-type',
496 473 help='EC2 instance type to use',
497 474 default='m6i.large',
498 475 )
499 476 sp.add_argument(
500 477 '--python-version',
501 478 help='Python version to use',
502 479 choices={'3.5', '3.6', '3.7', '3.8', '3.9', '3.10'},
503 480 default='3.9',
504 481 )
505 482 sp.add_argument(
506 483 '--arch',
507 484 help='Architecture to test',
508 485 choices={'x86', 'x64'},
509 486 default='x64',
510 487 )
511 488 sp.add_argument(
512 489 '--test-flags',
513 490 help='Extra command line flags to pass to run-tests.py',
514 491 )
515 492 sp.add_argument(
516 493 '--base-image-name',
517 494 help='AMI name of base image',
518 495 default=aws.WINDOWS_BASE_IMAGE_NAME,
519 496 )
520 497 sp.set_defaults(func=run_tests_windows)
521 498
522 499 sp = subparsers.add_parser(
523 500 'publish-windows-artifacts',
524 501 help='Publish built Windows artifacts (wheels, installers, etc)',
525 502 )
526 503 sp.add_argument(
527 504 '--no-pypi',
528 505 dest='pypi',
529 506 action='store_false',
530 507 default=True,
531 508 help='Skip uploading to PyPI',
532 509 )
533 510 sp.add_argument(
534 511 '--no-mercurial-scm-org',
535 512 dest='mercurial_scm_org',
536 513 action='store_false',
537 514 default=True,
538 515 help='Skip uploading to www.mercurial-scm.org',
539 516 )
540 517 sp.add_argument(
541 518 '--ssh-username',
542 519 help='SSH username for mercurial-scm.org',
543 520 )
544 521 sp.add_argument(
545 522 'version',
546 523 help='Mercurial version string to locate local packages',
547 524 )
548 525 sp.set_defaults(func=publish_windows_artifacts)
549 526
550 527 sp = subparsers.add_parser(
551 528 'try', help='Run CI automation against a custom changeset'
552 529 )
553 530 sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
554 531 sp.set_defaults(func=run_try)
555 532
556 533 return parser
557 534
558 535
559 536 def main():
560 537 parser = get_parser()
561 538 args = parser.parse_args()
562 539
563 540 local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
564 541 automation = HGAutomation(local_state_path)
565 542
566 543 if not hasattr(args, 'func'):
567 544 parser.print_help()
568 545 return
569 546
570 547 kwargs = dict(vars(args))
571 548 del kwargs['func']
572 549 del kwargs['state_path']
573 550
574 551 args.func(automation, **kwargs)
@@ -1,544 +1,539 b''
1 1 # windows.py - Automation specific to Windows
2 2 #
3 3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 # no-check-code because Python 3 native.
9 9
10 10 import datetime
11 11 import os
12 12 import paramiko
13 13 import pathlib
14 14 import re
15 15 import subprocess
16 16 import tempfile
17 17
18 18 from .pypi import upload as pypi_upload
19 19 from .winrm import run_powershell
20 20
21 21
22 22 HG_PURGE = r'''
23 23 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
24 24 Set-Location C:\hgdev\src
25 25 hg.exe --config extensions.purge= purge --all
26 26 if ($LASTEXITCODE -ne 0) {
27 27 throw "process exited non-0: $LASTEXITCODE"
28 28 }
29 29 Write-Output "purged Mercurial repo"
30 30 '''
31 31
32 32 HG_UPDATE_CLEAN = r'''
33 33 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
34 34 Set-Location C:\hgdev\src
35 35 hg.exe --config extensions.purge= purge --all
36 36 if ($LASTEXITCODE -ne 0) {{
37 37 throw "process exited non-0: $LASTEXITCODE"
38 38 }}
39 39 hg.exe update -C {revision}
40 40 if ($LASTEXITCODE -ne 0) {{
41 41 throw "process exited non-0: $LASTEXITCODE"
42 42 }}
43 43 hg.exe log -r .
44 44 Write-Output "updated Mercurial working directory to {revision}"
45 45 '''.lstrip()
46 46
47 47 BUILD_INNO_PYTHON3 = r'''
48 48 $Env:RUSTUP_HOME = "C:\hgdev\rustup"
49 49 $Env:CARGO_HOME = "C:\hgdev\cargo"
50 50 Set-Location C:\hgdev\src
51 51 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --pyoxidizer-target {pyoxidizer_target} --version {version}
52 52 if ($LASTEXITCODE -ne 0) {{
53 53 throw "process exited non-0: $LASTEXITCODE"
54 54 }}
55 55 '''
56 56
57 57
58 58 BUILD_WHEEL = r'''
59 59 Set-Location C:\hgdev\src
60 60 C:\hgdev\python{python_version}-{arch}\python.exe -m pip wheel --wheel-dir dist .
61 61 if ($LASTEXITCODE -ne 0) {{
62 62 throw "process exited non-0: $LASTEXITCODE"
63 63 }}
64 64 '''
65 65
66 66 BUILD_WIX_PYTHON3 = r'''
67 67 $Env:RUSTUP_HOME = "C:\hgdev\rustup"
68 68 $Env:CARGO_HOME = "C:\hgdev\cargo"
69 69 Set-Location C:\hgdev\src
70 70 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --pyoxidizer-target {pyoxidizer_target} --version {version}
71 71 if ($LASTEXITCODE -ne 0) {{
72 72 throw "process exited non-0: $LASTEXITCODE"
73 73 }}
74 74 '''
75 75
76 76
77 77 RUN_TESTS = r'''
78 78 C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
79 79 if ($LASTEXITCODE -ne 0) {{
80 80 throw "process exited non-0: $LASTEXITCODE"
81 81 }}
82 82 '''
83 83
84 84
85 85 WHEEL_FILENAME_PYTHON37_X86 = 'mercurial-{version}-cp37-cp37m-win32.whl'
86 86 WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl'
87 87 WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl'
88 88 WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl'
89 89 WHEEL_FILENAME_PYTHON39_X86 = 'mercurial-{version}-cp39-cp39-win32.whl'
90 90 WHEEL_FILENAME_PYTHON39_X64 = 'mercurial-{version}-cp39-cp39-win_amd64.whl'
91 91 WHEEL_FILENAME_PYTHON310_X86 = 'mercurial-{version}-cp310-cp310-win32.whl'
92 92 WHEEL_FILENAME_PYTHON310_X64 = 'mercurial-{version}-cp310-cp310-win_amd64.whl'
93 93
94 94 EXE_FILENAME_PYTHON3_X86 = 'Mercurial-{version}-x86.exe'
95 95 EXE_FILENAME_PYTHON3_X64 = 'Mercurial-{version}-x64.exe'
96 96
97 97 MSI_FILENAME_PYTHON3_X86 = 'mercurial-{version}-x86.msi'
98 98 MSI_FILENAME_PYTHON3_X64 = 'mercurial-{version}-x64.msi'
99 99
100 100 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
101 101
102 102 X86_USER_AGENT_PATTERN = '.*Windows.*'
103 103 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
104 104
105 105 # TODO remove Python version once Python 2 is dropped.
106 106 EXE_PYTHON3_X86_DESCRIPTION = (
107 107 'Mercurial {version} Inno Setup installer - x86 Windows (Python 3) '
108 108 '- does not require admin rights'
109 109 )
110 110 EXE_PYTHON3_X64_DESCRIPTION = (
111 111 'Mercurial {version} Inno Setup installer - x64 Windows (Python 3) '
112 112 '- does not require admin rights'
113 113 )
114 114 MSI_PYTHON3_X86_DESCRIPTION = (
115 115 'Mercurial {version} MSI installer - x86 Windows (Python 3) '
116 116 '- requires admin rights'
117 117 )
118 118 MSI_PYTHON3_X64_DESCRIPTION = (
119 119 'Mercurial {version} MSI installer - x64 Windows (Python 3) '
120 120 '- requires admin rights'
121 121 )
122 122
123 123
124 124 def fix_authorized_keys_permissions(winrm_client, path):
125 125 commands = [
126 126 '$ErrorActionPreference = "Stop"',
127 127 'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
128 128 r'icacls %s /remove:g "NT Service\sshd"' % path,
129 129 ]
130 130
131 131 run_powershell(winrm_client, '\n'.join(commands))
132 132
133 133
134 134 def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
135 135 """Synchronize local Mercurial repo to remote EC2 instance."""
136 136
137 137 winrm_client = ec2_instance.winrm_client
138 138
139 139 with tempfile.TemporaryDirectory() as temp_dir:
140 140 temp_dir = pathlib.Path(temp_dir)
141 141
142 142 ssh_dir = temp_dir / '.ssh'
143 143 ssh_dir.mkdir()
144 144 ssh_dir.chmod(0o0700)
145 145
146 146 # Generate SSH key to use for communication.
147 147 subprocess.run(
148 148 [
149 149 'ssh-keygen',
150 150 '-t',
151 151 'rsa',
152 152 '-b',
153 153 '4096',
154 154 '-N',
155 155 '',
156 156 '-f',
157 157 str(ssh_dir / 'id_rsa'),
158 158 ],
159 159 check=True,
160 160 capture_output=True,
161 161 )
162 162
163 163 # Add it to ~/.ssh/authorized_keys on remote.
164 164 # This assumes the file doesn't already exist.
165 165 authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
166 166 winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
167 167 winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
168 168 fix_authorized_keys_permissions(winrm_client, authorized_keys)
169 169
170 170 public_ip = ec2_instance.public_ip_address
171 171
172 172 ssh_config = temp_dir / '.ssh' / 'config'
173 173
174 174 with open(ssh_config, 'w', encoding='utf-8') as fh:
175 175 fh.write('Host %s\n' % public_ip)
176 176 fh.write(' User Administrator\n')
177 177 fh.write(' StrictHostKeyChecking no\n')
178 178 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
179 179 fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
180 180
181 181 if not (hg_repo / '.hg').is_dir():
182 182 raise Exception(
183 183 '%s is not a Mercurial repository; '
184 184 'synchronization not yet supported' % hg_repo
185 185 )
186 186
187 187 env = dict(os.environ)
188 188 env['HGPLAIN'] = '1'
189 189 env['HGENCODING'] = 'utf-8'
190 190
191 191 hg_bin = hg_repo / 'hg'
192 192
193 193 res = subprocess.run(
194 194 ['python3', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
195 195 cwd=str(hg_repo),
196 196 env=env,
197 197 check=True,
198 198 capture_output=True,
199 199 )
200 200
201 201 full_revision = res.stdout.decode('ascii')
202 202
203 203 args = [
204 204 'python3',
205 205 hg_bin,
206 206 '--config',
207 207 'ui.ssh=ssh -F %s' % ssh_config,
208 208 '--config',
209 209 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
210 210 # Also ensure .hgtags changes are present so auto version
211 211 # calculation works.
212 212 'push',
213 213 '-f',
214 214 '-r',
215 215 full_revision,
216 216 '-r',
217 217 'file(.hgtags)',
218 218 'ssh://%s/c:/hgdev/src' % public_ip,
219 219 ]
220 220
221 221 res = subprocess.run(args, cwd=str(hg_repo), env=env)
222 222
223 223 # Allow 1 (no-op) to not trigger error.
224 224 if res.returncode not in (0, 1):
225 225 res.check_returncode()
226 226
227 227 run_powershell(
228 228 winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
229 229 )
230 230
231 231 # TODO detect dirty local working directory and synchronize accordingly.
232 232
233 233
234 234 def purge_hg(winrm_client):
235 235 """Purge the Mercurial source repository on an EC2 instance."""
236 236 run_powershell(winrm_client, HG_PURGE)
237 237
238 238
239 239 def find_latest_dist(winrm_client, pattern):
240 240 """Find path to newest file in dist/ directory matching a pattern."""
241 241
242 242 res = winrm_client.execute_ps(
243 243 r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
244 244 '| Sort-Object LastWriteTime -Descending '
245 245 '| Select-Object -First 1\n'
246 246 '$v.name' % pattern
247 247 )
248 248 return res[0]
249 249
250 250
251 251 def copy_latest_dist(winrm_client, pattern, dest_path):
252 252 """Copy latest file matching pattern in dist/ directory.
253 253
254 254 Given a WinRM client and a file pattern, find the latest file on the remote
255 255 matching that pattern and copy it to the ``dest_path`` directory on the
256 256 local machine.
257 257 """
258 258 latest = find_latest_dist(winrm_client, pattern)
259 259 source = r'C:\hgdev\src\dist\%s' % latest
260 260 dest = dest_path / latest
261 261 print('copying %s to %s' % (source, dest))
262 262 winrm_client.fetch(source, str(dest))
263 263
264 264
265 265 def build_inno_installer(
266 266 winrm_client,
267 python_version: int,
268 267 arch: str,
269 268 dest_path: pathlib.Path,
270 269 version=None,
271 270 ):
272 271 """Build the Inno Setup installer on a remote machine.
273 272
274 273 Using a WinRM client, remote commands are executed to build
275 274 a Mercurial Inno Setup installer.
276 275 """
277 print(
278 'building Inno Setup installer for Python %d %s'
279 % (python_version, arch)
280 )
276 print('building Inno Setup installer for %s' % arch)
281 277
282 278 # TODO fix this limitation in packaging code
283 279 if not version:
284 280 raise Exception("version string is required when building for Python 3")
285 281
286 282 if arch == "x86":
287 283 target_triple = "i686-pc-windows-msvc"
288 284 elif arch == "x64":
289 285 target_triple = "x86_64-pc-windows-msvc"
290 286 else:
291 287 raise Exception("unhandled arch: %s" % arch)
292 288
293 289 ps = BUILD_INNO_PYTHON3.format(
294 290 pyoxidizer_target=target_triple,
295 291 version=version,
296 292 )
297 293
298 294 run_powershell(winrm_client, ps)
299 295 copy_latest_dist(winrm_client, '*.exe', dest_path)
300 296
301 297
302 298 def build_wheel(
303 299 winrm_client, python_version: str, arch: str, dest_path: pathlib.Path
304 300 ):
305 301 """Build Python wheels on a remote machine.
306 302
307 303 Using a WinRM client, remote commands are executed to build a Python wheel
308 304 for Mercurial.
309 305 """
310 306 print('Building Windows wheel for Python %s %s' % (python_version, arch))
311 307
312 308 ps = BUILD_WHEEL.format(
313 309 python_version=python_version.replace(".", ""), arch=arch
314 310 )
315 311
316 312 run_powershell(winrm_client, ps)
317 313 copy_latest_dist(winrm_client, '*.whl', dest_path)
318 314
319 315
320 316 def build_wix_installer(
321 317 winrm_client,
322 python_version: int,
323 318 arch: str,
324 319 dest_path: pathlib.Path,
325 320 version=None,
326 321 ):
327 322 """Build the WiX installer on a remote machine.
328 323
329 324 Using a WinRM client, remote commands are executed to build a WiX installer.
330 325 """
331 print('Building WiX installer for Python %d %s' % (python_version, arch))
326 print('Building WiX installer for %s' % arch)
332 327
333 328 # TODO fix this limitation in packaging code
334 329 if not version:
335 330 raise Exception("version string is required when building for Python 3")
336 331
337 332 if arch == "x86":
338 333 target_triple = "i686-pc-windows-msvc"
339 334 elif arch == "x64":
340 335 target_triple = "x86_64-pc-windows-msvc"
341 336 else:
342 337 raise Exception("unhandled arch: %s" % arch)
343 338
344 339 ps = BUILD_WIX_PYTHON3.format(
345 340 pyoxidizer_target=target_triple,
346 341 version=version,
347 342 )
348 343
349 344 run_powershell(winrm_client, ps)
350 345 copy_latest_dist(winrm_client, '*.msi', dest_path)
351 346
352 347
353 348 def run_tests(winrm_client, python_version, arch, test_flags=''):
354 349 """Run tests on a remote Windows machine.
355 350
356 351 ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
357 352 ``arch`` is ``x86`` or ``x64``.
358 353 ``test_flags`` is a str representing extra arguments to pass to
359 354 ``run-tests.py``.
360 355 """
361 356 if not re.match(r'\d\.\d', python_version):
362 357 raise ValueError(
363 358 r'python_version must be \d.\d; got %s' % python_version
364 359 )
365 360
366 361 if arch not in ('x86', 'x64'):
367 362 raise ValueError('arch must be x86 or x64; got %s' % arch)
368 363
369 364 python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
370 365
371 366 ps = RUN_TESTS.format(
372 367 python_path=python_path,
373 368 test_flags=test_flags or '',
374 369 )
375 370
376 371 run_powershell(winrm_client, ps)
377 372
378 373
379 374 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
380 375 return (
381 376 dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
382 377 dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
383 378 dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
384 379 dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
385 380 dist_path / WHEEL_FILENAME_PYTHON39_X86.format(version=version),
386 381 dist_path / WHEEL_FILENAME_PYTHON39_X64.format(version=version),
387 382 dist_path / WHEEL_FILENAME_PYTHON310_X86.format(version=version),
388 383 dist_path / WHEEL_FILENAME_PYTHON310_X64.format(version=version),
389 384 )
390 385
391 386
392 387 def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
393 388 return (
394 389 dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
395 390 dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
396 391 dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
397 392 dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
398 393 dist_path / WHEEL_FILENAME_PYTHON39_X86.format(version=version),
399 394 dist_path / WHEEL_FILENAME_PYTHON39_X64.format(version=version),
400 395 dist_path / WHEEL_FILENAME_PYTHON310_X86.format(version=version),
401 396 dist_path / WHEEL_FILENAME_PYTHON310_X64.format(version=version),
402 397 dist_path / EXE_FILENAME_PYTHON3_X86.format(version=version),
403 398 dist_path / EXE_FILENAME_PYTHON3_X64.format(version=version),
404 399 dist_path / MSI_FILENAME_PYTHON3_X86.format(version=version),
405 400 dist_path / MSI_FILENAME_PYTHON3_X64.format(version=version),
406 401 )
407 402
408 403
409 404 def generate_latest_dat(version: str):
410 405 python3_x86_exe_filename = EXE_FILENAME_PYTHON3_X86.format(version=version)
411 406 python3_x64_exe_filename = EXE_FILENAME_PYTHON3_X64.format(version=version)
412 407 python3_x86_msi_filename = MSI_FILENAME_PYTHON3_X86.format(version=version)
413 408 python3_x64_msi_filename = MSI_FILENAME_PYTHON3_X64.format(version=version)
414 409
415 410 entries = (
416 411 (
417 412 '10',
418 413 version,
419 414 X86_USER_AGENT_PATTERN,
420 415 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_exe_filename),
421 416 EXE_PYTHON3_X86_DESCRIPTION.format(version=version),
422 417 ),
423 418 (
424 419 '10',
425 420 version,
426 421 X64_USER_AGENT_PATTERN,
427 422 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_exe_filename),
428 423 EXE_PYTHON3_X64_DESCRIPTION.format(version=version),
429 424 ),
430 425 (
431 426 '10',
432 427 version,
433 428 X86_USER_AGENT_PATTERN,
434 429 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_msi_filename),
435 430 MSI_PYTHON3_X86_DESCRIPTION.format(version=version),
436 431 ),
437 432 (
438 433 '10',
439 434 version,
440 435 X64_USER_AGENT_PATTERN,
441 436 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_msi_filename),
442 437 MSI_PYTHON3_X64_DESCRIPTION.format(version=version),
443 438 ),
444 439 )
445 440
446 441 lines = ['\t'.join(e) for e in entries]
447 442
448 443 return '\n'.join(lines) + '\n'
449 444
450 445
451 446 def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
452 447 """Publish Windows release artifacts to PyPI."""
453 448
454 449 wheel_paths = resolve_wheel_artifacts(dist_path, version)
455 450
456 451 for p in wheel_paths:
457 452 if not p.exists():
458 453 raise Exception('%s not found' % p)
459 454
460 455 print('uploading wheels to PyPI (you may be prompted for credentials)')
461 456 pypi_upload(wheel_paths)
462 457
463 458
464 459 def publish_artifacts_mercurial_scm_org(
465 460 dist_path: pathlib.Path, version: str, ssh_username=None
466 461 ):
467 462 """Publish Windows release artifacts to mercurial-scm.org."""
468 463 all_paths = resolve_all_artifacts(dist_path, version)
469 464
470 465 for p in all_paths:
471 466 if not p.exists():
472 467 raise Exception('%s not found' % p)
473 468
474 469 client = paramiko.SSHClient()
475 470 client.load_system_host_keys()
476 471 # We assume the system SSH configuration knows how to connect.
477 472 print('connecting to mercurial-scm.org via ssh...')
478 473 try:
479 474 client.connect('mercurial-scm.org', username=ssh_username)
480 475 except paramiko.AuthenticationException:
481 476 print('error authenticating; is an SSH key available in an SSH agent?')
482 477 raise
483 478
484 479 print('SSH connection established')
485 480
486 481 print('opening SFTP client...')
487 482 sftp = client.open_sftp()
488 483 print('SFTP client obtained')
489 484
490 485 for p in all_paths:
491 486 dest_path = '/var/www/release/windows/%s' % p.name
492 487 print('uploading %s to %s' % (p, dest_path))
493 488
494 489 with p.open('rb') as fh:
495 490 data = fh.read()
496 491
497 492 with sftp.open(dest_path, 'wb') as fh:
498 493 fh.write(data)
499 494 fh.chmod(0o0664)
500 495
501 496 latest_dat_path = '/var/www/release/windows/latest.dat'
502 497
503 498 now = datetime.datetime.utcnow()
504 499 backup_path = dist_path / (
505 500 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
506 501 )
507 502 print('backing up %s to %s' % (latest_dat_path, backup_path))
508 503
509 504 with sftp.open(latest_dat_path, 'rb') as fh:
510 505 latest_dat_old = fh.read()
511 506
512 507 with backup_path.open('wb') as fh:
513 508 fh.write(latest_dat_old)
514 509
515 510 print('writing %s with content:' % latest_dat_path)
516 511 latest_dat_content = generate_latest_dat(version)
517 512 print(latest_dat_content)
518 513
519 514 with sftp.open(latest_dat_path, 'wb') as fh:
520 515 fh.write(latest_dat_content.encode('ascii'))
521 516
522 517
523 518 def publish_artifacts(
524 519 dist_path: pathlib.Path,
525 520 version: str,
526 521 pypi=True,
527 522 mercurial_scm_org=True,
528 523 ssh_username=None,
529 524 ):
530 525 """Publish Windows release artifacts.
531 526
532 527 Files are found in `dist_path`. We will look for files with version string
533 528 `version`.
534 529
535 530 `pypi` controls whether we upload to PyPI.
536 531 `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
537 532 """
538 533 if pypi:
539 534 publish_artifacts_pypi(dist_path, version)
540 535
541 536 if mercurial_scm_org:
542 537 publish_artifacts_mercurial_scm_org(
543 538 dist_path, version, ssh_username=ssh_username
544 539 )
General Comments 0
You need to be logged in to leave comments. Login now