# HG changeset patch # User Gregory Szorc # Date 2019-10-24 01:37:36 # Node ID 081a77df7bc65e18cce5c58dd3a879991d4d724e # Parent 5a0a4fa055ff9f3953939ca8a5cdb049eaf94f84 packaging: consolidate CLI functionality into packaging.py Consolidating functionality for invoking code in the hgpackaging package through a single CLI entry point will make things simpler when we add more complexity to that package. For example, it will allow us to run things out of a virtualenv with third party packages. This commit consolidates functionality from the Inno and WiX build.py scripts into a new packaging.py script. That script simply creates a virtualenv and runs the CLI functionality in it. The new virtualenv is populated with jinja2 because I felt it easier to incorporate requirements file processing in this commit and we will soon use jinja2 in an upcoming commit. The unified CLI functionality will also make it easier to script other packaging workflows going forward. e.g. RPM, Debian, and macOS packaging. Differential Revision: https://phab.mercurial-scm.org/D7156 diff --git a/contrib/automation/hgautomation/windows.py b/contrib/automation/hgautomation/windows.py --- a/contrib/automation/hgautomation/windows.py +++ b/contrib/automation/hgautomation/windows.py @@ -71,7 +71,7 @@ Write-Output "updated Mercurial working BUILD_INNO = r''' Set-Location C:\hgdev\src $python = "C:\hgdev\python27-{arch}\python.exe" -C:\hgdev\python37-x64\python.exe contrib\packaging\inno\build.py --python $python +C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python if ($LASTEXITCODE -ne 0) {{ throw "process exited non-0: $LASTEXITCODE" }} @@ -88,7 +88,7 @@ if ($LASTEXITCODE -ne 0) {{ BUILD_WIX = r''' Set-Location C:\hgdev\src $python = "C:\hgdev\python27-{arch}\python.exe" -C:\hgdev\python37-x64\python.exe contrib\packaging\wix\build.py --python $python {extra_args} +C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args} if ($LASTEXITCODE -ne 0) {{ throw "process exited non-0: $LASTEXITCODE" }} diff --git a/contrib/packaging/hgpackaging/cli.py b/contrib/packaging/hgpackaging/cli.py new file mode 100644 --- /dev/null +++ b/contrib/packaging/hgpackaging/cli.py @@ -0,0 +1,153 @@ +# cli.py - Command line interface for automation +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import argparse +import os +import pathlib + +from . import ( + inno, + wix, +) + +HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) +SOURCE_DIR = HERE.parent.parent.parent + + +def build_inno(python=None, iscc=None, version=None): + if not os.path.isabs(python): + raise Exception("--python arg must be an absolute path") + + if iscc: + iscc = pathlib.Path(iscc) + else: + iscc = ( + pathlib.Path(os.environ["ProgramFiles(x86)"]) + / "Inno Setup 5" + / "ISCC.exe" + ) + + build_dir = SOURCE_DIR / "build" + + inno.build( + SOURCE_DIR, build_dir, pathlib.Path(python), iscc, version=version, + ) + + +def build_wix( + name=None, + python=None, + version=None, + sign_sn=None, + sign_cert=None, + sign_password=None, + sign_timestamp_url=None, + extra_packages_script=None, + extra_wxs=None, + extra_features=None, +): + fn = wix.build_installer + kwargs = { + "source_dir": SOURCE_DIR, + "python_exe": pathlib.Path(python), + "version": version, + } + + if not os.path.isabs(python): + raise Exception("--python arg must be an absolute path") + + if extra_packages_script: + kwargs["extra_packages_script"] = extra_packages_script + if extra_wxs: + kwargs["extra_wxs"] = dict( + thing.split("=") for thing in extra_wxs.split(",") + ) + if extra_features: + kwargs["extra_features"] = extra_features.split(",") + + if sign_sn or sign_cert: + fn = wix.build_signed_installer + kwargs["name"] = name + kwargs["subject_name"] = sign_sn + kwargs["cert_path"] = sign_cert + kwargs["cert_password"] = sign_password + kwargs["timestamp_url"] = sign_timestamp_url + + fn(**kwargs) + + +def get_parser(): + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers() + + sp = subparsers.add_parser("inno", help="Build Inno Setup installer") + sp.add_argument("--python", required=True, help="path to python.exe to use") + sp.add_argument("--iscc", help="path to iscc.exe to use") + sp.add_argument( + "--version", + help="Mercurial version string to use " + "(detected from __version__.py if not defined", + ) + sp.set_defaults(func=build_inno) + + sp = subparsers.add_parser( + "wix", help="Build Windows installer with WiX Toolset" + ) + sp.add_argument("--name", help="Application name", default="Mercurial") + sp.add_argument( + "--python", help="Path to Python executable to use", required=True + ) + sp.add_argument( + "--sign-sn", + help="Subject name (or fragment thereof) of certificate " + "to use for signing", + ) + sp.add_argument( + "--sign-cert", help="Path to certificate to use for signing" + ) + sp.add_argument("--sign-password", help="Password for signing certificate") + sp.add_argument( + "--sign-timestamp-url", + help="URL of timestamp server to use for signing", + ) + sp.add_argument("--version", help="Version string to use") + sp.add_argument( + "--extra-packages-script", + help=( + "Script to execute to include extra packages in " "py2exe binary." + ), + ) + sp.add_argument( + "--extra-wxs", help="CSV of path_to_wxs_file=working_dir_for_wxs_file" + ) + sp.add_argument( + "--extra-features", + help=( + "CSV of extra feature names to include " + "in the installer from the extra wxs files" + ), + ) + sp.set_defaults(func=build_wix) + + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + if not hasattr(args, "func"): + parser.print_help() + return + + kwargs = dict(vars(args)) + del kwargs["func"] + + args.func(**kwargs) diff --git a/contrib/packaging/inno/build.py b/contrib/packaging/inno/build.py deleted file mode 100755 --- a/contrib/packaging/inno/build.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# build.py - Inno installer build script. -# -# Copyright 2019 Gregory Szorc -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -# This script automates the building of the Inno MSI installer for Mercurial. - -# no-check-code because Python 3 native. - -import argparse -import os -import pathlib -import sys - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument( - '--python', required=True, help='path to python.exe to use' - ) - parser.add_argument('--iscc', help='path to iscc.exe to use') - parser.add_argument( - '--version', - help='Mercurial version string to use ' - '(detected from __version__.py if not defined', - ) - - args = parser.parse_args() - - if not os.path.isabs(args.python): - raise Exception('--python arg must be an absolute path') - - if args.iscc: - iscc = pathlib.Path(args.iscc) - else: - iscc = ( - pathlib.Path(os.environ['ProgramFiles(x86)']) - / 'Inno Setup 5' - / 'ISCC.exe' - ) - - here = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) - source_dir = here.parent.parent.parent - build_dir = source_dir / 'build' - - sys.path.insert(0, str(source_dir / 'contrib' / 'packaging')) - - from hgpackaging.inno import build - - build( - source_dir, - build_dir, - pathlib.Path(args.python), - iscc, - version=args.version, - ) diff --git a/contrib/packaging/inno/readme.rst b/contrib/packaging/inno/readme.rst --- a/contrib/packaging/inno/readme.rst +++ b/contrib/packaging/inno/readme.rst @@ -11,12 +11,12 @@ The following system dependencies must b * Inno Setup (http://jrsoftware.org/isdl.php) version 5.4 or newer. Be sure to install the optional Inno Setup Preprocessor feature, which is required. -* Python 3.5+ (to run the ``build.py`` script) +* Python 3.5+ (to run the ``packaging.py`` script) Building ======== -The ``build.py`` script automates the process of producing an +The ``packaging.py`` script automates the process of producing an Inno installer. It manages fetching and configuring the non-system dependencies (such as py2exe, gettext, and various Python packages). @@ -31,11 +31,11 @@ either ``Visual C++ 2008 32-bit Command From the prompt, change to the Mercurial source directory. e.g. ``cd c:\src\hg``. -Next, invoke ``build.py`` to produce an Inno installer. You will +Next, invoke ``packaging.py`` to produce an Inno installer. You will need to supply the path to the Python interpreter to use.:: - $ python3.exe contrib\packaging\inno\build.py \ - --python c:\python27\python.exe + $ python3.exe contrib\packaging\packaging.py \ + inno --python c:\python27\python.exe .. note:: @@ -49,13 +49,13 @@ configured into the ``build`` sub-direct and an installer placed in the ``dist`` sub-directory. The final line of output should print the name of the generated installer. -Additional options may be configured. Run ``build.py --help`` to -see a list of program flags. +Additional options may be configured. Run +``packaging.py inno --help`` to see a list of program flags. MinGW ===== It is theoretically possible to generate an installer that uses -MinGW. This isn't well tested and ``build.py`` and may properly +MinGW. This isn't well tested and ``packaging.py`` and may properly support it. See old versions of this file in version control for potentially useful hints as to how to achieve this. diff --git a/contrib/packaging/packaging.py b/contrib/packaging/packaging.py new file mode 100755 --- /dev/null +++ b/contrib/packaging/packaging.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# packaging.py - Mercurial packaging functionality +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +import os +import pathlib +import subprocess +import sys +import venv + + +HERE = pathlib.Path(os.path.abspath(__file__)).parent +REQUIREMENTS_TXT = HERE / "requirements.txt" +SOURCE_DIR = HERE.parent.parent +VENV = SOURCE_DIR / "build" / "venv-packaging" + + +def bootstrap(): + venv_created = not VENV.exists() + + VENV.parent.mkdir(exist_ok=True) + + venv.create(VENV, with_pip=True) + + if os.name == "nt": + venv_bin = VENV / "Scripts" + pip = venv_bin / "pip.exe" + python = venv_bin / "python.exe" + else: + venv_bin = VENV / "bin" + pip = venv_bin / "pip" + python = venv_bin / "python" + + args = [ + str(pip), + "install", + "-r", + str(REQUIREMENTS_TXT), + "--disable-pip-version-check", + ] + + if not venv_created: + args.append("-q") + + subprocess.run(args, check=True) + + os.environ["HGPACKAGING_BOOTSTRAPPED"] = "1" + os.environ["PATH"] = "%s%s%s" % (venv_bin, os.pathsep, os.environ["PATH"]) + + subprocess.run([str(python), __file__] + sys.argv[1:], check=True) + + +def run(): + import hgpackaging.cli as cli + + # Need to strip off main Python executable. + cli.main() + + +if __name__ == "__main__": + try: + if "HGPACKAGING_BOOTSTRAPPED" not in os.environ: + bootstrap() + else: + run() + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + except KeyboardInterrupt: + sys.exit(1) diff --git a/contrib/packaging/requirements.txt b/contrib/packaging/requirements.txt new file mode 100644 --- /dev/null +++ b/contrib/packaging/requirements.txt @@ -0,0 +1,39 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file=contrib/packaging/requirements.txt contrib/packaging/requirements.txt.in +# +jinja2==2.10.3 \ + --hash=sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f \ + --hash=sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de +markupsafe==1.1.1 \ + --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \ + --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \ + --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \ + --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \ + --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \ + --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \ + --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \ + --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \ + --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \ + --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \ + --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \ + --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \ + --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \ + --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \ + --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \ + --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \ + --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \ + --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \ + --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \ + --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \ + --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \ + --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \ + --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \ + --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \ + --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \ + --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \ + --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \ + --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \ + # via jinja2 diff --git a/contrib/packaging/requirements.txt.in b/contrib/packaging/requirements.txt.in new file mode 100644 --- /dev/null +++ b/contrib/packaging/requirements.txt.in @@ -0,0 +1,1 @@ +jinja2 diff --git a/contrib/packaging/wix/build.py b/contrib/packaging/wix/build.py deleted file mode 100755 --- a/contrib/packaging/wix/build.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Gregory Szorc -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -# no-check-code because Python 3 native. - -"""Code to build Mercurial WiX installer.""" - -import argparse -import os -import pathlib -import sys - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--name', help='Application name', default='Mercurial') - parser.add_argument( - '--python', help='Path to Python executable to use', required=True - ) - parser.add_argument( - '--sign-sn', - help='Subject name (or fragment thereof) of certificate ' - 'to use for signing', - ) - parser.add_argument( - '--sign-cert', help='Path to certificate to use for signing' - ) - parser.add_argument( - '--sign-password', help='Password for signing certificate' - ) - parser.add_argument( - '--sign-timestamp-url', - help='URL of timestamp server to use for signing', - ) - parser.add_argument('--version', help='Version string to use') - parser.add_argument( - '--extra-packages-script', - help=( - 'Script to execute to include extra packages in ' 'py2exe binary.' - ), - ) - parser.add_argument( - '--extra-wxs', help='CSV of path_to_wxs_file=working_dir_for_wxs_file' - ) - parser.add_argument( - '--extra-features', - help=( - 'CSV of extra feature names to include ' - 'in the installer from the extra wxs files' - ), - ) - - args = parser.parse_args() - - here = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) - source_dir = here.parent.parent.parent - - sys.path.insert(0, str(source_dir / 'contrib' / 'packaging')) - - from hgpackaging.wix import ( - build_installer, - build_signed_installer, - ) - - fn = build_installer - kwargs = { - 'source_dir': source_dir, - 'python_exe': pathlib.Path(args.python), - 'version': args.version, - } - - if not os.path.isabs(args.python): - raise Exception('--python arg must be an absolute path') - - if args.extra_packages_script: - kwargs['extra_packages_script'] = args.extra_packages_script - if args.extra_wxs: - kwargs['extra_wxs'] = dict( - thing.split("=") for thing in args.extra_wxs.split(',') - ) - if args.extra_features: - kwargs['extra_features'] = args.extra_features.split(',') - - if args.sign_sn or args.sign_cert: - fn = build_signed_installer - kwargs['name'] = args.name - kwargs['subject_name'] = args.sign_sn - kwargs['cert_path'] = args.sign_cert - kwargs['cert_password'] = args.sign_password - kwargs['timestamp_url'] = args.sign_timestamp_url - - fn(**kwargs) diff --git a/contrib/packaging/wix/readme.rst b/contrib/packaging/wix/readme.rst --- a/contrib/packaging/wix/readme.rst +++ b/contrib/packaging/wix/readme.rst @@ -18,12 +18,12 @@ dependencies must be installed: * Python 2.7 (download from https://www.python.org/downloads/) * Microsoft Visual C++ Compiler for Python 2.7 (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -* Python 3.5+ (to run the ``build.py`` script) +* Python 3.5+ (to run the ``packaging.py`` script) Building ======== -The ``build.py`` script automates the process of producing an MSI +The ``packaging.py`` script automates the process of producing an MSI installer. It manages fetching and configuring non-system dependencies (such as py2exe, gettext, and various Python packages). @@ -37,11 +37,11 @@ launch either ``Visual C++ 2008 32-bit C From the prompt, change to the Mercurial source directory. e.g. ``cd c:\src\hg``. -Next, invoke ``build.py`` to produce an MSI installer. You will need +Next, invoke ``packaging.py`` to produce an MSI installer. You will need to supply the path to the Python interpreter to use.:: - $ python3 contrib\packaging\wix\build.py \ - --python c:\python27\python.exe + $ python3 contrib\packaging\packaging.py \ + wix --python c:\python27\python.exe .. note:: @@ -54,8 +54,8 @@ configured into the ``build`` sub-direct and an installer placed in the ``dist`` sub-directory. The final line of output should print the name of the generated installer. -Additional options may be configured. Run ``build.py --help`` to see -a list of program flags. +Additional options may be configured. Run ``packaging.py wix --help`` to +see a list of program flags. Relationship to TortoiseHG ========================== diff --git a/tests/test-check-code.t b/tests/test-check-code.t --- a/tests/test-check-code.t +++ b/tests/test-check-code.t @@ -21,13 +21,12 @@ New errors are not allowed. Warnings are Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob) + Skipping contrib/packaging/hgpackaging/cli.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/wix.py it has no-che?k-code (glob) - Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob) - Skipping contrib/packaging/wix/build.py it has no-che?k-code (glob) Skipping i18n/polib.py it has no-che?k-code (glob) Skipping mercurial/statprof.py it has no-che?k-code (glob) Skipping tests/badserverext.py it has no-che?k-code (glob) diff --git a/tests/test-check-py3-compat.t b/tests/test-check-py3-compat.t --- a/tests/test-check-py3-compat.t +++ b/tests/test-check-py3-compat.t @@ -8,6 +8,7 @@ > -X contrib/automation/ \ > -X contrib/packaging/hgpackaging/ \ > -X contrib/packaging/inno/ \ + > -X contrib/packaging/packaging.py \ > -X contrib/packaging/wix/ \ > -X hgdemandimport/demandimportpy2.py \ > -X mercurial/thirdparty/cbor \