packaging: add command line flag to add extra vars to pyoxidizer

This felt simpler than the previous incarnation of injecting
content into the WiX build. I decided the easiest way to pass
an arbitrary map into the process was some json - I may regret this,
but time will tell on that.

Differential Revision: https://phab.mercurial-scm.org/D11093

contrib/packaging/hgpackaging/cli.py

@@ -1,187 +1,194 b''
 # cli.py - Command line interface for automation
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import os
 import pathlib
 
 from . import (
     inno,
     wix,
 )
 
 HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
 SOURCE_DIR = HERE.parent.parent.parent
 
 
 def build_inno(pyoxidizer_target=None, python=None, iscc=None, version=None):
     if not pyoxidizer_target and not python:
         raise Exception("--python required unless building with PyOxidizer")
 
     if python and not os.path.isabs(python):
         raise Exception("--python arg must be an absolute path")
 
     if iscc:
         iscc = pathlib.Path(iscc)
     else:
         iscc = (
             pathlib.Path(os.environ["ProgramFiles(x86)"])
             / "Inno Setup 5"
             / "ISCC.exe"
         )
 
     build_dir = SOURCE_DIR / "build"
 
     if pyoxidizer_target:
         inno.build_with_pyoxidizer(
             SOURCE_DIR, build_dir, pyoxidizer_target, iscc, version=version
         )
     else:
         inno.build_with_py2exe(
             SOURCE_DIR,
             build_dir,
             pathlib.Path(python),
             iscc,
             version=version,
         )
 
 
 def build_wix(
     name=None,
     pyoxidizer_target=None,
     python=None,
     version=None,
     sign_sn=None,
     sign_cert=None,
     sign_password=None,
     sign_timestamp_url=None,
     extra_packages_script=None,
     extra_wxs=None,
     extra_features=None,
+    extra_pyoxidizer_vars=None,
 ):
     if not pyoxidizer_target and not python:
         raise Exception("--python required unless building with PyOxidizer")
 
     if python and not os.path.isabs(python):
         raise Exception("--python arg must be an absolute path")
 
     kwargs = {
         "source_dir": SOURCE_DIR,
         "version": version,
     }
 
     if pyoxidizer_target:
         fn = wix.build_installer_pyoxidizer
         kwargs["target_triple"] = pyoxidizer_target
     else:
         fn = wix.build_installer_py2exe
         kwargs["python_exe"] = pathlib.Path(python)
 
     if extra_packages_script:
         if pyoxidizer_target:
             raise Exception(
                 "pyoxidizer does not support --extra-packages-script"
             )
         kwargs["extra_packages_script"] = extra_packages_script
     if extra_wxs:
         kwargs["extra_wxs"] = dict(
             thing.split("=") for thing in extra_wxs.split(",")
         )
     if extra_features:
         kwargs["extra_features"] = extra_features.split(",")
 
     if sign_sn or sign_cert:
         kwargs["signing_info"] = {
             "name": name,
             "subject_name": sign_sn,
             "cert_path": sign_cert,
             "cert_password": sign_password,
             "timestamp_url": sign_timestamp_url,
         }
 
-    fn(**kwargs)
+    fn(**kwargs, extra_pyoxidizer_vars=extra_pyoxidizer_vars)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
 
     subparsers = parser.add_subparsers()
 
     sp = subparsers.add_parser("inno", help="Build Inno Setup installer")
     sp.add_argument(
         "--pyoxidizer-target",
         choices={"i686-pc-windows-msvc", "x86_64-pc-windows-msvc"},
         help="Build with PyOxidizer targeting this host triple",
     )
     sp.add_argument("--python", help="path to python.exe to use")
     sp.add_argument("--iscc", help="path to iscc.exe to use")
     sp.add_argument(
         "--version",
         help="Mercurial version string to use "
         "(detected from __version__.py if not defined",
     )
     sp.set_defaults(func=build_inno)
 
     sp = subparsers.add_parser(
         "wix", help="Build Windows installer with WiX Toolset"
     )
     sp.add_argument("--name", help="Application name", default="Mercurial")
     sp.add_argument(
         "--pyoxidizer-target",
         choices={"i686-pc-windows-msvc", "x86_64-pc-windows-msvc"},
         help="Build with PyOxidizer targeting this host triple",
     )
     sp.add_argument("--python", help="Path to Python executable to use")
     sp.add_argument(
         "--sign-sn",
         help="Subject name (or fragment thereof) of certificate "
         "to use for signing",
     )
     sp.add_argument(
         "--sign-cert", help="Path to certificate to use for signing"
     )
     sp.add_argument("--sign-password", help="Password for signing certificate")
     sp.add_argument(
         "--sign-timestamp-url",
         help="URL of timestamp server to use for signing",
     )
     sp.add_argument("--version", help="Version string to use")
     sp.add_argument(
         "--extra-packages-script",
         help=(
             "Script to execute to include extra packages in " "py2exe binary."
         ),
     )
     sp.add_argument(
         "--extra-wxs", help="CSV of path_to_wxs_file=working_dir_for_wxs_file"
     )
     sp.add_argument(
         "--extra-features",
         help=(
             "CSV of extra feature names to include "
             "in the installer from the extra wxs files"
         ),
     )
+
+    sp.add_argument(
+        "--extra-pyoxidizer-vars",
+        help="json map of extra variables to pass to pyoxidizer",
+    )
+
     sp.set_defaults(func=build_wix)
 
     return parser
 
 
 def main():
     parser = get_parser()
     args = parser.parse_args()
 
     if not hasattr(args, "func"):
         parser.print_help()
         return
 
     kwargs = dict(vars(args))
     del kwargs["func"]
 
     args.func(**kwargs)

contrib/packaging/hgpackaging/wix.py

@@ -1,581 +1,586 b''
 # wix.py - WiX installer functionality
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import collections
+import json
 import os
 import pathlib
 import re
 import shutil
 import subprocess
 import typing
 import uuid
 import xml.dom.minidom
 
 from .downloads import download_entry
 from .py2exe import (
     build_py2exe,
     stage_install,
 )
 from .pyoxidizer import (
     build_docs_html,
     create_pyoxidizer_install_layout,
     run_pyoxidizer,
 )
 from .util import (
     extract_zip_to_directory,
     normalize_windows_version,
     process_install_rules,
     sign_with_signtool,
 )
 
 
 EXTRA_PACKAGES = {
     'dulwich',
     'distutils',
     'keyring',
     'pygments',
     'win32ctypes',
 }
 
 EXTRA_INCLUDES = {
     '_curses',
     '_curses_panel',
 }
 
 EXTRA_INSTALL_RULES = [
     ('contrib/packaging/wix/COPYING.rtf', 'COPYING.rtf'),
     ('contrib/win32/mercurial.ini', 'defaultrc/mercurial.rc'),
 ]
 
 STAGING_REMOVE_FILES = [
     # We use the RTF variant.
     'copying.txt',
 ]
 
 SHORTCUTS = {
     # hg.1.html'
     'hg.file.5d3e441c_28d9_5542_afd0_cdd4234f12d5': {
         'Name': 'Mercurial Command Reference',
     },
     # hgignore.5.html
     'hg.file.5757d8e0_f207_5e10_a2ec_3ba0a062f431': {
         'Name': 'Mercurial Ignore Files',
     },
     # hgrc.5.html
     'hg.file.92e605fd_1d1a_5dc6_9fc0_5d2998eb8f5e': {
         'Name': 'Mercurial Configuration Files',
     },
 }
 
 
 def find_version(source_dir: pathlib.Path):
     version_py = source_dir / 'mercurial' / '__version__.py'
 
     with version_py.open('r', encoding='utf-8') as fh:
         source = fh.read().strip()
 
     m = re.search('version = b"(.*)"', source)
     return m.group(1)
 
 
 def ensure_vc90_merge_modules(build_dir):
     x86 = (
         download_entry(
             'vc9-crt-x86-msm',
             build_dir,
             local_name='microsoft.vcxx.crt.x86_msm.msm',
         )[0],
         download_entry(
             'vc9-crt-x86-msm-policy',
             build_dir,
             local_name='policy.x.xx.microsoft.vcxx.crt.x86_msm.msm',
         )[0],
     )
 
     x64 = (
         download_entry(
             'vc9-crt-x64-msm',
             build_dir,
             local_name='microsoft.vcxx.crt.x64_msm.msm',
         )[0],
         download_entry(
             'vc9-crt-x64-msm-policy',
             build_dir,
             local_name='policy.x.xx.microsoft.vcxx.crt.x64_msm.msm',
         )[0],
     )
     return {
         'x86': x86,
         'x64': x64,
     }
 
 
 def run_candle(wix, cwd, wxs, source_dir, defines=None):
     args = [
         str(wix / 'candle.exe'),
         '-nologo',
         str(wxs),
         '-dSourceDir=%s' % source_dir,
     ]
 
     if defines:
         args.extend('-d%s=%s' % define for define in sorted(defines.items()))
 
     subprocess.run(args, cwd=str(cwd), check=True)
 
 
 def make_files_xml(staging_dir: pathlib.Path, is_x64) -> str:
     """Create XML string listing every file to be installed."""
 
     # We derive GUIDs from a deterministic file path identifier.
     # We shoehorn the name into something that looks like a URL because
     # the UUID namespaces are supposed to work that way (even though
     # the input data probably is never validated).
 
     doc = xml.dom.minidom.parseString(
         '<?xml version="1.0" encoding="utf-8"?>'
         '<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">'
         '</Wix>'
     )
 
     # Assemble the install layout by directory. This makes it easier to
     # emit XML, since each directory has separate entities.
     manifest = collections.defaultdict(dict)
 
     for root, dirs, files in os.walk(staging_dir):
         dirs.sort()
 
         root = pathlib.Path(root)
         rel_dir = root.relative_to(staging_dir)
 
         for i in range(len(rel_dir.parts)):
             parent = '/'.join(rel_dir.parts[0 : i + 1])
             manifest.setdefault(parent, {})
 
         for f in sorted(files):
             full = root / f
             manifest[str(rel_dir).replace('\\', '/')][full.name] = full
 
     component_groups = collections.defaultdict(list)
 
     # Now emit a <Fragment> for each directory.
     # Each directory is composed of a <DirectoryRef> pointing to its parent
     # and defines child <Directory>'s and a <Component> with all the files.
     for dir_name, entries in sorted(manifest.items()):
         # The directory id is derived from the path. But the root directory
         # is special.
         if dir_name == '.':
             parent_directory_id = 'INSTALLDIR'
         else:
             parent_directory_id = 'hg.dir.%s' % dir_name.replace(
                 '/', '.'
             ).replace('-', '_')
 
         fragment = doc.createElement('Fragment')
         directory_ref = doc.createElement('DirectoryRef')
         directory_ref.setAttribute('Id', parent_directory_id)
 
         # Add <Directory> entries for immediate children directories.
         for possible_child in sorted(manifest.keys()):
             if (
                 dir_name == '.'
                 and '/' not in possible_child
                 and possible_child != '.'
             ):
                 child_directory_id = ('hg.dir.%s' % possible_child).replace(
                     '-', '_'
                 )
                 name = possible_child
             else:
                 if not possible_child.startswith('%s/' % dir_name):
                     continue
                 name = possible_child[len(dir_name) + 1 :]
                 if '/' in name:
                     continue
 
                 child_directory_id = 'hg.dir.%s' % possible_child.replace(
                     '/', '.'
                 ).replace('-', '_')
 
             directory = doc.createElement('Directory')
             directory.setAttribute('Id', child_directory_id)
             directory.setAttribute('Name', name)
             directory_ref.appendChild(directory)
 
         # Add <Component>s for files in this directory.
         for rel, source_path in sorted(entries.items()):
             if dir_name == '.':
                 full_rel = rel
             else:
                 full_rel = '%s/%s' % (dir_name, rel)
 
             component_unique_id = (
                 'https://www.mercurial-scm.org/wix-installer/0/component/%s'
                 % full_rel
             )
             component_guid = uuid.uuid5(uuid.NAMESPACE_URL, component_unique_id)
             component_id = 'hg.component.%s' % str(component_guid).replace(
                 '-', '_'
             )
 
             component = doc.createElement('Component')
 
             component.setAttribute('Id', component_id)
             component.setAttribute('Guid', str(component_guid).upper())
             component.setAttribute('Win64', 'yes' if is_x64 else 'no')
 
             # Assign this component to a top-level group.
             if dir_name == '.':
                 component_groups['ROOT'].append(component_id)
             elif '/' in dir_name:
                 component_groups[dir_name[0 : dir_name.index('/')]].append(
                     component_id
                 )
             else:
                 component_groups[dir_name].append(component_id)
 
             unique_id = (
                 'https://www.mercurial-scm.org/wix-installer/0/%s' % full_rel
             )
             file_guid = uuid.uuid5(uuid.NAMESPACE_URL, unique_id)
 
             # IDs have length limits. So use GUID to derive them.
             file_guid_normalized = str(file_guid).replace('-', '_')
             file_id = 'hg.file.%s' % file_guid_normalized
 
             file_element = doc.createElement('File')
             file_element.setAttribute('Id', file_id)
             file_element.setAttribute('Source', str(source_path))
             file_element.setAttribute('KeyPath', 'yes')
             file_element.setAttribute('ReadOnly', 'yes')
 
             component.appendChild(file_element)
             directory_ref.appendChild(component)
 
         fragment.appendChild(directory_ref)
         doc.documentElement.appendChild(fragment)
 
     for group, component_ids in sorted(component_groups.items()):
         fragment = doc.createElement('Fragment')
         component_group = doc.createElement('ComponentGroup')
         component_group.setAttribute('Id', 'hg.group.%s' % group)
 
         for component_id in component_ids:
             component_ref = doc.createElement('ComponentRef')
             component_ref.setAttribute('Id', component_id)
             component_group.appendChild(component_ref)
 
         fragment.appendChild(component_group)
         doc.documentElement.appendChild(fragment)
 
     # Add <Shortcut> to files that have it defined.
     for file_id, metadata in sorted(SHORTCUTS.items()):
         els = doc.getElementsByTagName('File')
         els = [el for el in els if el.getAttribute('Id') == file_id]
 
         if not els:
             raise Exception('could not find File[Id=%s]' % file_id)
 
         for el in els:
             shortcut = doc.createElement('Shortcut')
             shortcut.setAttribute('Id', 'hg.shortcut.%s' % file_id)
             shortcut.setAttribute('Directory', 'ProgramMenuDir')
             shortcut.setAttribute('Icon', 'hgIcon.ico')
             shortcut.setAttribute('IconIndex', '0')
             shortcut.setAttribute('Advertise', 'yes')
             for k, v in sorted(metadata.items()):
                 shortcut.setAttribute(k, v)
 
             el.appendChild(shortcut)
 
     return doc.toprettyxml()
 
 
 def build_installer_py2exe(
     source_dir: pathlib.Path,
     python_exe: pathlib.Path,
     msi_name='mercurial',
     version=None,
     extra_packages_script=None,
     extra_wxs: typing.Optional[typing.Dict[str, str]] = None,
     extra_features: typing.Optional[typing.List[str]] = None,
     signing_info: typing.Optional[typing.Dict[str, str]] = None,
 ):
     """Build a WiX MSI installer using py2exe.
 
     ``source_dir`` is the path to the Mercurial source tree to use.
     ``arch`` is the target architecture. either ``x86`` or ``x64``.
     ``python_exe`` is the path to the Python executable to use/bundle.
     ``version`` is the Mercurial version string. If not defined,
     ``mercurial/__version__.py`` will be consulted.
     ``extra_packages_script`` is a command to be run to inject extra packages
     into the py2exe binary. It should stage packages into the virtualenv and
     print a null byte followed by a newline-separated list of packages that
     should be included in the exe.
     ``extra_wxs`` is a dict of {wxs_name: working_dir_for_wxs_build}.
     ``extra_features`` is a list of additional named Features to include in
     the build. These must match Feature names in one of the wxs scripts.
     """
     arch = 'x64' if r'\x64' in os.environ.get('LIB', '') else 'x86'
 
     hg_build_dir = source_dir / 'build'
 
     requirements_txt = (
         source_dir / 'contrib' / 'packaging' / 'requirements-windows-py2.txt'
     )
 
     build_py2exe(
         source_dir,
         hg_build_dir,
         python_exe,
         'wix',
         requirements_txt,
         extra_packages=EXTRA_PACKAGES,
         extra_packages_script=extra_packages_script,
         extra_includes=EXTRA_INCLUDES,
     )
 
     build_dir = hg_build_dir / ('wix-%s' % arch)
     staging_dir = build_dir / 'stage'
 
     build_dir.mkdir(exist_ok=True)
 
     # Purge the staging directory for every build so packaging is pristine.
     if staging_dir.exists():
         print('purging %s' % staging_dir)
         shutil.rmtree(staging_dir)
 
     stage_install(source_dir, staging_dir, lower_case=True)
 
     # We also install some extra files.
     process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir)
 
     # And remove some files we don't want.
     for f in STAGING_REMOVE_FILES:
         p = staging_dir / f
         if p.exists():
             print('removing %s' % p)
             p.unlink()
 
     return run_wix_packaging(
         source_dir,
         build_dir,
         staging_dir,
         arch,
         version=version,
         python2=True,
         msi_name=msi_name,
         suffix="-python2",
         extra_wxs=extra_wxs,
         extra_features=extra_features,
         signing_info=signing_info,
     )
 
 
 def build_installer_pyoxidizer(
     source_dir: pathlib.Path,
     target_triple: str,
     msi_name='mercurial',
     version=None,
     extra_wxs: typing.Optional[typing.Dict[str, str]] = None,
     extra_features: typing.Optional[typing.List[str]] = None,
     signing_info: typing.Optional[typing.Dict[str, str]] = None,
+    extra_pyoxidizer_vars=None,
 ):
     """Build a WiX MSI installer using PyOxidizer."""
     hg_build_dir = source_dir / "build"
     build_dir = hg_build_dir / ("wix-%s" % target_triple)
 
     build_dir.mkdir(parents=True, exist_ok=True)
 
     # Need to ensure docs HTML is built because this isn't done as part of
     # `pip install Mercurial`.
     build_docs_html(source_dir)
 
     build_vars = {}
 
     if msi_name:
         build_vars["MSI_NAME"] = msi_name
 
     if version:
         build_vars["VERSION"] = version
 
     if extra_features:
         build_vars["EXTRA_MSI_FEATURES"] = ";".join(extra_features)
 
     if signing_info:
         if signing_info["cert_path"]:
             build_vars["SIGNING_PFX_PATH"] = signing_info["cert_path"]
         if signing_info["cert_password"]:
             build_vars["SIGNING_PFX_PASSWORD"] = signing_info["cert_password"]
         if signing_info["subject_name"]:
             build_vars["SIGNING_SUBJECT_NAME"] = signing_info["subject_name"]
         if signing_info["timestamp_url"]:
             build_vars["TIME_STAMP_SERVER_URL"] = signing_info["timestamp_url"]
 
+    if extra_pyoxidizer_vars:
+        build_vars.update(json.loads(extra_pyoxidizer_vars))
+
     if extra_wxs:
         raise Exception(
             "support for extra .wxs files has been temporarily dropped"
         )
 
     out_dir = run_pyoxidizer(
         source_dir,
         build_dir,
         target_triple,
         build_vars=build_vars,
         target="msi",
     )
 
     msi_dir = out_dir / "msi"
     msi_files = [f for f in os.listdir(msi_dir) if f.endswith(".msi")]
 
     if len(msi_files) != 1:
         raise Exception("expected exactly 1 .msi file; got %d" % len(msi_files))
 
     msi_filename = msi_files[0]
 
     msi_path = msi_dir / msi_filename
     dist_path = source_dir / "dist" / msi_filename
 
     dist_path.parent.mkdir(parents=True, exist_ok=True)
 
     shutil.copyfile(msi_path, dist_path)
 
     return {
         "msi_path": dist_path,
     }
 
 
 def run_wix_packaging(
     source_dir: pathlib.Path,
     build_dir: pathlib.Path,
     staging_dir: pathlib.Path,
     arch: str,
     version: str,
     python2: bool,
     msi_name: typing.Optional[str] = "mercurial",
     suffix: str = "",
     extra_wxs: typing.Optional[typing.Dict[str, str]] = None,
     extra_features: typing.Optional[typing.List[str]] = None,
     signing_info: typing.Optional[typing.Dict[str, str]] = None,
 ):
     """Invokes WiX to package up a built Mercurial.
 
     ``signing_info`` is a dict defining properties to facilitate signing the
     installer. Recognized keys include ``name``, ``subject_name``,
     ``cert_path``, ``cert_password``, and ``timestamp_url``. If populated,
     we will sign both the hg.exe and the .msi using the signing credentials
     specified.
     """
 
     orig_version = version or find_version(source_dir)
     version = normalize_windows_version(orig_version)
     print('using version string: %s' % version)
     if version != orig_version:
         print('(normalized from: %s)' % orig_version)
 
     if signing_info:
         sign_with_signtool(
             staging_dir / "hg.exe",
             "%s %s" % (signing_info["name"], version),
             subject_name=signing_info["subject_name"],
             cert_path=signing_info["cert_path"],
             cert_password=signing_info["cert_password"],
             timestamp_url=signing_info["timestamp_url"],
         )
 
     wix_dir = source_dir / 'contrib' / 'packaging' / 'wix'
 
     wix_pkg, wix_entry = download_entry('wix', build_dir)
     wix_path = build_dir / ('wix-%s' % wix_entry['version'])
 
     if not wix_path.exists():
         extract_zip_to_directory(wix_pkg, wix_path)
 
     if python2:
         ensure_vc90_merge_modules(build_dir)
 
     source_build_rel = pathlib.Path(os.path.relpath(source_dir, build_dir))
 
     defines = {'Platform': arch}
 
     # Derive a .wxs file with the staged files.
     manifest_wxs = build_dir / 'stage.wxs'
     with manifest_wxs.open('w', encoding='utf-8') as fh:
         fh.write(make_files_xml(staging_dir, is_x64=arch == 'x64'))
 
     run_candle(wix_path, build_dir, manifest_wxs, staging_dir, defines=defines)
 
     for source, rel_path in sorted((extra_wxs or {}).items()):
         run_candle(wix_path, build_dir, source, rel_path, defines=defines)
 
     source = wix_dir / 'mercurial.wxs'
     defines['Version'] = version
     defines['Comments'] = 'Installs Mercurial version %s' % version
 
     if python2:
         defines["PythonVersion"] = "2"
         defines['VCRedistSrcDir'] = str(build_dir)
     else:
         defines["PythonVersion"] = "3"
 
     if (staging_dir / "lib").exists():
         defines["MercurialHasLib"] = "1"
 
     if extra_features:
         assert all(';' not in f for f in extra_features)
         defines['MercurialExtraFeatures'] = ';'.join(extra_features)
 
     run_candle(wix_path, build_dir, source, source_build_rel, defines=defines)
 
     msi_path = (
         source_dir
         / 'dist'
         / ('%s-%s-%s%s.msi' % (msi_name, orig_version, arch, suffix))
     )
 
     args = [
         str(wix_path / 'light.exe'),
         '-nologo',
         '-ext',
         'WixUIExtension',
         '-sw1076',
         '-spdb',
         '-o',
         str(msi_path),
     ]
 
     for source, rel_path in sorted((extra_wxs or {}).items()):
         assert source.endswith('.wxs')
         source = os.path.basename(source)
         args.append(str(build_dir / ('%s.wixobj' % source[:-4])))
 
     args.extend(
         [
             str(build_dir / 'stage.wixobj'),
             str(build_dir / 'mercurial.wixobj'),
         ]
     )
 
     subprocess.run(args, cwd=str(source_dir), check=True)
 
     print('%s created' % msi_path)
 
     if signing_info:
         sign_with_signtool(
             msi_path,
             "%s %s" % (signing_info["name"], version),
             subject_name=signing_info["subject_name"],
             cert_path=signing_info["cert_path"],
             cert_password=signing_info["cert_password"],
             timestamp_url=signing_info["timestamp_url"],
         )
 
     return {
         'msi_path': msi_path,
     }