packaging: establish hgpackaging package

Previously, contrib/packaging behaved as a root to a
package directory and we had a "packagingutil" module. As I
work more on packaging code, we'll want to have more code
shared between different packaging tools. I think it makes
sense to have a single package containing multiple modules
than multiple top-level modules.

This commit establishes an "hgpackaging" package by moving
the existing packagingutil code to it.

Differential Revision: https://phab.mercurial-scm.org/D6083

contrib/packaging/hgpackaging/util.py

@@ -1,128 +1,128
-# packagingutil.py - Common packaging utility code.
+# util.py - Common packaging utility code.
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # no-check-code because Python 3 native.
 
 import gzip
 import hashlib
 import pathlib
 import tarfile
 import urllib.request
 import zipfile
 
 
 def hash_path(p: pathlib.Path):
     h = hashlib.sha256()
 
     with p.open('rb') as fh:
         while True:
             chunk = fh.read(65536)
             if not chunk:
                 break
 
             h.update(chunk)
 
     return h.hexdigest()
 
 
 class IntegrityError(Exception):
     """Represents an integrity error when downloading a URL."""
 
 
 def secure_download_stream(url, size, sha256):
     """Securely download a URL to a stream of chunks.
 
     If the integrity of the download fails, an IntegrityError is
     raised.
     """
     h = hashlib.sha256()
     length = 0
 
     with urllib.request.urlopen(url) as fh:
         if not url.endswith('.gz') and fh.info().get('Content-Encoding') == 'gzip':
             fh = gzip.GzipFile(fileobj=fh)
 
         while True:
             chunk = fh.read(65536)
             if not chunk:
                 break
 
             h.update(chunk)
             length += len(chunk)
 
             yield chunk
 
     digest = h.hexdigest()
 
     if length != size:
         raise IntegrityError('size mismatch on %s: wanted %d; got %d' % (
             url, size, length))
 
     if digest != sha256:
         raise IntegrityError('sha256 mismatch on %s: wanted %s; got %s' % (
             url, sha256, digest))
 
 
 def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str):
     """Download a URL to a filesystem path, possibly with verification."""
 
     # We download to a temporary file and rename at the end so there's
     # no chance of the final file being partially written or containing
     # bad data.
     print('downloading %s to %s' % (url, path))
 
     if path.exists():
         good = True
 
         if path.stat().st_size != size:
             print('existing file size is wrong; removing')
             good = False
 
         if good:
             if hash_path(path) != sha256:
                 print('existing file hash is wrong; removing')
                 good = False
 
         if good:
             print('%s exists and passes integrity checks' % path)
             return
 
         path.unlink()
 
     tmp = path.with_name('%s.tmp' % path.name)
 
     try:
         with tmp.open('wb') as fh:
             for chunk in secure_download_stream(url, size, sha256):
                 fh.write(chunk)
     except IntegrityError:
         tmp.unlink()
         raise
 
     tmp.rename(path)
     print('successfully downloaded %s' % url)
 
 
 def download_entry(entry: dict, dest_path: pathlib.Path, local_name=None) -> pathlib.Path:
     url = entry['url']
 
     local_name = local_name or url[url.rindex('/') + 1:]
 
     local_path = dest_path / local_name
     download_to_path(url, local_path, entry['size'], entry['sha256'])
 
     return local_path
 
 
 def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path):
     with tarfile.open(source, 'r') as tf:
         tf.extractall(dest)
 
 
 def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path):
     with zipfile.ZipFile(source, 'r') as zf:
         zf.extractall(dest)

contrib/packaging/inno/build.py

@@ -1,253 +1,253
 #!/usr/bin/env python3
 # build.py - Inno installer build script.
 #
 # Copyright 2019 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This script automates the building of the Inno MSI installer for Mercurial.
 
 # no-check-code because Python 3 native.
 
 import argparse
 import os
 import pathlib
 import shutil
 import subprocess
 import sys
 import tempfile
 
 
 DOWNLOADS = {
     'gettext': {
         'url': 'https://versaweb.dl.sourceforge.net/project/gnuwin32/gettext/0.14.4/gettext-0.14.4-bin.zip',
         'size': 1606131,
         'sha256': '60b9ef26bc5cceef036f0424e542106cf158352b2677f43a01affd6d82a1d641',
         'version': '0.14.4',
     },
     'gettext-dep': {
         'url': 'https://versaweb.dl.sourceforge.net/project/gnuwin32/gettext/0.14.4/gettext-0.14.4-dep.zip',
         'size': 715086,
         'sha256': '411f94974492fd2ecf52590cb05b1023530aec67e64154a88b1e4ebcd9c28588',
     },
     'py2exe': {
         'url': 'https://versaweb.dl.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.zip',
         'size': 149687,
         'sha256': '6bd383312e7d33eef2e43a5f236f9445e4f3e0f6b16333c6f183ed445c44ddbd',
         'version': '0.6.9',
     },
     'virtualenv': {
         'url': 'https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/virtualenv-16.4.3.tar.gz',
         'size': 3713208,
         'sha256': '984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39',
         'version': '16.4.3',
     },
 }
 
 
 PRINT_PYTHON_INFO = '''
 import platform, sys; print("%s:%d" % (platform.architecture()[0], sys.version_info[0]))
 '''.strip()
 
 
 def find_vc_runtime_files(x64=False):
     """Finds Visual C++ Runtime DLLs to include in distribution."""
     winsxs = pathlib.Path(os.environ['SYSTEMROOT']) / 'WinSxS'
 
     prefix = 'amd64' if x64 else 'x86'
 
     candidates = sorted(p for p in os.listdir(winsxs)
                   if p.lower().startswith('%s_microsoft.vc90.crt_' % prefix))
 
     for p in candidates:
         print('found candidate VC runtime: %s' % p)
 
     # Take the newest version.
     version = candidates[-1]
 
     d = winsxs / version
 
     return [
         d / 'msvcm90.dll',
         d / 'msvcp90.dll',
         d / 'msvcr90.dll',
         winsxs / 'Manifests' / ('%s.manifest' % version),
     ]
 
 
 def build(source_dir: pathlib.Path, build_dir: pathlib.Path,
           python_exe: pathlib.Path, iscc_exe: pathlib.Path,
           version=None):
     """Build the Inno installer.
 
     Build files will be placed in ``build_dir``.
 
     py2exe's setup.py doesn't use setuptools. It doesn't have modern logic
     for finding the Python 2.7 toolchain. So, we require the environment
     to already be configured with an active toolchain.
     """
-    from packagingutil import (
+    from hgpackaging.util import (
         download_entry,
         extract_tar_to_directory,
         extract_zip_to_directory,
     )
 
     if not iscc.exists():
         raise Exception('%s does not exist' % iscc)
 
     if 'VCINSTALLDIR' not in os.environ:
         raise Exception('not running from a Visual C++ build environment; '
                         'execute the "Visual C++ <version> Command Prompt" '
                         'application shortcut or a vcsvarsall.bat file')
 
     # Identity x86/x64 and validate the environment matches the Python
     # architecture.
     vc_x64 = r'\x64' in os.environ['LIB']
 
     res = subprocess.run(
         [str(python_exe), '-c', PRINT_PYTHON_INFO],
         capture_output=True, check=True)
 
     py_arch, py_version = res.stdout.decode('utf-8').split(':')
     py_version = int(py_version)
 
     if vc_x64:
         if py_arch != '64bit':
             raise Exception('architecture mismatch: Visual C++ environment '
                             'is configured for 64-bit but Python is 32-bit')
     else:
         if py_arch != '32bit':
             raise Exception('architecture mismatch: Visual C++ environment '
                             'is configured for 32-bit but Python is 64-bit')
 
     if py_version != 2:
         raise Exception('Only Python 2 is currently supported')
 
     build_dir.mkdir(exist_ok=True)
 
     gettext_pkg = download_entry(DOWNLOADS['gettext'], build_dir)
     gettext_dep_pkg = download_entry(DOWNLOADS['gettext-dep'], build_dir)
     virtualenv_pkg = download_entry(DOWNLOADS['virtualenv'], build_dir)
     py2exe_pkg = download_entry(DOWNLOADS['py2exe'], build_dir)
 
     venv_path = build_dir / ('venv-inno-%s' % ('x64' if vc_x64 else 'x86'))
 
     gettext_root = build_dir / (
         'gettext-win-%s' % DOWNLOADS['gettext']['version'])
 
     if not gettext_root.exists():
         extract_zip_to_directory(gettext_pkg, gettext_root)
         extract_zip_to_directory(gettext_dep_pkg, gettext_root)
 
     with tempfile.TemporaryDirectory() as td:
         td = pathlib.Path(td)
 
         # This assumes Python 2.
         extract_tar_to_directory(virtualenv_pkg, td)
         extract_zip_to_directory(py2exe_pkg, td)
 
         virtualenv_src_path = td / ('virtualenv-%s' %
             DOWNLOADS['virtualenv']['version'])
         py2exe_source_path = td / ('py2exe-%s' %
             DOWNLOADS['py2exe']['version'])
 
         virtualenv_py = virtualenv_src_path / 'virtualenv.py'
 
         if not venv_path.exists():
             print('creating virtualenv with dependencies')
             subprocess.run(
                 [str(python_exe), str(virtualenv_py), str(venv_path)],
                 check=True)
 
         venv_python = venv_path / 'Scripts' / 'python.exe'
         venv_pip = venv_path / 'Scripts' / 'pip.exe'
 
         requirements_txt = (source_dir / 'contrib' / 'packaging' /
                             'inno' / 'requirements.txt')
         subprocess.run([str(venv_pip), 'install', '-r', str(requirements_txt)],
                        check=True)
 
         # Force distutils to use VC++ settings from environment, which was
         # validated above.
         env = dict(os.environ)
         env['DISTUTILS_USE_SDK'] = '1'
         env['MSSdk'] = '1'
 
         py2exe_py_path = venv_path / 'Lib' / 'site-packages' / 'py2exe'
         if not py2exe_py_path.exists():
             print('building py2exe')
             subprocess.run([str(venv_python), 'setup.py', 'install'],
                            cwd=py2exe_source_path,
                            env=env,
                            check=True)
 
         # Register location of msgfmt and other binaries.
         env['PATH'] = '%s%s%s' % (
             env['PATH'], os.pathsep, str(gettext_root / 'bin'))
 
         print('building Mercurial')
         subprocess.run(
             [str(venv_python), 'setup.py',
              'py2exe', '-b', '3' if vc_x64 else '2',
              'build_doc', '--html'],
             cwd=str(source_dir),
             env=env,
             check=True)
 
         # hg.exe depends on VC9 runtime DLLs. Copy those into place.
         for f in find_vc_runtime_files(vc_x64):
             if f.name.endswith('.manifest'):
                 basename = 'Microsoft.VC90.CRT.manifest'
             else:
                 basename = f.name
 
             dest_path = source_dir / 'dist' / basename
 
             print('copying %s to %s' % (f, dest_path))
             shutil.copyfile(f, dest_path)
 
         print('creating installer')
 
         args = [str(iscc_exe)]
 
         if vc_x64:
             args.append('/dARCH=x64')
 
         if version:
             args.append('/dVERSION=%s' % version)
 
         args.append('/Odist')
         args.append('contrib/packaging/inno/mercurial.iss')
 
         subprocess.run(args, cwd=str(source_dir), check=True)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 
     parser.add_argument('--python',
                         required=True,
                         help='path to python.exe to use')
     parser.add_argument('--iscc',
                         help='path to iscc.exe to use')
     parser.add_argument('--version',
                         help='Mercurial version string to use '
                              '(detected from __version__.py if not defined')
 
     args = parser.parse_args()
 
     if args.iscc:
         iscc = pathlib.Path(args.iscc)
     else:
         iscc = (pathlib.Path(os.environ['ProgramFiles(x86)']) / 'Inno Setup 5' /
             'ISCC.exe')
 
     here = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
     source_dir = here.parent.parent.parent
     build_dir = source_dir / 'build'
 
     sys.path.insert(0, str(source_dir / 'contrib' / 'packaging'))
 
     build(source_dir, build_dir, pathlib.Path(args.python), iscc,
           version=args.version)

tests/test-check-code.t

@@ -1,71 +1,71
 #require test-repo
 
   $ . "$TESTDIR/helpers-testrepo.sh"
   $ check_code="$TESTDIR"/../contrib/check-code.py
   $ cd "$TESTDIR"/..
 
 New errors are not allowed. Warnings are strongly discouraged.
 (The writing "no-che?k-code" is for not skipping this file when checking.)
 
   $ testrepohg locate \
   > -X contrib/python-zstandard \
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/thirdparty \
   > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false
+  Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob)
   Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob)
-  Skipping contrib/packaging/packagingutil.py it has no-che?k-code (glob)
   Skipping i18n/polib.py it has no-che?k-code (glob)
   Skipping mercurial/statprof.py it has no-che?k-code (glob)
   Skipping tests/badserverext.py it has no-che?k-code (glob)
 
 @commands in debugcommands.py should be in alphabetical order.
 
   >>> import re
   >>> commands = []
   >>> with open('mercurial/debugcommands.py', 'rb') as fh:
   ...     for line in fh:
   ...         m = re.match(br"^@command\('([a-z]+)", line)
   ...         if m:
   ...             commands.append(m.group(1))
   >>> scommands = list(sorted(commands))
   >>> for i, command in enumerate(scommands):
   ...     if command != commands[i]:
   ...         print('commands in debugcommands.py not sorted; first differing '
   ...               'command is %s; expected %s' % (commands[i], command))
   ...         break
 
 Prevent adding new files in the root directory accidentally.
 
   $ testrepohg files 'glob:*'
   .arcconfig
   .clang-format
   .editorconfig
   .hgignore
   .hgsigs
   .hgtags
   .jshintrc
   CONTRIBUTING
   CONTRIBUTORS
   COPYING
   Makefile
   README.rst
   hg
   hgeditor
   hgweb.cgi
   setup.py
 
 Prevent adding modules which could be shadowed by ancient .so/.dylib.
 
   $ testrepohg files \
   > mercurial/base85.py \
   > mercurial/bdiff.py \
   > mercurial/diffhelpers.py \
   > mercurial/mpatch.py \
   > mercurial/osutil.py \
   > mercurial/parsers.py \
   > mercurial/zstd.py
   [1]
 
 Keep python3 tests sorted:
   $ sort < contrib/python3-whitelist > $TESTTMP/py3sorted
   $ cmp contrib/python3-whitelist $TESTTMP/py3sorted || echo 'Please sort passing tests!'

tests/test-check-module-imports.t

@@ -1,38 +1,38
 #require test-repo
 
   $ . "$TESTDIR/helpers-testrepo.sh"
   $ import_checker="$TESTDIR"/../contrib/import-checker.py
 
   $ cd "$TESTDIR"/..
 
 There are a handful of cases here that require renaming a module so it
 doesn't overlap with a stdlib module name. There are also some cycles
 here that we should still endeavor to fix, and some cycles will be
 hidden by deduplication algorithm in the cycle detector, so fixing
 these may expose other cycles.
 
 Known-bad files are excluded by -X as some of them would produce unstable
 outputs, which should be fixed later.
 
   $ testrepohg locate 'set:**.py or grep(r"^#!.*?python")' \
   > 'tests/**.t' \
   > -X hgweb.cgi \
   > -X setup.py \
   > -X contrib/debugshell.py \
   > -X contrib/hgweb.fcgi \
   > -X contrib/packaging/hg-docker \
+  > -X contrib/packaging/hgpackaging/ \
   > -X contrib/packaging/inno/ \
-  > -X contrib/packaging/packagingutil.py \
   > -X contrib/python-zstandard/ \
   > -X contrib/win32/hgwebdir_wsgi.py \
   > -X contrib/perf-utils/perf-revlog-write-plot.py \
   > -X doc/gendoc.py \
   > -X doc/hgmanpage.py \
   > -X i18n/posplit \
   > -X mercurial/thirdparty \
   > -X tests/hypothesishelpers.py \
   > -X tests/test-check-interfaces.py \
   > -X tests/test-demandimport.py \
   > -X tests/test-imports-checker.t \
   > -X tests/test-verify-repo-operations.py \
   > | sed 's-\\-/-g' | "$PYTHON" "$import_checker" -

tests/test-check-py3-compat.t

@@ -1,58 +1,58
 #require test-repo
 
   $ . "$TESTDIR/helpers-testrepo.sh"
   $ cd "$TESTDIR"/..
 
 #if no-py3
   $ testrepohg files 'set:(**.py)' \
+  > -X contrib/packaging/hgpackaging/ \
   > -X contrib/packaging/inno/ \
-  > -X contrib/packaging/packagingutil.py \
   > -X hgdemandimport/demandimportpy2.py \
   > -X mercurial/thirdparty/cbor \
   > | sed 's|\\|/|g' | xargs "$PYTHON" contrib/check-py3-compat.py
   contrib/python-zstandard/setup.py not using absolute_import
   contrib/python-zstandard/setup_zstd.py not using absolute_import
   contrib/python-zstandard/tests/common.py not using absolute_import
   contrib/python-zstandard/tests/test_buffer_util.py not using absolute_import
   contrib/python-zstandard/tests/test_compressor.py not using absolute_import
   contrib/python-zstandard/tests/test_compressor_fuzzing.py not using absolute_import
   contrib/python-zstandard/tests/test_data_structures.py not using absolute_import
   contrib/python-zstandard/tests/test_data_structures_fuzzing.py not using absolute_import
   contrib/python-zstandard/tests/test_decompressor.py not using absolute_import
   contrib/python-zstandard/tests/test_decompressor_fuzzing.py not using absolute_import
   contrib/python-zstandard/tests/test_estimate_sizes.py not using absolute_import
   contrib/python-zstandard/tests/test_module_attributes.py not using absolute_import
   contrib/python-zstandard/tests/test_train_dictionary.py not using absolute_import
   setup.py not using absolute_import
 #endif
 
 #if py3
   $ testrepohg files 'set:(**.py) - grep(pygments)' \
   > -X hgdemandimport/demandimportpy2.py \
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/cffi \
   > -X mercurial/thirdparty \
   > | sed 's|\\|/|g' | xargs "$PYTHON" contrib/check-py3-compat.py \
   > | sed 's/[0-9][0-9]*)$/*)/'
   contrib/python-zstandard/tests/test_compressor.py:324: SyntaxWarning: invalid escape sequence \( (py38 !)
     with self.assertRaisesRegexp(zstd.ZstdError, 'cannot call compress\(\) after compressor'): (py38 !)
   contrib/python-zstandard/tests/test_compressor.py:1329: SyntaxWarning: invalid escape sequence \( (py38 !)
     'cannot call compress\(\) after compression finished'): (py38 !)
   contrib/python-zstandard/tests/test_compressor.py:1341: SyntaxWarning: invalid escape sequence \( (py38 !)
     'cannot call flush\(\) after compression finished'): (py38 !)
   contrib/python-zstandard/tests/test_compressor.py:1353: SyntaxWarning: invalid escape sequence \( (py38 !)
     'cannot call finish\(\) after compression finished'): (py38 !)
   hgext/convert/transport.py: error importing: <*Error> No module named 'svn.client' (error at transport.py:*) (glob) (?)
   hgext/infinitepush/sqlindexapi.py: error importing: <*Error> No module named 'mysql' (error at sqlindexapi.py:*) (glob) (?)
   mercurial/scmwindows.py: error importing: <ValueError> _type_ 'v' not supported (error at win32.py:*) (no-windows !)
   mercurial/win32.py: error importing: <ValueError> _type_ 'v' not supported (error at win32.py:*) (no-windows !)
   mercurial/windows.py: error importing: <ModuleNotFoundError> No module named 'msvcrt' (error at windows.py:*) (no-windows !)
   mercurial/posix.py: error importing: <ModuleNotFoundError> No module named 'fcntl' (error at posix.py:*) (windows !)
   mercurial/scmposix.py: error importing: <ModuleNotFoundError> No module named 'fcntl' (error at scmposix.py:*) (windows !)
 #endif
 
 #if py3 pygments
   $ testrepohg files 'set:(**.py) and grep(pygments)' | sed 's|\\|/|g' \
   > | xargs "$PYTHON" contrib/check-py3-compat.py \
   > | sed 's/[0-9][0-9]*)$/*)/'
 #endif