contrib: add a set of scripts to run pytype in Docker

Having a simple way to run pytype for developers can massively shorten
development cycle. Using the same Docker image and scripts that we use on our
CI guarantees that the result achieved locally will be very similar to (if not
the same as) the output of our CI runners.

Things to note: the Dockerfile needs to do a little dance around user
permissions inside /home/ci-runner/ because:

- on one hand, creating new files on the host (e.g. .pyi files inside .pytype/)
  should use host user's uid and gid

- on the other hand, when we run the image as uid:gid of host user, it needs to
  be able to read/execute files inside the image that are owned by ci-runner

Since local user's uid might be different from ci-runner's uid, we execute this
very broad chmod command inside /home/ci-runner/, but then run the image as the
host user's uid:gid.

There might be a better way to do this.

contrib/docker/pytype/Dockerfile

@@ -0,0 +1,14 b''
+FROM registry.heptapod.net/mercurial/ci-images/mercurial-core:v2.0
+
+USER ci-runner
+
+ENV PATH=/home/ci-runner/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV PYTHONPATH=/home/ci-runner/.local/lib/python3.11/site-packages
+
+RUN python3 -m pip install --user --break-system-packages --upgrade pytype==2023.11.21
+
+ADD --chown=ci-runner entrypoint.sh /home/ci-runner/entrypoint.sh
+
+RUN chmod -R a=rwX /home/ci-runner/.local/ /home/ci-runner/entrypoint.sh
+
+CMD /home/ci-runner/entrypoint.sh

contrib/docker/pytype/entrypoint.sh

@@ -0,0 +1,8 b''
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd /tmp/mercurial-ci/
+make local
+./contrib/setup-pytype.sh
+./contrib/check-pytype.sh

contrib/docker/pytype/recipe.sh

@@ -0,0 +1,28 b''
+#!/usr/bin/env bash
+
+# find repo-root without calling hg as this might be run with sudo
+THIS="$(readlink -m "$0")"
+HERE="$(dirname "$THIS")"
+HG_ROOT="$(readlink -m "$HERE"/../../..)"
+echo source mercurial repository: "$HG_ROOT"
+
+# find actual user as this might be run with sudo
+if [ -n "$SUDO_UID" ]; then
+    ACTUAL_UID="$SUDO_UID"
+else
+    ACTUAL_UID="$(id -u)"
+fi
+if [ -n "$SUDO_GID" ]; then
+    ACTUAL_GID="$SUDO_GID"
+else
+    ACTUAL_GID="$(id -g)"
+fi
+echo using user "$ACTUAL_UID:$ACTUAL_GID"
+if groups | egrep -q '\<(docker|root)\>' ; then
+    env DOCKER_BUILDKIT=1 docker build --tag mercurial-pytype-checker "$HERE"
+    docker run --rm -it --user "$ACTUAL_UID:$ACTUAL_GID" -v "$HG_ROOT:/tmp/mercurial-ci" mercurial-pytype-checker
+else
+    echo "user not in the docker group" >&2
+    echo "(consider running this with \`sudo\`)" >&2
+    exit 255
+fi

Makefile

@@ -1,305 +1,307 b''
 # If you want to change PREFIX, do not just edit it below. The changed
 # value wont get passed on to recursive make calls. You should instead
 # override the variable on the command like:
 #
 # % make PREFIX=/opt/ install
 
 export PREFIX=/usr/local
 
 # Default to Python 3.
 #
 # Windows ships Python 3 as `python.exe`, which may not be on PATH.  py.exe is.
 ifeq ($(OS),Windows_NT)
 PYTHON?=py -3
 else
 PYTHON?=python3
 endif
 
 PYOXIDIZER?=pyoxidizer
 
 $(eval HGROOT := $(shell pwd))
 HGPYTHONS ?= $(HGROOT)/build/pythons
 PURE=
 PYFILESCMD=find mercurial hgext doc -name '*.py'
 PYFILES:=$(shell $(PYFILESCMD))
 DOCFILES=mercurial/helptext/*.txt
 export LANGUAGE=C
 export LC_ALL=C
 TESTFLAGS ?= $(shell echo $$HGTESTFLAGS)
 OSXVERSIONFLAGS ?= $(shell echo $$OSXVERSIONFLAGS)
 CARGO = cargo
 
 # Set this to e.g. "mingw32" to use a non-default compiler.
 COMPILER=
 
 COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
 help:
 	@echo 'Commonly used make targets:'
 	@echo '  all          - build program and documentation'
 	@echo '  install      - install program and man pages to $$PREFIX ($(PREFIX))'
 	@echo '  install-home - install with setup.py install --home=$$HOME ($(HOME))'
 	@echo '  local        - build for inplace usage'
 	@echo '  tests        - run all tests in the automatic test suite'
 	@echo '  test-foo     - run only specified tests (e.g. test-merge1.t)'
 	@echo '  dist         - run all tests and create a source tarball in dist/'
 	@echo '  clean        - remove files created by other targets'
 	@echo '                 (except installed files or dist source tarball)'
 	@echo '  update-pot   - update i18n/hg.pot'
 	@echo
 	@echo 'Example for a system-wide installation under /usr/local:'
 	@echo '  make all && su -c "make install" && hg version'
 	@echo
 	@echo 'Example for a local installation (usable in this directory):'
 	@echo '  make local && ./hg version'
 
 all: build doc
 
 local:
 	MERCURIAL_SETUP_MAKE_LOCAL=1 $(PYTHON) setup.py $(PURE) \
 	  build_py -c -d . \
 	  build_ext $(COMPILERFLAG) -i \
 	  build_hgexe $(COMPILERFLAG) -i \
 	  build_mo
 	env HGRCPATH= $(PYTHON) hg version
 
 build:
 	$(PYTHON) setup.py $(PURE) build $(COMPILERFLAG)
 
 build-chg:
 	make -C contrib/chg
 
 build-rhg:
 	(cd rust/rhg; cargo build --release)
 
 wheel:
 	FORCE_SETUPTOOLS=1 $(PYTHON) setup.py $(PURE) bdist_wheel $(COMPILERFLAG)
 
 doc:
 	$(MAKE) -C doc
 
 cleanbutpackages:
 	rm -f hg.exe
 	-$(PYTHON) setup.py clean --all # ignore errors from this command
 	find contrib doc hgext hgext3rd i18n mercurial tests hgdemandimport \
 		\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'
 	rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err
 	rm -f mercurial/__modulepolicy__.py
 	if test -d .hg; then rm -f mercurial/__version__.py; fi
 	rm -rf build mercurial/locale
 	$(MAKE) -C doc clean
 	$(MAKE) -C contrib/chg distclean
 	rm -rf rust/target
 	rm -f mercurial/rustext.so
 
 clean: cleanbutpackages
 	rm -rf packages
 
 install: install-bin install-doc
 
 install-bin: build
 	$(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force
 
 install-chg: build-chg
 	make -C contrib/chg install PREFIX="$(PREFIX)"
 
 install-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) install
 
 install-home: install-home-bin install-home-doc
 
 install-home-bin: build
 	$(PYTHON) setup.py $(PURE) install --home="$(HOME)" --prefix="" --force
 
 install-home-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) PREFIX="$(HOME)" install
 
 install-rhg: build-rhg
 	install -m 755 rust/target/release/rhg "$(PREFIX)"/bin/
 
 MANIFEST-doc:
 	$(MAKE) -C doc MANIFEST
 
 MANIFEST.in: MANIFEST-doc
 	hg manifest | sed -e 's/^/include /' > MANIFEST.in
 	echo include mercurial/__version__.py >> MANIFEST.in
 	sed -e 's/^/include /' < doc/MANIFEST >> MANIFEST.in
 
 dist:	tests dist-notests
 
 dist-notests:	doc MANIFEST.in
 	TAR_OPTIONS="--owner=root --group=root --mode=u+w,go-w,a+rX-s" $(PYTHON) setup.py -q sdist
 
 check: tests
 
 tests:
         # Run Rust tests if cargo is installed
 	if command -v $(CARGO) >/dev/null 2>&1; then \
 		$(MAKE) rust-tests; \
 		$(MAKE) cargo-clippy; \
 	fi
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS)
 
 test-%:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS) $@
 
 testpy-%:
 	@echo Looking for Python $* in $(HGPYTHONS)
 	[ -e $(HGPYTHONS)/$*/bin/python ] || ( \
 	cd $$(mktemp --directory --tmpdir) && \
         $(MAKE) -f $(HGROOT)/contrib/Makefile.python PYTHONVER=$* PREFIX=$(HGPYTHONS)/$* python )
 	cd tests && $(HGPYTHONS)/$*/bin/python run-tests.py $(TESTFLAGS)
 
 rust-tests:
 	cd $(HGROOT)/rust \
 		&& $(CARGO) test --quiet --all --features "$(HG_RUST_FEATURES)"
 
 cargo-clippy:
 	cd $(HGROOT)/rust \
 		&& $(CARGO) clippy --all --features "$(HG_RUST_FEATURES)" -- -D warnings
 
 check-code:
 	hg manifest | xargs python contrib/check-code.py
 
 format-c:
 	clang-format --style file -i \
 	  `hg files 'set:(**.c or **.cc or **.h) and not "listfile:contrib/clang-format-ignorelist"'`
 
 update-pot: i18n/hg.pot
 
 i18n/hg.pot: $(PYFILES) $(DOCFILES) i18n/posplit i18n/hggettext
 	$(PYTHON) i18n/hggettext mercurial/commands.py \
 	  hgext/*.py hgext/*/__init__.py \
 	  mercurial/fileset.py mercurial/revset.py \
 	  mercurial/templatefilters.py \
 	  mercurial/templatefuncs.py \
 	  mercurial/templatekw.py \
 	  mercurial/filemerge.py \
 	  mercurial/hgweb/webcommands.py \
 	  mercurial/util.py \
 	  $(DOCFILES) > i18n/hg.pot.tmp
         # All strings marked for translation in Mercurial contain
         # ASCII characters only. But some files contain string
         # literals like this '\037\213'. xgettext thinks it has to
         # parse them even though they are not marked for translation.
         # Extracting with an explicit encoding of ISO-8859-1 will make
         # xgettext "parse" and ignore them.
 	$(PYFILESCMD) | xargs \
 	  xgettext --package-name "Mercurial" \
 	  --msgid-bugs-address "<[e-mail]>" \
 	  --copyright-holder "Olivia Mackall <[e-mail]> and others" \
 	  --from-code ISO-8859-1 --join --sort-by-file --add-comments=i18n: \
 	  -d hg -p i18n -o hg.pot.tmp
 	$(PYTHON) i18n/posplit i18n/hg.pot.tmp
         # The target file is not created before the last step. So it never is in
         # an intermediate state.
 	mv -f i18n/hg.pot.tmp i18n/hg.pot
 
 %.po: i18n/hg.pot
         # work on a temporary copy for never having a half completed target
 	cp $@ $@.tmp
 	msgmerge --no-location --update $@.tmp $^
 	mv -f $@.tmp $@
 
 # Packaging targets
 
 packaging_targets := \
   rhel7 \
   rhel8 \
   rhel9 \
   deb \
   docker-rhel7 \
   docker-rhel8 \
   docker-rhel9 \
   docker-debian-bullseye \
   docker-debian-buster \
   docker-debian-stretch \
   docker-fedora \
   docker-ubuntu-xenial \
   docker-ubuntu-xenial-ppa \
   docker-ubuntu-bionic \
   docker-ubuntu-bionic-ppa \
   docker-ubuntu-focal \
   docker-ubuntu-focal-ppa \
   fedora \
   linux-wheels \
   linux-wheels-x86_64 \
   linux-wheels-i686 \
   ppa
 
 # Forward packaging targets for convenience.
 $(packaging_targets):
 	$(MAKE) -C contrib/packaging $@
 
 osx:
 	rm -rf build/mercurial
 	/usr/bin/python2.7 setup.py install --optimize=1 \
 	  --root=build/mercurial/ --prefix=/usr/local/ \
 	  --install-lib=/Library/Python/2.7/site-packages/
 	make -C doc all install DESTDIR="$(PWD)/build/mercurial/"
         # Place a bogon .DS_Store file in the target dir so we can be
         # sure it doesn't get included in the final package.
 	touch build/mercurial/.DS_Store
 	make -C contrib/chg \
 	  HGPATH=/usr/local/bin/hg \
 	  PYTHON=/usr/bin/python2.7 \
 	  DESTDIR=../../build/mercurial \
 	  PREFIX=/usr/local \
 	  clean install
 	mkdir -p $${OUTPUTDIR:-dist}
 	HGVER=$$(python contrib/genosxversion.py $(OSXVERSIONFLAGS) build/mercurial/Library/Python/2.7/site-packages/mercurial/__version__.py) && \
 	OSXVER=$$(sw_vers -productVersion | cut -d. -f1,2) && \
 	pkgbuild --filter \\.DS_Store --root build/mercurial/ \
 	  --identifier org.mercurial-scm.mercurial \
 	  --version "$${HGVER}" \
 	  build/mercurial.pkg && \
 	productbuild --distribution contrib/packaging/macosx/distribution.xml \
 	  --package-path build/ \
 	  --version "$${HGVER}" \
 	  --resources contrib/packaging/macosx/ \
 	  "$${OUTPUTDIR:-dist/}"/Mercurial-"$${HGVER}"-macosx"$${OSXVER}".pkg
 
 pyoxidizer:
 	$(PYOXIDIZER) build --path ./rust/hgcli --release
 
 
 # a temporary target to setup all we need for run-tests.py --pyoxidizer
 # (should go away as the run-tests implementation improves
 pyoxidizer-windows-tests: PYOX_DIR=build/pyoxidizer/x86_64-pc-windows-msvc/release/app
 pyoxidizer-windows-tests: pyoxidizer
 	rm -rf $(PYOX_DIR)/templates
 	cp -ar $(PYOX_DIR)/lib/mercurial/templates $(PYOX_DIR)/templates
 	rm -rf $(PYOX_DIR)/helptext
 	cp -ar $(PYOX_DIR)/lib/mercurial/helptext $(PYOX_DIR)/helptext
 	rm -rf $(PYOX_DIR)/defaultrc
 	cp -ar $(PYOX_DIR)/lib/mercurial/defaultrc $(PYOX_DIR)/defaultrc
 	rm -rf $(PYOX_DIR)/contrib
 	cp -ar contrib $(PYOX_DIR)/contrib
 	rm -rf $(PYOX_DIR)/doc
 	cp -ar doc $(PYOX_DIR)/doc
 
 
 # a temporary target to setup all we need for run-tests.py --pyoxidizer
 # (should go away as the run-tests implementation improves
 pyoxidizer-macos-tests: PYOX_DIR=build/pyoxidizer/x86_64-apple-darwin/release/app
 pyoxidizer-macos-tests: pyoxidizer
 	rm -rf $(PYOX_DIR)/templates
 	cp -a mercurial/templates $(PYOX_DIR)/templates
 	rm -rf $(PYOX_DIR)/helptext
 	cp -a mercurial/helptext $(PYOX_DIR)/helptext
 	rm -rf $(PYOX_DIR)/defaultrc
 	cp -a mercurial/defaultrc $(PYOX_DIR)/defaultrc
 	rm -rf $(PYOX_DIR)/contrib
 	cp -a contrib $(PYOX_DIR)/contrib
 	rm -rf $(PYOX_DIR)/doc
 	cp -a doc $(PYOX_DIR)/doc
 
+pytype-docker:
+	contrib/docker/pytype/recipe.sh
 
 .PHONY: help all local build doc cleanbutpackages clean install install-bin \
 	install-doc install-home install-home-bin install-home-doc \
 	dist dist-notests check tests rust-tests check-code format-c \
 	update-pot pyoxidizer pyoxidizer-windows-tests pyoxidizer-macos-tests \
 	$(packaging_targets) \
-	osx
+	osx pytype-docker