branching: merge stable into default

This show that the recent changes on default fixed the issue with transaction
overwriting content in `test-transaction-wc-rollback-race.t`

contrib/setup-pytype.sh

@@ -0,0 +1,33
+#!/bin/bash
+
+set -e
+set -u
+
+# Find the python3 setup that would run pytype
+PYTYPE=`which pytype`
+PYTHON3=`head -n1 ${PYTYPE} | sed -s 's/#!//'`
+
+# Existing stubs that pytype processes live here
+TYPESHED=$(${PYTHON3} -c "import pytype; print(pytype.__path__[0])")/typeshed/stubs
+HG_STUBS=${TYPESHED}/mercurial
+
+echo "Patching typeshed at $HG_STUBS"
+
+rm -rf ${HG_STUBS}
+mkdir -p ${HG_STUBS}
+
+cat > ${HG_STUBS}/METADATA.toml <<EOF
+version = "0.1"
+EOF
+
+
+mkdir -p ${HG_STUBS}/mercurial/cext ${HG_STUBS}/mercurial/thirdparty/attr
+
+touch ${HG_STUBS}/mercurial/__init__.pyi
+touch ${HG_STUBS}/mercurial/cext/__init__.pyi
+touch ${HG_STUBS}/mercurial/thirdparty/__init__.pyi
+
+ln -sf $(hg root)/mercurial/cext/*.{pyi,typed} \
+       ${HG_STUBS}/mercurial/cext
+ln -sf $(hg root)/mercurial/thirdparty/attr/*.{pyi,typed} \
+       ${HG_STUBS}/mercurial/thirdparty/attr

.gitlab/merge_request_templates/Default.md

@@ -1,13 +1,18
 /assign_reviewer @mercurial.review
 
+
+<!--
+
 Welcome to the Mercurial Merge Request creation process:
 
 * Set a simple title for your MR,
 * All important information should be contained in your changesets' content or description,
 * You can add some workflow-relevant information here (eg: when this depends on another MR),
 * If your changes are not ready for review yet, click `Start the title with Draft:` under the title.
 
 More details here:
 
 * https://www.mercurial-scm.org/wiki/ContributingChanges
 * https://www.mercurial-scm.org/wiki/Heptapod
+
+-->

Makefile

@@ -1,300 +1,305
 # If you want to change PREFIX, do not just edit it below. The changed
 # value wont get passed on to recursive make calls. You should instead
 # override the variable on the command like:
 #
 # % make PREFIX=/opt/ install
 
 export PREFIX=/usr/local
 
 # Default to Python 3.
 #
 # Windows ships Python 3 as `python.exe`, which may not be on PATH.  py.exe is.
 ifeq ($(OS),Windows_NT)
 PYTHON?=py -3
 else
 PYTHON?=python3
 endif
 
 PYOXIDIZER?=pyoxidizer
 
 $(eval HGROOT := $(shell pwd))
 HGPYTHONS ?= $(HGROOT)/build/pythons
 PURE=
 PYFILESCMD=find mercurial hgext doc -name '*.py'
 PYFILES:=$(shell $(PYFILESCMD))
 DOCFILES=mercurial/helptext/*.txt
 export LANGUAGE=C
 export LC_ALL=C
 TESTFLAGS ?= $(shell echo $$HGTESTFLAGS)
 OSXVERSIONFLAGS ?= $(shell echo $$OSXVERSIONFLAGS)
 CARGO = cargo
 
 # Set this to e.g. "mingw32" to use a non-default compiler.
 COMPILER=
 
 COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
 help:
 	@echo 'Commonly used make targets:'
 	@echo '  all          - build program and documentation'
 	@echo '  install      - install program and man pages to $$PREFIX ($(PREFIX))'
 	@echo '  install-home - install with setup.py install --home=$$HOME ($(HOME))'
 	@echo '  local        - build for inplace usage'
 	@echo '  tests        - run all tests in the automatic test suite'
 	@echo '  test-foo     - run only specified tests (e.g. test-merge1.t)'
 	@echo '  dist         - run all tests and create a source tarball in dist/'
 	@echo '  clean        - remove files created by other targets'
 	@echo '                 (except installed files or dist source tarball)'
 	@echo '  update-pot   - update i18n/hg.pot'
 	@echo
 	@echo 'Example for a system-wide installation under /usr/local:'
 	@echo '  make all && su -c "make install" && hg version'
 	@echo
 	@echo 'Example for a local installation (usable in this directory):'
 	@echo '  make local && ./hg version'
 
 all: build doc
 
 local:
 	MERCURIAL_SETUP_MAKE_LOCAL=1 $(PYTHON) setup.py $(PURE) \
 	  build_py -c -d . \
 	  build_ext $(COMPILERFLAG) -i \
 	  build_hgexe $(COMPILERFLAG) -i \
 	  build_mo
 	env HGRCPATH= $(PYTHON) hg version
 
 build:
 	$(PYTHON) setup.py $(PURE) build $(COMPILERFLAG)
 
 build-chg:
 	make -C contrib/chg
 
 build-rhg:
 	(cd rust/rhg; cargo build --release)
 
 wheel:
 	FORCE_SETUPTOOLS=1 $(PYTHON) setup.py $(PURE) bdist_wheel $(COMPILERFLAG)
 
 doc:
 	$(MAKE) -C doc
 
 cleanbutpackages:
 	rm -f hg.exe
 	-$(PYTHON) setup.py clean --all # ignore errors from this command
 	find contrib doc hgext hgext3rd i18n mercurial tests hgdemandimport \
 		\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'
 	rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err
 	rm -f mercurial/__modulepolicy__.py
 	if test -d .hg; then rm -f mercurial/__version__.py; fi
 	rm -rf build mercurial/locale
 	$(MAKE) -C doc clean
 	$(MAKE) -C contrib/chg distclean
 	rm -rf rust/target
 	rm -f mercurial/rustext.so
 
 clean: cleanbutpackages
 	rm -rf packages
 
 install: install-bin install-doc
 
 install-bin: build
 	$(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force
 
 install-chg: build-chg
 	make -C contrib/chg install PREFIX="$(PREFIX)"
 
 install-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) install
 
 install-home: install-home-bin install-home-doc
 
 install-home-bin: build
 	$(PYTHON) setup.py $(PURE) install --home="$(HOME)" --prefix="" --force
 
 install-home-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) PREFIX="$(HOME)" install
 
 install-rhg: build-rhg
 	install -m 755 rust/target/release/rhg "$(PREFIX)"/bin/
 
 MANIFEST-doc:
 	$(MAKE) -C doc MANIFEST
 
 MANIFEST.in: MANIFEST-doc
 	hg manifest | sed -e 's/^/include /' > MANIFEST.in
 	echo include mercurial/__version__.py >> MANIFEST.in
 	sed -e 's/^/include /' < doc/MANIFEST >> MANIFEST.in
 
 dist:	tests dist-notests
 
 dist-notests:	doc MANIFEST.in
 	TAR_OPTIONS="--owner=root --group=root --mode=u+w,go-w,a+rX-s" $(PYTHON) setup.py -q sdist
 
 check: tests
 
 tests:
         # Run Rust tests if cargo is installed
 	if command -v $(CARGO) >/dev/null 2>&1; then \
 		$(MAKE) rust-tests; \
+		$(MAKE) cargo-clippy; \
 	fi
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS)
 
 test-%:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS) $@
 
 testpy-%:
 	@echo Looking for Python $* in $(HGPYTHONS)
 	[ -e $(HGPYTHONS)/$*/bin/python ] || ( \
 	cd $$(mktemp --directory --tmpdir) && \
         $(MAKE) -f $(HGROOT)/contrib/Makefile.python PYTHONVER=$* PREFIX=$(HGPYTHONS)/$* python )
 	cd tests && $(HGPYTHONS)/$*/bin/python run-tests.py $(TESTFLAGS)
 
 rust-tests:
-	cd $(HGROOT)/rust/hg-cpython \
+	cd $(HGROOT)/rust \
 		&& $(CARGO) test --quiet --all --features "$(HG_RUST_FEATURES)"
 
+cargo-clippy:
+	cd $(HGROOT)/rust \
+		&& $(CARGO) clippy --all --features "$(HG_RUST_FEATURES)" -- -D warnings
+
 check-code:
 	hg manifest | xargs python contrib/check-code.py
 
 format-c:
 	clang-format --style file -i \
 	  `hg files 'set:(**.c or **.cc or **.h) and not "listfile:contrib/clang-format-ignorelist"'`
 
 update-pot: i18n/hg.pot
 
 i18n/hg.pot: $(PYFILES) $(DOCFILES) i18n/posplit i18n/hggettext
 	$(PYTHON) i18n/hggettext mercurial/commands.py \
 	  hgext/*.py hgext/*/__init__.py \
 	  mercurial/fileset.py mercurial/revset.py \
 	  mercurial/templatefilters.py \
 	  mercurial/templatefuncs.py \
 	  mercurial/templatekw.py \
 	  mercurial/filemerge.py \
 	  mercurial/hgweb/webcommands.py \
 	  mercurial/util.py \
 	  $(DOCFILES) > i18n/hg.pot.tmp
         # All strings marked for translation in Mercurial contain
         # ASCII characters only. But some files contain string
         # literals like this '\037\213'. xgettext thinks it has to
         # parse them even though they are not marked for translation.
         # Extracting with an explicit encoding of ISO-8859-1 will make
         # xgettext "parse" and ignore them.
 	$(PYFILESCMD) | xargs \
 	  xgettext --package-name "Mercurial" \
 	  --msgid-bugs-address "<[e-mail]>" \
 	  --copyright-holder "Olivia Mackall <[e-mail]> and others" \
 	  --from-code ISO-8859-1 --join --sort-by-file --add-comments=i18n: \
 	  -d hg -p i18n -o hg.pot.tmp
 	$(PYTHON) i18n/posplit i18n/hg.pot.tmp
         # The target file is not created before the last step. So it never is in
         # an intermediate state.
 	mv -f i18n/hg.pot.tmp i18n/hg.pot
 
 %.po: i18n/hg.pot
         # work on a temporary copy for never having a half completed target
 	cp $@ $@.tmp
 	msgmerge --no-location --update $@.tmp $^
 	mv -f $@.tmp $@
 
 # Packaging targets
 
 packaging_targets := \
   rhel7 \
   rhel8 \
   rhel9 \
   deb \
   docker-rhel7 \
   docker-rhel8 \
   docker-rhel9 \
   docker-debian-bullseye \
   docker-debian-buster \
   docker-debian-stretch \
   docker-fedora \
   docker-ubuntu-xenial \
   docker-ubuntu-xenial-ppa \
   docker-ubuntu-bionic \
   docker-ubuntu-bionic-ppa \
   docker-ubuntu-focal \
   docker-ubuntu-focal-ppa \
   fedora \
   linux-wheels \
   linux-wheels-x86_64 \
   linux-wheels-i686 \
   ppa
 
 # Forward packaging targets for convenience.
 $(packaging_targets):
 	$(MAKE) -C contrib/packaging $@
 
 osx:
 	rm -rf build/mercurial
 	/usr/bin/python2.7 setup.py install --optimize=1 \
 	  --root=build/mercurial/ --prefix=/usr/local/ \
 	  --install-lib=/Library/Python/2.7/site-packages/
 	make -C doc all install DESTDIR="$(PWD)/build/mercurial/"
         # Place a bogon .DS_Store file in the target dir so we can be
         # sure it doesn't get included in the final package.
 	touch build/mercurial/.DS_Store
 	make -C contrib/chg \
 	  HGPATH=/usr/local/bin/hg \
 	  PYTHON=/usr/bin/python2.7 \
 	  DESTDIR=../../build/mercurial \
 	  PREFIX=/usr/local \
 	  clean install
 	mkdir -p $${OUTPUTDIR:-dist}
 	HGVER=$$(python contrib/genosxversion.py $(OSXVERSIONFLAGS) build/mercurial/Library/Python/2.7/site-packages/mercurial/__version__.py) && \
 	OSXVER=$$(sw_vers -productVersion | cut -d. -f1,2) && \
 	pkgbuild --filter \\.DS_Store --root build/mercurial/ \
 	  --identifier org.mercurial-scm.mercurial \
 	  --version "$${HGVER}" \
 	  build/mercurial.pkg && \
 	productbuild --distribution contrib/packaging/macosx/distribution.xml \
 	  --package-path build/ \
 	  --version "$${HGVER}" \
 	  --resources contrib/packaging/macosx/ \
 	  "$${OUTPUTDIR:-dist/}"/Mercurial-"$${HGVER}"-macosx"$${OSXVER}".pkg
 
 pyoxidizer:
 	$(PYOXIDIZER) build --path ./rust/hgcli --release
 
 
 # a temporary target to setup all we need for run-tests.py --pyoxidizer
 # (should go away as the run-tests implementation improves
 pyoxidizer-windows-tests: PYOX_DIR=build/pyoxidizer/x86_64-pc-windows-msvc/release/app
 pyoxidizer-windows-tests: pyoxidizer
 	rm -rf $(PYOX_DIR)/templates
 	cp -ar $(PYOX_DIR)/lib/mercurial/templates $(PYOX_DIR)/templates
 	rm -rf $(PYOX_DIR)/helptext
 	cp -ar $(PYOX_DIR)/lib/mercurial/helptext $(PYOX_DIR)/helptext
 	rm -rf $(PYOX_DIR)/defaultrc
 	cp -ar $(PYOX_DIR)/lib/mercurial/defaultrc $(PYOX_DIR)/defaultrc
 	rm -rf $(PYOX_DIR)/contrib
 	cp -ar contrib $(PYOX_DIR)/contrib
 	rm -rf $(PYOX_DIR)/doc
 	cp -ar doc $(PYOX_DIR)/doc
 
 
 # a temporary target to setup all we need for run-tests.py --pyoxidizer
 # (should go away as the run-tests implementation improves
 pyoxidizer-macos-tests: PYOX_DIR=build/pyoxidizer/x86_64-apple-darwin/release/app
 pyoxidizer-macos-tests: pyoxidizer
 	rm -rf $(PYOX_DIR)/templates
 	cp -a mercurial/templates $(PYOX_DIR)/templates
 	rm -rf $(PYOX_DIR)/helptext
 	cp -a mercurial/helptext $(PYOX_DIR)/helptext
 	rm -rf $(PYOX_DIR)/defaultrc
 	cp -a mercurial/defaultrc $(PYOX_DIR)/defaultrc
 	rm -rf $(PYOX_DIR)/contrib
 	cp -a contrib $(PYOX_DIR)/contrib
 	rm -rf $(PYOX_DIR)/doc
 	cp -a doc $(PYOX_DIR)/doc
 
 
 .PHONY: help all local build doc cleanbutpackages clean install install-bin \
 	install-doc install-home install-home-bin install-home-doc \
 	dist dist-notests check tests rust-tests check-code format-c \
 	update-pot pyoxidizer pyoxidizer-windows-tests pyoxidizer-macos-tests \
 	$(packaging_targets) \
 	osx

contrib/check-code.py

@@ -1,1095 +1,1091
 #!/usr/bin/env python3
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """style and portability checker for Mercurial
 
 when a rule triggers wrong, do one of the following (prefer one from top):
  * do the work-around the rule suggests
  * doublecheck that it is a false match
  * improve the rule pattern
  * add an ignore pattern to the rule (3rd arg) which matches your good line
    (you can append a short comment and match this, like: #re-raises)
  * change the pattern to a warning and list the exception in test-check-code-hg
  * ONLY use no--check-code for skipping entire files from external sources
 """
 
 import glob
 import keyword
 import optparse
 import os
 import re
 import sys
 
 if sys.version_info[0] < 3:
     opentext = open
 else:
 
     def opentext(f):
         return open(f, encoding='latin1')
 
 
 try:
     xrange
 except NameError:
     xrange = range
 try:
     import re2
 except ImportError:
     re2 = None
 
 import testparseutil
 
 
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
     if re2:
         try:
             return re2.compile(pat)
         except re2.error:
             pass
     return re.compile(pat)
 
 
 # check "rules depending on implementation of repquote()" in each
 # patterns (especially pypats), before changing around repquote()
 _repquotefixedmap = {
     ' ': ' ',
     '\n': '\n',
     '.': 'p',
     ':': 'q',
     '%': '%',
     '\\': 'b',
     '*': 'A',
     '+': 'P',
     '-': 'M',
 }
 
 
 def _repquoteencodechr(i):
     if i > 255:
         return 'u'
     c = chr(i)
     if c in _repquotefixedmap:
         return _repquotefixedmap[c]
     if c.isalpha():
         return 'x'
     if c.isdigit():
         return 'n'
     return 'o'
 
 
 _repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256))
 
 
 def repquote(m):
     t = m.group('text')
     t = t.translate(_repquotett)
     return m.group('quote') + t + m.group('quote')
 
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         l = len(comment.rstrip())
         return "#" * l + comment[l:]
     return repquote(m)
 
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
     [
         (r'\b(push|pop)d\b', "don't use 'pushd' or 'popd', use 'cd'"),
         (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
         (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
         (r'(?<!hg )grep.* -a', "don't use 'grep -a', use in-line python"),
         (r'sed.*-i', "don't use 'sed -i', use a temporary file"),
         (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"),
         (r'echo -n', "don't use 'echo -n', use printf"),
         (r'(^|\|\s*)\bwc\b[^|]*$\n(?!.*\(re\))', "filter wc output"),
         (r'head -c', "don't use 'head -c', use 'dd'"),
         (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
         (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
         (r'\bls\b.*-\w*R', "don't use 'ls -R', use 'find'"),
         (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"),
         (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
         (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
         (
             r'\[[^\]]+==',
             '[ foo == bar ] is a bashism, use [ foo = bar ] instead',
         ),
         (
             r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
             "use egrep for extended grep syntax",
         ),
         (r'(^|\|\s*)e?grep .*\\S', "don't use \\S in regular expression"),
         (r'(?<!!)/bin/', "don't use explicit paths for tools"),
         (r'#!.*/bash', "don't use bash in shebang, use sh"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'export .*=', "don't export and assign at once"),
         (r'^source\b', "don't use 'source', use '.'"),
         (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
         (r'\bls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
         (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
         (r'^stop\(\)', "don't use 'stop' as a shell function name"),
         (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
         (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"),
         (r'^alias\b.*=', "don't use alias, use a function"),
         (r'if\s*!', "don't use '!' to negate exit status"),
         (r'/dev/u?random', "don't use entropy, use /dev/zero"),
         (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"),
         (
             r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)',
             "put a backslash-escaped newline after sed 'i' command",
         ),
         (r'^diff *-\w*[uU].*$\n(^  \$ |^$)', "prefix diff -u/-U with cmp"),
         (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"),
         (r'[\s="`\']python\s(?!bindings)', "don't use 'python', use '$PYTHON'"),
         (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (r'\|&', "don't use |&, use 2>&1"),
         (r'\w =  +\w', "only one space after = allowed"),
         (
             r'\bsed\b.*[^\\]\\n',
             "don't use 'sed ... \\n', use a \\ and a newline",
         ),
         (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"),
         (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"),
         (r'grep.* -[ABC]', "don't use grep's context flags"),
         (
             r'find.*-printf',
             "don't use 'find -printf', it doesn't exist on BSD find(1)",
         ),
         (r'\$RANDOM ', "don't use bash-only $RANDOM to generate random values"),
     ],
     # warnings
     [
         (r'^function', "don't use 'function', use old style"),
         (r'^diff.*-\w*N', "don't use 'diff -N'"),
         (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`", "no-pwd-check"),
         (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"),
         (r'kill (`|\$\()', "don't use kill, use killdaemons.py"),
     ],
 ]
 
 testfilters = [
     (r"( *)(#([^!][^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 uprefix = r"^  \$ "
 utestpats = [
     [
         (r'^(\S.*||  [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"),
         (
             uprefix + r'.*\|\s*sed[^|>\n]*\n',
             "use regex test output patterns instead of sed",
         ),
         (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
         (
             uprefix + r'.*\|\| echo.*(fail|error)',
             "explicit exit code checks unnecessary",
         ),
         (uprefix + r'set -e', "don't use set -e"),
         (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"),
         (
             uprefix + r'.*:\.\S*/',
             "x:.y in a path does not work on msys, rewrite "
             "as x://.y, or see `hg log -k msys` for alternatives",
             r'-\S+:\.|' '# no-msys',  # -Rxxx
         ),  # in test-pull.t which is skipped on windows
         (
             r'^  [^$>].*27\.0\.0\.1',
             'use $LOCALIP not an explicit loopback address',
         ),
         (
             r'^  (?![>$] ).*\$LOCALIP.*[^)]$',
             'mark $LOCALIP output lines with (glob) to help tests in BSD jails',
         ),
         (
             r'^  (cat|find): .*: \$ENOENT\$',
             'use test -f to test for file existence',
         ),
         (
             r'^  diff -[^ -]*p',
             "don't use (external) diff with -p for portability",
         ),
         (r' readlink ', 'use readlink.py instead of readlink'),
         (
             r'^  [-+][-+][-+] .* [-+]0000 \(glob\)',
             "glob timezone field in diff output for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+,[0-9]+ @@',
             "use '@@ -N* +N,n @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+,[0-9]+ [+][0-9]+ @@',
             "use '@@ -N,n +N* @@ (glob)' style chunk header for portability",
         ),
         (
             r'^  @@ -[0-9]+ [+][0-9]+ @@',
             "use '@@ -N* +N* @@ (glob)' style chunk header for portability",
         ),
         (
             uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff'
             r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$',
             "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)",
         ),
     ],
     # warnings
     [
         (
             r'^  (?!.*\$LOCALIP)[^*?/\n]* \(glob\)$',
             "glob match with no glob string (?, *, /, and $LOCALIP)",
         ),
     ],
 ]
 
 # transform plain test rules to unified test's
 for i in [0, 1]:
     for tp in testpats[i]:
         p = tp[0]
         m = tp[1]
         if p.startswith('^'):
             p = "^  [$>] (%s)" % p[1:]
         else:
             p = "^  [$>] .*(%s)" % p
         utestpats[i].append((p, m) + tp[2:])
 
 # don't transform the following rules:
 # "  > \t" and "  \t" should be allowed in unified tests
 testpats[0].append((r'^( *)\t', "don't use tabs to indent"))
 utestpats[0].append((r'^( ?)\t', "don't use tabs to indent"))
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
     (r"( +)(#([^!][^\n]*\S)?)", repcomment),
 ]
 
 # common patterns to check *.py
 commonpypats = [
     [
         (r'\\$', 'Use () to wrap long lines in Python, not \\'),
         (
             r'^\s*def\s*\w+\s*\(.*,\s*\(',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (
             r'lambda\s*\(.*,.*\)',
             "tuple parameter unpacking not available in Python 3+",
         ),
         (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
         (r'(?<!\.)\breduce\s*\(.*', "reduce is not available in Python 3+"),
         (
             r'\bdict\(.*=',
             'dict() is different in Py2 and 3 and is slower than {}',
             'dict-from-generator',
         ),
         (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
         (r'\s<>\s', '<> operator is not available in Python 3+, use !='),
         (r'^\s*\t', "don't use tabs"),
         (r'\S;\s*\n', "semicolon"),
         (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"),
         (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"),
         (r'(\w|\)),\w', "missing whitespace after ,"),
         (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (
             (
                 # a line ending with a colon, potentially with trailing comments
                 r':([ \t]*#[^\n]*)?\n'
                 # one that is not a pass and not only a comment
                 r'(?P<indent>[ \t]+)[^#][^\n]+\n'
                 # more lines at the same indent level
                 r'((?P=indent)[^\n]+\n)*'
                 # a pass at the same indent level, which is bogus
                 r'(?P=indent)pass[ \t\n#]'
             ),
             'omit superfluous pass',
         ),
         (r'[^\n]\Z', "no trailing newline"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (
             r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
             "linebreak after :",
         ),
         (
             r'\b(%s)\('
             % '|'.join(k for k in keyword.kwlist if k not in ('print', 'exec')),
             "Python keyword is not a function",
         ),
         #    (r'class\s[A-Z][^\(]*\((?!Exception)',
         #     "don't capitalize non-exception classes"),
         #    (r'in range\(', "use xrange"),
         #    (r'^\s*print\s+', "avoid using print in core and extensions"),
         (r'[\x80-\xff]', "non-ASCII character literal"),
         (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"),
         (
             r'([\(\[][ \t]\S)|(\S[ \t][\)\]])',
             "gratuitous whitespace in () or []",
         ),
         #    (r'\s\s=', "gratuitous whitespace before ="),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (
             r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s',
             "missing whitespace around operator",
         ),
         (
             r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
             "missing whitespace around operator",
         ),
         (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', "wrong whitespace around ="),
         (
-            r'\([^()]*( =[^=]|[^<>!=]= )',
-            "no whitespace around = for named parameters",
-        ),
-        (
             r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
             "don't use old-style two-argument raise, use Exception(message)",
         ),
         (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
         (
             r' [=!]=\s+(True|False|None)',
             "comparison with singleton, use 'is' or 'is not' instead",
         ),
         (
             r'^\s*(while|if) [01]:',
             "use True/False for constant Boolean expression",
         ),
         (r'^\s*if False(:| +and)', 'Remove code instead of using `if False`'),
         (
             r'(?:(?<!def)\s+|\()hasattr\(',
             'hasattr(foo, bar) is broken on py2, use util.safehasattr(foo, bar) '
             'instead',
             r'#.*hasattr-py3-only',
         ),
         (r'opener\([^)]*\).read\(', "use opener.read() instead"),
         (r'opener\([^)]*\).write\(', "use opener.write() instead"),
         (r'(?i)descend[e]nt', "the proper spelling is descendAnt"),
         (r'\.debug\(\_', "don't mark debug messages for translation"),
         (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
         (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
         (
             r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
             'legacy exception syntax; use "as" instead of ","',
         ),
         (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
         (r'\bdef\s+__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
         (
             r'os\.path\.join\(.*, *(""|\'\')\)',
             "use pathutil.normasprefix(path) instead of os.path.join(path, '')",
         ),
         (r'\s0[0-7]+\b', 'legacy octal syntax; use "0o" prefix instead of "0"'),
         # XXX only catch mutable arguments on the first line of the definition
         (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"),
         (r'\butil\.Abort\b', "directly use error.Abort"),
         (
             r'^@(\w*\.)?cachefunc',
             "module-level @cachefunc is risky, please avoid",
         ),
         (
             r'^(from|import) mercurial\.(cext|pure|cffi)',
             "use mercurial.policy.importmod instead",
         ),
         (r'\.next\(\)', "don't use .next(), use next(...)"),
         (
             r'([a-z]*).revision\(\1\.node\(',
             "don't convert rev to node before passing to revision(nodeorrev)",
         ),
         (r'platform\.system\(\)', "don't use platform.system(), use pycompat"),
     ],
     # warnings
     [],
 ]
 
 # patterns to check normal *.py files
 pypats = [
     [
         # Ideally, these should be placed in "commonpypats" for
         # consistency of coding rules in Mercurial source tree.
         # But on the other hand, these are not so seriously required for
         # python code fragments embedded in test scripts. Fixing test
         # scripts for these patterns requires many changes, and has less
         # profit than effort.
         (r'raise Exception', "don't raise generic exceptions"),
         (r'[\s\(](open|file)\([^)]*\)\.read\(', "use util.readfile() instead"),
         (
             r'[\s\(](open|file)\([^)]*\)\.write\(',
             "use util.writefile() instead",
         ),
         (
             r'^[\s\(]*(open(er)?|file)\([^)]*\)(?!\.close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (
             r'[\s\(](open|file)\([^)]*\)\.(?!close\(\))',
             "always assign an opened file to a variable, and close it afterwards",
         ),
         (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
         (r'^import atexit', "don't use atexit, use ui.atexit"),
         # rules depending on implementation of repquote()
         (
             r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
             'string join across lines with no space',
         ),
         (
             r'''(?x)ui\.(status|progress|write|note|warn)\(
          [ \t\n#]*
          (?# any strings/comments might precede a string, which
            # contains translatable message)
          b?((['"]|\'\'\'|""")[ \npq%bAPMxno]*(['"]|\'\'\'|""")[ \t\n#]+)*
          (?# sequence consisting of below might precede translatable message
            # - formatting string: "% 10s", "%05d", "% -3.2f", "%*s", "%%" ...
            # - escaped character: "\\", "\n", "\0" ...
            # - character other than '%', 'b' as '\', and 'x' as alphabet)
          (['"]|\'\'\'|""")
          ((%([ n]?[PM]?([np]+|A))?x)|%%|b[bnx]|[ \nnpqAPMo])*x
          (?# this regexp can't use [^...] style,
            # because _preparepats forcibly adds "\n" into [^...],
            # even though this regexp wants match it against "\n")''',
             "missing _() in ui message (use () to hide false-positives)",
         ),
     ]
     + commonpypats[0],
     # warnings
     [
         # rules depending on implementation of repquote()
         (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
     ]
     + commonpypats[1],
 ]
 
 # patterns to check *.py for embedded ones in test script
 embeddedpypats = [
     [] + commonpypats[0],
     # warnings
     [] + commonpypats[1],
 ]
 
 # common filters to convert *.py
 commonpyfilters = [
     (
         r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""",
         reppython,
     ),
 ]
 
 # pattern only for mercurial and extensions
 core_py_pats = [
     [
         # Windows tend to get confused about capitalization of the drive letter
         #
         # see mercurial.windows.abspath for details
         (
             r'os\.path\.abspath',
             "use util.abspath instead (windows)",
             r'#.*re-exports',
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert normal *.py files
 pyfilters = [] + commonpyfilters
 
 # non-filter patterns
 pynfpats = [
     [
         (r'pycompat\.osname\s*[=!]=\s*[\'"]nt[\'"]', "use pycompat.iswindows"),
         (r'pycompat\.osname\s*[=!]=\s*[\'"]posix[\'"]', "use pycompat.isposix"),
         (
             r'pycompat\.sysplatform\s*[!=]=\s*[\'"]darwin[\'"]',
             "use pycompat.isdarwin",
         ),
     ],
     # warnings
     [],
 ]
 
 # filters to convert *.py for embedded ones in test script
 embeddedpyfilters = [] + commonpyfilters
 
 # extension non-filter patterns
 pyextnfpats = [
     [(r'^"""\n?[A-Z]', "don't capitalize docstring title")],
     # warnings
     [],
 ]
 
 txtfilters = []
 
 txtpats = [
     [
         (r'\s$', 'trailing whitespace'),
         ('.. note::[ \n][^\n]', 'add two newlines after note::'),
     ],
     [],
 ]
 
 cpats = [
     [
         (r'//', "don't use //-style comments"),
         (r'\S\t', "don't use tabs except for indent"),
         (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
         (r'(while|if|do|for)\(', "use space after while/if/do/for"),
         (r'return\(', "return is not a function"),
         (r' ;', "no space before ;"),
         (r'[^;] \)', "no space before )"),
         (r'[)][{]', "space between ) and {"),
         (r'\w+\* \w+', "use int *foo, not int* foo"),
         (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
         (r'\w+ (\+\+|--)', "use foo++, not foo ++"),
         (r'\w,\w', "missing whitespace after ,"),
         (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
         (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
         (r'^#\s+\w', "use #foo, not # foo"),
         (r'[^\n]\Z', "no trailing newline"),
         (r'^\s*#import\b', "use only #include in standard C code"),
         (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"),
         (r'strcat\(', "don't use strcat"),
         # rules depending on implementation of repquote()
     ],
     # warnings
     [
         # rules depending on implementation of repquote()
     ],
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
     [
         (r'\bui\.', "don't use ui in util"),
     ],
     # warnings
     [],
 ]
 
 inrevlogpats = [
     [
         (r'\brepo\.', "don't use repo in revlog"),
     ],
     # warnings
     [],
 ]
 
 webtemplatefilters = []
 
 webtemplatepats = [
     [],
     [
         (
             r'{desc(\|(?!websub|firstline)[^\|]*)+}',
             'follow desc keyword with either firstline or websub',
         ),
     ],
 ]
 
 allfilesfilters = []
 
 allfilespats = [
     [
         (
             r'(http|https)://[a-zA-Z0-9./]*selenic.com/',
             'use mercurial-scm.org domain URL',
         ),
         (
             r'mercurial@selenic\.com',
             'use mercurial-scm.org domain for mercurial ML address',
         ),
         (
             r'mercurial-devel@selenic\.com',
             'use mercurial-scm.org domain for mercurial-devel ML address',
         ),
     ],
     # warnings
     [],
 ]
 
 py3pats = [
     [
         (
             r'os\.environ',
             "use encoding.environ instead (py3)",
             r'#.*re-exports',
         ),
         (r'os\.name', "use pycompat.osname instead (py3)"),
         (r'os\.getcwd', "use encoding.getcwd instead (py3)", r'#.*re-exports'),
         (r'os\.sep', "use pycompat.ossep instead (py3)"),
         (r'os\.pathsep', "use pycompat.ospathsep instead (py3)"),
         (r'os\.altsep', "use pycompat.osaltsep instead (py3)"),
         (r'sys\.platform', "use pycompat.sysplatform instead (py3)"),
         (r'getopt\.getopt', "use pycompat.getoptb instead (py3)"),
         (r'os\.getenv', "use encoding.environ.get instead"),
         (r'os\.setenv', "modifying the environ dict is not preferred"),
         (r'(?<!pycompat\.)xrange', "use pycompat.xrange instead (py3)"),
     ],
     # warnings
     [],
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', pyfilters, pypats),
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', [], pynfpats),
     ('python', r'.*hgext.*\.py$', '', [], pyextnfpats),
     (
         'python 3',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         py3pats,
     ),
     (
         'core files',
         r'.*(hgext|mercurial)/(?!demandimport|policy|pycompat).*\.py',
         '',
         pyfilters,
         core_py_pats,
     ),
     ('test script', r'(.*/)?test-[^.~]*$', '', testfilters, testpats),
     ('c', r'.*\.[ch]$', '', cfilters, cpats),
     ('unified test', r'.*\.t$', '', utestfilters, utestpats),
     (
         'layering violation repo in revlog',
         r'mercurial/revlog\.py',
         '',
         pyfilters,
         inrevlogpats,
     ),
     (
         'layering violation ui in util',
         r'mercurial/util\.py',
         '',
         pyfilters,
         inutilpats,
     ),
     ('txt', r'.*\.txt$', '', txtfilters, txtpats),
     (
         'web template',
         r'mercurial/templates/.*\.tmpl',
         '',
         webtemplatefilters,
         webtemplatepats,
     ),
     ('all except for .po', r'.*(?<!\.po)$', '', allfilesfilters, allfilespats),
 ]
 
 # (desc,
 #  func to pick up embedded code fragments,
 #  list of patterns to convert target files
 #  list of patterns to detect errors/warnings)
 embeddedchecks = [
     (
         'embedded python',
         testparseutil.pyembedded,
         embeddedpyfilters,
         embeddedpypats,
     )
 ]
 
 
 def _preparepats():
     def preparefailandwarn(failandwarn):
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
                 p = pseq[0]
                 # \s doesn't match \n (done in two steps)
                 # first, we replace \s that appears in a set already
                 p = re.sub(r'\[\\s', r'[ \\t', p)
                 # now we replace other \s instances.
                 p = re.sub(r'(?<!(\\|\[))\\s', r'[ \\t]', p)
                 # [^...] doesn't match newline
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
 
     def preparefilters(filters):
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
 
     for cs in (checks, embeddedchecks):
         for c in cs:
             failandwarn = c[-1]
             preparefailandwarn(failandwarn)
 
             filters = c[-2]
             preparefilters(filters)
 
 
 class norepeatlogger:
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print("%s:%d (%s):" % (fname, lineno, blame))
             else:
                 print("%s:%d:" % (fname, lineno))
             print(" > %s" % line)
             self._lastseen = msgid
         print(" " + msg)
 
 
 _defaultlogger = norepeatlogger()
 
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 
 def checkfile(
     f,
     logfunc=_defaultlogger.log,
     maxerr=None,
     warnings=False,
     blame=False,
     debug=False,
     lineno=True,
 ):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting.
              Set to false (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     result = True
 
     try:
         with opentext(f) as fp:
             try:
                 pre = fp.read()
             except UnicodeDecodeError as e:
                 print("%s while reading %s" % (e, f))
                 return result
     except IOError as e:
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
 
     # context information shared while single checkfile() invocation
     context = {'blamecache': None}
 
     for name, match, magic, filters, pats in checks:
         if debug:
             print(name, f)
         if not (re.match(match, f) or (magic and re.search(magic, pre))):
             if debug:
                 print(
                     "Skipping %s for %s it doesn't match %s" % (name, match, f)
                 )
             continue
         if "no-" "check-code" in pre:
             # If you're looking at this line, it's because a file has:
             # no- check- code
             # but the reason to output skipping is to make life for
             # tests easier. So, instead of writing it with a normal
             # spelling, we write it with the expected spelling from
             # tests/test-check-code.t
             print("Skipping %s it has no-che?k-code (glob)" % f)
             return "Skip"  # skip checking this file
 
         fc = _checkfiledata(
             name,
             f,
             pre,
             filters,
             pats,
             context,
             logfunc,
             maxerr,
             warnings,
             blame,
             debug,
             lineno,
         )
         if fc:
             result = False
 
     if f.endswith('.t') and "no-" "check-code" not in pre:
         if debug:
             print("Checking embedded code in %s" % f)
 
         prelines = pre.splitlines()
         embeddederros = []
         for name, embedded, filters, pats in embeddedchecks:
             # "reset curmax at each repetition" treats maxerr as "max
             # nubmer of errors in an actual file per entry of
             # (embedded)checks"
             curmaxerr = maxerr
 
             for found in embedded(f, prelines, embeddederros):
                 filename, starts, ends, code = found
                 fc = _checkfiledata(
                     name,
                     f,
                     code,
                     filters,
                     pats,
                     context,
                     logfunc,
                     curmaxerr,
                     warnings,
                     blame,
                     debug,
                     lineno,
                     offset=starts - 1,
                 )
                 if fc:
                     result = False
                     if curmaxerr:
                         if fc >= curmaxerr:
                             break
                         curmaxerr -= fc
 
     return result
 
 
 def _checkfiledata(
     name,
     f,
     filedata,
     filters,
     pats,
     context,
     logfunc,
     maxerr,
     warnings,
     blame,
     debug,
     lineno,
     offset=None,
 ):
     """Execute actual error check for file data
 
     :name: of the checking category
     :f: filepath
     :filedata: content of a file
     :filters: to be applied before checking
     :pats: to detect errors
     :context: a dict of information shared while single checkfile() invocation
               Valid keys: 'blamecache'.
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting, or False to
              report all errors
     :warnings: whether warning level checks should be applied
     :blame: whether blame information should be displayed at error reporting
     :debug: whether debug information should be displayed
     :lineno: whether lineno should be displayed at error reporting
     :offset: line number offset of 'filedata' in 'f' for checking
              an embedded code fragment, or None (offset=0 is different
              from offset=None)
 
     returns number of detected errors.
     """
     blamecache = context['blamecache']
     if offset is None:
         lineoffset = 0
     else:
         lineoffset = offset
 
     fc = 0
     pre = post = filedata
 
     if True:  # TODO: get rid of this redundant 'if' block
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0])  # nerr elements are errors
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
 
         if debug:
             print("Checking %s for %s" % (name, f))
 
         prelines = None
         errors = []
         for i, pat in enumerate(pats):
             if len(pat) == 3:
                 p, msg, ignore = pat
             else:
                 p, msg = pat
                 ignore = None
             if i >= nerrs:
                 msg = "warning: " + msg
 
             pos = 0
             n = 0
             for m in p.finditer(post):
                 if prelines is None:
                     prelines = pre.splitlines()
                     postlines = post.splitlines(True)
 
                 start = m.start()
                 while n < len(postlines):
                     step = len(postlines[n])
                     if pos + step > start:
                         break
                     pos += step
                     n += 1
                 l = prelines[n]
 
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print(
                             "Skipping %s for %s:%s (ignore pattern)"
                             % (name, f, (n + lineoffset))
                         )
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
                     if blamecache is None:
                         blamecache = getblame(f)
                         context['blamecache'] = blamecache
                     if (n + lineoffset) < len(blamecache):
                         bl, bu, br = blamecache[(n + lineoffset)]
                         if offset is None and bl == l:
                             bd = '%s@%s' % (bu, br)
                         elif offset is not None and bl.endswith(l):
                             # "offset is not None" means "checking
                             # embedded code fragment". In this case,
                             # "l" does not have information about the
                             # beginning of an *original* line in the
                             # file (e.g. '  > ').
                             # Therefore, use "str.endswith()", and
                             # show "maybe" for a little loose
                             # examination.
                             bd = '%s@%s, maybe' % (bu, br)
 
                 errors.append((f, lineno and (n + lineoffset + 1), l, msg, bd))
 
         errors.sort()
         for e in errors:
             logfunc(*e)
             fc += 1
             if maxerr and fc >= maxerr:
                 print(" (too many errors, giving up)")
                 break
 
     return fc
 
 
 def main():
     parser = optparse.OptionParser("%prog [options] [files | -]")
     parser.add_option(
         "-w",
         "--warnings",
         action="store_true",
         help="include warning-level checks",
     )
     parser.add_option(
         "-p", "--per-file", type="int", help="max warnings per file"
     )
     parser.add_option(
         "-b",
         "--blame",
         action="store_true",
         help="use annotate to generate blame info",
     )
     parser.add_option(
         "", "--debug", action="store_true", help="show debug information"
     )
     parser.add_option(
         "",
         "--nolineno",
         action="store_false",
         dest='lineno',
         help="don't show line numbers",
     )
 
     parser.set_defaults(
         per_file=15, warnings=False, blame=False, debug=False, lineno=True
     )
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     elif args == ['-']:
         # read file list from stdin
         check = sys.stdin.read().splitlines()
     else:
         check = args
 
     _preparepats()
 
     ret = 0
     for f in check:
         if not checkfile(
             f,
             maxerr=options.per_file,
             warnings=options.warnings,
             blame=options.blame,
             debug=options.debug,
             lineno=options.lineno,
         ):
             ret = 1
     return ret
 
 
 if __name__ == "__main__":
     sys.exit(main())

contrib/check-pytype.sh

@@ -1,72 +1,133
 #!/bin/sh
 
 set -e
 set -u
 
 cd `hg root`
 
 # Many of the individual files that are excluded here confuse pytype
 # because they do a mix of Python 2 and Python 3 things
 # conditionally. There's no good way to help it out with that as far as
 # I can tell, so let's just hide those files from it for now. We should
 # endeavor to empty this list out over time, as some of these are
 # probably hiding real problems.
 #
+# hgext/absorb.py               # [attribute-error]
+# hgext/bugzilla.py             # [pyi-error], [attribute-error]
+# hgext/convert/bzr.py          # [attribute-error]
+# hgext/convert/cvs.py          # [attribute-error], [wrong-arg-types]
+# hgext/convert/cvsps.py        # [attribute-error]
+# hgext/convert/p4.py           # [wrong-arg-types] (__file: mercurial.utils.procutil._pfile -> IO)
+# hgext/convert/subversion.py   # [attribute-error], [name-error], [pyi-error]
+# hgext/fastannotate/context.py # no linelog.copyfrom()
+# hgext/fastannotate/formatter.py  # [unsupported-operands]
+# hgext/fsmonitor/__init__.py   # [name-error]
+# hgext/git/__init__.py         # [attribute-error]
+# hgext/githelp.py              # [attribute-error] [wrong-arg-types]
+# hgext/hgk.py                  # [attribute-error]
+# hgext/histedit.py             # [attribute-error], [wrong-arg-types]
+# hgext/infinitepush            # using bytes for str literal; scheduled for removal
+# hgext/keyword.py              # [attribute-error]
+# hgext/largefiles/storefactory.py  # [attribute-error]
+# hgext/lfs/__init__.py         # [attribute-error]
+# hgext/narrow/narrowbundle2.py # [attribute-error]
+# hgext/narrow/narrowcommands.py    # [attribute-error], [name-error]
+# hgext/rebase.py               # [attribute-error]
+# hgext/remotefilelog/basepack.py   # [attribute-error], [wrong-arg-count]
+# hgext/remotefilelog/basestore.py  # [attribute-error]
+# hgext/remotefilelog/contentstore.py   # [missing-parameter], [wrong-keyword-args], [attribute-error]
+# hgext/remotefilelog/fileserverclient.py  # [attribute-error]
+# hgext/remotefilelog/shallowbundle.py     # [attribute-error]
+# hgext/remotefilelog/remotefilectx.py  # [module-attr] (This is an actual bug)
+# hgext/sqlitestore.py          # [attribute-error]
+# hgext/zeroconf/__init__.py    # bytes vs str; tests fail on macOS
+#
 # mercurial/bundlerepo.py       # no vfs and ui attrs on bundlerepo
 # mercurial/context.py          # many [attribute-error]
 # mercurial/crecord.py          # tons of [attribute-error], [module-attr]
 # mercurial/debugcommands.py    # [wrong-arg-types]
 # mercurial/dispatch.py         # initstdio: No attribute ... on TextIO [attribute-error]
 # mercurial/exchange.py         # [attribute-error]
 # mercurial/hgweb/hgweb_mod.py  # [attribute-error], [name-error], [wrong-arg-types]
 # mercurial/hgweb/server.py     # [attribute-error], [name-error], [module-attr]
 # mercurial/hgweb/wsgicgi.py    # confused values in os.environ
 # mercurial/httppeer.py         # [attribute-error], [wrong-arg-types]
 # mercurial/interfaces          # No attribute 'capabilities' on peer [attribute-error]
 # mercurial/keepalive.py        # [attribute-error]
 # mercurial/localrepo.py        # [attribute-error]
 # mercurial/manifest.py         # [unsupported-operands], [wrong-arg-types]
 # mercurial/minirst.py          # [unsupported-operands], [attribute-error]
 # mercurial/pure/osutil.py      # [invalid-typevar], [not-callable]
 # mercurial/pure/parsers.py     # [attribute-error]
 # mercurial/repoview.py         # [attribute-error]
 # mercurial/testing/storage.py  # tons of [attribute-error]
-# mercurial/ui.py               # [attribute-error], [wrong-arg-types]
 # mercurial/unionrepo.py        # ui, svfs, unfiltered [attribute-error]
 # mercurial/win32.py            # [not-callable]
 # mercurial/wireprotoframing.py # [unsupported-operands], [attribute-error], [import-error]
 # mercurial/wireprotov1peer.py  # [attribute-error]
 # mercurial/wireprotov1server.py  # BUG?: BundleValueError handler accesses subclass's attrs
 
 # TODO: use --no-cache on test server?  Caching the files locally helps during
 #       development, but may be a hinderance for CI testing.
 
 # TODO: include hgext and hgext3rd
 
-pytype -V 3.7 --keep-going --jobs auto mercurial \
+pytype -V 3.7 --keep-going --jobs auto \
+    doc/check-seclevel.py hgdemandimport hgext mercurial \
+    -x hgext/absorb.py \
+    -x hgext/bugzilla.py \
+    -x hgext/convert/bzr.py \
+    -x hgext/convert/cvs.py \
+    -x hgext/convert/cvsps.py \
+    -x hgext/convert/p4.py \
+    -x hgext/convert/subversion.py \
+    -x hgext/fastannotate/context.py \
+    -x hgext/fastannotate/formatter.py \
+    -x hgext/fsmonitor/__init__.py \
+    -x hgext/git/__init__.py \
+    -x hgext/githelp.py \
+    -x hgext/hgk.py \
+    -x hgext/histedit.py \
+    -x hgext/infinitepush \
+    -x hgext/keyword.py \
+    -x hgext/largefiles/storefactory.py \
+    -x hgext/lfs/__init__.py \
+    -x hgext/narrow/narrowbundle2.py \
+    -x hgext/narrow/narrowcommands.py \
+    -x hgext/rebase.py \
+    -x hgext/remotefilelog/basepack.py \
+    -x hgext/remotefilelog/basestore.py \
+    -x hgext/remotefilelog/contentstore.py \
+    -x hgext/remotefilelog/fileserverclient.py \
+    -x hgext/remotefilelog/remotefilectx.py \
+    -x hgext/remotefilelog/shallowbundle.py \
+    -x hgext/sqlitestore.py \
+    -x hgext/zeroconf/__init__.py \
     -x mercurial/bundlerepo.py \
     -x mercurial/context.py \
     -x mercurial/crecord.py \
     -x mercurial/debugcommands.py \
     -x mercurial/dispatch.py \
     -x mercurial/exchange.py \
     -x mercurial/hgweb/hgweb_mod.py \
     -x mercurial/hgweb/server.py \
     -x mercurial/hgweb/wsgicgi.py \
     -x mercurial/httppeer.py \
     -x mercurial/interfaces \
     -x mercurial/keepalive.py \
     -x mercurial/localrepo.py \
     -x mercurial/manifest.py \
     -x mercurial/minirst.py \
     -x mercurial/pure/osutil.py \
     -x mercurial/pure/parsers.py \
     -x mercurial/repoview.py \
     -x mercurial/testing/storage.py \
     -x mercurial/thirdparty \
-    -x mercurial/ui.py \
     -x mercurial/unionrepo.py \
     -x mercurial/win32.py \
     -x mercurial/wireprotoframing.py \
     -x mercurial/wireprotov1peer.py \
     -x mercurial/wireprotov1server.py
+
+echo 'pytype crashed while generating the following type stubs:'
+find .pytype/pyi -name '*.pyi' | xargs grep -l '# Caught error' | sort

contrib/fuzz/revlog.cc

@@ -1,61 +1,61
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include <string>
 
 #include "pyutil.h"
 
 extern "C" {
 
 static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
 	code = (PYCODETYPE *)Py_CompileString(R"py(
 for inline in (True, False):
     try:
         index, cache = parsers.parse_index2(data, inline)
         index.slicechunktodensity(list(range(len(index))), 0.5, 262144)
         index.stats()
-        index.findsnapshots({}, 0)
+        index.findsnapshots({}, 0, len(index) - 1)
         10 in index
         for rev in range(len(index)):
             index.reachableroots(0, [len(index)-1], [rev])
             node = index[rev][7]
             partial = index.shortest(node)
             index.partialmatch(node[:partial])
             index.deltachain(rev, None, True)
     except Exception as e:
         pass
         # uncomment this print if you're editing this Python code
         # to debug failures.
         # print e
 )py",
 	                                      "fuzzer", Py_file_input);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	// Don't allow fuzzer inputs larger than 60k, since we'll just bog
 	// down and not accomplish much.
 	if (Size > 60000) {
 		return 0;
 	}
 	PyObject *text =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "data", text);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(text);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/heptapod-ci.yml

@@ -1,141 +1,143
 stages:
   - tests
 
 image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG
 
 variables:
     PYTHON: python
     TEST_HGMODULEPOLICY: "allow"
     HG_CI_IMAGE_TAG: "v1.0"
     TEST_HGTESTS_ALLOW_NETIO: "0"
 
 .all_template: &all
   when: on_success
 
 .runtests_template: &runtests
     <<: *all
     stage: tests
     # The runner made a clone as root.
     # We make a new clone owned by user used to run the step.
     before_script:
       - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
       - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
       - cd /tmp/mercurial-ci/
       - ls -1 tests/test-check-*.* > /tmp/check-tests.txt
       - black --version
       - clang-format --version
     script:
         - echo "python used, $PYTHON"
         - echo "$RUNTEST_ARGS"
         - HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" "$PYTHON" tests/run-tests.py --color=always $RUNTEST_ARGS
 
 checks:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
         PYTHON: python3
         CI_CLEVER_CLOUD_FLAVOR: S
 
 rust-cargo-test:
     <<: *all
     stage: tests
     script:
         - echo "python used, $PYTHON"
         - make rust-tests
+        - make cargo-clippy
     variables:
         PYTHON: python3
         CI_CLEVER_CLOUD_FLAVOR: S
 
 test-c:
     <<: *runtests
     variables:
         RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
         TEST_HGTESTS_ALLOW_NETIO: "1"
 
 test-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "py"
 
 test-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-rhg:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
         RUNTEST_ARGS: "--rust --rhg --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-chg:
     <<: *runtests
     variables:
         PYTHON: python3
         RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
         TEST_HGMODULEPOLICY: "c"
 
 check-pytype:
     extends: .runtests_template
     before_script:
       - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
       - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
       - cd /tmp/mercurial-ci/
       - make local PYTHON=$PYTHON
-      - $PYTHON -m pip install --user -U libcst==0.3.20 pytype==2022.03.29
+      - $PYTHON -m pip install --user -U libcst==0.3.20 pytype==2022.11.18
+      - ./contrib/setup-pytype.sh
     script:
       - echo "Entering script section"
       - sh contrib/check-pytype.sh
     variables:
         PYTHON: python3
 
 # `sh.exe --login` sets a couple of extra environment variables that are defined
 # in the MinGW shell, but switches CWD to /home/$username.  The previous value
 # is stored in OLDPWD.  Of the added variables, MSYSTEM is crucial to running
 # run-tests.py- it is needed to make run-tests.py generate a `python3` script
 # that satisfies the various shebang lines and delegates to `py -3`.
 .window_runtests_template: &windows_runtests
     <<: *all
     when: manual  # we don't have any Windows runners anymore at the moment
     stage: tests
     before_script:
       - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && ls -1 tests/test-check-*.* > C:/Temp/check-tests.txt'
       # TODO: find/install cvs, bzr, perforce, gpg, sqlite3
 
     script:
         - echo "Entering script section"
         - echo "python used, $Env:PYTHON"
         - Invoke-Expression "$Env:PYTHON -V"
         - Invoke-Expression "$Env:PYTHON -m black --version"
         - echo "$Env:RUNTEST_ARGS"
         - echo "$Env:TMP"
         - echo "$Env:TEMP"
 
         - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" $PYTHON tests/run-tests.py --color=always $RUNTEST_ARGS'
 
 windows:
     <<: *windows_runtests
     tags:
       - windows
     variables:
         TEST_HGMODULEPOLICY: "c"
         RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt"
         PYTHON: py -3
 
 windows-pyox:
     <<: *windows_runtests
     tags:
       - windows
     variables:
         TEST_HGMODULEPOLICY: "c"
         RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt --pyoxidized"
         PYTHON: py -3

contrib/perf.py

@@ -1,4230 +1,4245
 # perf.py - performance test routines
 '''helper extension to measure performance
 
 Configurations
 ==============
 
 ``perf``
 --------
 
 ``all-timing``
     When set, additional statistics will be reported for each benchmark: best,
     worst, median average. If not set only the best timing is reported
     (default: off).
 
 ``presleep``
   number of second to wait before any group of runs (default: 1)
 
 ``pre-run``
   number of run to perform before starting measurement.
 
 ``profile-benchmark``
   Enable profiling for the benchmarked section.
   (The first iteration is benchmarked)
 
 ``run-limits``
   Control the number of runs each benchmark will perform. The option value
   should be a list of `<time>-<numberofrun>` pairs. After each run the
   conditions are considered in order with the following logic:
 
       If benchmark has been running for <time> seconds, and we have performed
       <numberofrun> iterations, stop the benchmark,
 
   The default value is: `3.0-100, 10.0-3`
 
 ``stub``
     When set, benchmarks will only be run once, useful for testing
     (default: off)
 '''
 
 # "historical portability" policy of perf.py:
 #
 # We have to do:
 # - make perf.py "loadable" with as wide Mercurial version as possible
 #   This doesn't mean that perf commands work correctly with that Mercurial.
 #   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
 # - make historical perf command work correctly with as wide Mercurial
 #   version as possible
 #
 # We have to do, if possible with reasonable cost:
 # - make recent perf command for historical feature work correctly
 #   with early Mercurial
 #
 # We don't have to do:
 # - make perf command for recent feature work correctly with early
 #   Mercurial
 
 import contextlib
 import functools
 import gc
 import os
 import random
 import shutil
 import struct
 import sys
 import tempfile
 import threading
 import time
 
 import mercurial.revlog
 from mercurial import (
     changegroup,
     cmdutil,
     commands,
     copies,
     error,
     extensions,
     hg,
     mdiff,
     merge,
     util,
 )
 
 # for "historical portability":
 # try to import modules separately (in dict order), and ignore
 # failure, because these aren't available with early Mercurial
 try:
     from mercurial import branchmap  # since 2.5 (or bcee63733aad)
 except ImportError:
     pass
 try:
     from mercurial import obsolete  # since 2.3 (or ad0d6c2b3279)
 except ImportError:
     pass
 try:
     from mercurial import registrar  # since 3.7 (or 37d50250b696)
 
     dir(registrar)  # forcibly load it
 except ImportError:
     registrar = None
 try:
     from mercurial import repoview  # since 2.5 (or 3a6ddacb7198)
 except ImportError:
     pass
 try:
     from mercurial.utils import repoviewutil  # since 5.0
 except ImportError:
     repoviewutil = None
 try:
     from mercurial import scmutil  # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
 try:
     from mercurial import setdiscovery  # since 1.9 (or cb98fed52495)
 except ImportError:
     pass
 
 try:
     from mercurial import profiling
 except ImportError:
     profiling = None
 
 try:
     from mercurial.revlogutils import constants as revlog_constants
 
     perf_rl_kind = (revlog_constants.KIND_OTHER, b'created-by-perf')
 
     def revlog(opener, *args, **kwargs):
         return mercurial.revlog.revlog(opener, perf_rl_kind, *args, **kwargs)
 
 
 except (ImportError, AttributeError):
     perf_rl_kind = None
 
     def revlog(opener, *args, **kwargs):
         return mercurial.revlog.revlog(opener, *args, **kwargs)
 
 
 def identity(a):
     return a
 
 
 try:
     from mercurial import pycompat
 
     getargspec = pycompat.getargspec  # added to module after 4.5
     _byteskwargs = pycompat.byteskwargs  # since 4.1 (or fbc3f73dc802)
     _sysstr = pycompat.sysstr  # since 4.0 (or 2219f4f82ede)
     _bytestr = pycompat.bytestr  # since 4.2 (or b70407bd84d5)
     _xrange = pycompat.xrange  # since 4.8 (or 7eba8f83129b)
     fsencode = pycompat.fsencode  # since 3.9 (or f4a5e0e86a7e)
     if pycompat.ispy3:
         _maxint = sys.maxsize  # per py3 docs for replacing maxint
     else:
         _maxint = sys.maxint
 except (NameError, ImportError, AttributeError):
     import inspect
 
     getargspec = inspect.getargspec
     _byteskwargs = identity
     _bytestr = str
     fsencode = identity  # no py3 support
     _maxint = sys.maxint  # no py3 support
     _sysstr = lambda x: x  # no py3 support
     _xrange = xrange
 
 try:
     # 4.7+
     queue = pycompat.queue.Queue
 except (NameError, AttributeError, ImportError):
     # <4.7.
     try:
         queue = pycompat.queue
     except (NameError, AttributeError, ImportError):
         import Queue as queue
 
 try:
     from mercurial import logcmdutil
 
     makelogtemplater = logcmdutil.maketemplater
 except (AttributeError, ImportError):
     try:
         makelogtemplater = cmdutil.makelogtemplater
     except (AttributeError, ImportError):
         makelogtemplater = None
 
 # for "historical portability":
 # define util.safehasattr forcibly, because util.safehasattr has been
 # available since 1.9.3 (or 94b200a11cf7)
 _undefined = object()
 
 
 def safehasattr(thing, attr):
     return getattr(thing, _sysstr(attr), _undefined) is not _undefined
 
 
 setattr(util, 'safehasattr', safehasattr)
 
 # for "historical portability":
 # define util.timer forcibly, because util.timer has been available
 # since ae5d60bb70c9
 if safehasattr(time, 'perf_counter'):
     util.timer = time.perf_counter
 elif os.name == b'nt':
     util.timer = time.clock
 else:
     util.timer = time.time
 
 # for "historical portability":
 # use locally defined empty option list, if formatteropts isn't
 # available, because commands.formatteropts has been available since
 # 3.2 (or 7a7eed5176a4), even though formatting itself has been
 # available since 2.2 (or ae5f92e154d3)
 formatteropts = getattr(
     cmdutil, "formatteropts", getattr(commands, "formatteropts", [])
 )
 
 # for "historical portability":
 # use locally defined option list, if debugrevlogopts isn't available,
 # because commands.debugrevlogopts has been available since 3.7 (or
 # 5606f7d0d063), even though cmdutil.openrevlog() has been available
 # since 1.9 (or a79fea6b3e77).
 revlogopts = getattr(
     cmdutil,
     "debugrevlogopts",
     getattr(
         commands,
         "debugrevlogopts",
         [
             (b'c', b'changelog', False, b'open changelog'),
             (b'm', b'manifest', False, b'open manifest'),
             (b'', b'dir', False, b'open directory manifest'),
         ],
     ),
 )
 
 cmdtable = {}
 
+
 # for "historical portability":
 # define parsealiases locally, because cmdutil.parsealiases has been
 # available since 1.5 (or 6252852b4332)
 def parsealiases(cmd):
     return cmd.split(b"|")
 
 
 if safehasattr(registrar, 'command'):
     command = registrar.command(cmdtable)
 elif safehasattr(cmdutil, 'command'):
     command = cmdutil.command(cmdtable)
     if 'norepo' not in getargspec(command).args:
         # for "historical portability":
         # wrap original cmdutil.command, because "norepo" option has
         # been available since 3.1 (or 75a96326cecb)
         _command = command
 
         def command(name, options=(), synopsis=None, norepo=False):
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return _command(name, list(options), synopsis)
 
 
 else:
     # for "historical portability":
     # define "@command" annotation locally, because cmdutil.command
     # has been available since 1.9 (or 2daa5179e73f)
     def command(name, options=(), synopsis=None, norepo=False):
         def decorator(func):
             if synopsis:
                 cmdtable[name] = func, list(options), synopsis
             else:
                 cmdtable[name] = func, list(options)
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return func
 
         return decorator
 
 
 try:
     import mercurial.registrar
     import mercurial.configitems
 
     configtable = {}
     configitem = mercurial.registrar.configitem(configtable)
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'pre-run',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
 except (ImportError, AttributeError):
     pass
 except TypeError:
     # compatibility fix for a11fd395e83f
     # hg version: 5.2
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'pre-run',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
     )
 
 
 def getlen(ui):
     if ui.configbool(b"perf", b"stub", False):
         return lambda x: 1
     return len
 
 
 class noop:
     """dummy context manager"""
 
     def __enter__(self):
         pass
 
     def __exit__(self, *args):
         pass
 
 
 NOOPCTX = noop()
 
 
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
     This function exists to gather the creation of formatter in a single
     place instead of duplicating it in all performance commands."""
 
     # enforce an idle period before execution to counteract power management
     # experimental config: perf.presleep
     time.sleep(getint(ui, b"perf", b"presleep", 1))
 
     if opts is None:
         opts = {}
     # redirect all to stderr unless buffer api is in use
     if not ui._buffers:
         ui = ui.copy()
         uifout = safeattrsetter(ui, b'fout', ignoremissing=True)
         if uifout:
             # for "historical portability":
             # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d)
             uifout.set(ui.ferr)
 
     # get a formatter
     uiformatter = getattr(ui, 'formatter', None)
     if uiformatter:
         fm = uiformatter(b'perf', opts)
     else:
         # for "historical portability":
         # define formatter locally, because ui.formatter has been
         # available since 2.2 (or ae5f92e154d3)
         from mercurial import node
 
         class defaultformatter:
             """Minimized composition of baseformatter and plainformatter"""
 
             def __init__(self, ui, topic, opts):
                 self._ui = ui
                 if ui.debugflag:
                     self.hexfunc = node.hex
                 else:
                     self.hexfunc = node.short
 
             def __nonzero__(self):
                 return False
 
             __bool__ = __nonzero__
 
             def startitem(self):
                 pass
 
             def data(self, **data):
                 pass
 
             def write(self, fields, deftext, *fielddata, **opts):
                 self._ui.write(deftext % fielddata, **opts)
 
             def condwrite(self, cond, fields, deftext, *fielddata, **opts):
                 if cond:
                     self._ui.write(deftext % fielddata, **opts)
 
             def plain(self, text, **opts):
                 self._ui.write(text, **opts)
 
             def end(self):
                 pass
 
         fm = defaultformatter(ui, b'perf', opts)
 
     # stub function, runs code only once instead of in a loop
     # experimental config: perf.stub
     if ui.configbool(b"perf", b"stub", False):
         return functools.partial(stub_timer, fm), fm
 
     # experimental config: perf.all-timing
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # experimental config: perf.run-limits
     limitspec = ui.configlist(b"perf", b"run-limits", [])
     limits = []
     for item in limitspec:
         parts = item.split(b'-', 1)
         if len(parts) < 2:
             ui.warn((b'malformatted run limit entry, missing "-": %s\n' % item))
             continue
         try:
             time_limit = float(_sysstr(parts[0]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         try:
             run_limit = int(_sysstr(parts[1]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         limits.append((time_limit, run_limit))
     if not limits:
         limits = DEFAULTLIMITS
 
     profiler = None
     if profiling is not None:
         if ui.configbool(b"perf", b"profile-benchmark", False):
             profiler = profiling.profile(ui)
 
     prerun = getint(ui, b"perf", b"pre-run", 0)
     t = functools.partial(
         _timer,
         fm,
         displayall=displayall,
         limits=limits,
         prerun=prerun,
         profiler=profiler,
     )
     return t, fm
 
 
 def stub_timer(fm, func, setup=None, title=None):
     if setup is not None:
         setup()
     func()
 
 
 @contextlib.contextmanager
 def timeone():
     r = []
     ostart = os.times()
     cstart = util.timer()
     yield r
     cstop = util.timer()
     ostop = os.times()
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1] - a[1]))
 
 
 # list of stop condition (elapsed time, minimal run count)
 DEFAULTLIMITS = (
     (3.0, 100),
     (10.0, 3),
 )
 
 
 def _timer(
     fm,
     func,
     setup=None,
     title=None,
     displayall=False,
     limits=DEFAULTLIMITS,
     prerun=0,
     profiler=None,
 ):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
     if profiler is None:
         profiler = NOOPCTX
     for i in range(prerun):
         if setup is not None:
             setup()
         func()
     keepgoing = True
     while keepgoing:
         if setup is not None:
             setup()
         with profiler:
             with timeone() as item:
                 r = func()
         profiler = NOOPCTX
         count += 1
         results.append(item[0])
         cstop = util.timer()
         # Look for a stop condition.
         elapsed = cstop - begin
         for t, mincount in limits:
             if elapsed >= t and count >= mincount:
                 keepgoing = False
                 break
 
     formatone(fm, results, title=title, result=r, displayall=displayall)
 
 
 def formatone(fm, timings, title=None, result=None, displayall=False):
-
     count = len(timings)
 
     fm.startitem()
 
     if title:
         fm.write(b'title', b'! %s\n', title)
     if result:
         fm.write(b'result', b'! result: %s\n', result)
 
     def display(role, entry):
         prefix = b''
         if role != b'best':
             prefix = b'%s.' % role
         fm.plain(b'!')
         fm.write(prefix + b'wall', b' wall %f', entry[0])
         fm.write(prefix + b'comb', b' comb %f', entry[1] + entry[2])
         fm.write(prefix + b'user', b' user %f', entry[1])
         fm.write(prefix + b'sys', b' sys %f', entry[2])
         fm.write(prefix + b'count', b' (%s of %%d)' % role, count)
         fm.plain(b'\n')
 
     timings.sort()
     min_val = timings[0]
     display(b'best', min_val)
     if displayall:
         max_val = timings[-1]
         display(b'max', max_val)
         avg = tuple([sum(x) / count for x in zip(*timings)])
         display(b'avg', avg)
         median = timings[len(timings) // 2]
         display(b'median', median)
 
 
 # utilities for historical portability
 
 
 def getint(ui, section, name, default):
     # for "historical portability":
     # ui.configint has been available since 1.9 (or fa2b596db182)
     v = ui.config(section, name, None)
     if v is None:
         return default
     try:
         return int(v)
     except ValueError:
         raise error.ConfigError(
             b"%s.%s is not an integer ('%s')" % (section, name, v)
         )
 
 
 def safeattrsetter(obj, name, ignoremissing=False):
     """Ensure that 'obj' has 'name' attribute before subsequent setattr
 
     This function is aborted, if 'obj' doesn't have 'name' attribute
     at runtime. This avoids overlooking removal of an attribute, which
     breaks assumption of performance measurement, in the future.
 
     This function returns the object to (1) assign a new value, and
     (2) restore an original value to the attribute.
 
     If 'ignoremissing' is true, missing 'name' attribute doesn't cause
     abortion, and this function returns None. This is useful to
     examine an attribute, which isn't ensured in all Mercurial
     versions.
     """
     if not util.safehasattr(obj, name):
         if ignoremissing:
             return None
         raise error.Abort(
             (
                 b"missing attribute %s of %s might break assumption"
                 b" of performance measurement"
             )
             % (name, obj)
         )
 
     origvalue = getattr(obj, _sysstr(name))
 
     class attrutil:
         def set(self, newvalue):
             setattr(obj, _sysstr(name), newvalue)
 
         def restore(self):
             setattr(obj, _sysstr(name), origvalue)
 
     return attrutil()
 
 
 # utilities to examine each internal API changes
 
 
 def getbranchmapsubsettable():
     # for "historical portability":
     # subsettable is defined in:
     # - branchmap since 2.9 (or 175c6fd8cacc)
     # - repoview since 2.5 (or 59a9f18d4587)
     # - repoviewutil since 5.0
     for mod in (branchmap, repoview, repoviewutil):
         subsettable = getattr(mod, 'subsettable', None)
         if subsettable:
             return subsettable
 
     # bisecting in bcee63733aad::59a9f18d4587 can reach here (both
     # branchmap and repoview modules exist, but subsettable attribute
     # doesn't)
     raise error.Abort(
         b"perfbranchmap not available with this Mercurial",
         hint=b"use 2.5 or later",
     )
 
 
 def getsvfs(repo):
     """Return appropriate object to access files under .hg/store"""
     # for "historical portability":
     # repo.svfs has been available since 2.3 (or 7034365089bf)
     svfs = getattr(repo, 'svfs', None)
     if svfs:
         return svfs
     else:
         return getattr(repo, 'sopener')
 
 
 def getvfs(repo):
     """Return appropriate object to access files under .hg"""
     # for "historical portability":
     # repo.vfs has been available since 2.3 (or 7034365089bf)
     vfs = getattr(repo, 'vfs', None)
     if vfs:
         return vfs
     else:
         return getattr(repo, 'opener')
 
 
 def repocleartagscachefunc(repo):
     """Return the function to clear tags cache according to repo internal API"""
     if util.safehasattr(repo, b'_tagscache'):  # since 2.0 (or 9dca7653b525)
         # in this case, setattr(repo, '_tagscache', None) or so isn't
         # correct way to clear tags cache, because existing code paths
         # expect _tagscache to be a structured object.
         def clearcache():
             # _tagscache has been filteredpropertycache since 2.5 (or
             # 98c867ac1330), and delattr() can't work in such case
             if '_tagscache' in vars(repo):
                 del repo.__dict__['_tagscache']
 
         return clearcache
 
     repotags = safeattrsetter(repo, b'_tags', ignoremissing=True)
     if repotags:  # since 1.4 (or 5614a628d173)
         return lambda: repotags.set(None)
 
     repotagscache = safeattrsetter(repo, b'tagscache', ignoremissing=True)
     if repotagscache:  # since 0.6 (or d7df759d0e97)
         return lambda: repotagscache.set(None)
 
     # Mercurial earlier than 0.6 (or d7df759d0e97) logically reaches
     # this point, but it isn't so problematic, because:
     # - repo.tags of such Mercurial isn't "callable", and repo.tags()
     #   in perftags() causes failure soon
     # - perf.py itself has been available since 1.1 (or eb240755386d)
     raise error.Abort(b"tags API of this hg command is unknown")
 
 
 # utilities to clear cache
 
 
 def clearfilecache(obj, attrname):
     unfiltered = getattr(obj, 'unfiltered', None)
     if unfiltered is not None:
         obj = obj.unfiltered()
     if attrname in vars(obj):
         delattr(obj, attrname)
     obj._filecache.pop(attrname, None)
 
 
 def clearchangelog(repo):
     if repo is not repo.unfiltered():
         object.__setattr__(repo, '_clcachekey', None)
         object.__setattr__(repo, '_clcache', None)
     clearfilecache(repo.unfiltered(), 'changelog')
 
 
 # perf commands
 
 
 @command(b'perf::walk|perfwalk', formatteropts)
 def perfwalk(ui, repo, *pats, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     m = scmutil.match(repo[None], pats, {})
     timer(
         lambda: len(
             list(
                 repo.dirstate.walk(m, subrepos=[], unknown=True, ignored=False)
             )
         )
     )
     fm.end()
 
 
 @command(b'perf::annotate|perfannotate', formatteropts)
 def perfannotate(ui, repo, f, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     fc = repo[b'.'][f]
     timer(lambda: len(fc.annotate(True)))
     fm.end()
 
 
 @command(
     b'perf::status|perfstatus',
     [
         (b'u', b'unknown', False, b'ask status to look for unknown files'),
         (b'', b'dirstate', False, b'benchmark the internal dirstate call'),
     ]
     + formatteropts,
 )
 def perfstatus(ui, repo, **opts):
     """benchmark the performance of a single status call
 
     The repository data are preserved between each call.
 
     By default, only the status of the tracked file are requested. If
     `--unknown` is passed, the "unknown" files are also tracked.
     """
     opts = _byteskwargs(opts)
     # m = match.always(repo.root, repo.getcwd())
     # timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer, fm = gettimer(ui, opts)
     if opts[b'dirstate']:
         dirstate = repo.dirstate
         m = scmutil.matchall(repo)
         unknown = opts[b'unknown']
 
         def status_dirstate():
             s = dirstate.status(
                 m, subrepos=[], ignored=False, clean=False, unknown=unknown
             )
             sum(map(bool, s))
 
-        timer(status_dirstate)
+        if util.safehasattr(dirstate, 'running_status'):
+            with dirstate.running_status(repo):
+                timer(status_dirstate)
+                dirstate.invalidate()
+        else:
+            timer(status_dirstate)
     else:
         timer(lambda: sum(map(len, repo.status(unknown=opts[b'unknown']))))
     fm.end()
 
 
 @command(b'perf::addremove|perfaddremove', formatteropts)
 def perfaddremove(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     try:
         oldquiet = repo.ui.quiet
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         opts[b'dry_run'] = True
         if 'uipathfn' in getargspec(scmutil.addremove).args:
             uipathfn = scmutil.getuipathfn(repo)
             timer(lambda: scmutil.addremove(repo, matcher, b"", uipathfn, opts))
         else:
             timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
 
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, b'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, b'_nodecache'):
         # <= hg-5.2
         from mercurial.node import nullid, nullrev
 
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 
 @command(b'perf::heads|perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
     """benchmark the computation of a changelog heads"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def s():
         clearcaches(cl)
 
     def d():
         len(cl.headrevs())
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::tags|perftags',
     formatteropts
     + [
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ],
 )
 def perftags(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repocleartagscache = repocleartagscachefunc(repo)
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
             clearfilecache(repo.unfiltered(), 'manifest')
         repocleartagscache()
 
     def t():
         return len(repo.tags())
 
     timer(t, setup=s)
     fm.end()
 
 
 @command(b'perf::ancestors|perfancestors', formatteropts)
 def perfancestors(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     heads = repo.changelog.headrevs()
 
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ancestorset|perfancestorset', formatteropts)
 def perfancestorset(ui, repo, revset, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
 
     def d():
         s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::delta-find',
     revlogopts + formatteropts,
     b'-c|-m|FILE REV',
 )
 def perf_delta_find(ui, repo, arg_1, arg_2=None, **opts):
     """benchmark the process of finding a valid delta for a revlog revision
 
     When a revlog receives a new revision (e.g. from a commit, or from an
     incoming bundle), it searches for a suitable delta-base to produce a delta.
     This perf command measures how much time we spend in this process. It
     operates on an already stored revision.
 
     See `hg help debug-delta-find` for another related command.
     """
     from mercurial import revlogutils
     import mercurial.revlogutils.deltas as deltautil
 
     opts = _byteskwargs(opts)
     if arg_2 is None:
         file_ = None
         rev = arg_1
     else:
         file_ = arg_1
         rev = arg_2
 
     repo = repo.unfiltered()
 
     timer, fm = gettimer(ui, opts)
 
     rev = int(rev)
 
     revlog = cmdutil.openrevlog(repo, b'perf::delta-find', file_, opts)
 
     deltacomputer = deltautil.deltacomputer(revlog)
 
     node = revlog.node(rev)
     p1r, p2r = revlog.parentrevs(rev)
     p1 = revlog.node(p1r)
     p2 = revlog.node(p2r)
     full_text = revlog.revision(rev)
     textlen = len(full_text)
     cachedelta = None
     flags = revlog.flags(rev)
 
     revinfo = revlogutils.revisioninfo(
         node,
         p1,
         p2,
         [full_text],  # btext
         textlen,
         cachedelta,
         flags,
     )
 
     # Note: we should probably purge the potential caches (like the full
     # manifest cache) between runs.
     def find_one():
         with revlog._datafp() as fh:
             deltacomputer.finddeltainfo(revinfo, fh, target_rev=rev)
 
     timer(find_one)
     fm.end()
 
 
 @command(b'perf::discovery|perfdiscovery', formatteropts, b'PATH')
 def perfdiscovery(ui, repo, path, **opts):
     """benchmark discovery between local repo and the peer at given path"""
     repos = [repo, None]
     timer, fm = gettimer(ui, opts)
 
     try:
-        from mercurial.utils.urlutil import get_unique_pull_path
-
-        path = get_unique_pull_path(b'perfdiscovery', repo, ui, path)[0]
+        from mercurial.utils.urlutil import get_unique_pull_path_obj
+
+        path = get_unique_pull_path_obj(b'perfdiscovery', ui, path)
     except ImportError:
-        path = ui.expandpath(path)
+        try:
+            from mercurial.utils.urlutil import get_unique_pull_path
+
+            path = get_unique_pull_path(b'perfdiscovery', repo, ui, path)[0]
+        except ImportError:
+            path = ui.expandpath(path)
 
     def s():
         repos[1] = hg.peer(ui, opts, path)
 
     def d():
         setdiscovery.findcommonheads(ui, *repos)
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::bookmarks|perfbookmarks',
     formatteropts
     + [
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ],
 )
 def perfbookmarks(ui, repo, **opts):
     """benchmark parsing bookmarks from disk to memory"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
         clearfilecache(repo, b'_bookmarks')
 
     def d():
         repo._bookmarks
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perf::bundle',
     [
         (
             b'r',
             b'rev',
             [],
             b'changesets to bundle',
             b'REV',
         ),
         (
             b't',
             b'type',
             b'none',
             b'bundlespec to use (see `hg help bundlespec`)',
             b'TYPE',
         ),
     ]
     + formatteropts,
     b'REVS',
 )
 def perfbundle(ui, repo, *revs, **opts):
     """benchmark the creation of a bundle from a repository
 
     For now, this only supports "none" compression.
     """
     try:
         from mercurial import bundlecaches
 
         parsebundlespec = bundlecaches.parsebundlespec
     except ImportError:
         from mercurial import exchange
 
         parsebundlespec = exchange.parsebundlespec
 
     from mercurial import discovery
     from mercurial import bundle2
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     cl = repo.changelog
     revs = list(revs)
     revs.extend(opts.get(b'rev', ()))
     revs = scmutil.revrange(repo, revs)
     if not revs:
         raise error.Abort(b"not revision specified")
     # make it a consistent set (ie: without topological gaps)
     old_len = len(revs)
     revs = list(repo.revs(b"%ld::%ld", revs, revs))
     if old_len != len(revs):
         new_count = len(revs) - old_len
         msg = b"add %d new revisions to make it a consistent set\n"
         ui.write_err(msg % new_count)
 
     targets = [cl.node(r) for r in repo.revs(b"heads(::%ld)", revs)]
     bases = [cl.node(r) for r in repo.revs(b"heads(::%ld - %ld)", revs, revs)]
     outgoing = discovery.outgoing(repo, bases, targets)
 
     bundle_spec = opts.get(b'type')
 
     bundle_spec = parsebundlespec(repo, bundle_spec, strict=False)
 
     cgversion = bundle_spec.params.get(b"cg.version")
     if cgversion is None:
         if bundle_spec.version == b'v1':
             cgversion = b'01'
         if bundle_spec.version == b'v2':
             cgversion = b'02'
     if cgversion not in changegroup.supportedoutgoingversions(repo):
         err = b"repository does not support bundle version %s"
         raise error.Abort(err % cgversion)
 
     if cgversion == b'01':  # bundle1
         bversion = b'HG10' + bundle_spec.wirecompression
         bcompression = None
     elif cgversion in (b'02', b'03'):
         bversion = b'HG20'
         bcompression = bundle_spec.wirecompression
     else:
         err = b'perf::bundle: unexpected changegroup version %s'
         raise error.ProgrammingError(err % cgversion)
 
     if bcompression is None:
         bcompression = b'UN'
 
     if bcompression != b'UN':
         err = b'perf::bundle: compression currently unsupported: %s'
         raise error.ProgrammingError(err % bcompression)
 
     def do_bundle():
         bundle2.writenewbundle(
             ui,
             repo,
             b'perf::bundle',
             os.devnull,
             bversion,
             outgoing,
             bundle_spec.params,
         )
 
     timer(do_bundle)
     fm.end()
 
 
 @command(b'perf::bundleread|perfbundleread', formatteropts, b'BUNDLE')
 def perfbundleread(ui, repo, bundlepath, **opts):
     """Benchmark reading of bundle files.
 
     This command is meant to isolate the I/O part of bundle reading as
     much as possible.
     """
     from mercurial import (
         bundle2,
         exchange,
         streamclone,
     )
 
     opts = _byteskwargs(opts)
 
     def makebench(fn):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 fn(bundle)
 
         return run
 
     def makereadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 while bundle.read(size):
                     pass
 
         return run
 
     def makestdioread(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 while fh.read(size):
                     pass
 
         return run
 
     # bundle1
 
     def deltaiter(bundle):
         for delta in bundle.deltaiter():
             pass
 
     def iterchunks(bundle):
         for chunk in bundle.getchunks():
             pass
 
     # bundle2
 
     def forwardchunks(bundle):
         for chunk in bundle._forwardchunks():
             pass
 
     def iterparts(bundle):
         for part in bundle.iterparts():
             pass
 
     def iterpartsseekable(bundle):
         for part in bundle.iterparts(seekable=True):
             pass
 
     def seek(bundle):
         for part in bundle.iterparts(seekable=True):
             part.seek(0, os.SEEK_END)
 
     def makepartreadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 for part in bundle.iterparts():
                     while part.read(size):
                         pass
 
         return run
 
     benches = [
         (makestdioread(8192), b'read(8k)'),
         (makestdioread(16384), b'read(16k)'),
         (makestdioread(32768), b'read(32k)'),
         (makestdioread(131072), b'read(128k)'),
     ]
 
     with open(bundlepath, b'rb') as fh:
         bundle = exchange.readbundle(ui, fh, bundlepath)
 
         if isinstance(bundle, changegroup.cg1unpacker):
             benches.extend(
                 [
                     (makebench(deltaiter), b'cg1 deltaiter()'),
                     (makebench(iterchunks), b'cg1 getchunks()'),
                     (makereadnbytes(8192), b'cg1 read(8k)'),
                     (makereadnbytes(16384), b'cg1 read(16k)'),
                     (makereadnbytes(32768), b'cg1 read(32k)'),
                     (makereadnbytes(131072), b'cg1 read(128k)'),
                 ]
             )
         elif isinstance(bundle, bundle2.unbundle20):
             benches.extend(
                 [
                     (makebench(forwardchunks), b'bundle2 forwardchunks()'),
                     (makebench(iterparts), b'bundle2 iterparts()'),
                     (
                         makebench(iterpartsseekable),
                         b'bundle2 iterparts() seekable',
                     ),
                     (makebench(seek), b'bundle2 part seek()'),
                     (makepartreadnbytes(8192), b'bundle2 part read(8k)'),
                     (makepartreadnbytes(16384), b'bundle2 part read(16k)'),
                     (makepartreadnbytes(32768), b'bundle2 part read(32k)'),
                     (makepartreadnbytes(131072), b'bundle2 part read(128k)'),
                 ]
             )
         elif isinstance(bundle, streamclone.streamcloneapplier):
             raise error.Abort(b'stream clone bundles not supported')
         else:
             raise error.Abort(b'unhandled bundle type: %s' % type(bundle))
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::changegroupchangelog|perfchangegroupchangelog',
     formatteropts
     + [
         (b'', b'cgversion', b'02', b'changegroup version'),
         (b'r', b'rev', b'', b'revisions to add to changegroup'),
     ],
 )
 def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
     """Benchmark producing a changelog group for a changegroup.
 
     This measures the time spent processing the changelog during a
     bundle operation. This occurs during `hg bundle` and on a server
     processing a `getbundle` wire protocol request (handles clones
     and pull requests).
 
     By default, all revisions are added to the changegroup.
     """
     opts = _byteskwargs(opts)
     cl = repo.changelog
     nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
     bundler = changegroup.getbundler(cgversion, repo)
 
     def d():
         state, chunks = bundler._generatechangelog(cl, nodes)
         for chunk in chunks:
             pass
 
     timer, fm = gettimer(ui, opts)
 
     # Terminal printing can interfere with timing. So disable it.
     with ui.configoverride({(b'progress', b'disable'): True}):
         timer(d)
 
     fm.end()
 
 
 @command(b'perf::dirs|perfdirs', formatteropts)
 def perfdirs(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
 
     def d():
         dirstate.hasdir(b'a')
         try:
             del dirstate._map._dirs
         except AttributeError:
             pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::dirstate|perfdirstate',
     [
         (
             b'',
             b'iteration',
             None,
             b'benchmark a full iteration for the dirstate',
         ),
         (
             b'',
             b'contains',
             None,
             b'benchmark a large amount of `nf in dirstate` calls',
         ),
     ]
     + formatteropts,
 )
 def perfdirstate(ui, repo, **opts):
     """benchmap the time of various distate operations
 
     By default benchmark the time necessary to load a dirstate from scratch.
     The dirstate is loaded to the point were a "contains" request can be
     answered.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     b"a" in repo.dirstate
 
     if opts[b'iteration'] and opts[b'contains']:
         msg = b'only specify one of --iteration or --contains'
         raise error.Abort(msg)
 
     if opts[b'iteration']:
         setup = None
         dirstate = repo.dirstate
 
         def d():
             for f in dirstate:
                 pass
 
     elif opts[b'contains']:
         setup = None
         dirstate = repo.dirstate
         allfiles = list(dirstate)
         # also add file path that will be "missing" from the dirstate
         allfiles.extend([f[::-1] for f in allfiles])
 
         def d():
             for f in allfiles:
                 f in dirstate
 
     else:
 
         def setup():
             repo.dirstate.invalidate()
 
         def d():
             b"a" in repo.dirstate
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatedirs|perfdirstatedirs', formatteropts)
 def perfdirstatedirs(ui, repo, **opts):
     """benchmap a 'dirstate.hasdir' call from an empty `dirs` cache"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo.dirstate.hasdir(b"a")
 
     def setup():
         try:
             del repo.dirstate._map._dirs
         except AttributeError:
             pass
 
     def d():
         repo.dirstate.hasdir(b"a")
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatefoldmap|perfdirstatefoldmap', formatteropts)
 def perfdirstatefoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.filefoldmap.get()` request
 
     The dirstate filefoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.filefoldmap.get(b'a')
 
     def setup():
         del dirstate._map.filefoldmap
 
     def d():
         dirstate._map.filefoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirfoldmap|perfdirfoldmap', formatteropts)
 def perfdirfoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.dirfoldmap.get()` request
 
     The dirstate dirfoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.dirfoldmap.get(b'a')
 
     def setup():
         del dirstate._map.dirfoldmap
         try:
             del dirstate._map._dirs
         except AttributeError:
             pass
 
     def d():
         dirstate._map.dirfoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::dirstatewrite|perfdirstatewrite', formatteropts)
 def perfdirstatewrite(ui, repo, **opts):
     """benchmap the time it take to write a dirstate on disk"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ds = repo.dirstate
     b"a" in ds
 
     def setup():
         ds._dirty = True
 
     def d():
         ds.write(repo.currenttransaction())
 
-    timer(d, setup=setup)
+    with repo.wlock():
+        timer(d, setup=setup)
     fm.end()
 
 
 def _getmergerevs(repo, opts):
     """parse command argument to return rev involved in merge
 
     input: options dictionnary with `rev`, `from` and `bse`
     output: (localctx, otherctx, basectx)
     """
     if opts[b'from']:
         fromrev = scmutil.revsingle(repo, opts[b'from'])
         wctx = repo[fromrev]
     else:
         wctx = repo[None]
         # we don't want working dir files to be stat'd in the benchmark, so
         # prime that cache
         wctx.dirty()
     rctx = scmutil.revsingle(repo, opts[b'rev'], opts[b'rev'])
     if opts[b'base']:
         fromrev = scmutil.revsingle(repo, opts[b'base'])
         ancestor = repo[fromrev]
     else:
         ancestor = wctx.ancestor(rctx)
     return (wctx, rctx, ancestor)
 
 
 @command(
     b'perf::mergecalculate|perfmergecalculate',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecalculate(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         merge.calculateupdates(
             repo,
             wctx,
             rctx,
             [ancestor],
             branchmerge=False,
             force=False,
             acceptremote=True,
             followcopies=True,
         )
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::mergecopies|perfmergecopies',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecopies(ui, repo, **opts):
     """measure runtime of `copies.mergecopies`"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         copies.mergecopies(repo, wctx, rctx, ancestor)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::pathcopies|perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
     """benchmark the copy tracing logic"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
     ctx2 = scmutil.revsingle(repo, rev2, rev2)
 
     def d():
         copies.pathcopies(ctx1, ctx2)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::phases|perfphases',
     [
         (b'', b'full', False, b'include file reading time too'),
     ],
     b"",
 )
 def perfphases(ui, repo, **opts):
     """benchmark phasesets computation"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     _phases = repo._phasecache
     full = opts.get(b'full')
 
     def d():
         phases = _phases
         if full:
             clearfilecache(repo, b'_phasecache')
             phases = repo._phasecache
         phases.invalidate()
         phases.loadphaserevs(repo)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::phasesremote|perfphasesremote', [], b"[DEST]")
 def perfphasesremote(ui, repo, dest=None, **opts):
     """benchmark time needed to analyse phases of the remote server"""
     from mercurial.node import bin
     from mercurial import (
         exchange,
         hg,
         phases,
     )
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     path = ui.getpath(dest, default=(b'default-push', b'default'))
     if not path:
         raise error.Abort(
             b'default repository not configured!',
             hint=b"see 'hg help config.paths'",
         )
-    dest = path.pushloc or path.loc
+    if util.safehasattr(path, 'main_path'):
+        path = path.get_push_variant()
+        dest = path.loc
+    else:
+        dest = path.pushloc or path.loc
     ui.statusnoi18n(b'analysing phase of %s\n' % util.hidepassword(dest))
     other = hg.peer(repo, opts, dest)
 
     # easier to perform discovery through the operation
     op = exchange.pushoperation(repo, other)
     exchange._pushdiscoverychangeset(op)
 
     remotesubset = op.fallbackheads
 
     with other.commandexecutor() as e:
         remotephases = e.callcommand(
             b'listkeys', {b'namespace': b'phases'}
         ).result()
     del other
     publishing = remotephases.get(b'publishing', False)
     if publishing:
         ui.statusnoi18n(b'publishing: yes\n')
     else:
         ui.statusnoi18n(b'publishing: no\n')
 
     has_node = getattr(repo.changelog.index, 'has_node', None)
     if has_node is None:
         has_node = repo.changelog.nodemap.__contains__
     nonpublishroots = 0
     for nhex, phase in remotephases.iteritems():
         if nhex == b'publishing':  # ignore data related to publish option
             continue
         node = bin(nhex)
         if has_node(node) and int(phase):
             nonpublishroots += 1
     ui.statusnoi18n(b'number of roots: %d\n' % len(remotephases))
     ui.statusnoi18n(b'number of known non public roots: %d\n' % nonpublishroots)
 
     def d():
         phases.remotephasessummary(repo, remotesubset, remotephases)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::manifest|perfmanifest',
     [
         (b'm', b'manifest-rev', False, b'Look up a manifest node revision'),
         (b'', b'clear-disk', False, b'clear on-disk caches too'),
     ]
     + formatteropts,
     b'REV|NODE',
 )
 def perfmanifest(ui, repo, rev, manifest_rev=False, clear_disk=False, **opts):
     """benchmark the time to read a manifest from disk and return a usable
     dict-like object
 
     Manifest caches are cleared before retrieval."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     if not manifest_rev:
         ctx = scmutil.revsingle(repo, rev, rev)
         t = ctx.manifestnode()
     else:
         from mercurial.node import bin
 
         if len(rev) == 40:
             t = bin(rev)
         else:
             try:
                 rev = int(rev)
 
                 if util.safehasattr(repo.manifestlog, b'getstorage'):
                     t = repo.manifestlog.getstorage(b'').node(rev)
                 else:
                     t = repo.manifestlog._revlog.lookup(rev)
             except ValueError:
                 raise error.Abort(
                     b'manifest revision must be integer or full node'
                 )
 
     def d():
         repo.manifestlog.clearcaches(clear_persisted_data=clear_disk)
         repo.manifestlog[t].read()
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::changeset|perfchangeset', formatteropts)
 def perfchangeset(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     n = scmutil.revsingle(repo, rev).node()
 
     def d():
         repo.changelog.read(n)
         # repo.changelog._cache = None
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ignore|perfignore', formatteropts)
 def perfignore(ui, repo, **opts):
     """benchmark operation related to computing ignore"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
 
     def setupone():
         dirstate.invalidate()
         clearfilecache(dirstate, b'_ignore')
 
     def runone():
         dirstate._ignore
 
     timer(runone, setup=setupone, title=b"load")
     fm.end()
 
 
 @command(
     b'perf::index|perfindex',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'no-lookup', None, b'do not revision lookup post creation'),
     ]
     + formatteropts,
 )
 def perfindex(ui, repo, **opts):
     """benchmark index creation time followed by a lookup
 
     The default is to look `tip` up. Depending on the index implementation,
     the revision looked up can matters. For example, an implementation
     scanning the index will have a faster lookup time for `--rev tip` than for
     `--rev 0`. The number of looked up revisions and their order can also
     matters.
 
     Example of useful set to test:
 
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     It is not currently possible to check for lookup of a missing node. For
     deeper lookup benchmarking, checkout the `perfnodemap` command."""
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     if opts[b'no_lookup']:
         if opts['rev']:
             raise error.Abort('--no-lookup and --rev are mutually exclusive')
         nodes = []
     elif not opts[b'rev']:
         nodes = [repo[b"tip"].node()]
     else:
         revs = scmutil.revrange(repo, opts[b'rev'])
         cl = repo.changelog
         nodes = [cl.node(r) for r in revs]
 
     unfi = repo.unfiltered()
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
 
     def setup():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
 
     def d():
         cl = makecl(unfi)
         for n in nodes:
             cl.rev(n)
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(
     b'perf::nodemap|perfnodemap',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ]
     + formatteropts,
 )
 def perfnodemap(ui, repo, **opts):
     """benchmark the time necessary to look up revision from a cold nodemap
 
     Depending on the implementation, the amount and order of revision we look
     up can varies. Example of useful set to test:
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     The command currently focus on valid binary lookup. Benchmarking for
     hexlookup, prefix lookup and missing lookup would also be valuable.
     """
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
 
     unfi = repo.unfiltered()
     clearcaches = opts[b'clear_caches']
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
     if not opts[b'rev']:
         raise error.Abort(b'use --rev to specify revisions to look up')
     revs = scmutil.revrange(repo, opts[b'rev'])
     cl = repo.changelog
     nodes = [cl.node(r) for r in revs]
 
     # use a list to pass reference to a nodemap from one closure to the next
     nodeget = [None]
 
     def setnodeget():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
         cl = makecl(unfi)
         if util.safehasattr(cl.index, 'get_rev'):
             nodeget[0] = cl.index.get_rev
         else:
             nodeget[0] = cl.nodemap.get
 
     def d():
         get = nodeget[0]
         for n in nodes:
             get(n)
 
     setup = None
     if clearcaches:
 
         def setup():
             setnodeget()
 
     else:
         setnodeget()
         d()  # prewarm the data structure
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perf::startup|perfstartup', formatteropts)
 def perfstartup(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def d():
         if os.name != 'nt':
             os.system(
                 b"HGRCPATH= %s version -q > /dev/null" % fsencode(sys.argv[0])
             )
         else:
             os.environ['HGRCPATH'] = r' '
             os.system("%s version -q > NUL" % sys.argv[0])
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::parents|perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
     """benchmark the time necessary to fetch one changeset's parents.
 
     The fetch is done using the `node identifier`, traversing all object layers
     from the repository object. The first N revisions will be used for this
     benchmark. N is controlled by the ``perf.parentscount`` config option
     (default: 1000).
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
     # experimental config: perf.parentscount
     count = getint(ui, b"perf", b"parentscount", 1000)
     if len(repo.changelog) < count:
         raise error.Abort(b"repo needs %d commits for this test" % count)
     repo = repo.unfiltered()
     nl = [repo.changelog.node(i) for i in _xrange(count)]
 
     def d():
         for n in nl:
             repo.changelog.parents(n)
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::ctxfiles|perfctxfiles', formatteropts)
 def perfctxfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
 
     def d():
         len(repo[x].files())
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::rawfiles|perfrawfiles', formatteropts)
 def perfrawfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def d():
         len(cl.read(x)[3])
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::lookup|perflookup', formatteropts)
 def perflookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: len(repo.lookup(rev)))
     fm.end()
 
 
 @command(
     b'perf::linelogedits|perflinelogedits',
     [
         (b'n', b'edits', 10000, b'number of edits'),
         (b'', b'max-hunk-lines', 10, b'max lines in a hunk'),
     ],
     norepo=True,
 )
 def perflinelogedits(ui, **opts):
     from mercurial import linelog
 
     opts = _byteskwargs(opts)
 
     edits = opts[b'edits']
     maxhunklines = opts[b'max_hunk_lines']
 
     maxb1 = 100000
     random.seed(0)
     randint = random.randint
     currentlines = 0
     arglist = []
     for rev in _xrange(edits):
         a1 = randint(0, currentlines)
         a2 = randint(a1, min(currentlines, a1 + maxhunklines))
         b1 = randint(0, maxb1)
         b2 = randint(b1, b1 + maxhunklines)
         currentlines += (b2 - b1) - (a2 - a1)
         arglist.append((rev, a1, a2, b1, b2))
 
     def d():
         ll = linelog.linelog()
         for args in arglist:
             ll.replacelines(*args)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perf::revrange|perfrevrange', formatteropts)
 def perfrevrange(ui, repo, *specs, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
     fm.end()
 
 
 @command(b'perf::nodelookup|perfnodelookup', formatteropts)
 def perfnodelookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     import mercurial.revlog
 
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     n = scmutil.revsingle(repo, rev).node()
 
     try:
         cl = revlog(getsvfs(repo), radix=b"00changelog")
     except TypeError:
         cl = revlog(getsvfs(repo), indexfile=b"00changelog.i")
 
     def d():
         cl.rev(n)
         clearcaches(cl)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::log|perflog',
     [(b'', b'rename', False, b'ask log to follow renames')] + formatteropts,
 )
 def perflog(ui, repo, rev=None, **opts):
     opts = _byteskwargs(opts)
     if rev is None:
         rev = []
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(
         lambda: commands.log(
             ui, repo, rev=rev, date=b'', user=b'', copies=opts.get(b'rename')
         )
     )
     ui.popbuffer()
     fm.end()
 
 
 @command(b'perf::moonwalk|perfmoonwalk', formatteropts)
 def perfmoonwalk(ui, repo, **opts):
     """benchmark walking the changelog backwards
 
     This also loads the changelog data for each revision in the changelog.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def moonwalk():
         for i in repo.changelog.revs(start=(len(repo) - 1), stop=-1):
             ctx = repo[i]
             ctx.branch()  # read changelog data (in addition to the index)
 
     timer(moonwalk)
     fm.end()
 
 
 @command(
     b'perf::templating|perftemplating',
     [
         (b'r', b'rev', [], b'revisions to run the template on'),
     ]
     + formatteropts,
 )
 def perftemplating(ui, repo, testedtemplate=None, **opts):
     """test the rendering time of a given template"""
     if makelogtemplater is None:
         raise error.Abort(
             b"perftemplating not available with this Mercurial",
             hint=b"use 4.3 or later",
         )
 
     opts = _byteskwargs(opts)
 
     nullui = ui.copy()
     nullui.fout = open(os.devnull, 'wb')
     nullui.disablepager()
     revs = opts.get(b'rev')
     if not revs:
         revs = [b'all()']
     revs = list(scmutil.revrange(repo, revs))
 
     defaulttemplate = (
         b'{date|shortdate} [{rev}:{node|short}]'
         b' {author|person}: {desc|firstline}\n'
     )
     if testedtemplate is None:
         testedtemplate = defaulttemplate
     displayer = makelogtemplater(nullui, repo, testedtemplate)
 
     def format():
         for r in revs:
             ctx = repo[r]
             displayer.show(ctx)
             displayer.flush(ctx)
 
     timer, fm = gettimer(ui, opts)
     timer(format)
     fm.end()
 
 
 def _displaystats(ui, opts, entries, data):
     # use a second formatter because the data are quite different, not sure
     # how it flies with the templater.
     fm = ui.formatter(b'perf-stats', opts)
     for key, title in entries:
         values = data[key]
         nbvalues = len(data)
         values.sort()
         stats = {
             'key': key,
             'title': title,
             'nbitems': len(values),
             'min': values[0][0],
             '10%': values[(nbvalues * 10) // 100][0],
             '25%': values[(nbvalues * 25) // 100][0],
             '50%': values[(nbvalues * 50) // 100][0],
             '75%': values[(nbvalues * 75) // 100][0],
             '80%': values[(nbvalues * 80) // 100][0],
             '85%': values[(nbvalues * 85) // 100][0],
             '90%': values[(nbvalues * 90) // 100][0],
             '95%': values[(nbvalues * 95) // 100][0],
             '99%': values[(nbvalues * 99) // 100][0],
             'max': values[-1][0],
         }
         fm.startitem()
         fm.data(**stats)
         # make node pretty for the human output
         fm.plain('### %s (%d items)\n' % (title, len(values)))
         lines = [
             'min',
             '10%',
             '25%',
             '50%',
             '75%',
             '80%',
             '85%',
             '90%',
             '95%',
             '99%',
             'max',
         ]
         for l in lines:
             fm.plain('%s: %s\n' % (l, stats[l]))
     fm.end()
 
 
 @command(
     b'perf::helper-mergecopies|perfhelper-mergecopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelpermergecopies(ui, repo, revs=[], **opts):
     """find statistics about potential parameters for `perfmergecopies`
 
     This command find (base, p1, p2) triplet relevant for copytracing
     benchmarking in the context of a merge.  It reports values for some of the
     parameters that impact merge copy tracing time during merge.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details come at the cost of slower command
     execution.
 
     Since rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision triplets are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     output_template = [
         ("base", "%(base)12s"),
         ("p1", "%(p1.node)12s"),
         ("p2", "%(p2.node)12s"),
         ("p1.nb-revs", "%(p1.nbrevs)12d"),
         ("p1.nb-files", "%(p1.nbmissingfiles)12d"),
         ("p1.renames", "%(p1.renamedfiles)12d"),
         ("p1.time", "%(p1.time)12.3f"),
         ("p2.nb-revs", "%(p2.nbrevs)12d"),
         ("p2.nb-files", "%(p2.nbmissingfiles)12d"),
         ("p2.renames", "%(p2.renamedfiles)12d"),
         ("p2.time", "%(p2.time)12.3f"),
         ("renames", "%(nbrenamedfiles)12d"),
         ("total.time", "%(time)12.3f"),
     ]
     if not dotiming:
         output_template = [
             i
             for i in output_template
             if not ('time' in i[0] or 'renames' in i[0])
         ]
     header_names = [h for (h, v) in output_template]
     output = ' '.join([v for (h, v) in output_template]) + '\n'
     header = ' '.join(['%12s'] * len(header_names)) + '\n'
     fm.plain(header % tuple(header_names))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['parentnbrenames'] = []
             alldata['totalnbrenames'] = []
             alldata['parenttime'] = []
             alldata['totaltime'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1()
         p2 = ctx.p2()
         bases = repo.changelog._commonancestorsheads(p1.rev(), p2.rev())
         for b in bases:
             b = repo[b]
             p1missing = copies._computeforwardmissing(b, p1)
             p2missing = copies._computeforwardmissing(b, p2)
             data = {
                 b'base': b.hex(),
                 b'p1.node': p1.hex(),
                 b'p1.nbrevs': len(repo.revs('only(%d, %d)', p1.rev(), b.rev())),
                 b'p1.nbmissingfiles': len(p1missing),
                 b'p2.node': p2.hex(),
                 b'p2.nbrevs': len(repo.revs('only(%d, %d)', p2.rev(), b.rev())),
                 b'p2.nbmissingfiles': len(p2missing),
             }
             if dostats:
                 if p1missing:
                     alldata['nbrevs'].append(
                         (data['p1.nbrevs'], b.hex(), p1.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p1.nbmissingfiles'], b.hex(), p1.hex())
                     )
                 if p2missing:
                     alldata['nbrevs'].append(
                         (data['p2.nbrevs'], b.hex(), p2.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p2.nbmissingfiles'], b.hex(), p2.hex())
                     )
             if dotiming:
                 begin = util.timer()
                 mergedata = copies.mergecopies(repo, p1, p2, b)
                 end = util.timer()
                 # not very stable timing since we did only one run
                 data['time'] = end - begin
                 # mergedata contains five dicts: "copy", "movewithdir",
                 # "diverge", "renamedelete" and "dirmove".
                 # The first 4 are about renamed file so lets count that.
                 renames = len(mergedata[0])
                 renames += len(mergedata[1])
                 renames += len(mergedata[2])
                 renames += len(mergedata[3])
                 data['nbrenamedfiles'] = renames
                 begin = util.timer()
                 p1renames = copies.pathcopies(b, p1)
                 end = util.timer()
                 data['p1.time'] = end - begin
                 begin = util.timer()
                 p2renames = copies.pathcopies(b, p2)
                 end = util.timer()
                 data['p2.time'] = end - begin
                 data['p1.renamedfiles'] = len(p1renames)
                 data['p2.renamedfiles'] = len(p2renames)
 
                 if dostats:
                     if p1missing:
                         alldata['parentnbrenames'].append(
                             (data['p1.renamedfiles'], b.hex(), p1.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p1.time'], b.hex(), p1.hex())
                         )
                     if p2missing:
                         alldata['parentnbrenames'].append(
                             (data['p2.renamedfiles'], b.hex(), p2.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p2.time'], b.hex(), p2.hex())
                         )
                     if p1missing or p2missing:
                         alldata['totalnbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 b.hex(),
                                 p1.hex(),
                                 p2.hex(),
                             )
                         )
                         alldata['totaltime'].append(
                             (data['time'], b.hex(), p1.hex(), p2.hex())
                         )
             fm.startitem()
             fm.data(**data)
             # make node pretty for the human output
             out = data.copy()
             out['base'] = fm.hexfunc(b.node())
             out['p1.node'] = fm.hexfunc(p1.node())
             out['p2.node'] = fm.hexfunc(p2.node())
             fm.plain(output % out)
 
     fm.end()
     if dostats:
         # use a second formatter because the data are quite different, not sure
         # how it flies with the templater.
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(
                 ('parentnbrenames', 'rename from one parent to base')
             )
             entries.append(('totalnbrenames', 'total number of renames'))
             entries.append(('parenttime', 'time for one parent'))
             entries.append(('totaltime', 'time for both parents'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(
     b'perf::helper-pathcopies|perfhelper-pathcopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelperpathcopies(ui, repo, revs=[], **opts):
     """find statistic about potential parameters for the `perftracecopies`
 
     This command find source-destination pair relevant for copytracing testing.
     It report value for some of the parameters that impact copy tracing time.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details comes at the cost of a slower command
     execution.
 
     Since the rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision pairs are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     if dotiming:
         header = '%12s %12s %12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d "
             "%(nbrenamedfiles)12d %(time)18.5f\n"
         )
         header_names = (
             "source",
             "destination",
             "nb-revs",
             "nb-files",
             "nb-renames",
             "time",
         )
         fm.plain(header % header_names)
     else:
         header = '%12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d\n"
         )
         fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['nbrenames'] = []
             alldata['time'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1().rev()
         p2 = ctx.p2().rev()
         bases = repo.changelog._commonancestorsheads(p1, p2)
         for p in (p1, p2):
             for b in bases:
                 base = repo[b]
                 parent = repo[p]
                 missing = copies._computeforwardmissing(base, parent)
                 if not missing:
                     continue
                 data = {
                     b'source': base.hex(),
                     b'destination': parent.hex(),
                     b'nbrevs': len(repo.revs('only(%d, %d)', p, b)),
                     b'nbmissingfiles': len(missing),
                 }
                 if dostats:
                     alldata['nbrevs'].append(
                         (
                             data['nbrevs'],
                             base.hex(),
                             parent.hex(),
                         )
                     )
                     alldata['nbmissingfiles'].append(
                         (
                             data['nbmissingfiles'],
                             base.hex(),
                             parent.hex(),
                         )
                     )
                 if dotiming:
                     begin = util.timer()
                     renames = copies.pathcopies(base, parent)
                     end = util.timer()
                     # not very stable timing since we did only one run
                     data['time'] = end - begin
                     data['nbrenamedfiles'] = len(renames)
                     if dostats:
                         alldata['time'].append(
                             (
                                 data['time'],
                                 base.hex(),
                                 parent.hex(),
                             )
                         )
                         alldata['nbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 base.hex(),
                                 parent.hex(),
                             )
                         )
                 fm.startitem()
                 fm.data(**data)
                 out = data.copy()
                 out['source'] = fm.hexfunc(base.node())
                 out['destination'] = fm.hexfunc(parent.node())
                 fm.plain(output % out)
 
     fm.end()
     if dostats:
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(('nbrenames', 'renamed files'))
             entries.append(('time', 'time'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(b'perf::cca|perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
     fm.end()
 
 
 @command(b'perf::fncacheload|perffncacheload', formatteropts)
 def perffncacheload(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
 
     def d():
         s.fncache._load()
 
     timer(d)
     fm.end()
 
 
 @command(b'perf::fncachewrite|perffncachewrite', formatteropts)
 def perffncachewrite(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     lock = repo.lock()
     s.fncache._load()
     tr = repo.transaction(b'perffncachewrite')
     tr.addbackup(b'fncache')
 
     def d():
         s.fncache._dirty = True
         s.fncache.write(tr)
 
     timer(d)
     tr.close()
     lock.release()
     fm.end()
 
 
 @command(b'perf::fncacheencode|perffncacheencode', formatteropts)
 def perffncacheencode(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
 
     def d():
         for p in s.fncache.entries:
             s.encode(p)
 
     timer(d)
     fm.end()
 
 
 def _bdiffworker(q, blocks, xdiff, ready, done):
     while not done.is_set():
         pair = q.get()
         while pair is not None:
             if xdiff:
                 mdiff.bdiff.xdiffblocks(*pair)
             elif blocks:
                 mdiff.bdiff.blocks(*pair)
             else:
                 mdiff.textdiff(*pair)
             q.task_done()
             pair = q.get()
         q.task_done()  # for the None one
         with ready:
             ready.wait()
 
 
 def _manifestrevision(repo, mnode):
     ml = repo.manifestlog
 
     if util.safehasattr(ml, b'getstorage'):
         store = ml.getstorage(b'')
     else:
         store = ml._revlog
 
     return store.revision(mnode)
 
 
 @command(
     b'perf::bdiff|perfbdiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test bdiffs for all associated revisions'),
         (b'', b'threads', 0, b'number of thread to use (disable with 0)'),
         (b'', b'blocks', False, b'test computing diffs into blocks'),
         (b'', b'xdiff', False, b'use xdiff algorithm'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfbdiff(ui, repo, file_, rev=None, count=None, threads=0, **opts):
     """benchmark a bdiff between revisions
 
     By default, benchmark a bdiff between its delta parent and itself.
 
     With ``--count``, benchmark bdiffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure bdiffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
 
     if opts[b'xdiff'] and not opts[b'blocks']:
         raise error.CommandError(b'perfbdiff', b'--xdiff requires --blocks')
 
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfbdiff', b'invalid arguments')
 
     blocks = opts[b'blocks']
     xdiff = opts[b'xdiff']
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfbdiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     withthreads = threads > 0
     if not withthreads:
 
         def d():
             for pair in textpairs:
                 if xdiff:
                     mdiff.bdiff.xdiffblocks(*pair)
                 elif blocks:
                     mdiff.bdiff.blocks(*pair)
                 else:
                     mdiff.textdiff(*pair)
 
     else:
         q = queue()
         for i in _xrange(threads):
             q.put(None)
         ready = threading.Condition()
         done = threading.Event()
         for i in _xrange(threads):
             threading.Thread(
                 target=_bdiffworker, args=(q, blocks, xdiff, ready, done)
             ).start()
         q.join()
 
         def d():
             for pair in textpairs:
                 q.put(pair)
             for i in _xrange(threads):
                 q.put(None)
             with ready:
                 ready.notify_all()
             q.join()
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
     if withthreads:
         done.set()
         for i in _xrange(threads):
             q.put(None)
         with ready:
             ready.notify_all()
 
 
 @command(
     b'perf::unbundle',
     formatteropts,
     b'BUNDLE_FILE',
 )
 def perf_unbundle(ui, repo, fname, **opts):
     """benchmark application of a bundle in a repository.
 
     This does not include the final transaction processing"""
 
     from mercurial import exchange
     from mercurial import bundle2
     from mercurial import transaction
 
     opts = _byteskwargs(opts)
 
     ###  some compatibility hotfix
     #
     # the data attribute is dropped in 63edc384d3b7 a changeset introducing a
     # critical regression that break transaction rollback for files that are
     # de-inlined.
     method = transaction.transaction._addentry
     pre_63edc384d3b7 = "data" in getargspec(method).args
     # the `detailed_exit_code` attribute is introduced in 33c0c25d0b0f
     # a changeset that is a close descendant of 18415fc918a1, the changeset
     # that conclude the fix run for the bug introduced in 63edc384d3b7.
     args = getargspec(error.Abort.__init__).args
     post_18415fc918a1 = "detailed_exit_code" in args
 
     old_max_inline = None
     try:
         if not (pre_63edc384d3b7 or post_18415fc918a1):
             # disable inlining
             old_max_inline = mercurial.revlog._maxinline
             # large enough to never happen
             mercurial.revlog._maxinline = 2 ** 50
 
         with repo.lock():
             bundle = [None, None]
             orig_quiet = repo.ui.quiet
             try:
                 repo.ui.quiet = True
                 with open(fname, mode="rb") as f:
 
                     def noop_report(*args, **kwargs):
                         pass
 
                     def setup():
                         gen, tr = bundle
                         if tr is not None:
                             tr.abort()
                         bundle[:] = [None, None]
                         f.seek(0)
                         bundle[0] = exchange.readbundle(ui, f, fname)
                         bundle[1] = repo.transaction(b'perf::unbundle')
                         # silence the transaction
                         bundle[1]._report = noop_report
 
                     def apply():
                         gen, tr = bundle
                         bundle2.applybundle(
                             repo,
                             gen,
                             tr,
                             source=b'perf::unbundle',
                             url=fname,
                         )
 
                     timer, fm = gettimer(ui, opts)
                     timer(apply, setup=setup)
                     fm.end()
             finally:
                 repo.ui.quiet == orig_quiet
                 gen, tr = bundle
                 if tr is not None:
                     tr.abort()
     finally:
         if old_max_inline is not None:
             mercurial.revlog._maxinline = old_max_inline
 
 
 @command(
     b'perf::unidiff|perfunidiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test unidiffs for all associated revisions'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfunidiff(ui, repo, file_, rev=None, count=None, **opts):
     """benchmark a unified diff between revisions
 
     This doesn't include any copy tracing - it's just a unified diff
     of the texts.
 
     By default, benchmark a diff between its delta parent and itself.
 
     With ``--count``, benchmark diffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure diffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfunidiff', b'invalid arguments')
 
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfunidiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     def d():
         for left, right in textpairs:
             # The date strings don't matter, so we pass empty strings.
             headerlines, hunks = mdiff.unidiff(
                 left, b'', right, b'', b'left', b'right', binary=False
             )
             # consume iterators in roughly the way patch.py does
             b'\n'.join(headerlines)
             b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perf::diffwd|perfdiffwd', formatteropts)
 def perfdiffwd(ui, repo, **opts):
     """Profile diff of working directory changes"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
     }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
         opts = {options[c]: b'1' for c in diffopt}
 
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
 
         diffopt = diffopt.encode('ascii')
         title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
         timer(d, title=title)
     fm.end()
 
 
 @command(
     b'perf::revlogindex|perfrevlogindex',
     revlogopts + formatteropts,
     b'-c|-m|FILE',
 )
 def perfrevlogindex(ui, repo, file_=None, **opts):
     """Benchmark operations against a revlog index.
 
     This tests constructing a revlog instance, reading index data,
     parsing index data, and performing various operations related to
     index data.
     """
 
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
 
     opener = getattr(rl, 'opener')  # trick linter
     # compat with hg <= 5.8
     radix = getattr(rl, 'radix', None)
     indexfile = getattr(rl, '_indexfile', None)
     if indexfile is None:
         # compatibility with <= hg-5.8
         indexfile = getattr(rl, 'indexfile')
     data = opener.read(indexfile)
 
     header = struct.unpack(b'>I', data[0:4])[0]
     version = header & 0xFFFF
     if version == 1:
         inline = header & (1 << 16)
     else:
         raise error.Abort(b'unsupported revlog version: %d' % version)
 
     parse_index_v1 = getattr(mercurial.revlog, 'parse_index_v1', None)
     if parse_index_v1 is None:
         parse_index_v1 = mercurial.revlog.revlogio().parseindex
 
     rllen = len(rl)
 
     node0 = rl.node(0)
     node25 = rl.node(rllen // 4)
     node50 = rl.node(rllen // 2)
     node75 = rl.node(rllen // 4 * 3)
     node100 = rl.node(rllen - 1)
 
     allrevs = range(rllen)
     allrevsrev = list(reversed(allrevs))
     allnodes = [rl.node(rev) for rev in range(rllen)]
     allnodesrev = list(reversed(allnodes))
 
     def constructor():
         if radix is not None:
             revlog(opener, radix=radix)
         else:
             # hg <= 5.8
             revlog(opener, indexfile=indexfile)
 
     def read():
         with opener(indexfile) as fh:
             fh.read()
 
     def parseindex():
         parse_index_v1(data, inline)
 
     def getentry(revornode):
         index = parse_index_v1(data, inline)[0]
         index[revornode]
 
     def getentries(revs, count=1):
         index = parse_index_v1(data, inline)[0]
 
         for i in range(count):
             for rev in revs:
                 index[rev]
 
     def resolvenode(node):
         index = parse_index_v1(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
             nodemap = getattr(parse_index_v1(data, inline)[0], 'nodemap', None)
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         try:
             rev(node)
         except error.RevlogError:
             pass
 
     def resolvenodes(nodes, count=1):
         index = parse_index_v1(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
             nodemap = getattr(parse_index_v1(data, inline)[0], 'nodemap', None)
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         for i in range(count):
             for node in nodes:
                 try:
                     rev(node)
                 except error.RevlogError:
                     pass
 
     benches = [
         (constructor, b'revlog constructor'),
         (read, b'read'),
         (parseindex, b'create index object'),
         (lambda: getentry(0), b'retrieve index entry for rev 0'),
         (lambda: resolvenode(b'a' * 20), b'look up missing node'),
         (lambda: resolvenode(node0), b'look up node at rev 0'),
         (lambda: resolvenode(node25), b'look up node at 1/4 len'),
         (lambda: resolvenode(node50), b'look up node at 1/2 len'),
         (lambda: resolvenode(node75), b'look up node at 3/4 len'),
         (lambda: resolvenode(node100), b'look up node at tip'),
         # 2x variation is to measure caching impact.
         (lambda: resolvenodes(allnodes), b'look up all nodes (forward)'),
         (lambda: resolvenodes(allnodes, 2), b'look up all nodes 2x (forward)'),
         (lambda: resolvenodes(allnodesrev), b'look up all nodes (reverse)'),
         (
             lambda: resolvenodes(allnodesrev, 2),
             b'look up all nodes 2x (reverse)',
         ),
         (lambda: getentries(allrevs), b'retrieve all index entries (forward)'),
         (
             lambda: getentries(allrevs, 2),
             b'retrieve all index entries 2x (forward)',
         ),
         (
             lambda: getentries(allrevsrev),
             b'retrieve all index entries (reverse)',
         ),
         (
             lambda: getentries(allrevsrev, 2),
             b'retrieve all index entries 2x (reverse)',
         ),
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::revlogrevisions|perfrevlogrevisions',
     revlogopts
     + formatteropts
     + [
         (b'd', b'dist', 100, b'distance between the revisions'),
         (b's', b'startrev', 0, b'revision to start reading at'),
         (b'', b'reverse', False, b'read in reverse'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogrevisions(
     ui, repo, file_=None, startrev=0, reverse=False, **opts
 ):
     """Benchmark reading a series of revisions from a revlog.
 
     By default, we read every ``-d/--dist`` revision from 0 to tip of
     the specified revlog.
 
     The start revision can be defined via ``-s/--startrev``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogrevisions', file_, opts)
     rllen = getlen(ui)(rl)
 
     if startrev < 0:
         startrev = rllen + startrev
 
     def d():
         rl.clearcaches()
 
         beginrev = startrev
         endrev = rllen
         dist = opts[b'dist']
 
         if reverse:
             beginrev, endrev = endrev - 1, beginrev - 1
             dist = -1 * dist
 
         for x in _xrange(beginrev, endrev, dist):
             # Old revisions don't support passing int.
             n = rl.node(x)
             rl.revision(n)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::revlogwrite|perfrevlogwrite',
     revlogopts
     + formatteropts
     + [
         (b's', b'startrev', 1000, b'revision to start writing at'),
         (b'', b'stoprev', -1, b'last revision to write'),
         (b'', b'count', 3, b'number of passes to perform'),
         (b'', b'details', False, b'print timing for every revisions tested'),
         (b'', b'source', b'full', b'the kind of data feed in the revlog'),
         (b'', b'lazydeltabase', True, b'try the provided delta first'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
     """Benchmark writing a series of revisions to a revlog.
 
     Possible source values are:
     * `full`: add from a full text (default).
     * `parent-1`: add from a delta to the first parent
     * `parent-2`: add from a delta to the second parent if it exists
                   (use a delta from the first parent otherwise)
     * `parent-smallest`: add from the smallest delta (either p1 or p2)
     * `storage`: add from the existing precomputed deltas
 
     Note: This performance command measures performance in a custom way. As a
     result some of the global configuration of the 'perf' command does not
     apply to it:
 
     * ``pre-run``: disabled
 
     * ``profile-benchmark``: disabled
 
     * ``run-limits``: disabled use --count instead
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
     rllen = getlen(ui)(rl)
     if startrev < 0:
         startrev = rllen + startrev
     if stoprev < 0:
         stoprev = rllen + stoprev
 
     lazydeltabase = opts['lazydeltabase']
     source = opts['source']
     clearcaches = opts['clear_caches']
     validsource = (
         b'full',
         b'parent-1',
         b'parent-2',
         b'parent-smallest',
         b'storage',
     )
     if source not in validsource:
         raise error.Abort('invalid source type: %s' % source)
 
     ### actually gather results
     count = opts['count']
     if count <= 0:
         raise error.Abort('invalide run count: %d' % count)
     allresults = []
     for c in range(count):
         timing = _timeonewrite(
             ui,
             rl,
             source,
             startrev,
             stoprev,
             c + 1,
             lazydeltabase=lazydeltabase,
             clearcaches=clearcaches,
         )
         allresults.append(timing)
 
     ### consolidate the results in a single list
     results = []
     for idx, (rev, t) in enumerate(allresults[0]):
         ts = [t]
         for other in allresults[1:]:
             orev, ot = other[idx]
             assert orev == rev
             ts.append(ot)
         results.append((rev, ts))
     resultcount = len(results)
 
     ### Compute and display relevant statistics
 
     # get a formatter
     fm = ui.formatter(b'perf', opts)
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # print individual details if requested
     if opts['details']:
         for idx, item in enumerate(results, 1):
             rev, data = item
             title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
             formatone(fm, data, title=title, displayall=displayall)
 
     # sorts results by median time
     results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
     # list of (name, index) to display)
     relevants = [
         ("min", 0),
         ("10%", resultcount * 10 // 100),
         ("25%", resultcount * 25 // 100),
         ("50%", resultcount * 70 // 100),
         ("75%", resultcount * 75 // 100),
         ("90%", resultcount * 90 // 100),
         ("95%", resultcount * 95 // 100),
         ("99%", resultcount * 99 // 100),
         ("99.9%", resultcount * 999 // 1000),
         ("99.99%", resultcount * 9999 // 10000),
         ("99.999%", resultcount * 99999 // 100000),
         ("max", -1),
     ]
     if not ui.quiet:
         for name, idx in relevants:
             data = results[idx]
             title = '%s of %d, rev %d' % (name, resultcount, data[0])
             formatone(fm, data[1], title=title, displayall=displayall)
 
     # XXX summing that many float will not be very precise, we ignore this fact
     # for now
     totaltime = []
     for item in allresults:
         totaltime.append(
             (
                 sum(x[1][0] for x in item),
                 sum(x[1][1] for x in item),
                 sum(x[1][2] for x in item),
             )
         )
     formatone(
         fm,
         totaltime,
         title="total time (%d revs)" % resultcount,
         displayall=displayall,
     )
     fm.end()
 
 
 class _faketr:
     def add(s, x, y, z=None):
         return None
 
 
 def _timeonewrite(
     ui,
     orig,
     source,
     startrev,
     stoprev,
     runidx=None,
     lazydeltabase=True,
     clearcaches=True,
 ):
     timings = []
     tr = _faketr()
     with _temprevlog(ui, orig, startrev) as dest:
         dest._lazydeltabase = lazydeltabase
         revs = list(orig.revs(startrev, stoprev))
         total = len(revs)
         topic = 'adding'
         if runidx is not None:
             topic += ' (run #%d)' % runidx
         # Support both old and new progress API
         if util.safehasattr(ui, 'makeprogress'):
             progress = ui.makeprogress(topic, unit='revs', total=total)
 
             def updateprogress(pos):
                 progress.update(pos)
 
             def completeprogress():
                 progress.complete()
 
         else:
 
             def updateprogress(pos):
                 ui.progress(topic, pos, unit='revs', total=total)
 
             def completeprogress():
                 ui.progress(topic, None, unit='revs', total=total)
 
         for idx, rev in enumerate(revs):
             updateprogress(idx)
             addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
             if clearcaches:
                 dest.index.clearcaches()
                 dest.clearcaches()
             with timeone() as r:
                 dest.addrawrevision(*addargs, **addkwargs)
             timings.append((rev, r[0]))
         updateprogress(total)
         completeprogress()
     return timings
 
 
 def _getrevisionseed(orig, rev, tr, source):
     from mercurial.node import nullid
 
     linkrev = orig.linkrev(rev)
     node = orig.node(rev)
     p1, p2 = orig.parents(node)
     flags = orig.flags(rev)
     cachedelta = None
     text = None
 
     if source == b'full':
         text = orig.revision(rev)
     elif source == b'parent-1':
         baserev = orig.rev(p1)
         cachedelta = (baserev, orig.revdiff(p1, rev))
     elif source == b'parent-2':
         parent = p2
         if p2 == nullid:
             parent = p1
         baserev = orig.rev(parent)
         cachedelta = (baserev, orig.revdiff(parent, rev))
     elif source == b'parent-smallest':
         p1diff = orig.revdiff(p1, rev)
         parent = p1
         diff = p1diff
         if p2 != nullid:
             p2diff = orig.revdiff(p2, rev)
             if len(p1diff) > len(p2diff):
                 parent = p2
                 diff = p2diff
         baserev = orig.rev(parent)
         cachedelta = (baserev, diff)
     elif source == b'storage':
         baserev = orig.deltaparent(rev)
         cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
 
     return (
         (text, tr, linkrev, p1, p2),
         {'node': node, 'flags': flags, 'cachedelta': cachedelta},
     )
 
 
 @contextlib.contextmanager
 def _temprevlog(ui, orig, truncaterev):
     from mercurial import vfs as vfsmod
 
     if orig._inline:
         raise error.Abort('not supporting inline revlog (yet)')
     revlogkwargs = {}
     k = 'upperboundcomp'
     if util.safehasattr(orig, k):
         revlogkwargs[k] = getattr(orig, k)
 
     indexfile = getattr(orig, '_indexfile', None)
     if indexfile is None:
         # compatibility with <= hg-5.8
         indexfile = getattr(orig, 'indexfile')
     origindexpath = orig.opener.join(indexfile)
 
     datafile = getattr(orig, '_datafile', getattr(orig, 'datafile'))
     origdatapath = orig.opener.join(datafile)
     radix = b'revlog'
     indexname = b'revlog.i'
     dataname = b'revlog.d'
 
     tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
     try:
         # copy the data file in a temporary directory
         ui.debug('copying data in %s\n' % tmpdir)
         destindexpath = os.path.join(tmpdir, 'revlog.i')
         destdatapath = os.path.join(tmpdir, 'revlog.d')
         shutil.copyfile(origindexpath, destindexpath)
         shutil.copyfile(origdatapath, destdatapath)
 
         # remove the data we want to add again
         ui.debug('truncating data to be rewritten\n')
         with open(destindexpath, 'ab') as index:
             index.seek(0)
             index.truncate(truncaterev * orig._io.size)
         with open(destdatapath, 'ab') as data:
             data.seek(0)
             data.truncate(orig.start(truncaterev))
 
         # instantiate a new revlog from the temporary copy
         ui.debug('truncating adding to be rewritten\n')
         vfs = vfsmod.vfs(tmpdir)
         vfs.options = getattr(orig.opener, 'options', None)
 
         try:
             dest = revlog(vfs, radix=radix, **revlogkwargs)
         except TypeError:
             dest = revlog(
                 vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
             )
         if dest._inline:
             raise error.Abort('not supporting inline revlog (yet)')
         # make sure internals are initialized
         dest.revision(len(dest) - 1)
         yield dest
         del dest, vfs
     finally:
         shutil.rmtree(tmpdir, True)
 
 
 @command(
     b'perf::revlogchunks|perfrevlogchunks',
     revlogopts
     + formatteropts
     + [
         (b'e', b'engines', b'', b'compression engines to use'),
         (b's', b'startrev', 0, b'revision to start at'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
     stored within each revlog are "chunks" of possibly compressed data. This
     data needs to be read and decompressed or compressed and written.
 
     This command measures the time it takes to read+decompress and recompress
     chunks in a revlog. It effectively isolates I/O and compression performance.
     For measurements of higher-level operations like resolving revisions,
     see ``perfrevlogrevisions`` and ``perfrevlogrevision``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogchunks', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = rl._getsegmentforrevs
     except AttributeError:
         segmentforrevs = rl._chunkraw
 
     # Verify engines argument.
     if engines:
         engines = {e.strip() for e in engines.split(b',')}
         for engine in engines:
             try:
                 util.compressionengines[engine]
             except KeyError:
                 raise error.Abort(b'unknown compression engine: %s' % engine)
     else:
         engines = []
         for e in util.compengines:
             engine = util.compengines[e]
             try:
                 if engine.available():
                     engine.revlogcompressor().compress(b'dummy')
                     engines.append(e)
             except NotImplementedError:
                 pass
 
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
         if rl._inline:
             indexfile = getattr(rl, '_indexfile', None)
             if indexfile is None:
                 # compatibility with <= hg-5.8
                 indexfile = getattr(rl, 'indexfile')
             return getsvfs(repo)(indexfile)
         else:
             datafile = getattr(rl, 'datafile', getattr(rl, 'datafile'))
             return getsvfs(repo)(datafile)
 
     def doread():
         rl.clearcaches()
         for rev in revs:
             segmentforrevs(rev, rev)
 
     def doreadcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             segmentforrevs(rev, rev, df=fh)
 
     def doreadbatch():
         rl.clearcaches()
         segmentforrevs(revs[0], revs[-1])
 
     def doreadbatchcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         segmentforrevs(revs[0], revs[-1], df=fh)
 
     def dochunk():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             rl._chunk(rev, df=fh)
 
     chunks = [None]
 
     def dochunkbatch():
         rl.clearcaches()
         fh = rlfh(rl)
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
     def docompress(compressor):
         rl.clearcaches()
 
         try:
             # Swap in the requested compression engine.
             oldcompressor = rl._compressor
             rl._compressor = compressor
             for chunk in chunks[0]:
                 rl.compress(chunk)
         finally:
             rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), b'read'),
         (lambda: doreadcachedfh(), b'read w/ reused fd'),
         (lambda: doreadbatch(), b'read batch'),
         (lambda: doreadbatchcachedfh(), b'read batch w/ reused fd'),
         (lambda: dochunk(), b'chunk'),
         (lambda: dochunkbatch(), b'chunk batch'),
     ]
 
     for engine in sorted(engines):
         compressor = util.compengines[engine].revlogcompressor()
         benches.append(
             (
                 functools.partial(docompress, compressor),
                 b'compress w/ %s' % engine,
             )
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::revlogrevision|perfrevlogrevision',
     revlogopts
     + formatteropts
     + [(b'', b'cache', False, b'use caches instead of clearing')],
     b'-c|-m|FILE REV',
 )
 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
     """Benchmark obtaining a revlog revision.
 
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
     2. Slice the delta chain if applicable
     3. Obtain the raw chunks for that delta chain
     4. Decompress each raw chunk
     5. Apply binary patches to obtain fulltext
     6. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
     opts = _byteskwargs(opts)
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfrevlogrevision', b'invalid arguments')
 
     r = cmdutil.openrevlog(repo, b'perfrevlogrevision', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = r._getsegmentforrevs
     except AttributeError:
         segmentforrevs = r._chunkraw
 
     node = r.lookup(rev)
     rev = r.rev(node)
 
     def getrawchunks(data, chain):
         start = r.start
         length = r.length
         inline = r._inline
         try:
             iosize = r.index.entry_size
         except AttributeError:
             iosize = r._io.size
         buffer = util.buffer
 
         chunks = []
         ladd = chunks.append
         for idx, item in enumerate(chain):
             offset = start(item[0])
             bits = data[idx]
             for rev in item:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(buffer(bits, chunkstart - offset, chunklength))
 
         return chunks
 
     def dodeltachain(rev):
         if not cache:
             r.clearcaches()
         r._deltachain(rev)
 
     def doread(chain):
         if not cache:
             r.clearcaches()
         for item in slicedchain:
             segmentforrevs(item[0], item[-1])
 
     def doslice(r, chain, size):
         for s in slicechunk(r, chain, targetsize=size):
             pass
 
     def dorawchunks(data, chain):
         if not cache:
             r.clearcaches()
         getrawchunks(data, chain)
 
     def dodecompress(chunks):
         decomp = r.decompress
         for chunk in chunks:
             decomp(chunk)
 
     def dopatch(text, bins):
         if not cache:
             r.clearcaches()
         mdiff.patches(text, bins)
 
     def dohash(text):
         if not cache:
             r.clearcaches()
         r.checkhash(text, node, rev=rev)
 
     def dorevision():
         if not cache:
             r.clearcaches()
         r.revision(node)
 
     try:
         from mercurial.revlogutils.deltas import slicechunk
     except ImportError:
         slicechunk = getattr(revlog, '_slicechunk', None)
 
     size = r.length(rev)
     chain = r._deltachain(rev)[0]
     if not getattr(r, '_withsparseread', False):
         slicedchain = (chain,)
     else:
         slicedchain = tuple(slicechunk(r, chain, targetsize=size))
     data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
     rawchunks = getrawchunks(data, slicedchain)
     bins = r._chunks(chain)
     text = bytes(bins[0])
     bins = bins[1:]
     text = mdiff.patches(text, bins)
 
     benches = [
         (lambda: dorevision(), b'full'),
         (lambda: dodeltachain(rev), b'deltachain'),
         (lambda: doread(chain), b'read'),
     ]
 
     if getattr(r, '_withsparseread', False):
         slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
         benches.append(slicing)
 
     benches.extend(
         [
             (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
             (lambda: dodecompress(rawchunks), b'decompress'),
             (lambda: dopatch(text, bins), b'patch'),
             (lambda: dohash(text), b'hash'),
         ]
     )
 
     timer, fm = gettimer(ui, opts)
     for fn, title in benches:
         timer(fn, title=title)
     fm.end()
 
 
 @command(
     b'perf::revset|perfrevset',
     [
         (b'C', b'clear', False, b'clear volatile cache between each call.'),
         (b'', b'contexts', False, b'obtain changectx for each revision'),
     ]
     + formatteropts,
     b"REVSET",
 )
 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
     """benchmark the execution time of a revset
 
     Use the --clean option if need to evaluate the impact of build volatile
     revisions set cache on the revset execution. Volatile cache hold filtered
     and obsolete related cache."""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def d():
         if clear:
             repo.invalidatevolatilesets()
         if contexts:
             for ctx in repo.set(expr):
                 pass
         else:
             for r in repo.revs(expr):
                 pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perf::volatilesets|perfvolatilesets',
     [
         (b'', b'clear-obsstore', False, b'drop obsstore between each call.'),
     ]
     + formatteropts,
 )
 def perfvolatilesets(ui, repo, *names, **opts):
     """benchmark the computation of various volatile set
 
     Volatile set computes element related to filtering and obsolescence."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo = repo.unfiltered()
 
     def getobs(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             obsolete.getrevs(repo, name)
 
         return d
 
     allobs = sorted(obsolete.cachefuncs)
     if names:
         allobs = [n for n in allobs if n in names]
 
     for name in allobs:
         timer(getobs(name), title=name)
 
     def getfiltered(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             repoview.filterrevs(repo, name)
 
         return d
 
     allfilter = sorted(repoview.filtertable)
     if names:
         allfilter = [n for n in allfilter if n in names]
 
     for name in allfilter:
         timer(getfiltered(name), title=name)
     fm.end()
 
 
 @command(
     b'perf::branchmap|perfbranchmap',
     [
         (b'f', b'full', False, b'Includes build time of subset'),
         (
             b'',
             b'clear-revbranch',
             False,
             b'purge the revbranch cache between computation',
         ),
     ]
     + formatteropts,
 )
 def perfbranchmap(ui, repo, *filternames, **opts):
     """benchmark the update of a branchmap
 
     This benchmarks the full repo.branchmap() call with read and write disabled
     """
     opts = _byteskwargs(opts)
     full = opts.get(b"full", False)
     clear_revbranch = opts.get(b"clear_revbranch", False)
     timer, fm = gettimer(ui, opts)
 
     def getbranchmap(filtername):
         """generate a benchmark function for the filtername"""
         if filtername is None:
             view = repo
         else:
             view = repo.filtered(filtername)
         if util.safehasattr(view._branchcaches, '_per_filter'):
             filtered = view._branchcaches._per_filter
         else:
             # older versions
             filtered = view._branchcaches
 
         def d():
             if clear_revbranch:
                 repo.revbranchcache()._clear()
             if full:
                 view._branchcaches.clear()
             else:
                 filtered.pop(filtername, None)
             view.branchmap()
 
         return d
 
     # add filter in smaller subset to bigger subset
     possiblefilters = set(repoview.filtertable)
     if filternames:
         possiblefilters &= set(filternames)
     subsettable = getbranchmapsubsettable()
     allfilters = []
     while possiblefilters:
         for name in possiblefilters:
             subset = subsettable.get(name)
             if subset not in possiblefilters:
                 break
         else:
             assert False, b'subset cycle %s!' % possiblefilters
         allfilters.append(name)
         possiblefilters.remove(name)
 
     # warm the cache
     if not full:
         for name in allfilters:
             repo.filtered(name).branchmap()
     if not filternames or b'unfiltered' in filternames:
         # add unfiltered
         allfilters.append(None)
 
     if util.safehasattr(branchmap.branchcache, 'fromfile'):
         branchcacheread = safeattrsetter(branchmap.branchcache, b'fromfile')
         branchcacheread.set(classmethod(lambda *args: None))
     else:
         # older versions
         branchcacheread = safeattrsetter(branchmap, b'read')
         branchcacheread.set(lambda *args: None)
     branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
     branchcachewrite.set(lambda *args: None)
     try:
         for name in allfilters:
             printname = name
             if name is None:
                 printname = b'unfiltered'
             timer(getbranchmap(name), title=printname)
     finally:
         branchcacheread.restore()
         branchcachewrite.restore()
     fm.end()
 
 
 @command(
     b'perf::branchmapupdate|perfbranchmapupdate',
     [
         (b'', b'base', [], b'subset of revision to start from'),
         (b'', b'target', [], b'subset of revision to end with'),
         (b'', b'clear-caches', False, b'clear cache between each runs'),
     ]
     + formatteropts,
 )
 def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
     """benchmark branchmap update from for <base> revs to <target> revs
 
     If `--clear-caches` is passed, the following items will be reset before
     each update:
         * the changelog instance and associated indexes
         * the rev-branch-cache instance
 
     Examples:
 
        # update for the one last revision
        $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
 
        $ update for change coming with a new branch
        $ hg perfbranchmapupdate --base 'stable' --target 'default'
     """
     from mercurial import branchmap
     from mercurial import repoview
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     clearcaches = opts[b'clear_caches']
     unfi = repo.unfiltered()
     x = [None]  # used to pass data between closure
 
     # we use a `list` here to avoid possible side effect from smartset
     baserevs = list(scmutil.revrange(repo, base))
     targetrevs = list(scmutil.revrange(repo, target))
     if not baserevs:
         raise error.Abort(b'no revisions selected for --base')
     if not targetrevs:
         raise error.Abort(b'no revisions selected for --target')
 
     # make sure the target branchmap also contains the one in the base
     targetrevs = list(set(baserevs) | set(targetrevs))
     targetrevs.sort()
 
     cl = repo.changelog
     allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
     allbaserevs.sort()
     alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
 
     newrevs = list(alltargetrevs.difference(allbaserevs))
     newrevs.sort()
 
     allrevs = frozenset(unfi.changelog.revs())
     basefilterrevs = frozenset(allrevs.difference(allbaserevs))
     targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
 
     def basefilter(repo, visibilityexceptions=None):
         return basefilterrevs
 
     def targetfilter(repo, visibilityexceptions=None):
         return targetfilterrevs
 
     msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
     ui.status(msg % (len(allbaserevs), len(newrevs)))
     if targetfilterrevs:
         msg = b'(%d revisions still filtered)\n'
         ui.status(msg % len(targetfilterrevs))
 
     try:
         repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
         repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
 
         baserepo = repo.filtered(b'__perf_branchmap_update_base')
         targetrepo = repo.filtered(b'__perf_branchmap_update_target')
 
         # try to find an existing branchmap to reuse
         subsettable = getbranchmapsubsettable()
         candidatefilter = subsettable.get(None)
         while candidatefilter is not None:
             candidatebm = repo.filtered(candidatefilter).branchmap()
             if candidatebm.validfor(baserepo):
                 filtered = repoview.filterrevs(repo, candidatefilter)
                 missing = [r for r in allbaserevs if r in filtered]
                 base = candidatebm.copy()
                 base.update(baserepo, missing)
                 break
             candidatefilter = subsettable.get(candidatefilter)
         else:
             # no suitable subset where found
             base = branchmap.branchcache()
             base.update(baserepo, allbaserevs)
 
         def setup():
             x[0] = base.copy()
             if clearcaches:
                 unfi._revbranchcache = None
                 clearchangelog(repo)
 
         def bench():
             x[0].update(targetrepo, newrevs)
 
         timer(bench, setup=setup)
         fm.end()
     finally:
         repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
         repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
 
 
 @command(
     b'perf::branchmapload|perfbranchmapload',
     [
         (b'f', b'filter', b'', b'Specify repoview filter'),
         (b'', b'list', False, b'List brachmap filter caches'),
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ]
     + formatteropts,
 )
 def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
     """benchmark reading the branchmap"""
     opts = _byteskwargs(opts)
     clearrevlogs = opts[b'clear_revlogs']
 
     if list:
         for name, kind, st in repo.cachevfs.readdir(stat=True):
             if name.startswith(b'branch2'):
                 filtername = name.partition(b'-')[2] or b'unfiltered'
                 ui.status(
                     b'%s - %s\n' % (filtername, util.bytecount(st.st_size))
                 )
         return
     if not filter:
         filter = None
     subsettable = getbranchmapsubsettable()
     if filter is None:
         repo = repo.unfiltered()
     else:
         repo = repoview.repoview(repo, filter)
 
     repo.branchmap()  # make sure we have a relevant, up to date branchmap
 
     try:
         fromfile = branchmap.branchcache.fromfile
     except AttributeError:
         # older versions
         fromfile = branchmap.read
 
     currentfilter = filter
     # try once without timer, the filter may not be cached
     while fromfile(repo) is None:
         currentfilter = subsettable.get(currentfilter)
         if currentfilter is None:
             raise error.Abort(
                 b'No branchmap cached for %s repo' % (filter or b'unfiltered')
             )
         repo = repo.filtered(currentfilter)
     timer, fm = gettimer(ui, opts)
 
     def setup():
         if clearrevlogs:
             clearchangelog(repo)
 
     def bench():
         fromfile(repo)
 
     timer(bench, setup=setup)
     fm.end()
 
 
 @command(b'perf::loadmarkers|perfloadmarkers')
 def perfloadmarkers(ui, repo):
     """benchmark the time to parse the on-disk markers for a repo
 
     Result is the number of markers in the repo."""
     timer, fm = gettimer(ui)
     svfs = getsvfs(repo)
     timer(lambda: len(obsolete.obsstore(repo, svfs)))
     fm.end()
 
 
 @command(
     b'perf::lrucachedict|perflrucachedict',
     formatteropts
     + [
         (b'', b'costlimit', 0, b'maximum total cost of items in cache'),
         (b'', b'mincost', 0, b'smallest cost of items in cache'),
         (b'', b'maxcost', 100, b'maximum cost of items in cache'),
         (b'', b'size', 4, b'size of cache'),
         (b'', b'gets', 10000, b'number of key lookups'),
         (b'', b'sets', 10000, b'number of key sets'),
         (b'', b'mixed', 10000, b'number of mixed mode operations'),
         (
             b'',
             b'mixedgetfreq',
             50,
             b'frequency of get vs set ops in mixed mode',
         ),
     ],
     norepo=True,
 )
 def perflrucache(
     ui,
     mincost=0,
     maxcost=100,
     costlimit=0,
     size=4,
     gets=10000,
     sets=10000,
     mixed=10000,
     mixedgetfreq=50,
     **opts
 ):
     opts = _byteskwargs(opts)
 
     def doinit():
         for i in _xrange(10000):
             util.lrucachedict(size)
 
     costrange = list(range(mincost, maxcost + 1))
 
     values = []
     for i in _xrange(size):
         values.append(random.randint(0, _maxint))
 
     # Get mode fills the cache and tests raw lookup performance with no
     # eviction.
     getseq = []
     for i in _xrange(gets):
         getseq.append(random.choice(values))
 
     def dogets():
         d = util.lrucachedict(size)
         for v in values:
             d[v] = v
         for key in getseq:
             value = d[key]
             value  # silence pyflakes warning
 
     def dogetscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(values):
             d.insert(v, v, cost=costs[i])
         for key in getseq:
             try:
                 value = d[key]
                 value  # silence pyflakes warning
             except KeyError:
                 pass
 
     # Set mode tests insertion speed with cache eviction.
     setseq = []
     costs = []
     for i in _xrange(sets):
         setseq.append(random.randint(0, _maxint))
         costs.append(random.choice(costrange))
 
     def doinserts():
         d = util.lrucachedict(size)
         for v in setseq:
             d.insert(v, v)
 
     def doinsertscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(setseq):
             d.insert(v, v, cost=costs[i])
 
     def dosets():
         d = util.lrucachedict(size)
         for v in setseq:
             d[v] = v
 
     # Mixed mode randomly performs gets and sets with eviction.
     mixedops = []
     for i in _xrange(mixed):
         r = random.randint(0, 100)
         if r < mixedgetfreq:
             op = 0
         else:
             op = 1
 
         mixedops.append(
             (op, random.randint(0, size * 2), random.choice(costrange))
         )
 
     def domixed():
         d = util.lrucachedict(size)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d[v] = v
 
     def domixedcost():
         d = util.lrucachedict(size, maxcost=costlimit)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d.insert(v, v, cost=cost)
 
     benches = [
         (doinit, b'init'),
     ]
 
     if costlimit:
         benches.extend(
             [
                 (dogetscost, b'gets w/ cost limit'),
                 (doinsertscost, b'inserts w/ cost limit'),
                 (domixedcost, b'mixed w/ cost limit'),
             ]
         )
     else:
         benches.extend(
             [
                 (dogets, b'gets'),
                 (doinserts, b'inserts'),
                 (dosets, b'sets'),
                 (domixed, b'mixed'),
             ]
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perf::write|perfwrite',
     formatteropts
     + [
         (b'', b'write-method', b'write', b'ui write method'),
         (b'', b'nlines', 100, b'number of lines'),
         (b'', b'nitems', 100, b'number of items (per line)'),
         (b'', b'item', b'x', b'item that is written'),
         (b'', b'batch-line', None, b'pass whole line to write method at once'),
         (b'', b'flush-line', None, b'flush after each line'),
     ],
 )
 def perfwrite(ui, repo, **opts):
     """microbenchmark ui.write (and others)"""
     opts = _byteskwargs(opts)
 
     write = getattr(ui, _sysstr(opts[b'write_method']))
     nlines = int(opts[b'nlines'])
     nitems = int(opts[b'nitems'])
     item = opts[b'item']
     batch_line = opts.get(b'batch_line')
     flush_line = opts.get(b'flush_line')
 
     if batch_line:
         line = item * nitems + b'\n'
 
     def benchmark():
         for i in pycompat.xrange(nlines):
             if batch_line:
                 write(line)
             else:
                 for i in pycompat.xrange(nitems):
                     write(item)
                 write(b'\n')
             if flush_line:
                 ui.flush()
         ui.flush()
 
     timer, fm = gettimer(ui, opts)
     timer(benchmark)
     fm.end()
 
 
 def uisetup(ui):
     if util.safehasattr(cmdutil, b'openrevlog') and not util.safehasattr(
         commands, b'debugrevlogopts'
     ):
         # for "historical portability":
         # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
         # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
         # openrevlog() should cause failure, because it has been
         # available since 3.5 (or 49c583ca48c4).
         def openrevlog(orig, repo, cmd, file_, opts):
             if opts.get(b'dir') and not util.safehasattr(repo, b'dirlog'):
                 raise error.Abort(
                     b"This version doesn't support --dir option",
                     hint=b"use 3.5 or later",
                 )
             return orig(repo, cmd, file_, opts)
 
         extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
 
 
 @command(
     b'perf::progress|perfprogress',
     formatteropts
     + [
         (b'', b'topic', b'topic', b'topic for progress messages'),
         (b'c', b'total', 1000000, b'total value we are progressing to'),
     ],
     norepo=True,
 )
 def perfprogress(ui, topic=None, total=None, **opts):
     """printing of progress bars"""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def doprogress():
         with ui.makeprogress(topic, total=total) as progress:
             for i in _xrange(total):
                 progress.increment()
 
     timer(doprogress)
     fm.end()

contrib/testparseutil.py

@@ -1,667 +1,657
 # testparseutil.py - utilities to parse test script for check tools
 #
 #  Copyright 2018 FUJIWARA Katsunori <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import abc
+import builtins
 import re
-import sys
 
 ####################
 # for Python3 compatibility (almost comes from mercurial/pycompat.py)
 
-ispy3 = sys.version_info[0] >= 3
-
 
 def identity(a):
     return a
 
 
 def _rapply(f, xs):
     if xs is None:
         # assume None means non-value of optional data
         return xs
     if isinstance(xs, (list, set, tuple)):
         return type(xs)(_rapply(f, x) for x in xs)
     if isinstance(xs, dict):
         return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items())
     return f(xs)
 
 
 def rapply(f, xs):
     if f is identity:
         # fast path mainly for py2
         return xs
     return _rapply(f, xs)
 
 
-if ispy3:
-    import builtins
-
-    def bytestr(s):
-        # tiny version of pycompat.bytestr
-        return s.encode('latin1')
-
-    def sysstr(s):
-        if isinstance(s, builtins.str):
-            return s
-        return s.decode('latin-1')
-
-    def opentext(f):
-        return open(f, 'r')
+def bytestr(s):
+    # tiny version of pycompat.bytestr
+    return s.encode('latin1')
 
 
-else:
-    bytestr = str
-    sysstr = identity
+def sysstr(s):
+    if isinstance(s, builtins.str):
+        return s
+    return s.decode('latin-1')
 
-    opentext = open
+
+def opentext(f):
+    return open(f, 'r')
 
 
 def b2s(x):
     # convert BYTES elements in "x" to SYSSTR recursively
     return rapply(sysstr, x)
 
 
 def writeout(data):
     # write "data" in BYTES into stdout
     sys.stdout.write(data)
 
 
 def writeerr(data):
     # write "data" in BYTES into stderr
     sys.stderr.write(data)
 
 
 ####################
 
 
 class embeddedmatcher:  # pytype: disable=ignored-metaclass
     """Base class to detect embedded code fragments in *.t test script"""
 
     __metaclass__ = abc.ABCMeta
 
     def __init__(self, desc):
         self.desc = desc
 
     @abc.abstractmethod
     def startsat(self, line):
         """Examine whether embedded code starts at line
 
         This can return arbitrary object, and it is used as 'ctx' for
         subsequent method invocations.
         """
 
     @abc.abstractmethod
     def endsat(self, ctx, line):
         """Examine whether embedded code ends at line"""
 
     @abc.abstractmethod
     def isinside(self, ctx, line):
         """Examine whether line is inside embedded code, if not yet endsat"""
 
     @abc.abstractmethod
     def ignores(self, ctx):
         """Examine whether detected embedded code should be ignored"""
 
     @abc.abstractmethod
     def filename(self, ctx):
         """Return filename of embedded code
 
         If filename isn't specified for embedded code explicitly, this
         returns None.
         """
 
     @abc.abstractmethod
     def codeatstart(self, ctx, line):
         """Return actual code at the start line of embedded code
 
         This might return None, if the start line doesn't contain
         actual code.
         """
 
     @abc.abstractmethod
     def codeatend(self, ctx, line):
         """Return actual code at the end line of embedded code
 
         This might return None, if the end line doesn't contain actual
         code.
         """
 
     @abc.abstractmethod
     def codeinside(self, ctx, line):
         """Return actual code at line inside embedded code"""
 
 
 def embedded(basefile, lines, errors, matchers):
     """pick embedded code fragments up from given lines
 
     This is common parsing logic, which examines specified matchers on
     given lines.
 
     :basefile: a name of a file, from which lines to be parsed come.
     :lines: to be parsed (might be a value returned by "open(basefile)")
     :errors: an array, into which messages for detected error are stored
     :matchers: an array of embeddedmatcher objects
 
     This function yields '(filename, starts, ends, code)' tuple.
 
     :filename: a name of embedded code, if it is explicitly specified
                (e.g.  "foobar" of "cat >> foobar <<EOF").
                Otherwise, this is None
     :starts: line number (1-origin), at which embedded code starts (inclusive)
     :ends: line number (1-origin), at which embedded code ends (exclusive)
     :code: extracted embedded code, which is single-stringified
 
     >>> class ambigmatcher:
     ...     # mock matcher class to examine implementation of
     ...     # "ambiguous matching" corner case
     ...     def __init__(self, desc, matchfunc):
     ...         self.desc = desc
     ...         self.matchfunc = matchfunc
     ...     def startsat(self, line):
     ...         return self.matchfunc(line)
     >>> ambig1 = ambigmatcher('ambiguous #1',
     ...                       lambda l: l.startswith('  $ cat '))
     >>> ambig2 = ambigmatcher('ambiguous #2',
     ...                       lambda l: l.endswith('<< EOF\\n'))
     >>> lines = ['  $ cat > foo.py << EOF\\n']
     >>> errors = []
     >>> matchers = [ambig1, ambig2]
     >>> list(t for t in embedded('<dummy>', lines, errors, matchers))
     []
     >>> b2s(errors)
     ['<dummy>:1: ambiguous line for "ambiguous #1", "ambiguous #2"']
 
     """
     matcher = None
     ctx = filename = code = startline = None  # for pyflakes
 
     for lineno, line in enumerate(lines, 1):
         if not line.endswith('\n'):
             line += '\n'  # to normalize EOF line
         if matcher:  # now, inside embedded code
             if matcher.endsat(ctx, line):
                 codeatend = matcher.codeatend(ctx, line)
                 if codeatend is not None:
                     code.append(codeatend)
                 if not matcher.ignores(ctx):
                     yield (filename, startline, lineno, ''.join(code))
                 matcher = None
                 # DO NOT "continue", because line might start next fragment
             elif not matcher.isinside(ctx, line):
                 # this is an error of basefile
                 # (if matchers are implemented correctly)
                 errors.append(
                     '%s:%d: unexpected line for "%s"'
                     % (basefile, lineno, matcher.desc)
                 )
                 # stop extracting embedded code by current 'matcher',
                 # because appearance of unexpected line might mean
                 # that expected end-of-embedded-code line might never
                 # appear
                 matcher = None
                 # DO NOT "continue", because line might start next fragment
             else:
                 code.append(matcher.codeinside(ctx, line))
                 continue
 
         # examine whether current line starts embedded code or not
         assert not matcher
 
         matched = []
         for m in matchers:
             ctx = m.startsat(line)
             if ctx:
                 matched.append((m, ctx))
         if matched:
             if len(matched) > 1:
                 # this is an error of matchers, maybe
                 errors.append(
                     '%s:%d: ambiguous line for %s'
                     % (
                         basefile,
                         lineno,
                         ', '.join(['"%s"' % m.desc for m, c in matched]),
                     )
                 )
                 # omit extracting embedded code, because choosing
                 # arbitrary matcher from matched ones might fail to
                 # detect the end of embedded code as expected.
                 continue
             matcher, ctx = matched[0]
             filename = matcher.filename(ctx)
             code = []
             codeatstart = matcher.codeatstart(ctx, line)
             if codeatstart is not None:
                 code.append(codeatstart)
                 startline = lineno
             else:
                 startline = lineno + 1
 
     if matcher:
         # examine whether EOF ends embedded code, because embedded
         # code isn't yet ended explicitly
         if matcher.endsat(ctx, '\n'):
             codeatend = matcher.codeatend(ctx, '\n')
             if codeatend is not None:
                 code.append(codeatend)
             if not matcher.ignores(ctx):
                 yield (filename, startline, lineno + 1, ''.join(code))
         else:
             # this is an error of basefile
             # (if matchers are implemented correctly)
             errors.append(
                 '%s:%d: unexpected end of file for "%s"'
                 % (basefile, lineno, matcher.desc)
             )
 
 
 # heredoc limit mark to ignore embedded code at check-code.py or so
 heredocignorelimit = 'NO_CHECK_EOF'
 
 # the pattern to match against cases below, and to return a limit mark
 # string as 'lname' group
 #
 # - << LIMITMARK
 # - << "LIMITMARK"
 # - << 'LIMITMARK'
 heredoclimitpat = r'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)'
 
 
 class fileheredocmatcher(embeddedmatcher):
     """Detect "cat > FILE << LIMIT" style embedded code
 
     >>> matcher = fileheredocmatcher('heredoc .py file', r'[^<]+\\.py')
     >>> b2s(matcher.startsat('  $ cat > file.py << EOF\\n'))
     ('file.py', '  > EOF\\n')
     >>> b2s(matcher.startsat('  $ cat   >>file.py   <<EOF\\n'))
     ('file.py', '  > EOF\\n')
     >>> b2s(matcher.startsat('  $ cat>  \\x27any file.py\\x27<<  "EOF"\\n'))
     ('any file.py', '  > EOF\\n')
     >>> b2s(matcher.startsat("  $ cat > file.py << 'ANYLIMIT'\\n"))
     ('file.py', '  > ANYLIMIT\\n')
     >>> b2s(matcher.startsat('  $ cat<<ANYLIMIT>"file.py"\\n'))
     ('file.py', '  > ANYLIMIT\\n')
     >>> start = '  $ cat > file.py << EOF\\n'
     >>> ctx = matcher.startsat(start)
     >>> matcher.codeatstart(ctx, start)
     >>> b2s(matcher.filename(ctx))
     'file.py'
     >>> matcher.ignores(ctx)
     False
     >>> inside = '  > foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> end = '  > EOF\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     >>> matcher.endsat(ctx, '  > EOFEOF\\n')
     False
     >>> ctx = matcher.startsat('  $ cat > file.py << NO_CHECK_EOF\\n')
     >>> matcher.ignores(ctx)
     True
     """
 
     _prefix = '  > '
 
     def __init__(self, desc, namepat):
         super(fileheredocmatcher, self).__init__(desc)
 
         # build the pattern to match against cases below (and ">>"
         # variants), and to return a target filename string as 'name'
         # group
         #
         # - > NAMEPAT
         # - > "NAMEPAT"
         # - > 'NAMEPAT'
         namepat = (
             r'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)' % namepat
         )
         self._fileres = [
             # "cat > NAME << LIMIT" case
             re.compile(r' {2}\$ \s*cat' + namepat + heredoclimitpat),
             # "cat << LIMIT > NAME" case
             re.compile(r' {2}\$ \s*cat' + heredoclimitpat + namepat),
         ]
 
     def startsat(self, line):
         # ctx is (filename, END-LINE-OF-EMBEDDED-CODE) tuple
         for filere in self._fileres:
             matched = filere.match(line)
             if matched:
                 return (
                     matched.group('name'),
                     '  > %s\n' % matched.group('limit'),
                 )
 
     def endsat(self, ctx, line):
         return ctx[1] == line
 
     def isinside(self, ctx, line):
         return line.startswith(self._prefix)
 
     def ignores(self, ctx):
         return '  > %s\n' % heredocignorelimit == ctx[1]
 
     def filename(self, ctx):
         return ctx[0]
 
     def codeatstart(self, ctx, line):
         return None  # no embedded code at start line
 
     def codeatend(self, ctx, line):
         return None  # no embedded code at end line
 
     def codeinside(self, ctx, line):
         return line[len(self._prefix) :]  # strip prefix
 
 
 ####
 # for embedded python script
 
 
 class pydoctestmatcher(embeddedmatcher):
     """Detect ">>> code" style embedded python code
 
     >>> matcher = pydoctestmatcher()
     >>> startline = '  >>> foo = 1\\n'
     >>> matcher.startsat(startline)
     True
     >>> matcher.startsat('  ... foo = 1\\n')
     False
     >>> ctx = matcher.startsat(startline)
     >>> matcher.filename(ctx)
     >>> matcher.ignores(ctx)
     False
     >>> b2s(matcher.codeatstart(ctx, startline))
     'foo = 1\\n'
     >>> inside = '  >>> foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> inside = '  ... foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> inside = '  expected output\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     '\\n'
     >>> inside = '  \\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     '\\n'
     >>> end = '  $ foo bar\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     >>> end = '\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     """
 
     _prefix = '  >>> '
     _prefixre = re.compile(r' {2}(>>>|\.\.\.) ')
 
     # If a line matches against not _prefixre but _outputre, that line
     # is "an expected output line" (= not a part of code fragment).
     #
     # Strictly speaking, a line matching against "(#if|#else|#endif)"
     # is also treated similarly in "inline python code" semantics by
     # run-tests.py. But "directive line inside inline python code"
     # should be rejected by Mercurial reviewers. Therefore, this
     # regexp does not matche against such directive lines.
     _outputre = re.compile(r' {2}$| {2}[^$]')
 
     def __init__(self):
         super(pydoctestmatcher, self).__init__("doctest style python code")
 
     def startsat(self, line):
         # ctx is "True"
         return line.startswith(self._prefix)
 
     def endsat(self, ctx, line):
         return not (self._prefixre.match(line) or self._outputre.match(line))
 
     def isinside(self, ctx, line):
         return True  # always true, if not yet ended
 
     def ignores(self, ctx):
         return False  # should be checked always
 
     def filename(self, ctx):
         return None  # no filename
 
     def codeatstart(self, ctx, line):
         return line[len(self._prefix) :]  # strip prefix '  >>> '/'  ... '
 
     def codeatend(self, ctx, line):
         return None  # no embedded code at end line
 
     def codeinside(self, ctx, line):
         if self._prefixre.match(line):
             return line[len(self._prefix) :]  # strip prefix '  >>> '/'  ... '
         return '\n'  # an expected output line is treated as an empty line
 
 
 class pyheredocmatcher(embeddedmatcher):
     """Detect "python << LIMIT" style embedded python code
 
     >>> matcher = pyheredocmatcher()
     >>> b2s(matcher.startsat('  $ python << EOF\\n'))
     '  > EOF\\n'
     >>> b2s(matcher.startsat('  $ $PYTHON   <<EOF\\n'))
     '  > EOF\\n'
     >>> b2s(matcher.startsat('  $ "$PYTHON"<<  "EOF"\\n'))
     '  > EOF\\n'
     >>> b2s(matcher.startsat("  $ $PYTHON << 'ANYLIMIT'\\n"))
     '  > ANYLIMIT\\n'
     >>> matcher.startsat('  $ "$PYTHON" < EOF\\n')
     >>> start = '  $ python << EOF\\n'
     >>> ctx = matcher.startsat(start)
     >>> matcher.codeatstart(ctx, start)
     >>> matcher.filename(ctx)
     >>> matcher.ignores(ctx)
     False
     >>> inside = '  > foo = 1\\n'
     >>> matcher.endsat(ctx, inside)
     False
     >>> matcher.isinside(ctx, inside)
     True
     >>> b2s(matcher.codeinside(ctx, inside))
     'foo = 1\\n'
     >>> end = '  > EOF\\n'
     >>> matcher.endsat(ctx, end)
     True
     >>> matcher.codeatend(ctx, end)
     >>> matcher.endsat(ctx, '  > EOFEOF\\n')
     False
     >>> ctx = matcher.startsat('  $ python << NO_CHECK_EOF\\n')
     >>> matcher.ignores(ctx)
     True
     """
 
     _prefix = '  > '
 
     _startre = re.compile(
         r' {2}\$ (\$PYTHON|"\$PYTHON"|python).*' + heredoclimitpat
     )
 
     def __init__(self):
         super(pyheredocmatcher, self).__init__("heredoc python invocation")
 
     def startsat(self, line):
         # ctx is END-LINE-OF-EMBEDDED-CODE
         matched = self._startre.match(line)
         if matched:
             return '  > %s\n' % matched.group('limit')
 
     def endsat(self, ctx, line):
         return ctx == line
 
     def isinside(self, ctx, line):
         return line.startswith(self._prefix)
 
     def ignores(self, ctx):
         return '  > %s\n' % heredocignorelimit == ctx
 
     def filename(self, ctx):
         return None  # no filename
 
     def codeatstart(self, ctx, line):
         return None  # no embedded code at start line
 
     def codeatend(self, ctx, line):
         return None  # no embedded code at end line
 
     def codeinside(self, ctx, line):
         return line[len(self._prefix) :]  # strip prefix
 
 
 _pymatchers = [
     pydoctestmatcher(),
     pyheredocmatcher(),
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
     fileheredocmatcher('heredoc .py file', r'[^<]+\.py'),
 ]
 
 
 def pyembedded(basefile, lines, errors):
     return embedded(basefile, lines, errors, _pymatchers)
 
 
 ####
 # for embedded shell script
 
 _shmatchers = [
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
     fileheredocmatcher('heredoc .sh file', r'[^<]+\.sh'),
 ]
 
 
 def shembedded(basefile, lines, errors):
     return embedded(basefile, lines, errors, _shmatchers)
 
 
 ####
 # for embedded hgrc configuration
 
 _hgrcmatchers = [
     # use '[^<]+' instead of '\S+', in order to match against
     # paths including whitespaces
     fileheredocmatcher(
         'heredoc hgrc file', r'(([^/<]+/)+hgrc|\$HGRCPATH|\${HGRCPATH})'
     ),
 ]
 
 
 def hgrcembedded(basefile, lines, errors):
     return embedded(basefile, lines, errors, _hgrcmatchers)
 
 
 ####
 
 if __name__ == "__main__":
     import optparse
     import sys
 
     def showembedded(basefile, lines, embeddedfunc, opts):
         errors = []
         for name, starts, ends, code in embeddedfunc(basefile, lines, errors):
             if not name:
                 name = '<anonymous>'
             writeout("%s:%d: %s starts\n" % (basefile, starts, name))
             if opts.verbose and code:
                 writeout("  |%s\n" % "\n  |".join(l for l in code.splitlines()))
             writeout("%s:%d: %s ends\n" % (basefile, ends, name))
         for e in errors:
             writeerr("%s\n" % e)
         return len(errors)
 
     def applyembedded(args, embeddedfunc, opts):
         ret = 0
         if args:
             for f in args:
                 with opentext(f) as fp:
                     if showembedded(f, fp, embeddedfunc, opts):
                         ret = 1
         else:
             lines = [l for l in sys.stdin.readlines()]
             if showembedded('<stdin>', lines, embeddedfunc, opts):
                 ret = 1
         return ret
 
     commands = {}
 
     def command(name, desc):
         def wrap(func):
             commands[name] = (desc, func)
 
         return wrap
 
     @command("pyembedded", "detect embedded python script")
     def pyembeddedcmd(args, opts):
         return applyembedded(args, pyembedded, opts)
 
     @command("shembedded", "detect embedded shell script")
     def shembeddedcmd(args, opts):
         return applyembedded(args, shembedded, opts)
 
     @command("hgrcembedded", "detect embedded hgrc configuration")
     def hgrcembeddedcmd(args, opts):
         return applyembedded(args, hgrcembedded, opts)
 
     availablecommands = "\n".join(
         ["  - %s: %s" % (key, value[0]) for key, value in commands.items()]
     )
 
     parser = optparse.OptionParser(
         """%prog COMMAND [file ...]
 
 Pick up embedded code fragments from given file(s) or stdin, and list
 up start/end lines of them in standard compiler format
 ("FILENAME:LINENO:").
 
 Available commands are:
 """
         + availablecommands
         + """
 """
     )
     parser.add_option(
         "-v",
         "--verbose",
         help="enable additional output (e.g. actual code)",
         action="store_true",
     )
     (opts, args) = parser.parse_args()
 
     if not args or args[0] not in commands:
         parser.print_help()
         sys.exit(255)
 
     sys.exit(commands[args[0]][1](args[1:], opts))

doc/check-seclevel.py

@@ -1,232 +1,242
 #!/usr/bin/env python3
 #
 # checkseclevel - checking section title levels in each online help document
 
 
 import optparse
 import os
 import sys
 
 # import from the live mercurial repo
 os.environ['HGMODULEPOLICY'] = 'py'
 sys.path.insert(0, os.path.abspath(".."))
 from mercurial import demandimport
 
 demandimport.enable()
 from mercurial import (
     commands,
     extensions,
     help,
     minirst,
     ui as uimod,
 )
 
 table = commands.table
 helptable = help.helptable
 
 level2mark = [b'"', b'=', b'-', b'.', b'#']
 reservedmarks = [b'"']
 
 mark2level = {}
 for m, l in zip(level2mark, range(len(level2mark))):
     if m not in reservedmarks:
         mark2level[m] = l
 
 initlevel_topic = 0
 initlevel_cmd = 1
 initlevel_ext = 1
 initlevel_ext_cmd = 3
 
 
 def showavailables(ui, initlevel):
     avail = '    available marks and order of them in this help: %s\n' % (
         ', '.join(['%r' % (m * 4) for m in level2mark[initlevel + 1 :]])
     )
     ui.warn(avail.encode('utf-8'))
 
 
 def checkseclevel(ui, doc, name, initlevel):
-    ui.notenoi18n('checking "%s"\n' % name)
+    ui.notenoi18n(('checking "%s"\n' % name).encode('utf-8'))
     if not isinstance(doc, bytes):
         doc = doc.encode('utf-8')
     blocks, pruned = minirst.parse(doc, 0, ['verbose'])
     errorcnt = 0
     curlevel = initlevel
     for block in blocks:
         if block[b'type'] != b'section':
             continue
         mark = block[b'underline']
         title = block[b'lines'][0]
         if (mark not in mark2level) or (mark2level[mark] <= initlevel):
             ui.warn(
                 (
                     'invalid section mark %r for "%s" of %s\n'
                     % (mark * 4, title, name)
                 ).encode('utf-8')
             )
             showavailables(ui, initlevel)
             errorcnt += 1
             continue
         nextlevel = mark2level[mark]
         if curlevel < nextlevel and curlevel + 1 != nextlevel:
             ui.warnnoi18n(
-                'gap of section level at "%s" of %s\n' % (title, name)
+                ('gap of section level at "%s" of %s\n' % (title, name)).encode(
+                    'utf-8'
+                )
             )
             showavailables(ui, initlevel)
             errorcnt += 1
             continue
         ui.notenoi18n(
-            'appropriate section level for "%s %s"\n'
-            % (mark * (nextlevel * 2), title)
+            (
+                'appropriate section level for "%s %s"\n'
+                % (mark * (nextlevel * 2), title)
+            ).encode('utf-8')
         )
         curlevel = nextlevel
 
     return errorcnt
 
 
 def checkcmdtable(ui, cmdtable, namefmt, initlevel):
     errorcnt = 0
     for k, entry in cmdtable.items():
         name = k.split(b"|")[0].lstrip(b"^")
         if not entry[0].__doc__:
             ui.notenoi18n(
-                'skip checking %s: no help document\n' % (namefmt % name)
+                (
+                    'skip checking %s: no help document\n' % (namefmt % name)
+                ).encode('utf-8')
             )
             continue
         errorcnt += checkseclevel(
             ui, entry[0].__doc__, namefmt % name, initlevel
         )
     return errorcnt
 
 
 def checkhghelps(ui):
     errorcnt = 0
     for h in helptable:
         names, sec, doc = h[0:3]
         if callable(doc):
             doc = doc(ui)
         errorcnt += checkseclevel(
             ui, doc, '%s help topic' % names[0], initlevel_topic
         )
 
     errorcnt += checkcmdtable(ui, table, '%s command', initlevel_cmd)
 
     for name in sorted(
         list(extensions.enabled()) + list(extensions.disabled())
     ):
         mod = extensions.load(ui, name, None)
         if not mod.__doc__:
             ui.notenoi18n(
-                'skip checking %s extension: no help document\n' % name
+                (
+                    'skip checking %s extension: no help document\n' % name
+                ).encode('utf-8')
             )
             continue
         errorcnt += checkseclevel(
             ui, mod.__doc__, '%s extension' % name, initlevel_ext
         )
 
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
             errorcnt += checkcmdtable(
                 ui,
                 cmdtable,
                 '%%s command of %s extension' % name,
                 initlevel_ext_cmd,
             )
     return errorcnt
 
 
 def checkfile(ui, filename, initlevel):
     if filename == '-':
         filename = 'stdin'
         doc = sys.stdin.read()
     else:
         with open(filename) as fp:
             doc = fp.read()
 
     ui.notenoi18n(
-        'checking input from %s with initlevel %d\n' % (filename, initlevel)
+        (
+            'checking input from %s with initlevel %d\n' % (filename, initlevel)
+        ).encode('utf-8')
     )
     return checkseclevel(ui, doc, 'input from %s' % filename, initlevel)
 
 
 def main():
     optparser = optparse.OptionParser(
         """%prog [options]
 
 This checks all help documents of Mercurial (topics, commands,
 extensions and commands of them), if no file is specified by --file
 option.
 """
     )
     optparser.add_option(
         "-v", "--verbose", help="enable additional output", action="store_true"
     )
     optparser.add_option(
         "-d", "--debug", help="debug mode", action="store_true"
     )
     optparser.add_option(
         "-f",
         "--file",
         help="filename to read in (or '-' for stdin)",
         action="store",
         default="",
     )
 
     optparser.add_option(
         "-t",
         "--topic",
         help="parse file as help topic",
         action="store_const",
         dest="initlevel",
         const=0,
     )
     optparser.add_option(
         "-c",
         "--command",
         help="parse file as help of core command",
         action="store_const",
         dest="initlevel",
         const=1,
     )
     optparser.add_option(
         "-e",
         "--extension",
         help="parse file as help of extension",
         action="store_const",
         dest="initlevel",
         const=1,
     )
     optparser.add_option(
         "-C",
         "--extension-command",
         help="parse file as help of extension command",
         action="store_const",
         dest="initlevel",
         const=3,
     )
 
     optparser.add_option(
         "-l",
         "--initlevel",
         help="set initial section level manually",
         action="store",
         type="int",
         default=0,
     )
 
     (options, args) = optparser.parse_args()
 
     ui = uimod.ui.load()
     ui.setconfig(b'ui', b'verbose', options.verbose, b'--verbose')
     ui.setconfig(b'ui', b'debug', options.debug, b'--debug')
 
     if options.file:
         if checkfile(ui, options.file, options.initlevel):
             sys.exit(1)
     else:
         if checkhghelps(ui):
             sys.exit(1)
 
 
 if __name__ == "__main__":
     main()

hgdemandimport/demandimportpy3.py

@@ -1,174 +1,178
 # demandimportpy3 - global demand-loading of modules for Mercurial
 #
 # Copyright 2017 Facebook Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Lazy loading for Python 3.6 and above.
 
 This uses the new importlib finder/loader functionality available in Python 3.5
 and up. The code reuses most of the mechanics implemented inside importlib.util,
 but with a few additions:
 
 * Allow excluding certain modules from lazy imports.
 * Expose an interface that's substantially the same as demandimport for
   Python 2.
 
 This also has some limitations compared to the Python 2 implementation:
 
 * Much of the logic is per-package, not per-module, so any packages loaded
   before demandimport is enabled will not be lazily imported in the future. In
   practice, we only expect builtins to be loaded before demandimport is
   enabled.
 """
 
-# This line is unnecessary, but it satisfies test-check-py3-compat.t.
-
 import contextlib
 import importlib.util
 import sys
 
 from . import tracing
 
 _deactivated = False
 
 
 class _lazyloaderex(importlib.util.LazyLoader):
     """This is a LazyLoader except it also follows the _deactivated global and
     the ignore list.
     """
 
+    _HAS_DYNAMIC_ATTRIBUTES = True  # help pytype not flag self.loader
+
     def exec_module(self, module):
         """Make the module load lazily."""
         with tracing.log('demandimport %s', module):
             if _deactivated or module.__name__ in ignores:
+                # Reset the loader on the module as super() does (issue6725)
+                module.__spec__.loader = self.loader
+                module.__loader__ = self.loader
+
                 self.loader.exec_module(module)
             else:
                 super().exec_module(module)
 
 
 class LazyFinder:
     """A wrapper around a ``MetaPathFinder`` that makes loaders lazy.
 
     ``sys.meta_path`` finders have their ``find_spec()`` called to locate a
     module. This returns a ``ModuleSpec`` if found or ``None``. The
     ``ModuleSpec`` has a ``loader`` attribute, which is called to actually
     load a module.
 
     Our class wraps an existing finder and overloads its ``find_spec()`` to
     replace the ``loader`` with our lazy loader proxy.
 
     We have to use __getattribute__ to proxy the instance because some meta
     path finders don't support monkeypatching.
     """
 
     __slots__ = ("_finder",)
 
     def __init__(self, finder):
         object.__setattr__(self, "_finder", finder)
 
     def __repr__(self):
         return "<LazyFinder for %r>" % object.__getattribute__(self, "_finder")
 
     # __bool__ is canonical Python 3. But check-code insists on __nonzero__ being
     # defined via `def`.
     def __nonzero__(self):
         return bool(object.__getattribute__(self, "_finder"))
 
     __bool__ = __nonzero__
 
     def __getattribute__(self, name):
         if name in ("_finder", "find_spec"):
             return object.__getattribute__(self, name)
 
         return getattr(object.__getattribute__(self, "_finder"), name)
 
     def __delattr__(self, name):
         return delattr(object.__getattribute__(self, "_finder"), name)
 
     def __setattr__(self, name, value):
         return setattr(object.__getattribute__(self, "_finder"), name, value)
 
     def find_spec(self, fullname, path, target=None):
         finder = object.__getattribute__(self, "_finder")
         try:
             find_spec = finder.find_spec
         except AttributeError:
             loader = finder.find_module(fullname, path)
             if loader is None:
                 spec = None
             else:
                 spec = importlib.util.spec_from_loader(fullname, loader)
         else:
             spec = find_spec(fullname, path, target)
 
         # Lazy loader requires exec_module().
         if (
             spec is not None
             and spec.loader is not None
             and getattr(spec.loader, "exec_module", None)
         ):
             spec.loader = _lazyloaderex(spec.loader)
 
         return spec
 
 
 ignores = set()
 
 
 def init(ignoreset):
     global ignores
     ignores = ignoreset
 
 
 def isenabled():
     return not _deactivated and any(
         isinstance(finder, LazyFinder) for finder in sys.meta_path
     )
 
 
 def disable():
     new_finders = []
     for finder in sys.meta_path:
         new_finders.append(
             finder._finder if isinstance(finder, LazyFinder) else finder
         )
     sys.meta_path[:] = new_finders
 
 
 def enable():
     new_finders = []
     for finder in sys.meta_path:
         new_finders.append(
             LazyFinder(finder) if not isinstance(finder, LazyFinder) else finder
         )
     sys.meta_path[:] = new_finders
 
 
 @contextlib.contextmanager
 def deactivated():
     # This implementation is a bit different from Python 2's. Python 3
     # maintains a per-package finder cache in sys.path_importer_cache (see
     # PEP 302). This means that we can't just call disable + enable.
     # If we do that, in situations like:
     #
     #   demandimport.enable()
     #   ...
     #   from foo.bar import mod1
     #   with demandimport.deactivated():
     #       from foo.bar import mod2
     #
     # mod2 will be imported lazily. (The converse also holds -- whatever finder
     # first gets cached will be used.)
     #
     # Instead, have a global flag the LazyLoader can use.
     global _deactivated
     demandenabled = isenabled()
     if demandenabled:
         _deactivated = True
     try:
         yield
     finally:
         if demandenabled:
             _deactivated = False

hgext/absorb.py

@@ -1,1165 +1,1165
 # absorb.py
 #
 # Copyright 2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """apply working directory changes to changesets (EXPERIMENTAL)
 
 The absorb extension provides a command to use annotate information to
 amend modified chunks into the corresponding non-public changesets.
 
 ::
 
     [absorb]
     # only check 50 recent non-public changesets at most
     max-stack-size = 50
     # whether to add noise to new commits to avoid obsolescence cycle
     add-noise = 1
     # make `amend --correlated` a shortcut to the main command
     amend-flag = correlated
 
     [color]
     absorb.description = yellow
     absorb.node = blue bold
     absorb.path = bold
 """
 
 # TODO:
 #  * Rename config items to [commands] namespace
 #  * Converge getdraftstack() with other code in core
 #  * move many attributes on fixupstate to be private
 
 
 import collections
 
 from mercurial.i18n import _
 from mercurial.node import (
     hex,
     short,
 )
 from mercurial import (
     cmdutil,
     commands,
     context,
     crecord,
     error,
     linelog,
     mdiff,
     obsolete,
     patch,
     phases,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
 )
 from mercurial.utils import stringutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(b'absorb', b'add-noise', default=True)
 configitem(b'absorb', b'amend-flag', default=None)
 configitem(b'absorb', b'max-stack-size', default=50)
 
 colortable = {
     b'absorb.description': b'yellow',
     b'absorb.node': b'blue bold',
     b'absorb.path': b'bold',
 }
 
 defaultdict = collections.defaultdict
 
 
 class nullui:
     """blank ui object doing nothing"""
 
     debugflag = False
     verbose = False
     quiet = True
 
     def __getitem__(name):
         def nullfunc(*args, **kwds):
             return
 
         return nullfunc
 
 
 class emptyfilecontext:
     """minimal filecontext representing an empty file"""
 
     def __init__(self, repo):
         self._repo = repo
 
     def data(self):
         return b''
 
     def node(self):
         return self._repo.nullid
 
 
 def uniq(lst):
     """list -> list. remove duplicated items without changing the order"""
     seen = set()
     result = []
     for x in lst:
         if x not in seen:
             seen.add(x)
             result.append(x)
     return result
 
 
 def getdraftstack(headctx, limit=None):
     """(ctx, int?) -> [ctx]. get a linear stack of non-public changesets.
 
     changesets are sorted in topo order, oldest first.
     return at most limit items, if limit is a positive number.
 
     merges are considered as non-draft as well. i.e. every commit
     returned has and only has 1 parent.
     """
     ctx = headctx
     result = []
     while ctx.phase() != phases.public:
         if limit and len(result) >= limit:
             break
         parents = ctx.parents()
         if len(parents) != 1:
             break
         result.append(ctx)
         ctx = parents[0]
     result.reverse()
     return result
 
 
 def getfilestack(stack, path, seenfctxs=None):
     """([ctx], str, set) -> [fctx], {ctx: fctx}
 
     stack is a list of contexts, from old to new. usually they are what
     "getdraftstack" returns.
 
     follows renames, but not copies.
 
     seenfctxs is a set of filecontexts that will be considered "immutable".
     they are usually what this function returned in earlier calls, useful
     to avoid issues that a file was "moved" to multiple places and was then
     modified differently, like: "a" was copied to "b", "a" was also copied to
     "c" and then "a" was deleted, then both "b" and "c" were "moved" from "a"
     and we enforce only one of them to be able to affect "a"'s content.
 
     return an empty list and an empty dict, if the specified path does not
     exist in stack[-1] (the top of the stack).
 
     otherwise, return a list of de-duplicated filecontexts, and the map to
     convert ctx in the stack to fctx, for possible mutable fctxs. the first item
     of the list would be outside the stack and should be considered immutable.
     the remaining items are within the stack.
 
     for example, given the following changelog and corresponding filelog
     revisions:
 
       changelog: 3----4----5----6----7
       filelog:   x    0----1----1----2 (x: no such file yet)
 
     - if stack = [5, 6, 7], returns ([0, 1, 2], {5: 1, 6: 1, 7: 2})
     - if stack = [3, 4, 5], returns ([e, 0, 1], {4: 0, 5: 1}), where "e" is a
       dummy empty filecontext.
     - if stack = [2], returns ([], {})
     - if stack = [7], returns ([1, 2], {7: 2})
     - if stack = [6, 7], returns ([1, 2], {6: 1, 7: 2}), although {6: 1} can be
       removed, since 1 is immutable.
     """
     if seenfctxs is None:
         seenfctxs = set()
     assert stack
 
     if path not in stack[-1]:
         return [], {}
 
     fctxs = []
     fctxmap = {}
 
     pctx = stack[0].p1()  # the public (immutable) ctx we stop at
     for ctx in reversed(stack):
         if path not in ctx:  # the file is added in the next commit
             pctx = ctx
             break
         fctx = ctx[path]
         fctxs.append(fctx)
         if fctx in seenfctxs:  # treat fctx as the immutable one
             pctx = None  # do not add another immutable fctx
             break
         fctxmap[ctx] = fctx  # only for mutable fctxs
         copy = fctx.copysource()
         if copy:
             path = copy  # follow rename
             if path in ctx:  # but do not follow copy
                 pctx = ctx.p1()
                 break
 
     if pctx is not None:  # need an extra immutable fctx
         if path in pctx:
             fctxs.append(pctx[path])
         else:
             fctxs.append(emptyfilecontext(pctx.repo()))
 
     fctxs.reverse()
     # note: we rely on a property of hg: filerev is not reused for linear
     # history. i.e. it's impossible to have:
     #   changelog:  4----5----6 (linear, no merges)
     #   filelog:    1----2----1
     #                         ^ reuse filerev (impossible)
     # because parents are part of the hash. if that's not true, we need to
     # remove uniq and find a different way to identify fctxs.
     return uniq(fctxs), fctxmap
 
 
 class overlaystore(patch.filestore):
     """read-only, hybrid store based on a dict and ctx.
     memworkingcopy: {path: content}, overrides file contents.
     """
 
     def __init__(self, basectx, memworkingcopy):
         self.basectx = basectx
         self.memworkingcopy = memworkingcopy
 
     def getfile(self, path):
         """comply with mercurial.patch.filestore.getfile"""
         if path not in self.basectx:
             return None, None, None
         fctx = self.basectx[path]
         if path in self.memworkingcopy:
             content = self.memworkingcopy[path]
         else:
             content = fctx.data()
         mode = (fctx.islink(), fctx.isexec())
         copy = fctx.copysource()
         return content, mode, copy
 
 
 def overlaycontext(memworkingcopy, ctx, parents=None, extra=None, desc=None):
     """({path: content}, ctx, (p1node, p2node)?, {}?) -> memctx
     memworkingcopy overrides file contents.
     """
     # parents must contain 2 items: (node1, node2)
     if parents is None:
         parents = ctx.repo().changelog.parents(ctx.node())
     if extra is None:
         extra = ctx.extra()
     if desc is None:
         desc = ctx.description()
     date = ctx.date()
     user = ctx.user()
     files = set(ctx.files()).union(memworkingcopy)
     store = overlaystore(ctx, memworkingcopy)
     return context.memctx(
         repo=ctx.repo(),
         parents=parents,
         text=desc,
         files=files,
         filectxfn=store,
         user=user,
         date=date,
         branch=None,
         extra=extra,
     )
 
 
 class filefixupstate:
     """state needed to apply fixups to a single file
 
     internally, it keeps file contents of several revisions and a linelog.
 
     the linelog uses odd revision numbers for original contents (fctxs passed
     to __init__), and even revision numbers for fixups, like:
 
         linelog rev 1: self.fctxs[0] (from an immutable "public" changeset)
         linelog rev 2: fixups made to self.fctxs[0]
         linelog rev 3: self.fctxs[1] (a child of fctxs[0])
         linelog rev 4: fixups made to self.fctxs[1]
         ...
 
     a typical use is like:
 
         1. call diffwith, to calculate self.fixups
         2. (optionally), present self.fixups to the user, or change it
         3. call apply, to apply changes
         4. read results from "finalcontents", or call getfinalcontent
     """
 
     def __init__(self, fctxs, path, ui=None, opts=None):
         """([fctx], ui or None) -> None
 
         fctxs should be linear, and sorted by topo order - oldest first.
         fctxs[0] will be considered as "immutable" and will not be changed.
         """
         self.fctxs = fctxs
         self.path = path
         self.ui = ui or nullui()
         self.opts = opts or {}
 
         # following fields are built from fctxs. they exist for perf reason
         self.contents = [f.data() for f in fctxs]
         self.contentlines = pycompat.maplist(mdiff.splitnewlines, self.contents)
         self.linelog = self._buildlinelog()
         if self.ui.debugflag:
             assert self._checkoutlinelog() == self.contents
 
         # following fields will be filled later
         self.chunkstats = [0, 0]  # [adopted, total : int]
         self.targetlines = []  # [str]
         self.fixups = []  # [(linelog rev, a1, a2, b1, b2)]
         self.finalcontents = []  # [str]
         self.ctxaffected = set()
 
     def diffwith(self, targetfctx, fm=None):
         """calculate fixups needed by examining the differences between
         self.fctxs[-1] and targetfctx, chunk by chunk.
 
         targetfctx is the target state we move towards. we may or may not be
         able to get there because not all modified chunks can be amended into
         a non-public fctx unambiguously.
 
         call this only once, before apply().
 
         update self.fixups, self.chunkstats, and self.targetlines.
         """
         a = self.contents[-1]
         alines = self.contentlines[-1]
         b = targetfctx.data()
         blines = mdiff.splitnewlines(b)
         self.targetlines = blines
 
         self.linelog.annotate(self.linelog.maxrev)
         annotated = self.linelog.annotateresult  # [(linelog rev, linenum)]
         assert len(annotated) == len(alines)
         # add a dummy end line to make insertion at the end easier
         if annotated:
             dummyendline = (annotated[-1][0], annotated[-1][1] + 1)
             annotated.append(dummyendline)
 
         # analyse diff blocks
         for chunk in self._alldiffchunks(a, b, alines, blines):
             newfixups = self._analysediffchunk(chunk, annotated)
             self.chunkstats[0] += bool(newfixups)  # 1 or 0
             self.chunkstats[1] += 1
             self.fixups += newfixups
             if fm is not None:
                 self._showchanges(fm, alines, blines, chunk, newfixups)
 
     def apply(self):
         """apply self.fixups. update self.linelog, self.finalcontents.
 
         call this only once, before getfinalcontent(), after diffwith().
         """
         # the following is unnecessary, as it's done by "diffwith":
         #   self.linelog.annotate(self.linelog.maxrev)
         for rev, a1, a2, b1, b2 in reversed(self.fixups):
             blines = self.targetlines[b1:b2]
             if self.ui.debugflag:
                 idx = (max(rev - 1, 0)) // 2
                 self.ui.write(
                     _(b'%s: chunk %d:%d -> %d lines\n')
                     % (short(self.fctxs[idx].node()), a1, a2, len(blines))
                 )
             self.linelog.replacelines(rev, a1, a2, b1, b2)
         if self.opts.get(b'edit_lines', False):
             self.finalcontents = self._checkoutlinelogwithedits()
         else:
             self.finalcontents = self._checkoutlinelog()
 
     def getfinalcontent(self, fctx):
         """(fctx) -> str. get modified file content for a given filecontext"""
         idx = self.fctxs.index(fctx)
         return self.finalcontents[idx]
 
     def _analysediffchunk(self, chunk, annotated):
         """analyse a different chunk and return new fixups found
 
         return [] if no lines from the chunk can be safely applied.
 
         the chunk (or lines) cannot be safely applied, if, for example:
           - the modified (deleted) lines belong to a public changeset
             (self.fctxs[0])
           - the chunk is a pure insertion and the adjacent lines (at most 2
             lines) belong to different non-public changesets, or do not belong
             to any non-public changesets.
           - the chunk is modifying lines from different changesets.
             in this case, if the number of lines deleted equals to the number
             of lines added, assume it's a simple 1:1 map (could be wrong).
             otherwise, give up.
           - the chunk is modifying lines from a single non-public changeset,
             but other revisions touch the area as well. i.e. the lines are
             not continuous as seen from the linelog.
         """
         a1, a2, b1, b2 = chunk
         # find involved indexes from annotate result
         involved = annotated[a1:a2]
         if not involved and annotated:  # a1 == a2 and a is not empty
             # pure insertion, check nearby lines. ignore lines belong
             # to the public (first) changeset (i.e. annotated[i][0] == 1)
             nearbylinenums = {a2, max(0, a1 - 1)}
             involved = [
                 annotated[i] for i in nearbylinenums if annotated[i][0] != 1
             ]
         involvedrevs = list({r for r, l in involved})
         newfixups = []
         if len(involvedrevs) == 1 and self._iscontinuous(a1, a2 - 1, True):
             # chunk belongs to a single revision
             rev = involvedrevs[0]
             if rev > 1:
                 fixuprev = rev + 1
                 newfixups.append((fixuprev, a1, a2, b1, b2))
         elif a2 - a1 == b2 - b1 or b1 == b2:
             # 1:1 line mapping, or chunk was deleted
             for i in range(a1, a2):
                 rev, linenum = annotated[i]
                 if rev > 1:
                     if b1 == b2:  # deletion, simply remove that single line
                         nb1 = nb2 = 0
                     else:  # 1:1 line mapping, change the corresponding rev
                         nb1 = b1 + i - a1
                         nb2 = nb1 + 1
                     fixuprev = rev + 1
                     newfixups.append((fixuprev, i, i + 1, nb1, nb2))
         return self._optimizefixups(newfixups)
 
     @staticmethod
     def _alldiffchunks(a, b, alines, blines):
         """like mdiff.allblocks, but only care about differences"""
         blocks = mdiff.allblocks(a, b, lines1=alines, lines2=blines)
         for chunk, btype in blocks:
             if btype != b'!':
                 continue
             yield chunk
 
     def _buildlinelog(self):
         """calculate the initial linelog based on self.content{,line}s.
         this is similar to running a partial "annotate".
         """
         llog = linelog.linelog()
         a, alines = b'', []
         for i in range(len(self.contents)):
             b, blines = self.contents[i], self.contentlines[i]
             llrev = i * 2 + 1
             chunks = self._alldiffchunks(a, b, alines, blines)
             for a1, a2, b1, b2 in reversed(list(chunks)):
                 llog.replacelines(llrev, a1, a2, b1, b2)
             a, alines = b, blines
         return llog
 
     def _checkoutlinelog(self):
         """() -> [str]. check out file contents from linelog"""
         contents = []
         for i in range(len(self.contents)):
             rev = (i + 1) * 2
             self.linelog.annotate(rev)
             content = b''.join(map(self._getline, self.linelog.annotateresult))
             contents.append(content)
         return contents
 
     def _checkoutlinelogwithedits(self):
         """() -> [str]. prompt all lines for edit"""
         alllines = self.linelog.getalllines()
         # header
         editortext = (
             _(
                 b'HG: editing %s\nHG: "y" means the line to the right '
                 b'exists in the changeset to the top\nHG:\n'
             )
             % self.fctxs[-1].path()
         )
         # [(idx, fctx)]. hide the dummy emptyfilecontext
         visiblefctxs = [
             (i, f)
             for i, f in enumerate(self.fctxs)
             if not isinstance(f, emptyfilecontext)
         ]
         for i, (j, f) in enumerate(visiblefctxs):
             editortext += _(b'HG: %s/%s %s %s\n') % (
                 b'|' * i,
                 b'-' * (len(visiblefctxs) - i + 1),
                 short(f.node()),
                 f.description().split(b'\n', 1)[0],
             )
         editortext += _(b'HG: %s\n') % (b'|' * len(visiblefctxs))
         # figure out the lifetime of a line, this is relatively inefficient,
         # but probably fine
         lineset = defaultdict(lambda: set())  # {(llrev, linenum): {llrev}}
         for i, f in visiblefctxs:
             self.linelog.annotate((i + 1) * 2)
             for l in self.linelog.annotateresult:
                 lineset[l].add(i)
         # append lines
         for l in alllines:
             editortext += b'    %s : %s' % (
                 b''.join(
                     [
                         (b'y' if i in lineset[l] else b' ')
                         for i, _f in visiblefctxs
                     ]
                 ),
                 self._getline(l),
             )
         # run editor
         editedtext = self.ui.edit(editortext, b'', action=b'absorb')
         if not editedtext:
             raise error.InputError(_(b'empty editor text'))
         # parse edited result
         contents = [b''] * len(self.fctxs)
         leftpadpos = 4
         colonpos = leftpadpos + len(visiblefctxs) + 1
         for l in mdiff.splitnewlines(editedtext):
             if l.startswith(b'HG:'):
                 continue
             if l[colonpos - 1 : colonpos + 2] != b' : ':
                 raise error.InputError(_(b'malformed line: %s') % l)
             linecontent = l[colonpos + 2 :]
             for i, ch in enumerate(
                 pycompat.bytestr(l[leftpadpos : colonpos - 1])
             ):
                 if ch == b'y':
                     contents[visiblefctxs[i][0]] += linecontent
         # chunkstats is hard to calculate if anything changes, therefore
         # set them to just a simple value (1, 1).
         if editedtext != editortext:
             self.chunkstats = [1, 1]
         return contents
 
     def _getline(self, lineinfo):
         """((rev, linenum)) -> str. convert rev+line number to line content"""
         rev, linenum = lineinfo
         if rev & 1:  # odd: original line taken from fctxs
             return self.contentlines[rev // 2][linenum]
         else:  # even: fixup line from targetfctx
             return self.targetlines[linenum]
 
     def _iscontinuous(self, a1, a2, closedinterval=False):
         """(a1, a2 : int) -> bool
 
         check if these lines are continuous. i.e. no other insertions or
         deletions (from other revisions) among these lines.
 
         closedinterval decides whether a2 should be included or not. i.e. is
         it [a1, a2), or [a1, a2] ?
         """
         if a1 >= a2:
             return True
         llog = self.linelog
         offset1 = llog.getoffset(a1)
         offset2 = llog.getoffset(a2) + int(closedinterval)
         linesinbetween = llog.getalllines(offset1, offset2)
         return len(linesinbetween) == a2 - a1 + int(closedinterval)
 
     def _optimizefixups(self, fixups):
         """[(rev, a1, a2, b1, b2)] -> [(rev, a1, a2, b1, b2)].
         merge adjacent fixups to make them less fragmented.
         """
         result = []
         pcurrentchunk = [[-1, -1, -1, -1, -1]]
 
         def pushchunk():
             if pcurrentchunk[0][0] != -1:
                 result.append(tuple(pcurrentchunk[0]))
 
         for i, chunk in enumerate(fixups):
             rev, a1, a2, b1, b2 = chunk
             lastrev = pcurrentchunk[0][0]
             lasta2 = pcurrentchunk[0][2]
             lastb2 = pcurrentchunk[0][4]
             if (
                 a1 == lasta2
                 and b1 == lastb2
                 and rev == lastrev
                 and self._iscontinuous(max(a1 - 1, 0), a1)
             ):
                 # merge into currentchunk
                 pcurrentchunk[0][2] = a2
                 pcurrentchunk[0][4] = b2
             else:
                 pushchunk()
                 pcurrentchunk[0] = list(chunk)
         pushchunk()
         return result
 
     def _showchanges(self, fm, alines, blines, chunk, fixups):
         def trim(line):
             if line.endswith(b'\n'):
                 line = line[:-1]
             return line
 
         # this is not optimized for perf but _showchanges only gets executed
         # with an extra command-line flag.
         a1, a2, b1, b2 = chunk
         aidxs, bidxs = [0] * (a2 - a1), [0] * (b2 - b1)
         for idx, fa1, fa2, fb1, fb2 in fixups:
             for i in range(fa1, fa2):
                 aidxs[i - a1] = (max(idx, 1) - 1) // 2
             for i in range(fb1, fb2):
                 bidxs[i - b1] = (max(idx, 1) - 1) // 2
 
         fm.startitem()
         fm.write(
             b'hunk',
             b'        %s\n',
             b'@@ -%d,%d +%d,%d @@' % (a1, a2 - a1, b1, b2 - b1),
             label=b'diff.hunk',
         )
         fm.data(path=self.path, linetype=b'hunk')
 
         def writeline(idx, diffchar, line, linetype, linelabel):
             fm.startitem()
             node = b''
             if idx:
                 ctx = self.fctxs[idx]
                 fm.context(fctx=ctx)
                 node = ctx.hex()
                 self.ctxaffected.add(ctx.changectx())
             fm.write(b'node', b'%-7.7s ', node, label=b'absorb.node')
             fm.write(
                 b'diffchar ' + linetype,
                 b'%s%s\n',
                 diffchar,
                 line,
                 label=linelabel,
             )
             fm.data(path=self.path, linetype=linetype)
 
         for i in range(a1, a2):
             writeline(
                 aidxs[i - a1],
                 b'-',
                 trim(alines[i]),
                 b'deleted',
                 b'diff.deleted',
             )
         for i in range(b1, b2):
             writeline(
                 bidxs[i - b1],
                 b'+',
                 trim(blines[i]),
                 b'inserted',
                 b'diff.inserted',
             )
 
 
 class fixupstate:
     """state needed to run absorb
 
     internally, it keeps paths and filefixupstates.
 
     a typical use is like filefixupstates:
 
         1. call diffwith, to calculate fixups
         2. (optionally), present fixups to the user, or edit fixups
         3. call apply, to apply changes to memory
         4. call commit, to commit changes to hg database
     """
 
     def __init__(self, stack, ui=None, opts=None):
         """([ctx], ui or None) -> None
 
         stack: should be linear, and sorted by topo order - oldest first.
         all commits in stack are considered mutable.
         """
         assert stack
         self.ui = ui or nullui()
         self.opts = opts or {}
         self.stack = stack
         self.repo = stack[-1].repo().unfiltered()
 
         # following fields will be filled later
         self.paths = []  # [str]
         self.status = None  # ctx.status output
         self.fctxmap = {}  # {path: {ctx: fctx}}
         self.fixupmap = {}  # {path: filefixupstate}
         self.replacemap = {}  # {oldnode: newnode or None}
         self.finalnode = None  # head after all fixups
         self.ctxaffected = set()  # ctx that will be absorbed into
 
     def diffwith(self, targetctx, match=None, fm=None):
         """diff and prepare fixups. update self.fixupmap, self.paths"""
         # only care about modified files
         self.status = self.stack[-1].status(targetctx, match)
         self.paths = []
         # but if --edit-lines is used, the user may want to edit files
         # even if they are not modified
         editopt = self.opts.get(b'edit_lines')
         if not self.status.modified and editopt and match:
             interestingpaths = match.files()
         else:
             interestingpaths = self.status.modified
         # prepare the filefixupstate
         seenfctxs = set()
         # sorting is necessary to eliminate ambiguity for the "double move"
         # case: "hg cp A B; hg cp A C; hg rm A", then only "B" can affect "A".
         for path in sorted(interestingpaths):
             self.ui.debug(b'calculating fixups for %s\n' % path)
             targetfctx = targetctx[path]
             fctxs, ctx2fctx = getfilestack(self.stack, path, seenfctxs)
             # ignore symbolic links or binary, or unchanged files
             if any(
                 f.islink() or stringutil.binary(f.data())
                 for f in [targetfctx] + fctxs
                 if not isinstance(f, emptyfilecontext)
             ):
                 continue
             if targetfctx.data() == fctxs[-1].data() and not editopt:
                 continue
             seenfctxs.update(fctxs[1:])
             self.fctxmap[path] = ctx2fctx
             fstate = filefixupstate(fctxs, path, ui=self.ui, opts=self.opts)
             if fm is not None:
                 fm.startitem()
                 fm.plain(b'showing changes for ')
                 fm.write(b'path', b'%s\n', path, label=b'absorb.path')
                 fm.data(linetype=b'path')
             fstate.diffwith(targetfctx, fm)
             self.fixupmap[path] = fstate
             self.paths.append(path)
             self.ctxaffected.update(fstate.ctxaffected)
 
     def apply(self):
         """apply fixups to individual filefixupstates"""
         for path, state in self.fixupmap.items():
             if self.ui.debugflag:
                 self.ui.write(_(b'applying fixups to %s\n') % path)
             state.apply()
 
     @property
     def chunkstats(self):
         """-> {path: chunkstats}. collect chunkstats from filefixupstates"""
         return {path: state.chunkstats for path, state in self.fixupmap.items()}
 
     def commit(self):
         """commit changes. update self.finalnode, self.replacemap"""
         with self.repo.transaction(b'absorb') as tr:
             self._commitstack()
             self._movebookmarks(tr)
             if self.repo[b'.'].node() in self.replacemap:
                 self._moveworkingdirectoryparent()
             self._cleanupoldcommits()
         return self.finalnode
 
     def printchunkstats(self):
         """print things like '1 of 2 chunk(s) applied'"""
         ui = self.ui
         chunkstats = self.chunkstats
         if ui.verbose:
             # chunkstats for each file
             for path, stat in chunkstats.items():
                 if stat[0]:
                     ui.write(
                         _(b'%s: %d of %d chunk(s) applied\n')
                         % (path, stat[0], stat[1])
                     )
         elif not ui.quiet:
             # a summary for all files
             stats = chunkstats.values()
             applied, total = (sum(s[i] for s in stats) for i in (0, 1))
             ui.write(_(b'%d of %d chunk(s) applied\n') % (applied, total))
 
     def _commitstack(self):
         """make new commits. update self.finalnode, self.replacemap.
         it is splitted from "commit" to avoid too much indentation.
         """
         # last node (20-char) committed by us
         lastcommitted = None
         # p1 which overrides the parent of the next commit, "None" means use
         # the original parent unchanged
         nextp1 = None
         for ctx in self.stack:
             memworkingcopy = self._getnewfilecontents(ctx)
             if not memworkingcopy and not lastcommitted:
                 # nothing changed, nothing commited
                 nextp1 = ctx
                 continue
             willbecomenoop = ctx.files() and self._willbecomenoop(
                 memworkingcopy, ctx, nextp1
             )
             if self.skip_empty_successor and willbecomenoop:
                 # changeset is no longer necessary
                 self.replacemap[ctx.node()] = None
                 msg = _(b'became empty and was dropped')
             else:
                 # changeset needs re-commit
                 nodestr = self._commitsingle(memworkingcopy, ctx, p1=nextp1)
                 lastcommitted = self.repo[nodestr]
                 nextp1 = lastcommitted
                 self.replacemap[ctx.node()] = lastcommitted.node()
                 if memworkingcopy:
                     if willbecomenoop:
                         msg = _(b'%d file(s) changed, became empty as %s')
                     else:
                         msg = _(b'%d file(s) changed, became %s')
                     msg = msg % (
                         len(memworkingcopy),
                         self._ctx2str(lastcommitted),
                     )
                 else:
                     msg = _(b'became %s') % self._ctx2str(lastcommitted)
             if self.ui.verbose and msg:
                 self.ui.write(_(b'%s: %s\n') % (self._ctx2str(ctx), msg))
         self.finalnode = lastcommitted and lastcommitted.node()
 
     def _ctx2str(self, ctx):
         if self.ui.debugflag:
             return b'%d:%s' % (ctx.rev(), ctx.hex())
         else:
             return b'%d:%s' % (ctx.rev(), short(ctx.node()))
 
     def _getnewfilecontents(self, ctx):
         """(ctx) -> {path: str}
 
         fetch file contents from filefixupstates.
         return the working copy overrides - files different from ctx.
         """
         result = {}
         for path in self.paths:
             ctx2fctx = self.fctxmap[path]  # {ctx: fctx}
             if ctx not in ctx2fctx:
                 continue
             fctx = ctx2fctx[ctx]
             content = fctx.data()
             newcontent = self.fixupmap[path].getfinalcontent(fctx)
             if content != newcontent:
                 result[fctx.path()] = newcontent
         return result
 
     def _movebookmarks(self, tr):
         repo = self.repo
         needupdate = [
             (name, self.replacemap[hsh])
             for name, hsh in repo._bookmarks.items()
             if hsh in self.replacemap
         ]
         changes = []
         for name, hsh in needupdate:
             if hsh:
                 changes.append((name, hsh))
                 if self.ui.verbose:
                     self.ui.write(
                         _(b'moving bookmark %s to %s\n') % (name, hex(hsh))
                     )
             else:
                 changes.append((name, None))
                 if self.ui.verbose:
                     self.ui.write(_(b'deleting bookmark %s\n') % name)
         repo._bookmarks.applychanges(repo, tr, changes)
 
     def _moveworkingdirectoryparent(self):
         if not self.finalnode:
             # Find the latest not-{obsoleted,stripped} parent.
             revs = self.repo.revs(b'max(::. - %ln)', self.replacemap.keys())
             ctx = self.repo[revs.first()]
             self.finalnode = ctx.node()
         else:
             ctx = self.repo[self.finalnode]
 
         dirstate = self.repo.dirstate
         # dirstate.rebuild invalidates fsmonitorstate, causing "hg status" to
         # be slow. in absorb's case, no need to invalidate fsmonitorstate.
         noop = lambda: 0
         restore = noop
         if util.safehasattr(dirstate, '_fsmonitorstate'):
             bak = dirstate._fsmonitorstate.invalidate
 
             def restore():
                 dirstate._fsmonitorstate.invalidate = bak
 
             dirstate._fsmonitorstate.invalidate = noop
         try:
-            with dirstate.parentchange():
+            with dirstate.changing_parents(self.repo):
                 dirstate.rebuild(ctx.node(), ctx.manifest(), self.paths)
         finally:
             restore()
 
     @staticmethod
     def _willbecomenoop(memworkingcopy, ctx, pctx=None):
         """({path: content}, ctx, ctx) -> bool. test if a commit will be noop
 
         if it will become an empty commit (does not change anything, after the
         memworkingcopy overrides), return True. otherwise return False.
         """
         if not pctx:
             parents = ctx.parents()
             if len(parents) != 1:
                 return False
             pctx = parents[0]
         if ctx.branch() != pctx.branch():
             return False
         if ctx.extra().get(b'close'):
             return False
         # ctx changes more files (not a subset of memworkingcopy)
         if not set(ctx.files()).issubset(set(memworkingcopy)):
             return False
         for path, content in memworkingcopy.items():
             if path not in pctx or path not in ctx:
                 return False
             fctx = ctx[path]
             pfctx = pctx[path]
             if pfctx.flags() != fctx.flags():
                 return False
             if pfctx.data() != content:
                 return False
         return True
 
     def _commitsingle(self, memworkingcopy, ctx, p1=None):
         """(ctx, {path: content}, node) -> node. make a single commit
 
         the commit is a clone from ctx, with a (optionally) different p1, and
         different file contents replaced by memworkingcopy.
         """
         parents = p1 and (p1, self.repo.nullid)
         extra = ctx.extra()
         if self._useobsolete and self.ui.configbool(b'absorb', b'add-noise'):
             extra[b'absorb_source'] = ctx.hex()
 
         desc = rewriteutil.update_hash_refs(
             ctx.repo(),
             ctx.description(),
             {
                 oldnode: [newnode]
                 for oldnode, newnode in self.replacemap.items()
             },
         )
         mctx = overlaycontext(
             memworkingcopy, ctx, parents, extra=extra, desc=desc
         )
         return mctx.commit()
 
     @util.propertycache
     def _useobsolete(self):
         """() -> bool"""
         return obsolete.isenabled(self.repo, obsolete.createmarkersopt)
 
     def _cleanupoldcommits(self):
         replacements = {
             k: ([v] if v is not None else [])
             for k, v in self.replacemap.items()
         }
         if replacements:
             scmutil.cleanupnodes(
                 self.repo, replacements, operation=b'absorb', fixphase=True
             )
 
     @util.propertycache
     def skip_empty_successor(self):
         return rewriteutil.skip_empty_successor(self.ui, b'absorb')
 
 
 def _parsechunk(hunk):
     """(crecord.uihunk or patch.recordhunk) -> (path, (a1, a2, [bline]))"""
     if type(hunk) not in (crecord.uihunk, patch.recordhunk):
         return None, None
     path = hunk.header.filename()
     a1 = hunk.fromline + len(hunk.before) - 1
     # remove before and after context
     hunk.before = hunk.after = []
     buf = util.stringio()
     hunk.write(buf)
     patchlines = mdiff.splitnewlines(buf.getvalue())
     # hunk.prettystr() will update hunk.removed
     a2 = a1 + hunk.removed
     blines = [l[1:] for l in patchlines[1:] if not l.startswith(b'-')]
     return path, (a1, a2, blines)
 
 
 def overlaydiffcontext(ctx, chunks):
     """(ctx, [crecord.uihunk]) -> memctx
 
     return a memctx with some [1] patches (chunks) applied to ctx.
     [1]: modifications are handled. renames, mode changes, etc. are ignored.
     """
     # sadly the applying-patch logic is hardly reusable, and messy:
     # 1. the core logic "_applydiff" is too heavy - it writes .rej files, it
     #    needs a file stream of a patch and will re-parse it, while we have
     #    structured hunk objects at hand.
     # 2. a lot of different implementations about "chunk" (patch.hunk,
     #    patch.recordhunk, crecord.uihunk)
     # as we only care about applying changes to modified files, no mode
     # change, no binary diff, and no renames, it's probably okay to
     # re-invent the logic using much simpler code here.
     memworkingcopy = {}  # {path: content}
     patchmap = defaultdict(lambda: [])  # {path: [(a1, a2, [bline])]}
     for path, info in map(_parsechunk, chunks):
         if not path or not info:
             continue
         patchmap[path].append(info)
     for path, patches in patchmap.items():
         if path not in ctx or not patches:
             continue
         patches.sort(reverse=True)
         lines = mdiff.splitnewlines(ctx[path].data())
         for a1, a2, blines in patches:
             lines[a1:a2] = blines
         memworkingcopy[path] = b''.join(lines)
     return overlaycontext(memworkingcopy, ctx)
 
 
 def absorb(ui, repo, stack=None, targetctx=None, pats=None, opts=None):
     """pick fixup chunks from targetctx, apply them to stack.
 
     if targetctx is None, the working copy context will be used.
     if stack is None, the current draft stack will be used.
     return fixupstate.
     """
     if stack is None:
         limit = ui.configint(b'absorb', b'max-stack-size')
         headctx = repo[b'.']
         if len(headctx.parents()) > 1:
             raise error.InputError(_(b'cannot absorb into a merge'))
         stack = getdraftstack(headctx, limit)
         if limit and len(stack) >= limit:
             ui.warn(
                 _(
                     b'absorb: only the recent %d changesets will '
                     b'be analysed\n'
                 )
                 % limit
             )
     if not stack:
         raise error.InputError(_(b'no mutable changeset to change'))
     if targetctx is None:  # default to working copy
         targetctx = repo[None]
     if pats is None:
         pats = ()
     if opts is None:
         opts = {}
     state = fixupstate(stack, ui=ui, opts=opts)
     matcher = scmutil.match(targetctx, pats, opts)
     if opts.get(b'interactive'):
         diff = patch.diff(repo, stack[-1].node(), targetctx.node(), matcher)
         origchunks = patch.parsepatch(diff)
         chunks = cmdutil.recordfilter(ui, origchunks, matcher)[0]
         targetctx = overlaydiffcontext(stack[-1], chunks)
     if opts.get(b'edit_lines'):
         # If we're going to open the editor, don't ask the user to confirm
         # first
         opts[b'apply_changes'] = True
     fm = None
     if opts.get(b'print_changes') or not opts.get(b'apply_changes'):
         fm = ui.formatter(b'absorb', opts)
     state.diffwith(targetctx, matcher, fm)
     if fm is not None:
         fm.startitem()
         fm.write(
             b"count", b"\n%d changesets affected\n", len(state.ctxaffected)
         )
         fm.data(linetype=b'summary')
         for ctx in reversed(stack):
             if ctx not in state.ctxaffected:
                 continue
             fm.startitem()
             fm.context(ctx=ctx)
             fm.data(linetype=b'changeset')
             fm.write(b'node', b'%-7.7s ', ctx.hex(), label=b'absorb.node')
             descfirstline = stringutil.firstline(ctx.description())
             fm.write(
                 b'descfirstline',
                 b'%s\n',
                 descfirstline,
                 label=b'absorb.description',
             )
         fm.end()
     if not opts.get(b'dry_run'):
         if (
             not opts.get(b'apply_changes')
             and state.ctxaffected
             and ui.promptchoice(
                 b"apply changes (y/N)? $$ &Yes $$ &No", default=1
             )
         ):
             raise error.CanceledError(_(b'absorb cancelled\n'))
 
         state.apply()
         if state.commit():
             state.printchunkstats()
         elif not ui.quiet:
             ui.write(_(b'nothing applied\n'))
     return state
 
 
 @command(
     b'absorb',
     [
         (
             b'a',
             b'apply-changes',
             None,
             _(b'apply changes without prompting for confirmation'),
         ),
         (
             b'p',
             b'print-changes',
             None,
             _(b'always print which changesets are modified by which changes'),
         ),
         (
             b'i',
             b'interactive',
             None,
             _(b'interactively select which chunks to apply'),
         ),
         (
             b'e',
             b'edit-lines',
             None,
             _(
                 b'edit what lines belong to which changesets before commit '
                 b'(EXPERIMENTAL)'
             ),
         ),
     ]
     + commands.dryrunopts
     + commands.templateopts
     + commands.walkopts,
     _(b'hg absorb [OPTION] [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     helpbasic=True,
 )
 def absorbcmd(ui, repo, *pats, **opts):
     """incorporate corrections into the stack of draft changesets
 
     absorb analyzes each change in your working directory and attempts to
     amend the changed lines into the changesets in your stack that first
     introduced those lines.
 
     If absorb cannot find an unambiguous changeset to amend for a change,
     that change will be left in the working directory, untouched. They can be
     observed by :hg:`status` or :hg:`diff` afterwards. In other words,
     absorb does not write to the working directory.
 
     Changesets outside the revset `::. and not public() and not merge()` will
     not be changed.
 
     Changesets that become empty after applying the changes will be deleted.
 
     By default, absorb will show what it plans to do and prompt for
     confirmation.  If you are confident that the changes will be absorbed
     to the correct place, run :hg:`absorb -a` to apply the changes
     immediately.
 
     Returns 0 on success, 1 if all chunks were ignored and nothing amended.
     """
     opts = pycompat.byteskwargs(opts)
 
     with repo.wlock(), repo.lock():
         if not opts[b'dry_run']:
             cmdutil.checkunfinished(repo)
 
         state = absorb(ui, repo, pats=pats, opts=opts)
         if sum(s[0] for s in state.chunkstats.values()) == 0:
             return 1

hgext/amend.py

@@ -1,73 +1,75
 # amend.py - provide the amend command
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """provide the amend command (EXPERIMENTAL)
 
 This extension provides an ``amend`` command that is similar to
 ``commit --amend`` but does not prompt an editor.
 """
 
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     registrar,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 
 @command(
     b'amend',
     [
         (
             b'A',
             b'addremove',
             None,
             _(b'mark new/missing files as added/removed before committing'),
         ),
         (b'e', b'edit', None, _(b'invoke editor on commit messages')),
         (b'i', b'interactive', None, _(b'use interactive mode')),
         (
             b'',
             b'close-branch',
             None,
             _(b'mark a branch as closed, hiding it from the branch list'),
         ),
         (b's', b'secret', None, _(b'use the secret phase for committing')),
+        (b'', b'draft', None, _(b'use the draft phase for committing')),
         (b'n', b'note', b'', _(b'store a note on the amend')),
     ]
     + cmdutil.walkopts
     + cmdutil.commitopts
     + cmdutil.commitopts2
     + cmdutil.commitopts3,
     _(b'[OPTION]... [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     inferrepo=True,
 )
 def amend(ui, repo, *pats, **opts):
     """amend the working copy parent with all or specified outstanding changes
 
     Similar to :hg:`commit --amend`, but reuse the commit message without
     invoking editor, unless ``--edit`` was set.
 
     See :hg:`help commit` for more details.
     """
+    cmdutil.check_at_most_one_arg(opts, 'draft', 'secret')
     cmdutil.check_note_size(opts)
 
     with repo.wlock(), repo.lock():
         if not opts.get('logfile'):
             opts['message'] = opts.get('message') or repo[b'.'].description()
         opts['amend'] = True
         return commands._docommit(ui, repo, *pats, **opts)

hgext/automv.py

@@ -1,118 +1,126
 # automv.py
 #
 # Copyright 2013-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """check for unrecorded moves at commit time (EXPERIMENTAL)
 
 This extension checks at commit/amend time if any of the committed files
 comes from an unrecorded mv.
 
 The threshold at which a file is considered a move can be set with the
 ``automv.similarity`` config option. This option takes a percentage between 0
 (disabled) and 100 (files must be identical), the default is 95.
 
 """
 
 # Using 95 as a default similarity is based on an analysis of the mercurial
 # repositories of the cpython, mozilla-central & mercurial repositories, as
 # well as 2 very large facebook repositories. At 95 50% of all potential
 # missed moves would be caught, as well as correspond with 87% of all
 # explicitly marked moves.  Together, 80% of moved files are 95% similar or
 # more.
 #
 # See http://markmail.org/thread/5pxnljesvufvom57 for context.
 
 
 from mercurial.i18n import _
 from mercurial import (
     commands,
     copies,
     error,
     extensions,
     pycompat,
     registrar,
     scmutil,
     similar,
 )
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'automv',
     b'similarity',
     default=95,
 )
 
 
 def extsetup(ui):
     entry = extensions.wrapcommand(commands.table, b'commit', mvcheck)
     entry[1].append(
         (b'', b'no-automv', None, _(b'disable automatic file move detection'))
     )
 
 
 def mvcheck(orig, ui, repo, *pats, **opts):
     """Hook to check for moves at commit time"""
     opts = pycompat.byteskwargs(opts)
     renames = None
     disabled = opts.pop(b'no_automv', False)
-    if not disabled:
-        threshold = ui.configint(b'automv', b'similarity')
-        if not 0 <= threshold <= 100:
-            raise error.Abort(_(b'automv.similarity must be between 0 and 100'))
-        if threshold > 0:
-            match = scmutil.match(repo[None], pats, opts)
-            added, removed = _interestingfiles(repo, match)
-            uipathfn = scmutil.getuipathfn(repo, legacyrelativevalue=True)
-            renames = _findrenames(
-                repo, uipathfn, added, removed, threshold / 100.0
-            )
+    with repo.wlock():
+        if not disabled:
+            threshold = ui.configint(b'automv', b'similarity')
+            if not 0 <= threshold <= 100:
+                raise error.Abort(
+                    _(b'automv.similarity must be between 0 and 100')
+                )
+            if threshold > 0:
+                match = scmutil.match(repo[None], pats, opts)
+                added, removed = _interestingfiles(repo, match)
+                uipathfn = scmutil.getuipathfn(repo, legacyrelativevalue=True)
+                renames = _findrenames(
+                    repo, uipathfn, added, removed, threshold / 100.0
+                )
 
-    with repo.wlock():
         if renames is not None:
-            scmutil._markchanges(repo, (), (), renames)
+            with repo.dirstate.changing_files(repo):
+                # XXX this should be wider and integrated with the commit
+                # transaction. At the same time as we do the `addremove` logic
+                # for commit.  However we can't really do better with the
+                # current extension structure, and this is not worse than what
+                # happened before.
+                scmutil._markchanges(repo, (), (), renames)
         return orig(ui, repo, *pats, **pycompat.strkwargs(opts))
 
 
 def _interestingfiles(repo, matcher):
     """Find what files were added or removed in this commit.
 
     Returns a tuple of two lists: (added, removed). Only files not *already*
     marked as moved are included in the added list.
 
     """
     stat = repo.status(match=matcher)
     added = stat.added
     removed = stat.removed
 
     copy = copies.pathcopies(repo[b'.'], repo[None], matcher)
     # remove the copy files for which we already have copy info
     added = [f for f in added if f not in copy]
 
     return added, removed
 
 
 def _findrenames(repo, uipathfn, added, removed, similarity):
     """Find what files in added are really moved files.
 
     Any file named in removed that is at least similarity% similar to a file
     in added is seen as a rename.
 
     """
     renames = {}
     if similarity > 0:
         for src, dst, score in similar.findrenames(
             repo, added, removed, similarity
         ):
             if repo.ui.verbose:
                 repo.ui.status(
                     _(b'detected move of %s as %s (%d%% similar)\n')
                     % (uipathfn(src), uipathfn(dst), score * 100)
                 )
             renames[dst] = src
     if renames:
         repo.ui.status(_(b'detected move of %d files\n') % len(renames))
     return renames

hgext/blackbox.py

@@ -1,240 +1,242
 # blackbox.py - log repository events to a file for post-mortem debugging
 #
 # Copyright 2010 Nicolas Dumazet
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """log repository events to a blackbox for debugging
 
 Logs event information to .hg/blackbox.log to help debug and diagnose problems.
 The events that get logged can be configured via the blackbox.track and
 blackbox.ignore config keys.
 
 Examples::
 
   [blackbox]
   track = *
   ignore = pythonhook
   # dirty is *EXPENSIVE* (slow);
   # each log entry indicates `+` if the repository is dirty, like :hg:`id`.
   dirty = True
   # record the source of log messages
   logsource = True
 
   [blackbox]
   track = command, commandfinish, commandexception, exthook, pythonhook
 
   [blackbox]
   track = incoming
 
   [blackbox]
   # limit the size of a log file
   maxsize = 1.5 MB
   # rotate up to N log files when the current one gets too big
   maxfiles = 3
 
   [blackbox]
   # Include microseconds in log entries with %f (see Python function
   # datetime.datetime.strftime)
   date-format = %Y-%m-%d @ %H:%M:%S.%f
 
 """
 
 
 import re
 
 from mercurial.i18n import _
 from mercurial.node import hex
 
 from mercurial import (
     encoding,
     loggingutil,
     registrar,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'blackbox',
     b'dirty',
     default=False,
 )
 configitem(
     b'blackbox',
     b'maxsize',
     default=b'1 MB',
 )
 configitem(
     b'blackbox',
     b'logsource',
     default=False,
 )
 configitem(
     b'blackbox',
     b'maxfiles',
     default=7,
 )
 configitem(
     b'blackbox',
     b'track',
     default=lambda: [b'*'],
 )
 configitem(
     b'blackbox',
     b'ignore',
     default=lambda: [b'chgserver', b'cmdserver', b'extension'],
 )
 configitem(b'blackbox', b'date-format', default=b'')
 
 _lastlogger = loggingutil.proxylogger()
 
 
 class blackboxlogger:
     def __init__(self, ui, repo):
         self._repo = repo
         self._trackedevents = set(ui.configlist(b'blackbox', b'track'))
         self._ignoredevents = set(ui.configlist(b'blackbox', b'ignore'))
         self._maxfiles = ui.configint(b'blackbox', b'maxfiles')
         self._maxsize = ui.configbytes(b'blackbox', b'maxsize')
         self._inlog = False
 
     def tracked(self, event):
         return (
             b'*' in self._trackedevents and event not in self._ignoredevents
         ) or event in self._trackedevents
 
     def log(self, ui, event, msg, opts):
         # self._log() -> ctx.dirty() may create new subrepo instance, which
         # ui is derived from baseui. So the recursion guard in ui.log()
         # doesn't work as it's local to the ui instance.
         if self._inlog:
             return
         self._inlog = True
         try:
             self._log(ui, event, msg, opts)
         finally:
             self._inlog = False
 
     def _log(self, ui, event, msg, opts):
         default = ui.configdate(b'devel', b'default-date')
         dateformat = ui.config(b'blackbox', b'date-format')
         if dateformat:
             date = dateutil.datestr(default, dateformat)
         else:
             # We want to display milliseconds (more precision seems
             # unnecessary).  Since %.3f is not supported, use %f and truncate
             # microseconds.
             date = dateutil.datestr(default, b'%Y-%m-%d %H:%M:%S.%f')[:-3]
         user = procutil.getuser()
         pid = b'%d' % procutil.getpid()
         changed = b''
         ctx = self._repo[None]
         parents = ctx.parents()
         rev = b'+'.join([hex(p.node()) for p in parents])
         if ui.configbool(b'blackbox', b'dirty') and ctx.dirty(
             missing=True, merge=False, branch=False
         ):
             changed = b'+'
         if ui.configbool(b'blackbox', b'logsource'):
             src = b' [%s]' % event
         else:
             src = b''
         try:
             fmt = b'%s %s @%s%s (%s)%s> %s'
             args = (date, user, rev, changed, pid, src, msg)
             with loggingutil.openlogfile(
                 ui,
                 self._repo.vfs,
                 name=b'blackbox.log',
                 maxfiles=self._maxfiles,
                 maxsize=self._maxsize,
             ) as fp:
                 fp.write(fmt % args)
         except (IOError, OSError) as err:
             # deactivate this to avoid failed logging again
             self._trackedevents.clear()
             ui.debug(
                 b'warning: cannot write to blackbox.log: %s\n'
                 % encoding.strtolocal(err.strerror)
             )
             return
         _lastlogger.logger = self
 
 
 def uipopulate(ui):
     ui.setlogger(b'blackbox', _lastlogger)
 
 
 def reposetup(ui, repo):
     # During 'hg pull' a httppeer repo is created to represent the remote repo.
     # It doesn't have a .hg directory to put a blackbox in, so we don't do
     # the blackbox setup for it.
     if not repo.local():
         return
 
     # Since blackbox.log is stored in the repo directory, the logger should be
     # instantiated per repository.
     logger = blackboxlogger(ui, repo)
     ui.setlogger(b'blackbox', logger)
 
     # Set _lastlogger even if ui.log is not called. This gives blackbox a
     # fallback place to log
     if _lastlogger.logger is None:
         _lastlogger.logger = logger
 
     repo._wlockfreeprefix.add(b'blackbox.log')
 
 
 @command(
     b'blackbox',
     [
         (b'l', b'limit', 10, _(b'the number of events to show')),
     ],
     _(b'hg blackbox [OPTION]...'),
     helpcategory=command.CATEGORY_MAINTENANCE,
     helpbasic=True,
 )
 def blackbox(ui, repo, *revs, **opts):
     """view the recent repository events"""
 
     if not repo.vfs.exists(b'blackbox.log'):
         return
 
     limit = opts.get('limit')
+    assert limit is not None  # help pytype
+
     fp = repo.vfs(b'blackbox.log', b'r')
     lines = fp.read().split(b'\n')
 
     count = 0
     output = []
     for line in reversed(lines):
         if count >= limit:
             break
 
         # count the commands by matching lines like:
         # 2013/01/23 19:13:36 root>
         # 2013/01/23 19:13:36 root (1234)>
         # 2013/01/23 19:13:36 root @0000000000000000000000000000000000000000 (1234)>
         # 2013-01-23 19:13:36.000 root @0000000000000000000000000000000000000000 (1234)>
         if re.match(
             br'^\d{4}[-/]\d{2}[-/]\d{2} \d{2}:\d{2}:\d{2}(.\d*)? .*> .*', line
         ):
             count += 1
         output.append(line)
 
     ui.status(b'\n'.join(reversed(output)))

hgext/convert/bzr.py

@@ -1,343 +1,346
 # bzr.py - bzr support for the convert extension
 #
 #  Copyright 2008, 2009 Marek Kubica <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This module is for handling Breezy imports or `brz`, but it's also compatible
 # with Bazaar or `bzr`, that was formerly known as Bazaar-NG;
 # it cannot access `bar` repositories, but they were never used very much.
 
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     demandimport,
     error,
     util,
 )
 from . import common
 
 
 # these do not work with demandimport, blacklist
 demandimport.IGNORES.update(
     [
         'breezy.transactions',
         'breezy.urlutils',
         'ElementPath',
     ]
 )
 
 try:
     # bazaar imports
+    # pytype: disable=import-error
     import breezy.bzr.bzrdir
     import breezy.errors
     import breezy.revision
     import breezy.revisionspec
 
+    # pytype: enable=import-error
+
     bzrdir = breezy.bzr.bzrdir
     errors = breezy.errors
     revision = breezy.revision
     revisionspec = breezy.revisionspec
     revisionspec.RevisionSpec
 
     try:
         # brz 3.3.0 (revno: 7614.2.2)
         from breezy.transport import NoSuchFile
     except ImportError:
         from breezy.errors import NoSuchFile
 except ImportError:
     pass
 
 supportedkinds = ('file', 'symlink')
 
 
 class bzr_source(common.converter_source):
     """Reads Bazaar repositories by using the Bazaar Python libraries"""
 
     def __init__(self, ui, repotype, path, revs=None):
         super(bzr_source, self).__init__(ui, repotype, path, revs=revs)
 
         if not os.path.exists(os.path.join(path, b'.bzr')):
             raise common.NoRepo(
                 _(b'%s does not look like a Bazaar repository') % path
             )
 
         try:
             # access breezy stuff
             bzrdir
         except NameError:
             raise common.NoRepo(_(b'Bazaar modules could not be loaded'))
 
         path = util.abspath(path)
         self._checkrepotype(path)
         try:
             bzr_dir = bzrdir.BzrDir.open(path.decode())
             self.sourcerepo = bzr_dir.open_repository()
         except errors.NoRepositoryPresent:
             raise common.NoRepo(
                 _(b'%s does not look like a Bazaar repository') % path
             )
         self._parentids = {}
         self._saverev = ui.configbool(b'convert', b'bzr.saverev')
 
     def _checkrepotype(self, path):
         # Lightweight checkouts detection is informational but probably
         # fragile at API level. It should not terminate the conversion.
         try:
             dir = bzrdir.BzrDir.open_containing(path.decode())[0]
             try:
                 tree = dir.open_workingtree(recommend_upgrade=False)
                 branch = tree.branch
             except (errors.NoWorkingTree, errors.NotLocalUrl):
                 tree = None
                 branch = dir.open_branch()
             if (
                 tree is not None
                 and tree.controldir.root_transport.base
                 != branch.controldir.root_transport.base
             ):
                 self.ui.warn(
                     _(
                         b'warning: lightweight checkouts may cause '
                         b'conversion failures, try with a regular '
                         b'branch instead.\n'
                     )
                 )
         except Exception:
             self.ui.note(_(b'bzr source type could not be determined\n'))
 
     def before(self):
         """Before the conversion begins, acquire a read lock
         for all the operations that might need it. Fortunately
         read locks don't block other reads or writes to the
         repository, so this shouldn't have any impact on the usage of
         the source repository.
 
         The alternative would be locking on every operation that
         needs locks (there are currently two: getting the file and
         getting the parent map) and releasing immediately after,
         but this approach can take even 40% longer."""
         self.sourcerepo.lock_read()
 
     def after(self):
         self.sourcerepo.unlock()
 
     def _bzrbranches(self):
         return self.sourcerepo.find_branches(using=True)
 
     def getheads(self):
         if not self.revs:
             # Set using=True to avoid nested repositories (see issue3254)
             heads = sorted([b.last_revision() for b in self._bzrbranches()])
         else:
             revid = None
             for branch in self._bzrbranches():
                 try:
                     revspec = self.revs[0].decode()
                     r = revisionspec.RevisionSpec.from_string(revspec)
                     info = r.in_history(branch)
                 except errors.BzrError:
                     pass
                 revid = info.rev_id
             if revid is None:
                 raise error.Abort(
                     _(b'%s is not a valid revision') % self.revs[0]
                 )
             heads = [revid]
         # Empty repositories return 'null:', which cannot be retrieved
         heads = [h for h in heads if h != b'null:']
         return heads
 
     def getfile(self, name, rev):
         name = name.decode()
         revtree = self.sourcerepo.revision_tree(rev)
 
         try:
             kind = revtree.kind(name)
         except NoSuchFile:
             return None, None
         if kind not in supportedkinds:
             # the file is not available anymore - was deleted
             return None, None
         mode = self._modecache[(name.encode(), rev)]
         if kind == 'symlink':
             target = revtree.get_symlink_target(name)
             if target is None:
                 raise error.Abort(
                     _(b'%s.%s symlink has no target') % (name, rev)
                 )
             return target.encode(), mode
         else:
             sio = revtree.get_file(name)
             return sio.read(), mode
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_(b"convert from cvs does not support --full"))
         self._modecache = {}
         self._revtree = self.sourcerepo.revision_tree(version)
         # get the parentids from the cache
         parentids = self._parentids.pop(version)
         # only diff against first parent id
         prevtree = self.sourcerepo.revision_tree(parentids[0])
         files, changes = self._gettreechanges(self._revtree, prevtree)
         return files, changes, set()
 
     def getcommit(self, version):
         rev = self.sourcerepo.get_revision(version)
         # populate parent id cache
         if not rev.parent_ids:
             parents = []
             self._parentids[version] = (revision.NULL_REVISION,)
         else:
             parents = self._filterghosts(rev.parent_ids)
             self._parentids[version] = parents
 
         branch = rev.properties.get('branch-nick', 'default')
         if branch == 'trunk':
             branch = 'default'
         return common.commit(
             parents=parents,
             date=b'%d %d' % (rev.timestamp, -rev.timezone),
             author=self.recode(rev.committer),
             desc=self.recode(rev.message),
             branch=branch.encode('utf8'),
             rev=version,
             saverev=self._saverev,
         )
 
     def gettags(self):
         bytetags = {}
         for branch in self._bzrbranches():
             if not branch.supports_tags():
                 return {}
             tagdict = branch.tags.get_tag_dict()
             for name, rev in tagdict.items():
                 bytetags[self.recode(name)] = rev
         return bytetags
 
     def getchangedfiles(self, rev, i):
         self._modecache = {}
         curtree = self.sourcerepo.revision_tree(rev)
         if i is not None:
             parentid = self._parentids[rev][i]
         else:
             # no parent id, get the empty revision
             parentid = revision.NULL_REVISION
 
         prevtree = self.sourcerepo.revision_tree(parentid)
         changes = [e[0] for e in self._gettreechanges(curtree, prevtree)[0]]
         return changes
 
     def _gettreechanges(self, current, origin):
         revid = current._revision_id
         changes = []
         renames = {}
         seen = set()
 
         # Fall back to the deprecated attribute for legacy installations.
         try:
             inventory = origin.root_inventory
         except AttributeError:
             inventory = origin.inventory
 
         # Process the entries by reverse lexicographic name order to
         # handle nested renames correctly, most specific first.
 
         def key(c):
             return c.path[0] or c.path[1] or ""
 
         curchanges = sorted(
             current.iter_changes(origin),
             key=key,
             reverse=True,
         )
         for change in curchanges:
             paths = change.path
             kind = change.kind
             executable = change.executable
             if paths[0] == u'' or paths[1] == u'':
                 # ignore changes to tree root
                 continue
 
             # bazaar tracks directories, mercurial does not, so
             # we have to rename the directory contents
             if kind[1] == 'directory':
                 if kind[0] not in (None, 'directory'):
                     # Replacing 'something' with a directory, record it
                     # so it can be removed.
                     changes.append((self.recode(paths[0]), revid))
 
                 if kind[0] == 'directory' and None not in paths:
                     renaming = paths[0] != paths[1]
                     # neither an add nor an delete - a move
                     # rename all directory contents manually
                     subdir = inventory.path2id(paths[0])
                     # get all child-entries of the directory
                     for name, entry in inventory.iter_entries(subdir):
                         # hg does not track directory renames
                         if entry.kind == 'directory':
                             continue
                         frompath = self.recode(paths[0] + '/' + name)
                         if frompath in seen:
                             # Already handled by a more specific change entry
                             # This is important when you have:
                             # a => b
                             # a/c => a/c
                             # Here a/c must not be renamed into b/c
                             continue
                         seen.add(frompath)
                         if not renaming:
                             continue
                         topath = self.recode(paths[1] + '/' + name)
                         # register the files as changed
                         changes.append((frompath, revid))
                         changes.append((topath, revid))
                         # add to mode cache
                         mode = (
                             (entry.executable and b'x')
                             or (entry.kind == 'symlink' and b's')
                             or b''
                         )
                         self._modecache[(topath, revid)] = mode
                         # register the change as move
                         renames[topath] = frompath
 
                 # no further changes, go to the next change
                 continue
 
             # we got unicode paths, need to convert them
             path, topath = paths
             if path is not None:
                 path = self.recode(path)
             if topath is not None:
                 topath = self.recode(topath)
             seen.add(path or topath)
 
             if topath is None:
                 # file deleted
                 changes.append((path, revid))
                 continue
 
             # renamed
             if path and path != topath:
                 renames[topath] = path
                 changes.append((path, revid))
 
             # populate the mode cache
             kind, executable = [e[1] for e in (kind, executable)]
             mode = (executable and b'x') or (kind == 'symlink' and b'l') or b''
             self._modecache[(topath, revid)] = mode
             changes.append((topath, revid))
 
         return changes, renames
 
     def _filterghosts(self, ids):
         """Filters out ghost revisions which hg does not support, see
         <http://bazaar-vcs.org/GhostRevision>
         """
         parentmap = self.sourcerepo.get_parent_map(ids)
         parents = tuple([parent for parent in ids if parent in parentmap])
         return parents

hgext/convert/hg.py

@@ -1,731 +1,734
 # hg.py - hg backend for convert extension
 #
 #  Copyright 2005-2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # Notes for hg->hg conversion:
 #
 # * Old versions of Mercurial didn't trim the whitespace from the ends
 #   of commit messages, but new versions do.  Changesets created by
 #   those older versions, then converted, may thus have different
 #   hashes for changesets that are otherwise identical.
 #
 # * Using "--config convert.hg.saverev=true" will make the source
 #   identifier to be stored in the converted revision. This will cause
 #   the converted revision to have a different identity than the
 #   source.
 
 import os
 import re
 import time
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial.node import (
     bin,
     hex,
     sha1nodeconstants,
 )
 from mercurial import (
     bookmarks,
     context,
     error,
     exchange,
     hg,
     lock as lockmod,
     logcmdutil,
     merge as mergemod,
     mergestate,
     phases,
     util,
 )
 from mercurial.utils import dateutil
 
 stringio = util.stringio
 
 from . import common
 
 mapfile = common.mapfile
 NoRepo = common.NoRepo
 
 sha1re = re.compile(br'\b[0-9a-f]{12,40}\b')
 
 
 class mercurial_sink(common.converter_sink):
     def __init__(self, ui, repotype, path):
         common.converter_sink.__init__(self, ui, repotype, path)
         self.branchnames = ui.configbool(b'convert', b'hg.usebranchnames')
         self.clonebranches = ui.configbool(b'convert', b'hg.clonebranches')
         self.tagsbranch = ui.config(b'convert', b'hg.tagsbranch')
         self.lastbranch = None
         if os.path.isdir(path) and len(os.listdir(path)) > 0:
             try:
                 self.repo = hg.repository(self.ui, path)
                 if not self.repo.local():
                     raise NoRepo(
                         _(b'%s is not a local Mercurial repository') % path
                     )
             except error.RepoError as err:
                 ui.traceback()
                 raise NoRepo(err.args[0])
         else:
             try:
                 ui.status(_(b'initializing destination %s repository\n') % path)
                 self.repo = hg.repository(self.ui, path, create=True)
                 if not self.repo.local():
                     raise NoRepo(
                         _(b'%s is not a local Mercurial repository') % path
                     )
                 self.created.append(path)
             except error.RepoError:
                 ui.traceback()
                 raise NoRepo(
                     _(b"could not create hg repository %s as sink") % path
                 )
         self.lock = None
         self.wlock = None
         self.filemapmode = False
         self.subrevmaps = {}
 
     def before(self):
         self.ui.debug(b'run hg sink pre-conversion action\n')
         self.wlock = self.repo.wlock()
         self.lock = self.repo.lock()
 
     def after(self):
         self.ui.debug(b'run hg sink post-conversion action\n')
         if self.lock:
             self.lock.release()
         if self.wlock:
             self.wlock.release()
 
     def revmapfile(self):
         return self.repo.vfs.join(b"shamap")
 
     def authorfile(self):
         return self.repo.vfs.join(b"authormap")
 
     def setbranch(self, branch, pbranches):
         if not self.clonebranches:
             return
 
         setbranch = branch != self.lastbranch
         self.lastbranch = branch
         if not branch:
             branch = b'default'
         pbranches = [(b[0], b[1] and b[1] or b'default') for b in pbranches]
 
         branchpath = os.path.join(self.path, branch)
         if setbranch:
             self.after()
             try:
                 self.repo = hg.repository(self.ui, branchpath)
             except Exception:
                 self.repo = hg.repository(self.ui, branchpath, create=True)
             self.before()
 
         # pbranches may bring revisions from other branches (merge parents)
         # Make sure we have them, or pull them.
         missings = {}
         for b in pbranches:
             try:
                 self.repo.lookup(b[0])
             except Exception:
                 missings.setdefault(b[1], []).append(b[0])
 
         if missings:
             self.after()
             for pbranch, heads in sorted(missings.items()):
                 pbranchpath = os.path.join(self.path, pbranch)
                 prepo = hg.peer(self.ui, {}, pbranchpath)
                 self.ui.note(
                     _(b'pulling from %s into %s\n') % (pbranch, branch)
                 )
                 exchange.pull(
                     self.repo, prepo, heads=[prepo.lookup(h) for h in heads]
                 )
             self.before()
 
     def _rewritetags(self, source, revmap, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(b' ', 1)
             if len(s) != 2:
                 self.ui.warn(_(b'invalid tag entry: "%s"\n') % line)
                 fp.write(b'%s\n' % line)  # Bogus, but keep for hash stability
                 continue
             revid = revmap.get(source.lookuprev(s[0]))
             if not revid:
                 if s[0] == sha1nodeconstants.nullhex:
                     revid = s[0]
                 else:
                     # missing, but keep for hash stability
                     self.ui.warn(_(b'missing tag entry: "%s"\n') % line)
                     fp.write(b'%s\n' % line)
                     continue
             fp.write(b'%s %s\n' % (revid, s[1]))
         return fp.getvalue()
 
     def _rewritesubstate(self, source, data):
         fp = stringio()
         for line in data.splitlines():
             s = line.split(b' ', 1)
             if len(s) != 2:
                 continue
 
             revid = s[0]
             subpath = s[1]
             if revid != sha1nodeconstants.nullhex:
                 revmap = self.subrevmaps.get(subpath)
                 if revmap is None:
                     revmap = mapfile(
                         self.ui, self.repo.wjoin(subpath, b'.hg/shamap')
                     )
                     self.subrevmaps[subpath] = revmap
 
                     # It is reasonable that one or more of the subrepos don't
                     # need to be converted, in which case they can be cloned
                     # into place instead of converted.  Therefore, only warn
                     # once.
                     msg = _(b'no ".hgsubstate" updates will be made for "%s"\n')
                     if len(revmap) == 0:
                         sub = self.repo.wvfs.reljoin(subpath, b'.hg')
 
                         if self.repo.wvfs.exists(sub):
                             self.ui.warn(msg % subpath)
 
                 newid = revmap.get(revid)
                 if not newid:
                     if len(revmap) > 0:
                         self.ui.warn(
                             _(b"%s is missing from %s/.hg/shamap\n")
                             % (revid, subpath)
                         )
                 else:
                     revid = newid
 
             fp.write(b'%s %s\n' % (revid, subpath))
 
         return fp.getvalue()
 
     def _calculatemergedfiles(self, source, p1ctx, p2ctx):
         """Calculates the files from p2 that we need to pull in when merging p1
         and p2, given that the merge is coming from the given source.
 
         This prevents us from losing files that only exist in the target p2 and
         that don't come from the source repo (like if you're merging multiple
         repositories together).
         """
         anc = [p1ctx.ancestor(p2ctx)]
         # Calculate what files are coming from p2
         # TODO: mresult.commitinfo might be able to get that info
         mresult = mergemod.calculateupdates(
             self.repo,
             p1ctx,
             p2ctx,
             anc,
             branchmerge=True,
             force=True,
             acceptremote=False,
             followcopies=False,
         )
 
         for file, (action, info, msg) in mresult.filemap():
             if source.targetfilebelongstosource(file):
                 # If the file belongs to the source repo, ignore the p2
                 # since it will be covered by the existing fileset.
                 continue
 
             # If the file requires actual merging, abort. We don't have enough
             # context to resolve merges correctly.
             if action in mergestate.CONVERT_MERGE_ACTIONS:
                 raise error.Abort(
                     _(
                         b"unable to convert merge commit "
                         b"since target parents do not merge cleanly (file "
                         b"%s, parents %s and %s)"
                     )
                     % (file, p1ctx, p2ctx)
                 )
             elif action == mergestate.ACTION_KEEP:
                 # 'keep' means nothing changed from p1
                 continue
             else:
                 # Any other change means we want to take the p2 version
                 yield file
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         files = dict(files)
 
         def getfilectx(repo, memctx, f):
             if p2ctx and f in p2files and f not in copies:
                 self.ui.debug(b'reusing %s from p2\n' % f)
                 try:
                     return p2ctx[f]
                 except error.ManifestLookupError:
                     # If the file doesn't exist in p2, then we're syncing a
                     # delete, so just return None.
                     return None
             try:
                 v = files[f]
             except KeyError:
                 return None
             data, mode = source.getfile(f, v)
             if data is None:
                 return None
             if f == b'.hgtags':
                 data = self._rewritetags(source, revmap, data)
             if f == b'.hgsubstate':
                 data = self._rewritesubstate(source, data)
             return context.memfilectx(
                 self.repo,
                 memctx,
                 f,
                 data,
                 b'l' in mode,
                 b'x' in mode,
                 copies.get(f),
             )
 
         pl = []
         for p in parents:
             if p not in pl:
                 pl.append(p)
         parents = pl
         nparents = len(parents)
         if self.filemapmode and nparents == 1:
             m1node = self.repo.changelog.read(bin(parents[0]))[0]
             parent = parents[0]
 
         if len(parents) < 2:
             parents.append(self.repo.nullid)
         if len(parents) < 2:
             parents.append(self.repo.nullid)
         p2 = parents.pop(0)
 
         text = commit.desc
 
         sha1s = re.findall(sha1re, text)
         for sha1 in sha1s:
             oldrev = source.lookuprev(sha1)
             newrev = revmap.get(oldrev)
             if newrev is not None:
                 text = text.replace(sha1, newrev[: len(sha1)])
 
         extra = commit.extra.copy()
 
         sourcename = self.repo.ui.config(b'convert', b'hg.sourcename')
         if sourcename:
             extra[b'convert_source'] = sourcename
 
         for label in (
             b'source',
             b'transplant_source',
             b'rebase_source',
             b'intermediate-source',
         ):
             node = extra.get(label)
 
             if node is None:
                 continue
 
             # Only transplant stores its reference in binary
             if label == b'transplant_source':
                 node = hex(node)
 
             newrev = revmap.get(node)
             if newrev is not None:
                 if label == b'transplant_source':
                     newrev = bin(newrev)
 
                 extra[label] = newrev
 
         if self.branchnames and commit.branch:
             extra[b'branch'] = commit.branch
         if commit.rev and commit.saverev:
             extra[b'convert_revision'] = commit.rev
 
         while parents:
             p1 = p2
             p2 = parents.pop(0)
             p1ctx = self.repo[p1]
             p2ctx = None
             if p2 != self.repo.nullid:
                 p2ctx = self.repo[p2]
             fileset = set(files)
             if full:
                 fileset.update(self.repo[p1])
                 fileset.update(self.repo[p2])
 
             if p2ctx:
                 p2files = set(cleanp2)
                 for file in self._calculatemergedfiles(source, p1ctx, p2ctx):
                     p2files.add(file)
                     fileset.add(file)
 
             ctx = context.memctx(
                 self.repo,
                 (p1, p2),
                 text,
                 fileset,
                 getfilectx,
                 commit.author,
                 commit.date,
                 extra,
             )
 
             # We won't know if the conversion changes the node until after the
             # commit, so copy the source's phase for now.
             self.repo.ui.setconfig(
                 b'phases',
                 b'new-commit',
                 phases.phasenames[commit.phase],
                 b'convert',
             )
 
             with self.repo.transaction(b"convert") as tr:
                 if self.repo.ui.config(b'convert', b'hg.preserve-hash'):
                     origctx = commit.ctx
                 else:
                     origctx = None
                 node = hex(self.repo.commitctx(ctx, origctx=origctx))
 
                 # If the node value has changed, but the phase is lower than
                 # draft, set it back to draft since it hasn't been exposed
                 # anywhere.
                 if commit.rev != node:
                     ctx = self.repo[node]
                     if ctx.phase() < phases.draft:
                         phases.registernew(
                             self.repo, tr, phases.draft, [ctx.rev()]
                         )
 
             text = b"(octopus merge fixup)\n"
             p2 = node
 
         if self.filemapmode and nparents == 1:
             man = self.repo.manifestlog.getstorage(b'')
             mnode = self.repo.changelog.read(bin(p2))[0]
             closed = b'close' in commit.extra
             if not closed and not man.cmp(m1node, man.revision(mnode)):
                 self.ui.status(_(b"filtering out empty revision\n"))
                 self.repo.rollback(force=True)
                 return parent
         return p2
 
     def puttags(self, tags):
         tagparent = self.repo.branchtip(self.tagsbranch, ignoremissing=True)
         tagparent = tagparent or self.repo.nullid
 
         oldlines = set()
         for branch, heads in self.repo.branchmap().items():
             for h in heads:
                 if b'.hgtags' in self.repo[h]:
                     oldlines.update(
                         set(self.repo[h][b'.hgtags'].data().splitlines(True))
                     )
         oldlines = sorted(list(oldlines))
 
         newlines = sorted([(b"%s %s\n" % (tags[tag], tag)) for tag in tags])
         if newlines == oldlines:
             return None, None
 
         # if the old and new tags match, then there is nothing to update
         oldtags = set()
         newtags = set()
         for line in oldlines:
             s = line.strip().split(b' ', 1)
             if len(s) != 2:
                 continue
             oldtags.add(s[1])
         for line in newlines:
             s = line.strip().split(b' ', 1)
             if len(s) != 2:
                 continue
             if s[1] not in oldtags:
                 newtags.add(s[1].strip())
 
         if not newtags:
             return None, None
 
         data = b"".join(newlines)
 
         def getfilectx(repo, memctx, f):
             return context.memfilectx(repo, memctx, f, data, False, False, None)
 
         self.ui.status(_(b"updating tags\n"))
         date = b"%d 0" % int(time.mktime(time.gmtime()))
         extra = {b'branch': self.tagsbranch}
         ctx = context.memctx(
             self.repo,
             (tagparent, None),
             b"update tags",
             [b".hgtags"],
             getfilectx,
             b"convert-repo",
             date,
             extra,
         )
         node = self.repo.commitctx(ctx)
         return hex(node), hex(tagparent)
 
     def setfilemapmode(self, active):
         self.filemapmode = active
 
     def putbookmarks(self, updatedbookmark):
         if not len(updatedbookmark):
             return
         wlock = lock = tr = None
         try:
             wlock = self.repo.wlock()
             lock = self.repo.lock()
             tr = self.repo.transaction(b'bookmark')
             self.ui.status(_(b"updating bookmarks\n"))
             destmarks = self.repo._bookmarks
             changes = [
                 (bookmark, bin(updatedbookmark[bookmark]))
                 for bookmark in updatedbookmark
             ]
             destmarks.applychanges(self.repo, tr, changes)
             tr.close()
         finally:
             lockmod.release(lock, wlock, tr)
 
     def hascommitfrommap(self, rev):
         # the exact semantics of clonebranches is unclear so we can't say no
         return rev in self.repo or self.clonebranches
 
     def hascommitforsplicemap(self, rev):
         if rev not in self.repo and self.clonebranches:
             raise error.Abort(
                 _(
                     b'revision %s not found in destination '
                     b'repository (lookups with clonebranches=true '
                     b'are not implemented)'
                 )
                 % rev
             )
         return rev in self.repo
 
 
 class mercurial_source(common.converter_source):
     def __init__(self, ui, repotype, path, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         self.ignoreerrors = ui.configbool(b'convert', b'hg.ignoreerrors')
         self.ignored = set()
         self.saverev = ui.configbool(b'convert', b'hg.saverev')
         try:
             self.repo = hg.repository(self.ui, path)
             # try to provoke an exception if this isn't really a hg
             # repo, but some other bogus compatible-looking url
             if not self.repo.local():
                 raise error.RepoError
         except error.RepoError:
             ui.traceback()
             raise NoRepo(_(b"%s is not a local Mercurial repository") % path)
         self.lastrev = None
         self.lastctx = None
         self._changescache = None, None
         self.convertfp = None
         # Restrict converted revisions to startrev descendants
         startnode = ui.config(b'convert', b'hg.startrev')
         hgrevs = ui.config(b'convert', b'hg.revs')
         if hgrevs is None:
             if startnode is not None:
                 try:
                     startnode = self.repo.lookup(startnode)
                 except error.RepoError:
                     raise error.Abort(
                         _(b'%s is not a valid start revision') % startnode
                     )
                 startrev = self.repo.changelog.rev(startnode)
                 children = {startnode: 1}
                 for r in self.repo.changelog.descendants([startrev]):
                     children[self.repo.changelog.node(r)] = 1
                 self.keep = children.__contains__
             else:
                 self.keep = util.always
             if revs:
                 self._heads = [self.repo.lookup(r) for r in revs]
             else:
                 self._heads = self.repo.heads()
         else:
             if revs or startnode is not None:
                 raise error.Abort(
                     _(
                         b'hg.revs cannot be combined with '
                         b'hg.startrev or --rev'
                     )
                 )
             nodes = set()
             parents = set()
             for r in logcmdutil.revrange(self.repo, [hgrevs]):
                 ctx = self.repo[r]
                 nodes.add(ctx.node())
                 parents.update(p.node() for p in ctx.parents())
             self.keep = nodes.__contains__
             self._heads = nodes - parents
 
     def _changectx(self, rev):
         if self.lastrev != rev:
             self.lastctx = self.repo[rev]
             self.lastrev = rev
         return self.lastctx
 
     def _parents(self, ctx):
         return [p for p in ctx.parents() if p and self.keep(p.node())]
 
     def getheads(self):
         return [hex(h) for h in self._heads if self.keep(h)]
 
     def getfile(self, name, rev):
         try:
             fctx = self._changectx(rev)[name]
             return fctx.data(), fctx.flags()
         except error.LookupError:
             return None, None
 
     def _changedfiles(self, ctx1, ctx2):
         ma, r = [], []
         maappend = ma.append
         rappend = r.append
         d = ctx1.manifest().diff(ctx2.manifest())
         for f, ((node1, flag1), (node2, flag2)) in d.items():
             if node2 is None:
                 rappend(f)
             else:
                 maappend(f)
         return ma, r
 
     def getchanges(self, rev, full):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if full or not parents:
             files = copyfiles = ctx.manifest()
         if parents:
             if self._changescache[0] == rev:
-                ma, r = self._changescache[1]
+                # TODO: add type hints to avoid this warning, instead of
+                #  suppressing it:
+                #     No attribute '__iter__' on None [attribute-error]
+                ma, r = self._changescache[1]  # pytype: disable=attribute-error
             else:
                 ma, r = self._changedfiles(parents[0], ctx)
             if not full:
                 files = ma + r
             copyfiles = ma
         # _getcopies() is also run for roots and before filtering so missing
         # revlogs are detected early
         copies = self._getcopies(ctx, parents, copyfiles)
         cleanp2 = set()
         if len(parents) == 2:
             d = parents[1].manifest().diff(ctx.manifest(), clean=True)
             for f, value in d.items():
                 if value is None:
                     cleanp2.add(f)
         changes = [(f, rev) for f in files if f not in self.ignored]
         changes.sort()
         return changes, copies, cleanp2
 
     def _getcopies(self, ctx, parents, files):
         copies = {}
         for name in files:
             if name in self.ignored:
                 continue
             try:
                 copysource = ctx.filectx(name).copysource()
                 if copysource in self.ignored:
                     continue
                 # Ignore copy sources not in parent revisions
                 if not any(copysource in p for p in parents):
                     continue
                 copies[name] = copysource
             except TypeError:
                 pass
             except error.LookupError as e:
                 if not self.ignoreerrors:
                     raise
                 self.ignored.add(name)
                 self.ui.warn(_(b'ignoring: %s\n') % e)
         return copies
 
     def getcommit(self, rev):
         ctx = self._changectx(rev)
         _parents = self._parents(ctx)
         parents = [p.hex() for p in _parents]
         optparents = [p.hex() for p in ctx.parents() if p and p not in _parents]
         crev = rev
 
         return common.commit(
             author=ctx.user(),
             date=dateutil.datestr(ctx.date(), b'%Y-%m-%d %H:%M:%S %1%2'),
             desc=ctx.description(),
             rev=crev,
             parents=parents,
             optparents=optparents,
             branch=ctx.branch(),
             extra=ctx.extra(),
             sortkey=ctx.rev(),
             saverev=self.saverev,
             phase=ctx.phase(),
             ctx=ctx,
         )
 
     def numcommits(self):
         return len(self.repo)
 
     def gettags(self):
         # This will get written to .hgtags, filter non global tags out.
         tags = [
             t
             for t in self.repo.tagslist()
             if self.repo.tagtype(t[0]) == b'global'
         ]
         return {name: hex(node) for name, node in tags if self.keep(node)}
 
     def getchangedfiles(self, rev, i):
         ctx = self._changectx(rev)
         parents = self._parents(ctx)
         if not parents and i is None:
             i = 0
             ma, r = ctx.manifest().keys(), []
         else:
             i = i or 0
             ma, r = self._changedfiles(parents[i], ctx)
         ma, r = [[f for f in l if f not in self.ignored] for l in (ma, r)]
 
         if i == 0:
             self._changescache = (rev, (ma, r))
 
         return ma + r
 
     def converted(self, rev, destrev):
         if self.convertfp is None:
             self.convertfp = open(self.repo.vfs.join(b'shamap'), b'ab')
         self.convertfp.write(util.tonativeeol(b'%s %s\n' % (destrev, rev)))
         self.convertfp.flush()
 
     def before(self):
         self.ui.debug(b'run hg source pre-conversion action\n')
 
     def after(self):
         self.ui.debug(b'run hg source post-conversion action\n')
 
     def hasnativeorder(self):
         return True
 
     def hasnativeclose(self):
         return True
 
     def lookuprev(self, rev):
         try:
             return hex(self.repo.lookup(rev))
         except (error.RepoError, error.LookupError):
             return None
 
     def getbookmarks(self):
         return bookmarks.listbookmarks(self.repo)
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """Mercurial, revision string is a 40 byte hex"""
         self.checkhexformat(revstr, mapname)

hgext/convert/monotone.py

@@ -1,410 +1,411
 # monotone.py - monotone support for the convert extension
 #
 #  Copyright 2008, 2009 Mikkel Fahnoe Jorgensen <[e-mail]> and
 #  others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     error,
     pycompat,
 )
 from mercurial.utils import dateutil
 
 from . import common
 
 
 class monotone_source(common.converter_source, common.commandline):
     def __init__(self, ui, repotype, path=None, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         if revs and len(revs) > 1:
             raise error.Abort(
                 _(
                     b'monotone source does not support specifying '
                     b'multiple revs'
                 )
             )
         common.commandline.__init__(self, ui, b'mtn')
 
         self.ui = ui
         self.path = path
         self.automatestdio = False
         self.revs = revs
 
         norepo = common.NoRepo(
             _(b"%s does not look like a monotone repository") % path
         )
         if not os.path.exists(os.path.join(path, b'_MTN')):
             # Could be a monotone repository (SQLite db file)
             try:
                 f = open(path, b'rb')
                 header = f.read(16)
                 f.close()
             except IOError:
                 header = b''
             if header != b'SQLite format 3\x00':
                 raise norepo
 
         # regular expressions for parsing monotone output
         space = br'\s*'
         name = br'\s+"((?:\\"|[^"])*)"\s*'
         value = name
         revision = br'\s+\[(\w+)\]\s*'
         lines = br'(?:.|\n)+'
 
         self.dir_re = re.compile(space + b"dir" + name)
         self.file_re = re.compile(
             space + b"file" + name + b"content" + revision
         )
         self.add_file_re = re.compile(
             space + b"add_file" + name + b"content" + revision
         )
         self.patch_re = re.compile(
             space + b"patch" + name + b"from" + revision + b"to" + revision
         )
         self.rename_re = re.compile(space + b"rename" + name + b"to" + name)
         self.delete_re = re.compile(space + b"delete" + name)
         self.tag_re = re.compile(space + b"tag" + name + b"revision" + revision)
         self.cert_re = re.compile(
             lines + space + b"name" + name + b"value" + value
         )
 
         attr = space + b"file" + lines + space + b"attr" + space
         self.attr_execute_re = re.compile(
             attr + b'"mtn:execute"' + space + b'"true"'
         )
 
         # cached data
         self.manifest_rev = None
         self.manifest = None
         self.files = None
         self.dirs = None
 
         common.checktool(b'mtn', abort=False)
 
     def mtnrun(self, *args, **kwargs):
         if self.automatestdio:
             return self.mtnrunstdio(*args, **kwargs)
         else:
             return self.mtnrunsingle(*args, **kwargs)
 
     def mtnrunsingle(self, *args, **kwargs):
         kwargs['d'] = self.path
         return self.run0(b'automate', *args, **kwargs)
 
     def mtnrunstdio(self, *args, **kwargs):
         # Prepare the command in automate stdio format
         kwargs = pycompat.byteskwargs(kwargs)
         command = []
         for k, v in kwargs.items():
             command.append(b"%d:%s" % (len(k), k))
             if v:
                 command.append(b"%d:%s" % (len(v), v))
         if command:
             command.insert(0, b'o')
             command.append(b'e')
 
         command.append(b'l')
         for arg in args:
             command.append(b"%d:%s" % (len(arg), arg))
         command.append(b'e')
         command = b''.join(command)
 
         self.ui.debug(b"mtn: sending '%s'\n" % command)
         self.mtnwritefp.write(command)
         self.mtnwritefp.flush()
 
         return self.mtnstdioreadcommandoutput(command)
 
     def mtnstdioreadpacket(self):
         read = None
         commandnbr = b''
         while read != b':':
             read = self.mtnreadfp.read(1)
             if not read:
                 raise error.Abort(_(b'bad mtn packet - no end of commandnbr'))
             commandnbr += read
         commandnbr = commandnbr[:-1]
 
         stream = self.mtnreadfp.read(1)
         if stream not in b'mewptl':
             raise error.Abort(
                 _(b'bad mtn packet - bad stream type %s') % stream
             )
 
         read = self.mtnreadfp.read(1)
         if read != b':':
             raise error.Abort(_(b'bad mtn packet - no divider before size'))
 
         read = None
         lengthstr = b''
         while read != b':':
             read = self.mtnreadfp.read(1)
             if not read:
                 raise error.Abort(_(b'bad mtn packet - no end of packet size'))
             lengthstr += read
         try:
             length = int(lengthstr[:-1])
         except TypeError:
             raise error.Abort(
                 _(b'bad mtn packet - bad packet size %s') % lengthstr
             )
 
         read = self.mtnreadfp.read(length)
         if len(read) != length:
             raise error.Abort(
                 _(
                     b"bad mtn packet - unable to read full packet "
                     b"read %s of %s"
                 )
                 % (len(read), length)
             )
 
         return (commandnbr, stream, length, read)
 
     def mtnstdioreadcommandoutput(self, command):
         retval = []
         while True:
             commandnbr, stream, length, output = self.mtnstdioreadpacket()
             self.ui.debug(
                 b'mtn: read packet %s:%s:%d\n' % (commandnbr, stream, length)
             )
 
             if stream == b'l':
                 # End of command
                 if output != b'0':
                     raise error.Abort(
                         _(b"mtn command '%s' returned %s") % (command, output)
                     )
                 break
             elif stream in b'ew':
                 # Error, warning output
                 self.ui.warn(_(b'%s error:\n') % self.command)
                 self.ui.warn(output)
             elif stream == b'p':
                 # Progress messages
                 self.ui.debug(b'mtn: ' + output)
             elif stream == b'm':
                 # Main stream - command output
                 retval.append(output)
 
         return b''.join(retval)
 
     def mtnloadmanifest(self, rev):
         if self.manifest_rev == rev:
             return
         self.manifest = self.mtnrun(b"get_manifest_of", rev).split(b"\n\n")
         self.manifest_rev = rev
         self.files = {}
         self.dirs = {}
 
         for e in self.manifest:
             m = self.file_re.match(e)
             if m:
                 attr = b""
                 name = m.group(1)
                 node = m.group(2)
                 if self.attr_execute_re.match(e):
                     attr += b"x"
                 self.files[name] = (node, attr)
             m = self.dir_re.match(e)
             if m:
                 self.dirs[m.group(1)] = True
 
     def mtnisfile(self, name, rev):
         # a non-file could be a directory or a deleted or renamed file
         self.mtnloadmanifest(rev)
         return name in self.files
 
     def mtnisdir(self, name, rev):
         self.mtnloadmanifest(rev)
         return name in self.dirs
 
     def mtngetcerts(self, rev):
         certs = {
             b"author": b"<missing>",
             b"date": b"<missing>",
             b"changelog": b"<missing>",
             b"branch": b"<missing>",
         }
         certlist = self.mtnrun(b"certs", rev)
         # mtn < 0.45:
         #   key "[e-mail]"
         # mtn >= 0.45:
         #   key [ff58a7ffb771907c4ff68995eada1c4da068d328]
         certlist = re.split(br'\n\n {6}key ["\[]', certlist)
         for e in certlist:
             m = self.cert_re.match(e)
             if m:
                 name, value = m.groups()
+                assert value is not None  # help pytype
                 value = value.replace(br'\"', b'"')
                 value = value.replace(br'\\', b'\\')
                 certs[name] = value
         # Monotone may have subsecond dates: 2005-02-05T09:39:12.364306
         # and all times are stored in UTC
         certs[b"date"] = certs[b"date"].split(b'.')[0] + b" UTC"
         return certs
 
     # implement the converter_source interface:
 
     def getheads(self):
         if not self.revs:
             return self.mtnrun(b"leaves").splitlines()
         else:
             return self.revs
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(
                 _(b"convert from monotone does not support --full")
             )
         revision = self.mtnrun(b"get_revision", rev).split(b"\n\n")
         files = {}
         ignoremove = {}
         renameddirs = []
         copies = {}
         for e in revision:
             m = self.add_file_re.match(e)
             if m:
                 files[m.group(1)] = rev
                 ignoremove[m.group(1)] = rev
             m = self.patch_re.match(e)
             if m:
                 files[m.group(1)] = rev
             # Delete/rename is handled later when the convert engine
             # discovers an IOError exception from getfile,
             # but only if we add the "from" file to the list of changes.
             m = self.delete_re.match(e)
             if m:
                 files[m.group(1)] = rev
             m = self.rename_re.match(e)
             if m:
                 toname = m.group(2)
                 fromname = m.group(1)
                 if self.mtnisfile(toname, rev):
                     ignoremove[toname] = 1
                     copies[toname] = fromname
                     files[toname] = rev
                     files[fromname] = rev
                 elif self.mtnisdir(toname, rev):
                     renameddirs.append((fromname, toname))
 
         # Directory renames can be handled only once we have recorded
         # all new files
         for fromdir, todir in renameddirs:
             renamed = {}
             for tofile in self.files:
                 if tofile in ignoremove:
                     continue
                 if tofile.startswith(todir + b'/'):
                     renamed[tofile] = fromdir + tofile[len(todir) :]
                     # Avoid chained moves like:
                     # d1(/a) => d3/d1(/a)
                     # d2 => d3
                     ignoremove[tofile] = 1
             for tofile, fromfile in renamed.items():
                 self.ui.debug(
                     b"copying file in renamed directory from '%s' to '%s'"
                     % (fromfile, tofile),
                     b'\n',
                 )
                 files[tofile] = rev
                 copies[tofile] = fromfile
             for fromfile in renamed.values():
                 files[fromfile] = rev
 
         return (files.items(), copies, set())
 
     def getfile(self, name, rev):
         if not self.mtnisfile(name, rev):
             return None, None
         try:
             data = self.mtnrun(b"get_file_of", name, r=rev)
         except Exception:
             return None, None
         self.mtnloadmanifest(rev)
         node, attr = self.files.get(name, (None, b""))
         return data, attr
 
     def getcommit(self, rev):
         extra = {}
         certs = self.mtngetcerts(rev)
         if certs.get(b'suspend') == certs[b"branch"]:
             extra[b'close'] = b'1'
         dateformat = b"%Y-%m-%dT%H:%M:%S"
         return common.commit(
             author=certs[b"author"],
             date=dateutil.datestr(dateutil.strdate(certs[b"date"], dateformat)),
             desc=certs[b"changelog"],
             rev=rev,
             parents=self.mtnrun(b"parents", rev).splitlines(),
             branch=certs[b"branch"],
             extra=extra,
         )
 
     def gettags(self):
         tags = {}
         for e in self.mtnrun(b"tags").split(b"\n\n"):
             m = self.tag_re.match(e)
             if m:
                 tags[m.group(1)] = m.group(2)
         return tags
 
     def getchangedfiles(self, rev, i):
         # This function is only needed to support --filemap
         # ... and we don't support that
         raise NotImplementedError
 
     def before(self):
         # Check if we have a new enough version to use automate stdio
         try:
             versionstr = self.mtnrunsingle(b"interface_version")
             version = float(versionstr)
         except Exception:
             raise error.Abort(
                 _(b"unable to determine mtn automate interface version")
             )
 
         if version >= 12.0:
             self.automatestdio = True
             self.ui.debug(
                 b"mtn automate version %f - using automate stdio\n" % version
             )
 
             # launch the long-running automate stdio process
             self.mtnwritefp, self.mtnreadfp = self._run2(
                 b'automate', b'stdio', b'-d', self.path
             )
             # read the headers
             read = self.mtnreadfp.readline()
             if read != b'format-version: 2\n':
                 raise error.Abort(
                     _(b'mtn automate stdio header unexpected: %s') % read
                 )
             while read != b'\n':
                 read = self.mtnreadfp.readline()
                 if not read:
                     raise error.Abort(
                         _(
                             b"failed to reach end of mtn automate "
                             b"stdio headers"
                         )
                     )
         else:
             self.ui.debug(
                 b"mtn automate version %s - not using automate stdio "
                 b"(automate >= 12.0 - mtn >= 0.46 is needed)\n" % version
             )
 
     def after(self):
         if self.automatestdio:
             self.mtnwritefp.close()
             self.mtnwritefp = None
             self.mtnreadfp.close()
             self.mtnreadfp = None

hgext/convert/subversion.py

@@ -1,1721 +1,1730
 # Subversion 1.4/1.5 Python API backend
 #
 # Copyright(C) 2007 Daniel Holth et al
 
 import codecs
 import locale
 import os
 import pickle
 import re
 import xml.dom.minidom
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     pycompat,
     util,
     vfs as vfsmod,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
 from . import common
 
 stringio = util.stringio
 propertycache = util.propertycache
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 commandline = common.commandline
 commit = common.commit
 converter_sink = common.converter_sink
 converter_source = common.converter_source
 decodeargs = common.decodeargs
 encodeargs = common.encodeargs
 makedatetimestamp = common.makedatetimestamp
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 
 # Subversion stuff. Works best with very recent Python SVN bindings
 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing
 # these bindings.
 
 try:
+    # pytype: disable=import-error
     import svn
     import svn.client
     import svn.core
     import svn.ra
     import svn.delta
+
+    # pytype: enable=import-error
     from . import transport
     import warnings
 
     warnings.filterwarnings(
         'ignore', module='svn.core', category=DeprecationWarning
     )
     svn.core.SubversionException  # trigger import to catch error
 
 except ImportError:
     svn = None
 
 
 # In Subversion, paths and URLs are Unicode (encoded as UTF-8), which
 # Subversion converts from / to native strings when interfacing with the OS.
 # When passing paths and URLs to Subversion, we have to recode them such that
 # it roundstrips with what Subversion is doing.
 
 fsencoding = None
 
 
 def init_fsencoding():
     global fsencoding, fsencoding_is_utf8
     if fsencoding is not None:
         return
     if pycompat.iswindows:
         # On Windows, filenames are Unicode, but we store them using the MBCS
         # encoding.
         fsencoding = 'mbcs'
     else:
         # This is the encoding used to convert UTF-8 back to natively-encoded
         # strings in Subversion 1.14.0 or earlier with APR 1.7.0 or earlier.
         with util.with_lc_ctype():
             fsencoding = locale.nl_langinfo(locale.CODESET) or 'ISO-8859-1'
     fsencoding = codecs.lookup(fsencoding).name
     fsencoding_is_utf8 = fsencoding == codecs.lookup('utf-8').name
 
 
 def fs2svn(s):
     if fsencoding_is_utf8:
         return s
     else:
         return s.decode(fsencoding).encode('utf-8')
 
 
 def formatsvndate(date):
     return dateutil.datestr(date, b'%Y-%m-%dT%H:%M:%S.000000Z')
 
 
 def parsesvndate(s):
     # Example SVN datetime. Includes microseconds.
     # ISO-8601 conformant
     # '2007-01-04T17:35:00.902377Z'
     return dateutil.parsedate(s[:19] + b' UTC', [b'%Y-%m-%dT%H:%M:%S'])
 
 
 class SvnPathNotFound(Exception):
     pass
 
 
 def revsplit(rev):
     """Parse a revision string and return (uuid, path, revnum).
     >>> revsplit(b'svn:a2147622-4a9f-4db4-a8d3-13562ff547b2'
     ...          b'/proj%20B/mytrunk/mytrunk@1')
     ('a2147622-4a9f-4db4-a8d3-13562ff547b2', '/proj%20B/mytrunk/mytrunk', 1)
     >>> revsplit(b'svn:8af66a51-67f5-4354-b62c-98d67cc7be1d@1')
     ('', '', 1)
     >>> revsplit(b'@7')
     ('', '', 7)
     >>> revsplit(b'7')
     ('', '', 0)
     >>> revsplit(b'bad')
     ('', '', 0)
     """
     parts = rev.rsplit(b'@', 1)
     revnum = 0
     if len(parts) > 1:
         revnum = int(parts[1])
     parts = parts[0].split(b'/', 1)
     uuid = b''
     mod = b''
     if len(parts) > 1 and parts[0].startswith(b'svn:'):
         uuid = parts[0][4:]
         mod = b'/' + parts[1]
     return uuid, mod, revnum
 
 
 def quote(s):
     # As of svn 1.7, many svn calls expect "canonical" paths. In
     # theory, we should call svn.core.*canonicalize() on all paths
     # before passing them to the API.  Instead, we assume the base url
     # is canonical and copy the behaviour of svn URL encoding function
     # so we can extend it safely with new components. The "safe"
     # characters were taken from the "svn_uri__char_validity" table in
     # libsvn_subr/path.c.
     return urlreq.quote(s, b"!$&'()*+,-./:=@_~")
 
 
 def geturl(path):
     """Convert path or URL to a SVN URL, encoded in UTF-8.
 
     This can raise UnicodeDecodeError if the path or URL can't be converted to
     unicode using `fsencoding`.
     """
     try:
         return svn.client.url_from_path(
             svn.core.svn_path_canonicalize(fs2svn(path))
         )
     except svn.core.SubversionException:
         # svn.client.url_from_path() fails with local repositories
         pass
     if os.path.isdir(path):
         path = os.path.normpath(util.abspath(path))
         if pycompat.iswindows:
             path = b'/' + util.normpath(path)
         # Module URL is later compared with the repository URL returned
         # by svn API, which is UTF-8.
         path = fs2svn(path)
         path = b'file://%s' % quote(path)
     return svn.core.svn_path_canonicalize(path)
 
 
 def optrev(number):
     optrev = svn.core.svn_opt_revision_t()
     optrev.kind = svn.core.svn_opt_revision_number
     optrev.value.number = number
     return optrev
 
 
 class changedpath:
     def __init__(self, p):
         self.copyfrom_path = p.copyfrom_path
         self.copyfrom_rev = p.copyfrom_rev
         self.action = p.action
 
 
 def get_log_child(
     fp,
     url,
     paths,
     start,
     end,
     limit=0,
     discover_changed_paths=True,
     strict_node_history=False,
 ):
     protocol = -1
 
     def receiver(orig_paths, revnum, author, date, message, pool):
         paths = {}
         if orig_paths is not None:
             for k, v in orig_paths.items():
                 paths[k] = changedpath(v)
         pickle.dump((paths, revnum, author, date, message), fp, protocol)
 
     try:
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(
             t.ra,
             paths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
             receiver,
         )
     except IOError:
         # Caller may interrupt the iteration
         pickle.dump(None, fp, protocol)
     except Exception as inst:
         pickle.dump(stringutil.forcebytestr(inst), fp, protocol)
     else:
         pickle.dump(None, fp, protocol)
     fp.flush()
     # With large history, cleanup process goes crazy and suddenly
     # consumes *huge* amount of memory. The output file being closed,
     # there is no need for clean termination.
     os._exit(0)
 
 
 def debugsvnlog(ui, **opts):
     """Fetch SVN log in a subprocess and channel them back to parent to
     avoid memory collection issues.
     """
     with util.with_lc_ctype():
         if svn is None:
             raise error.Abort(
                 _(b'debugsvnlog could not load Subversion python bindings')
             )
 
         args = decodeargs(ui.fin.read())
         get_log_child(ui.fout, *args)
 
 
 class logstream:
     """Interruptible revision log iterator."""
 
     def __init__(self, stdout):
         self._stdout = stdout
 
     def __iter__(self):
         while True:
             try:
                 entry = pickle.load(self._stdout)
             except EOFError:
                 raise error.Abort(
                     _(
                         b'Mercurial failed to run itself, check'
                         b' hg executable is in PATH'
                     )
                 )
             try:
                 orig_paths, revnum, author, date, message = entry
             except (TypeError, ValueError):
                 if entry is None:
                     break
                 raise error.Abort(_(b"log stream exception '%s'") % entry)
             yield entry
 
     def close(self):
         if self._stdout:
             self._stdout.close()
             self._stdout = None
 
 
 class directlogstream(list):
     """Direct revision log iterator.
     This can be used for debugging and development but it will probably leak
     memory and is not suitable for real conversions."""
 
     def __init__(
         self,
         url,
         paths,
         start,
         end,
         limit=0,
         discover_changed_paths=True,
         strict_node_history=False,
     ):
         def receiver(orig_paths, revnum, author, date, message, pool):
             paths = {}
             if orig_paths is not None:
                 for k, v in orig_paths.items():
                     paths[k] = changedpath(v)
             self.append((paths, revnum, author, date, message))
 
         # Use an ra of our own so that our parent can consume
         # our results without confusing the server.
         t = transport.SvnRaTransport(url=url)
         svn.ra.get_log(
             t.ra,
             paths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
             receiver,
         )
 
     def close(self):
         pass
 
 
 # Check to see if the given path is a local Subversion repo. Verify this by
 # looking for several svn-specific files and directories in the given
 # directory.
 def filecheck(ui, path, proto):
     for x in (b'locks', b'hooks', b'format', b'db'):
         if not os.path.exists(os.path.join(path, x)):
             return False
     return True
 
 
 # Check to see if a given path is the root of an svn repo over http. We verify
 # this by requesting a version-controlled URL we know can't exist and looking
 # for the svn-specific "not found" XML.
 def httpcheck(ui, path, proto):
     try:
         opener = urlreq.buildopener()
         rsp = opener.open(
             pycompat.strurl(b'%s://%s/!svn/ver/0/.svn' % (proto, path)), b'rb'
         )
         data = rsp.read()
     except urlerr.httperror as inst:
         if inst.code != 404:
             # Except for 404 we cannot know for sure this is not an svn repo
             ui.warn(
                 _(
                     b'svn: cannot probe remote repository, assume it could '
                     b'be a subversion repository. Use --source-type if you '
                     b'know better.\n'
                 )
             )
             return True
         data = inst.fp.read()
     except Exception:
         # Could be urlerr.urlerror if the URL is invalid or anything else.
         return False
     return b'<m:human-readable errcode="160013">' in data
 
 
 protomap = {
     b'http': httpcheck,
     b'https': httpcheck,
     b'file': filecheck,
 }
 
 
 def issvnurl(ui, url):
     try:
         proto, path = url.split(b'://', 1)
         if proto == b'file':
             if (
                 pycompat.iswindows
                 and path[:1] == b'/'
                 and path[1:2].isalpha()
                 and path[2:6].lower() == b'%3a/'
             ):
                 path = path[:2] + b':/' + path[6:]
             try:
                 unicodepath = path.decode(fsencoding)
             except UnicodeDecodeError:
                 ui.warn(
                     _(
                         b'Subversion requires that file URLs can be converted '
                         b'to Unicode using the current locale encoding (%s)\n'
                     )
                     % pycompat.sysbytes(fsencoding)
                 )
                 return False
 
             # Subversion paths are Unicode. Since it does percent-decoding on
             # UTF-8-encoded strings, percent-encoded bytes are interpreted as
             # UTF-8.
             # On Python 3, we have to pass unicode to urlreq.url2pathname().
             # Percent-decoded bytes get decoded using UTF-8 and the 'replace'
             # error handler.
             unicodepath = urlreq.url2pathname(unicodepath)
             if u'\N{REPLACEMENT CHARACTER}' in unicodepath:
                 ui.warn(
                     _(
                         b'Subversion does not support non-UTF-8 '
                         b'percent-encoded bytes in file URLs\n'
                     )
                 )
                 return False
 
             # Below, we approximate how Subversion checks the path. On Unix, we
             # should therefore convert the path to bytes using `fsencoding`
             # (like Subversion does). On Windows, the right thing would
             # actually be to leave the path as unicode. For now, we restrict
             # the path to MBCS.
             path = unicodepath.encode(fsencoding)
     except ValueError:
         proto = b'file'
         path = util.abspath(url)
         try:
             path.decode(fsencoding)
         except UnicodeDecodeError:
             ui.warn(
                 _(
                     b'Subversion requires that paths can be converted to '
                     b'Unicode using the current locale encoding (%s)\n'
                 )
                 % pycompat.sysbytes(fsencoding)
             )
             return False
     if proto == b'file':
         path = util.pconvert(path)
     elif proto in (b'http', 'https'):
         if not encoding.isasciistr(path):
             ui.warn(
                 _(
                     b"Subversion sources don't support non-ASCII characters in "
                     b"HTTP(S) URLs. Please percent-encode them.\n"
                 )
             )
             return False
     check = protomap.get(proto, lambda *args: False)
     while b'/' in path:
         if check(ui, path, proto):
             return True
         path = path.rsplit(b'/', 1)[0]
     return False
 
 
 # SVN conversion code stolen from bzr-svn and tailor
 #
 # Subversion looks like a versioned filesystem, branches structures
 # are defined by conventions and not enforced by the tool. First,
 # we define the potential branches (modules) as "trunk" and "branches"
 # children directories. Revisions are then identified by their
 # module and revision number (and a repository identifier).
 #
 # The revision graph is really a tree (or a forest). By default, a
 # revision parent is the previous revision in the same module. If the
 # module directory is copied/moved from another module then the
 # revision is the module root and its parent the source revision in
 # the parent module. A revision has at most one parent.
 #
 class svn_source(converter_source):
     def __init__(self, ui, repotype, url, revs=None):
         super(svn_source, self).__init__(ui, repotype, url, revs=revs)
 
         init_fsencoding()
         if not (
             url.startswith(b'svn://')
             or url.startswith(b'svn+ssh://')
             or (
                 os.path.exists(url)
                 and os.path.exists(os.path.join(url, b'.svn'))
             )
             or issvnurl(ui, url)
         ):
             raise NoRepo(
                 _(b"%s does not look like a Subversion repository") % url
             )
         if svn is None:
             raise MissingTool(_(b'could not load Subversion python bindings'))
 
         try:
             version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
             if version < (1, 4):
                 raise MissingTool(
                     _(
                         b'Subversion python bindings %d.%d found, '
                         b'1.4 or later required'
                     )
                     % version
                 )
         except AttributeError:
             raise MissingTool(
                 _(
                     b'Subversion python bindings are too old, 1.4 '
                     b'or later required'
                 )
             )
 
         self.lastrevs = {}
 
         latest = None
         try:
             # Support file://path@rev syntax. Useful e.g. to convert
             # deleted branches.
             at = url.rfind(b'@')
             if at >= 0:
                 latest = int(url[at + 1 :])
                 url = url[:at]
         except ValueError:
             pass
         self.url = geturl(url)
         self.encoding = b'UTF-8'  # Subversion is always nominal UTF-8
         try:
             with util.with_lc_ctype():
                 self.transport = transport.SvnRaTransport(url=self.url)
                 self.ra = self.transport.ra
                 self.ctx = self.transport.client
                 self.baseurl = svn.ra.get_repos_root(self.ra)
                 # Module is either empty or a repository path starting with
                 # a slash and not ending with a slash.
                 self.module = urlreq.unquote(self.url[len(self.baseurl) :])
                 self.prevmodule = None
                 self.rootmodule = self.module
                 self.commits = {}
                 self.paths = {}
                 self.uuid = svn.ra.get_uuid(self.ra)
         except svn.core.SubversionException:
             ui.traceback()
             svnversion = b'%d.%d.%d' % (
                 svn.core.SVN_VER_MAJOR,
                 svn.core.SVN_VER_MINOR,
                 svn.core.SVN_VER_MICRO,
             )
             raise NoRepo(
                 _(
                     b"%s does not look like a Subversion repository "
                     b"to libsvn version %s"
                 )
                 % (self.url, svnversion)
             )
 
         if revs:
             if len(revs) > 1:
                 raise error.Abort(
                     _(
                         b'subversion source does not support '
                         b'specifying multiple revisions'
                     )
                 )
             try:
                 latest = int(revs[0])
             except ValueError:
                 raise error.Abort(
                     _(b'svn: revision %s is not an integer') % revs[0]
                 )
 
         trunkcfg = self.ui.config(b'convert', b'svn.trunk')
         if trunkcfg is None:
             trunkcfg = b'trunk'
         self.trunkname = trunkcfg.strip(b'/')
         self.startrev = self.ui.config(b'convert', b'svn.startrev')
         try:
             self.startrev = int(self.startrev)
             if self.startrev < 0:
                 self.startrev = 0
         except ValueError:
             raise error.Abort(
                 _(b'svn: start revision %s is not an integer') % self.startrev
             )
 
         try:
             with util.with_lc_ctype():
                 self.head = self.latest(self.module, latest)
         except SvnPathNotFound:
             self.head = None
         if not self.head:
             raise error.Abort(
                 _(b'no revision found in module %s') % self.module
             )
         self.last_changed = self.revnum(self.head)
 
         self._changescache = (None, None)
 
         if os.path.exists(os.path.join(url, b'.svn/entries')):
             self.wc = url
         else:
             self.wc = None
         self.convertfp = None
 
     def before(self):
         self.with_lc_ctype = util.with_lc_ctype()
         self.with_lc_ctype.__enter__()
 
     def after(self):
         self.with_lc_ctype.__exit__(None, None, None)
 
     def setrevmap(self, revmap):
         lastrevs = {}
         for revid in revmap:
             uuid, module, revnum = revsplit(revid)
             lastrevnum = lastrevs.setdefault(module, revnum)
             if revnum > lastrevnum:
                 lastrevs[module] = revnum
         self.lastrevs = lastrevs
 
     def exists(self, path, optrev):
         try:
             svn.client.ls(
                 self.url.rstrip(b'/') + b'/' + quote(path),
                 optrev,
                 False,
                 self.ctx,
             )
             return True
         except svn.core.SubversionException:
             return False
 
     def getheads(self):
         def isdir(path, revnum):
             kind = self._checkpath(path, revnum)
             return kind == svn.core.svn_node_dir
 
         def getcfgpath(name, rev):
             cfgpath = self.ui.config(b'convert', b'svn.' + name)
             if cfgpath is not None and cfgpath.strip() == b'':
                 return None
             path = (cfgpath or name).strip(b'/')
             if not self.exists(path, rev):
                 if self.module.endswith(path) and name == b'trunk':
                     # we are converting from inside this directory
                     return None
                 if cfgpath:
                     raise error.Abort(
                         _(b'expected %s to be at %r, but not found')
                         % (name, path)
                     )
                 return None
             self.ui.note(
                 _(b'found %s at %r\n') % (name, pycompat.bytestr(path))
             )
             return path
 
         rev = optrev(self.last_changed)
         oldmodule = b''
         trunk = getcfgpath(b'trunk', rev)
         self.tags = getcfgpath(b'tags', rev)
         branches = getcfgpath(b'branches', rev)
 
         # If the project has a trunk or branches, we will extract heads
         # from them. We keep the project root otherwise.
         if trunk:
             oldmodule = self.module or b''
             self.module += b'/' + trunk
             self.head = self.latest(self.module, self.last_changed)
             if not self.head:
                 raise error.Abort(
                     _(b'no revision found in module %s') % self.module
                 )
 
         # First head in the list is the module's head
         self.heads = [self.head]
         if self.tags is not None:
             self.tags = b'%s/%s' % (oldmodule, (self.tags or b'tags'))
 
         # Check if branches bring a few more heads to the list
         if branches:
             rpath = self.url.strip(b'/')
             branchnames = svn.client.ls(
                 rpath + b'/' + quote(branches), rev, False, self.ctx
             )
             for branch in sorted(branchnames):
                 module = b'%s/%s/%s' % (oldmodule, branches, branch)
                 if not isdir(module, self.last_changed):
                     continue
                 brevid = self.latest(module, self.last_changed)
                 if not brevid:
                     self.ui.note(_(b'ignoring empty branch %s\n') % branch)
                     continue
                 self.ui.note(
                     _(b'found branch %s at %d\n')
                     % (branch, self.revnum(brevid))
                 )
                 self.heads.append(brevid)
 
         if self.startrev and self.heads:
             if len(self.heads) > 1:
                 raise error.Abort(
                     _(
                         b'svn: start revision is not supported '
                         b'with more than one branch'
                     )
                 )
             revnum = self.revnum(self.heads[0])
             if revnum < self.startrev:
                 raise error.Abort(
                     _(b'svn: no revision found after start revision %d')
                     % self.startrev
                 )
 
         return self.heads
 
     def _getchanges(self, rev, full):
         (paths, parents) = self.paths[rev]
         copies = {}
         if parents:
             files, self.removed, copies = self.expandpaths(rev, paths, parents)
         if full or not parents:
             # Perform a full checkout on roots
             uuid, module, revnum = revsplit(rev)
             entries = svn.client.ls(
                 self.baseurl + quote(module), optrev(revnum), True, self.ctx
             )
             files = [
                 n
                 for n, e in entries.items()
                 if e.kind == svn.core.svn_node_file
             ]
             self.removed = set()
 
         files.sort()
         files = pycompat.ziplist(files, [rev] * len(files))
         return (files, copies)
 
     def getchanges(self, rev, full):
         # reuse cache from getchangedfiles
         if self._changescache[0] == rev and not full:
+            # TODO: add type hints to avoid this warning, instead of
+            #  suppressing it:
+            #     No attribute '__iter__' on None [attribute-error]
+
+            # pytype: disable=attribute-error
             (files, copies) = self._changescache[1]
+            # pytype: enable=attribute-error
         else:
             (files, copies) = self._getchanges(rev, full)
             # caller caches the result, so free it here to release memory
             del self.paths[rev]
         return (files, copies, set())
 
     def getchangedfiles(self, rev, i):
         # called from filemap - cache computed values for reuse in getchanges
         (files, copies) = self._getchanges(rev, False)
         self._changescache = (rev, (files, copies))
         return [f[0] for f in files]
 
     def getcommit(self, rev):
         if rev not in self.commits:
             uuid, module, revnum = revsplit(rev)
             self.module = module
             self.reparent(module)
             # We assume that:
             # - requests for revisions after "stop" come from the
             # revision graph backward traversal. Cache all of them
             # down to stop, they will be used eventually.
             # - requests for revisions before "stop" come to get
             # isolated branches parents. Just fetch what is needed.
             stop = self.lastrevs.get(module, 0)
             if revnum < stop:
                 stop = revnum + 1
             self._fetch_revisions(revnum, stop)
             if rev not in self.commits:
                 raise error.Abort(_(b'svn: revision %s not found') % revnum)
         revcommit = self.commits[rev]
         # caller caches the result, so free it here to release memory
         del self.commits[rev]
         return revcommit
 
     def checkrevformat(self, revstr, mapname=b'splicemap'):
         """fails if revision format does not match the correct format"""
         if not re.match(
             br'svn:[0-9a-f]{8,8}-[0-9a-f]{4,4}-'
             br'[0-9a-f]{4,4}-[0-9a-f]{4,4}-[0-9a-f]'
             br'{12,12}(.*)@[0-9]+$',
             revstr,
         ):
             raise error.Abort(
                 _(b'%s entry %s is not a valid revision identifier')
                 % (mapname, revstr)
             )
 
     def numcommits(self):
         return int(self.head.rsplit(b'@', 1)[1]) - self.startrev
 
     def gettags(self):
         tags = {}
         if self.tags is None:
             return tags
 
         # svn tags are just a convention, project branches left in a
         # 'tags' directory. There is no other relationship than
         # ancestry, which is expensive to discover and makes them hard
         # to update incrementally.  Worse, past revisions may be
         # referenced by tags far away in the future, requiring a deep
         # history traversal on every calculation.  Current code
         # performs a single backward traversal, tracking moves within
         # the tags directory (tag renaming) and recording a new tag
         # everytime a project is copied from outside the tags
         # directory. It also lists deleted tags, this behaviour may
         # change in the future.
         pendings = []
         tagspath = self.tags
         start = svn.ra.get_latest_revnum(self.ra)
         stream = self._getlog([self.tags], start, self.startrev)
         try:
             for entry in stream:
                 origpaths, revnum, author, date, message = entry
                 if not origpaths:
                     origpaths = []
                 copies = [
                     (e.copyfrom_path, e.copyfrom_rev, p)
                     for p, e in origpaths.items()
                     if e.copyfrom_path
                 ]
                 # Apply moves/copies from more specific to general
                 copies.sort(reverse=True)
 
                 srctagspath = tagspath
                 if copies and copies[-1][2] == tagspath:
                     # Track tags directory moves
                     srctagspath = copies.pop()[0]
 
                 for source, sourcerev, dest in copies:
                     if not dest.startswith(tagspath + b'/'):
                         continue
                     for tag in pendings:
                         if tag[0].startswith(dest):
                             tagpath = source + tag[0][len(dest) :]
                             tag[:2] = [tagpath, sourcerev]
                             break
                     else:
                         pendings.append([source, sourcerev, dest])
 
                 # Filter out tags with children coming from different
                 # parts of the repository like:
                 # /tags/tag.1 (from /trunk:10)
                 # /tags/tag.1/foo (from /branches/foo:12)
                 # Here/tags/tag.1 discarded as well as its children.
                 # It happens with tools like cvs2svn. Such tags cannot
                 # be represented in mercurial.
                 addeds = {
                     p: e.copyfrom_path
                     for p, e in origpaths.items()
                     if e.action == b'A' and e.copyfrom_path
                 }
                 badroots = set()
                 for destroot in addeds:
                     for source, sourcerev, dest in pendings:
                         if not dest.startswith(
                             destroot + b'/'
                         ) or source.startswith(addeds[destroot] + b'/'):
                             continue
                         badroots.add(destroot)
                         break
 
                 for badroot in badroots:
                     pendings = [
                         p
                         for p in pendings
                         if p[2] != badroot
                         and not p[2].startswith(badroot + b'/')
                     ]
 
                 # Tell tag renamings from tag creations
                 renamings = []
                 for source, sourcerev, dest in pendings:
                     tagname = dest.split(b'/')[-1]
                     if source.startswith(srctagspath):
                         renamings.append([source, sourcerev, tagname])
                         continue
                     if tagname in tags:
                         # Keep the latest tag value
                         continue
                     # From revision may be fake, get one with changes
                     try:
                         tagid = self.latest(source, sourcerev)
                         if tagid and tagname not in tags:
                             tags[tagname] = tagid
                     except SvnPathNotFound:
                         # It happens when we are following directories
                         # we assumed were copied with their parents
                         # but were really created in the tag
                         # directory.
                         pass
                 pendings = renamings
                 tagspath = srctagspath
         finally:
             stream.close()
         return tags
 
     def converted(self, rev, destrev):
         if not self.wc:
             return
         if self.convertfp is None:
             self.convertfp = open(
                 os.path.join(self.wc, b'.svn', b'hg-shamap'), b'ab'
             )
         self.convertfp.write(
             util.tonativeeol(b'%s %d\n' % (destrev, self.revnum(rev)))
         )
         self.convertfp.flush()
 
     def revid(self, revnum, module=None):
         return b'svn:%s%s@%d' % (self.uuid, module or self.module, revnum)
 
     def revnum(self, rev):
         return int(rev.split(b'@')[-1])
 
     def latest(self, path, stop=None):
         """Find the latest revid affecting path, up to stop revision
         number. If stop is None, default to repository latest
         revision. It may return a revision in a different module,
         since a branch may be moved without a change being
         reported. Return None if computed module does not belong to
         rootmodule subtree.
         """
 
         def findchanges(path, start, stop=None):
             stream = self._getlog([path], start, stop or 1)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if stop is None and paths:
                         # We do not know the latest changed revision,
                         # keep the first one with changed paths.
                         break
                     if stop is not None and revnum <= stop:
                         break
 
                     for p in paths:
                         if not path.startswith(p) or not paths[p].copyfrom_path:
                             continue
                         newpath = paths[p].copyfrom_path + path[len(p) :]
                         self.ui.debug(
                             b"branch renamed from %s to %s at %d\n"
                             % (path, newpath, revnum)
                         )
                         path = newpath
                         break
                 if not paths:
                     revnum = None
                 return revnum, path
             finally:
                 stream.close()
 
         if not path.startswith(self.rootmodule):
             # Requests on foreign branches may be forbidden at server level
             self.ui.debug(b'ignoring foreign branch %r\n' % path)
             return None
 
         if stop is None:
             stop = svn.ra.get_latest_revnum(self.ra)
         try:
             prevmodule = self.reparent(b'')
             dirent = svn.ra.stat(self.ra, path.strip(b'/'), stop)
             self.reparent(prevmodule)
         except svn.core.SubversionException:
             dirent = None
         if not dirent:
             raise SvnPathNotFound(
                 _(b'%s not found up to revision %d') % (path, stop)
             )
 
         # stat() gives us the previous revision on this line of
         # development, but it might be in *another module*. Fetch the
         # log and detect renames down to the latest revision.
         revnum, realpath = findchanges(path, stop, dirent.created_rev)
         if revnum is None:
             # Tools like svnsync can create empty revision, when
             # synchronizing only a subtree for instance. These empty
             # revisions created_rev still have their original values
             # despite all changes having disappeared and can be
             # returned by ra.stat(), at least when stating the root
             # module. In that case, do not trust created_rev and scan
             # the whole history.
             revnum, realpath = findchanges(path, stop)
             if revnum is None:
                 self.ui.debug(b'ignoring empty branch %r\n' % realpath)
                 return None
 
         if not realpath.startswith(self.rootmodule):
             self.ui.debug(b'ignoring foreign branch %r\n' % realpath)
             return None
         return self.revid(revnum, realpath)
 
     def reparent(self, module):
         """Reparent the svn transport and return the previous parent."""
         if self.prevmodule == module:
             return module
         svnurl = self.baseurl + quote(module)
         prevmodule = self.prevmodule
         if prevmodule is None:
             prevmodule = b''
         self.ui.debug(b"reparent to %s\n" % svnurl)
         svn.ra.reparent(self.ra, svnurl)
         self.prevmodule = module
         return prevmodule
 
     def expandpaths(self, rev, paths, parents):
         changed, removed = set(), set()
         copies = {}
 
         new_module, revnum = revsplit(rev)[1:]
         if new_module != self.module:
             self.module = new_module
             self.reparent(self.module)
 
         progress = self.ui.makeprogress(
             _(b'scanning paths'), unit=_(b'paths'), total=len(paths)
         )
         for i, (path, ent) in enumerate(paths):
             progress.update(i, item=path)
             entrypath = self.getrelpath(path)
 
             kind = self._checkpath(entrypath, revnum)
             if kind == svn.core.svn_node_file:
                 changed.add(self.recode(entrypath))
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrom_path:
                     continue
                 self.ui.debug(
                     b"copied to %s from %s@%d\n"
                     % (entrypath, copyfrom_path, ent.copyfrom_rev)
                 )
                 copies[self.recode(entrypath)] = self.recode(copyfrom_path)
             elif kind == 0:  # gone, but had better be a deleted *file*
                 self.ui.debug(b"gone from %d\n" % ent.copyfrom_rev)
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 parentpath = pmodule + b"/" + entrypath
                 fromkind = self._checkpath(entrypath, prevnum, pmodule)
 
                 if fromkind == svn.core.svn_node_file:
                     removed.add(self.recode(entrypath))
                 elif fromkind == svn.core.svn_node_dir:
                     oroot = parentpath.strip(b'/')
                     nroot = path.strip(b'/')
                     children = self._iterfiles(oroot, prevnum)
                     for childpath in children:
                         childpath = childpath.replace(oroot, nroot)
                         childpath = self.getrelpath(b"/" + childpath, pmodule)
                         if childpath:
                             removed.add(self.recode(childpath))
                 else:
                     self.ui.debug(
                         b'unknown path in revision %d: %s\n' % (revnum, path)
                     )
             elif kind == svn.core.svn_node_dir:
                 if ent.action == b'M':
                     # If the directory just had a prop change,
                     # then we shouldn't need to look for its children.
                     continue
                 if ent.action == b'R' and parents:
                     # If a directory is replacing a file, mark the previous
                     # file as deleted
                     pmodule, prevnum = revsplit(parents[0])[1:]
                     pkind = self._checkpath(entrypath, prevnum, pmodule)
                     if pkind == svn.core.svn_node_file:
                         removed.add(self.recode(entrypath))
                     elif pkind == svn.core.svn_node_dir:
                         # We do not know what files were kept or removed,
                         # mark them all as changed.
                         for childpath in self._iterfiles(pmodule, prevnum):
                             childpath = self.getrelpath(b"/" + childpath)
                             if childpath:
                                 changed.add(self.recode(childpath))
 
                 for childpath in self._iterfiles(path, revnum):
                     childpath = self.getrelpath(b"/" + childpath)
                     if childpath:
                         changed.add(self.recode(childpath))
 
                 # Handle directory copies
                 if not ent.copyfrom_path or not parents:
                     continue
                 # Copy sources not in parent revisions cannot be
                 # represented, ignore their origin for now
                 pmodule, prevnum = revsplit(parents[0])[1:]
                 if ent.copyfrom_rev < prevnum:
                     continue
                 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
                 if not copyfrompath:
                     continue
                 self.ui.debug(
                     b"mark %s came from %s:%d\n"
                     % (path, copyfrompath, ent.copyfrom_rev)
                 )
                 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
                 for childpath in children:
                     childpath = self.getrelpath(b"/" + childpath, pmodule)
                     if not childpath:
                         continue
                     copytopath = path + childpath[len(copyfrompath) :]
                     copytopath = self.getrelpath(copytopath)
                     copies[self.recode(copytopath)] = self.recode(childpath)
 
         progress.complete()
         changed.update(removed)
         return (list(changed), removed, copies)
 
     def _fetch_revisions(self, from_revnum, to_revnum):
         if from_revnum < to_revnum:
             from_revnum, to_revnum = to_revnum, from_revnum
 
         self.child_cset = None
 
         def parselogentry(orig_paths, revnum, author, date, message):
             """Return the parsed commit object or None, and True if
             the revision is a branch root.
             """
             self.ui.debug(
                 b"parsing revision %d (%d changes)\n"
                 % (revnum, len(orig_paths))
             )
 
             branched = False
             rev = self.revid(revnum)
             # branch log might return entries for a parent we already have
 
             if rev in self.commits or revnum < to_revnum:
                 return None, branched
 
             parents = []
             # check whether this revision is the start of a branch or part
             # of a branch renaming
             orig_paths = sorted(orig_paths.items())
             root_paths = [
                 (p, e) for p, e in orig_paths if self.module.startswith(p)
             ]
             if root_paths:
                 path, ent = root_paths[-1]
                 if ent.copyfrom_path:
                     branched = True
                     newpath = ent.copyfrom_path + self.module[len(path) :]
                     # ent.copyfrom_rev may not be the actual last revision
                     previd = self.latest(newpath, ent.copyfrom_rev)
                     if previd is not None:
                         prevmodule, prevnum = revsplit(previd)[1:]
                         if prevnum >= self.startrev:
                             parents = [previd]
                             self.ui.note(
                                 _(b'found parent of branch %s at %d: %s\n')
                                 % (self.module, prevnum, prevmodule)
                             )
                 else:
                     self.ui.debug(b"no copyfrom path, don't know what to do.\n")
 
             paths = []
             # filter out unrelated paths
             for path, ent in orig_paths:
                 if self.getrelpath(path) is None:
                     continue
                 paths.append((path, ent))
 
             date = parsesvndate(date)
             if self.ui.configbool(b'convert', b'localtimezone'):
                 date = makedatetimestamp(date[0])
 
             if message:
                 log = self.recode(message)
             else:
                 log = b''
 
             if author:
                 author = self.recode(author)
             else:
                 author = b''
 
             try:
                 branch = self.module.split(b"/")[-1]
                 if branch == self.trunkname:
                     branch = None
             except IndexError:
                 branch = None
 
             cset = commit(
                 author=author,
                 date=dateutil.datestr(date, b'%Y-%m-%d %H:%M:%S %1%2'),
                 desc=log,
                 parents=parents,
                 branch=branch,
                 rev=rev,
             )
 
             self.commits[rev] = cset
             # The parents list is *shared* among self.paths and the
             # commit object. Both will be updated below.
             self.paths[rev] = (paths, cset.parents)
             if self.child_cset and not self.child_cset.parents:
                 self.child_cset.parents[:] = [rev]
             self.child_cset = cset
             return cset, branched
 
         self.ui.note(
             _(b'fetching revision log for "%s" from %d to %d\n')
             % (self.module, from_revnum, to_revnum)
         )
 
         try:
             firstcset = None
             lastonbranch = False
             stream = self._getlog([self.module], from_revnum, to_revnum)
             try:
                 for entry in stream:
                     paths, revnum, author, date, message = entry
                     if revnum < self.startrev:
                         lastonbranch = True
                         break
                     if not paths:
                         self.ui.debug(b'revision %d has no entries\n' % revnum)
                         # If we ever leave the loop on an empty
                         # revision, do not try to get a parent branch
                         lastonbranch = lastonbranch or revnum == 0
                         continue
                     cset, lastonbranch = parselogentry(
                         paths, revnum, author, date, message
                     )
                     if cset:
                         firstcset = cset
                     if lastonbranch:
                         break
             finally:
                 stream.close()
 
             if not lastonbranch and firstcset and not firstcset.parents:
                 # The first revision of the sequence (the last fetched one)
                 # has invalid parents if not a branch root. Find the parent
                 # revision now, if any.
                 try:
                     firstrevnum = self.revnum(firstcset.rev)
                     if firstrevnum > 1:
                         latest = self.latest(self.module, firstrevnum - 1)
                         if latest:
                             firstcset.parents.append(latest)
                 except SvnPathNotFound:
                     pass
         except svn.core.SubversionException as xxx_todo_changeme:
             (inst, num) = xxx_todo_changeme.args
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise error.Abort(
                     _(b'svn: branch has no revision %s') % to_revnum
                 )
             raise
 
     def getfile(self, file, rev):
         # TODO: ra.get_file transmits the whole file instead of diffs.
         if file in self.removed:
             return None, None
         try:
             new_module, revnum = revsplit(rev)[1:]
             if self.module != new_module:
                 self.module = new_module
                 self.reparent(self.module)
             io = stringio()
             info = svn.ra.get_file(self.ra, file, revnum, io)
             data = io.getvalue()
             # ra.get_file() seems to keep a reference on the input buffer
             # preventing collection. Release it explicitly.
             io.close()
             if isinstance(info, list):
                 info = info[-1]
             mode = (b"svn:executable" in info) and b'x' or b''
             mode = (b"svn:special" in info) and b'l' or mode
         except svn.core.SubversionException as e:
             notfound = (
                 svn.core.SVN_ERR_FS_NOT_FOUND,
                 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND,
             )
             if e.apr_err in notfound:  # File not found
                 return None, None
             raise
         if mode == b'l':
             link_prefix = b"link "
             if data.startswith(link_prefix):
                 data = data[len(link_prefix) :]
         return data, mode
 
     def _iterfiles(self, path, revnum):
         """Enumerate all files in path at revnum, recursively."""
         path = path.strip(b'/')
         pool = svn.core.Pool()
         rpath = b'/'.join([self.baseurl, quote(path)]).strip(b'/')
         entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)
         if path:
             path += b'/'
         return (
             (path + p)
             for p, e in entries.items()
             if e.kind == svn.core.svn_node_file
         )
 
     def getrelpath(self, path, module=None):
         if module is None:
             module = self.module
         # Given the repository url of this wc, say
         #   "http://server/plone/CMFPlone/branches/Plone-2_0-branch"
         # extract the "entry" portion (a relative path) from what
         # svn log --xml says, i.e.
         #   "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"
         # that is to say "tests/PloneTestCase.py"
         if path.startswith(module):
             relative = path.rstrip(b'/')[len(module) :]
             if relative.startswith(b'/'):
                 return relative[1:]
             elif relative == b'':
                 return relative
 
         # The path is outside our tracked tree...
         self.ui.debug(
             b'%r is not under %r, ignoring\n'
             % (pycompat.bytestr(path), pycompat.bytestr(module))
         )
         return None
 
     def _checkpath(self, path, revnum, module=None):
         if module is not None:
             prevmodule = self.reparent(b'')
             path = module + b'/' + path
         try:
             # ra.check_path does not like leading slashes very much, it leads
             # to PROPFIND subversion errors
             return svn.ra.check_path(self.ra, path.strip(b'/'), revnum)
         finally:
             if module is not None:
                 self.reparent(prevmodule)
 
     def _getlog(
         self,
         paths,
         start,
         end,
         limit=0,
         discover_changed_paths=True,
         strict_node_history=False,
     ):
         # Normalize path names, svn >= 1.5 only wants paths relative to
         # supplied URL
         relpaths = []
         for p in paths:
             if not p.startswith(b'/'):
                 p = self.module + b'/' + p
             relpaths.append(p.strip(b'/'))
         args = [
             self.baseurl,
             relpaths,
             start,
             end,
             limit,
             discover_changed_paths,
             strict_node_history,
         ]
         # developer config: convert.svn.debugsvnlog
         if not self.ui.configbool(b'convert', b'svn.debugsvnlog'):
             return directlogstream(*args)
         arg = encodeargs(args)
         hgexe = procutil.hgexecutable()
         cmd = b'%s debugsvnlog' % procutil.shellquote(hgexe)
         stdin, stdout = procutil.popen2(cmd)
         stdin.write(arg)
         try:
             stdin.close()
         except IOError:
             raise error.Abort(
                 _(
                     b'Mercurial failed to run itself, check'
                     b' hg executable is in PATH'
                 )
             )
         return logstream(stdout)
 
 
 pre_revprop_change_template = b'''#!/bin/sh
 
 REPOS="$1"
 REV="$2"
 USER="$3"
 PROPNAME="$4"
 ACTION="$5"
 
 %(rules)s
 
 echo "Changing prohibited revision property" >&2
 exit 1
 '''
 
 
 def gen_pre_revprop_change_hook(prop_actions_allowed):
     rules = []
     for action, propname in prop_actions_allowed:
         rules.append(
             (
                 b'if [ "$ACTION" = "%s" -a "$PROPNAME" = "%s" ]; '
                 b'then exit 0; fi'
             )
             % (action, propname)
         )
     return pre_revprop_change_template % {b'rules': b'\n'.join(rules)}
 
 
 class svn_sink(converter_sink, commandline):
     commit_re = re.compile(br'Committed revision (\d+).', re.M)
     uuid_re = re.compile(br'Repository UUID:\s*(\S+)', re.M)
 
     def prerun(self):
         if self.wc:
             os.chdir(self.wc)
 
     def postrun(self):
         if self.wc:
             os.chdir(self.cwd)
 
     def join(self, name):
         return os.path.join(self.wc, b'.svn', name)
 
     def revmapfile(self):
         return self.join(b'hg-shamap')
 
     def authorfile(self):
         return self.join(b'hg-authormap')
 
     def __init__(self, ui, repotype, path):
 
         converter_sink.__init__(self, ui, repotype, path)
         commandline.__init__(self, ui, b'svn')
         self.delete = []
         self.setexec = []
         self.delexec = []
         self.copies = []
         self.wc = None
         self.cwd = encoding.getcwd()
 
         created = False
         if os.path.isfile(os.path.join(path, b'.svn', b'entries')):
             self.wc = os.path.realpath(path)
             self.run0(b'update')
         else:
             if not re.search(br'^(file|http|https|svn|svn\+ssh)://', path):
                 path = os.path.realpath(path)
                 if os.path.isdir(os.path.dirname(path)):
                     if not os.path.exists(
                         os.path.join(path, b'db', b'fs-type')
                     ):
                         ui.status(
                             _(b"initializing svn repository '%s'\n")
                             % os.path.basename(path)
                         )
                         commandline(ui, b'svnadmin').run0(b'create', path)
                         created = path
                     path = util.normpath(path)
                     if not path.startswith(b'/'):
                         path = b'/' + path
                     path = b'file://' + path
 
             wcpath = os.path.join(
                 encoding.getcwd(), os.path.basename(path) + b'-wc'
             )
             ui.status(
                 _(b"initializing svn working copy '%s'\n")
                 % os.path.basename(wcpath)
             )
             self.run0(b'checkout', path, wcpath)
 
             self.wc = wcpath
         self.opener = vfsmod.vfs(self.wc)
         self.wopener = vfsmod.vfs(self.wc)
         self.childmap = mapfile(ui, self.join(b'hg-childmap'))
         if util.checkexec(self.wc):
             self.is_exec = util.isexec
         else:
             self.is_exec = None
 
         if created:
             prop_actions_allowed = [
                 (b'M', b'svn:log'),
                 (b'A', b'hg:convert-branch'),
                 (b'A', b'hg:convert-rev'),
             ]
 
             if self.ui.configbool(
                 b'convert', b'svn.dangerous-set-commit-dates'
             ):
                 prop_actions_allowed.append((b'M', b'svn:date'))
 
             hook = os.path.join(created, b'hooks', b'pre-revprop-change')
             fp = open(hook, b'wb')
             fp.write(gen_pre_revprop_change_hook(prop_actions_allowed))
             fp.close()
             util.setflags(hook, False, True)
 
         output = self.run0(b'info')
         self.uuid = self.uuid_re.search(output).group(1).strip()
 
     def wjoin(self, *names):
         return os.path.join(self.wc, *names)
 
     @propertycache
     def manifest(self):
         # As of svn 1.7, the "add" command fails when receiving
         # already tracked entries, so we have to track and filter them
         # ourselves.
         m = set()
         output = self.run0(b'ls', recursive=True, xml=True)
         doc = xml.dom.minidom.parseString(output)
         for e in doc.getElementsByTagName('entry'):
             for n in e.childNodes:
                 if n.nodeType != n.ELEMENT_NODE or n.tagName != 'name':
                     continue
                 name = ''.join(
                     c.data for c in n.childNodes if c.nodeType == c.TEXT_NODE
                 )
                 # Entries are compared with names coming from
                 # mercurial, so bytes with undefined encoding. Our
                 # best bet is to assume they are in local
                 # encoding. They will be passed to command line calls
                 # later anyway, so they better be.
                 m.add(encoding.unitolocal(name))
                 break
         return m
 
     def putfile(self, filename, flags, data):
         if b'l' in flags:
             self.wopener.symlink(data, filename)
         else:
             try:
                 if os.path.islink(self.wjoin(filename)):
                     os.unlink(filename)
             except OSError:
                 pass
 
             if self.is_exec:
                 # We need to check executability of the file before the change,
                 # because `vfs.write` is able to reset exec bit.
                 wasexec = False
                 if os.path.exists(self.wjoin(filename)):
                     wasexec = self.is_exec(self.wjoin(filename))
 
             self.wopener.write(filename, data)
 
             if self.is_exec:
                 if wasexec:
                     if b'x' not in flags:
                         self.delexec.append(filename)
                 else:
                     if b'x' in flags:
                         self.setexec.append(filename)
                 util.setflags(self.wjoin(filename), False, b'x' in flags)
 
     def _copyfile(self, source, dest):
         # SVN's copy command pukes if the destination file exists, but
         # our copyfile method expects to record a copy that has
         # already occurred.  Cross the semantic gap.
         wdest = self.wjoin(dest)
         exists = os.path.lexists(wdest)
         if exists:
             fd, tempname = pycompat.mkstemp(
                 prefix=b'hg-copy-', dir=os.path.dirname(wdest)
             )
             os.close(fd)
             os.unlink(tempname)
             os.rename(wdest, tempname)
         try:
             self.run0(b'copy', source, dest)
         finally:
             self.manifest.add(dest)
             if exists:
                 try:
                     os.unlink(wdest)
                 except OSError:
                     pass
                 os.rename(tempname, wdest)
 
     def dirs_of(self, files):
         dirs = set()
         for f in files:
             if os.path.isdir(self.wjoin(f)):
                 dirs.add(f)
             i = len(f)
             for i in iter(lambda: f.rfind(b'/', 0, i), -1):
                 dirs.add(f[:i])
         return dirs
 
     def add_dirs(self, files):
         add_dirs = [
             d for d in sorted(self.dirs_of(files)) if d not in self.manifest
         ]
         if add_dirs:
             self.manifest.update(add_dirs)
             self.xargs(add_dirs, b'add', non_recursive=True, quiet=True)
         return add_dirs
 
     def add_files(self, files):
         files = [f for f in files if f not in self.manifest]
         if files:
             self.manifest.update(files)
             self.xargs(files, b'add', quiet=True)
         return files
 
     def addchild(self, parent, child):
         self.childmap[parent] = child
 
     def revid(self, rev):
         return b"svn:%s@%s" % (self.uuid, rev)
 
     def putcommit(
         self, files, copies, parents, commit, source, revmap, full, cleanp2
     ):
         for parent in parents:
             try:
                 return self.revid(self.childmap[parent])
             except KeyError:
                 pass
 
         # Apply changes to working copy
         for f, v in files:
             data, mode = source.getfile(f, v)
             if data is None:
                 self.delete.append(f)
             else:
                 self.putfile(f, mode, data)
                 if f in copies:
                     self.copies.append([copies[f], f])
         if full:
             self.delete.extend(sorted(self.manifest.difference(files)))
         files = [f[0] for f in files]
 
         entries = set(self.delete)
         files = frozenset(files)
         entries.update(self.add_dirs(files.difference(entries)))
         if self.copies:
             for s, d in self.copies:
                 self._copyfile(s, d)
             self.copies = []
         if self.delete:
             self.xargs(self.delete, b'delete')
             for f in self.delete:
                 self.manifest.remove(f)
             self.delete = []
         entries.update(self.add_files(files.difference(entries)))
         if self.delexec:
             self.xargs(self.delexec, b'propdel', b'svn:executable')
             self.delexec = []
         if self.setexec:
             self.xargs(self.setexec, b'propset', b'svn:executable', b'*')
             self.setexec = []
 
         fd, messagefile = pycompat.mkstemp(prefix=b'hg-convert-')
         fp = os.fdopen(fd, 'wb')
         fp.write(util.tonativeeol(commit.desc))
         fp.close()
         try:
             output = self.run0(
                 b'commit',
                 username=stringutil.shortuser(commit.author),
                 file=messagefile,
                 encoding=b'utf-8',
             )
             try:
                 rev = self.commit_re.search(output).group(1)
             except AttributeError:
                 if not files:
                     return parents[0] if parents else b'None'
                 self.ui.warn(_(b'unexpected svn output:\n'))
                 self.ui.warn(output)
                 raise error.Abort(_(b'unable to cope with svn output'))
             if commit.rev:
                 self.run(
                     b'propset',
                     b'hg:convert-rev',
                     commit.rev,
                     revprop=True,
                     revision=rev,
                 )
             if commit.branch and commit.branch != b'default':
                 self.run(
                     b'propset',
                     b'hg:convert-branch',
                     commit.branch,
                     revprop=True,
                     revision=rev,
                 )
 
             if self.ui.configbool(
                 b'convert', b'svn.dangerous-set-commit-dates'
             ):
                 # Subverson always uses UTC to represent date and time
                 date = dateutil.parsedate(commit.date)
                 date = (date[0], 0)
 
                 # The only way to set date and time for svn commit is to use propset after commit is done
                 self.run(
                     b'propset',
                     b'svn:date',
                     formatsvndate(date),
                     revprop=True,
                     revision=rev,
                 )
 
             for parent in parents:
                 self.addchild(parent, rev)
             return self.revid(rev)
         finally:
             os.unlink(messagefile)
 
     def puttags(self, tags):
         self.ui.warn(_(b'writing Subversion tags is not yet implemented\n'))
         return None, None
 
     def hascommitfrommap(self, rev):
         # We trust that revisions referenced in a map still is present
         # TODO: implement something better if necessary and feasible
         return True
 
     def hascommitforsplicemap(self, rev):
         # This is not correct as one can convert to an existing subversion
         # repository and childmap would not list all revisions. Too bad.
         if rev in self.childmap:
             return True
         raise error.Abort(
             _(
                 b'splice map revision %s not found in subversion '
                 b'child map (revision lookups are not implemented)'
             )
             % rev
         )

hgext/convert/transport.py

@@ -1,157 +1,160
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2007 Daniel Holth <[e-mail]>
 # This is a stripped-down version of the original bzr-svn transport.py,
 # Copyright (C) 2006 Jelmer Vernooij <[e-mail]>
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+# pytype: disable=import-error
 import svn.client
 import svn.core
 import svn.ra
 
+# pytype: enable=import-error
+
 Pool = svn.core.Pool
 SubversionException = svn.core.SubversionException
 
 from mercurial.pycompat import getattr
 from mercurial import util
 
 # Some older versions of the Python bindings need to be
 # explicitly initialized. But what we want to do probably
 # won't work worth a darn against those libraries anyway!
 svn.ra.initialize()
 
 svn_config = None
 
 
 def _create_auth_baton(pool):
     """Create a Subversion authentication baton."""
-    import svn.client
+    import svn.client  # pytype: disable=import-error
 
     # Give the client context baton a suite of authentication
     # providers.h
     providers = [
         svn.client.get_simple_provider(pool),
         svn.client.get_username_provider(pool),
         svn.client.get_ssl_client_cert_file_provider(pool),
         svn.client.get_ssl_client_cert_pw_file_provider(pool),
         svn.client.get_ssl_server_trust_file_provider(pool),
     ]
     # Platform-dependent authentication methods
     getprovider = getattr(
         svn.core, 'svn_auth_get_platform_specific_provider', None
     )
     if getprovider:
         # Available in svn >= 1.6
         for name in (b'gnome_keyring', b'keychain', b'kwallet', b'windows'):
             for type in (b'simple', b'ssl_client_cert_pw', b'ssl_server_trust'):
                 p = getprovider(name, type, pool)
                 if p:
                     providers.append(p)
     else:
         if util.safehasattr(svn.client, b'get_windows_simple_provider'):
             providers.append(svn.client.get_windows_simple_provider(pool))
 
     return svn.core.svn_auth_open(providers, pool)
 
 
 class NotBranchError(SubversionException):
     pass
 
 
 class SvnRaTransport:
     """
     Open an ra connection to a Subversion repository.
     """
 
     def __init__(self, url=b"", ra=None):
         self.pool = Pool()
         self.svn_url = url
         self.username = b''
         self.password = b''
 
         # Only Subversion 1.4 has reparent()
         if ra is None or not util.safehasattr(svn.ra, b'reparent'):
             self.client = svn.client.create_context(self.pool)
             ab = _create_auth_baton(self.pool)
             self.client.auth_baton = ab
             global svn_config
             if svn_config is None:
                 svn_config = svn.core.svn_config_get_config(None)
             self.client.config = svn_config
             try:
                 self.ra = svn.client.open_ra_session(
                     self.svn_url, self.client, self.pool
                 )
             except SubversionException as xxx_todo_changeme:
                 (inst, num) = xxx_todo_changeme.args
                 if num in (
                     svn.core.SVN_ERR_RA_ILLEGAL_URL,
                     svn.core.SVN_ERR_RA_LOCAL_REPOS_OPEN_FAILED,
                     svn.core.SVN_ERR_BAD_URL,
                 ):
                     raise NotBranchError(url)
                 raise
         else:
             self.ra = ra
             svn.ra.reparent(self.ra, self.svn_url.encode('utf8'))
 
     class Reporter:
         def __init__(self, reporter_data):
             self._reporter, self._baton = reporter_data
 
         def set_path(self, path, revnum, start_empty, lock_token, pool=None):
             svn.ra.reporter2_invoke_set_path(
                 self._reporter,
                 self._baton,
                 path,
                 revnum,
                 start_empty,
                 lock_token,
                 pool,
             )
 
         def delete_path(self, path, pool=None):
             svn.ra.reporter2_invoke_delete_path(
                 self._reporter, self._baton, path, pool
             )
 
         def link_path(
             self, path, url, revision, start_empty, lock_token, pool=None
         ):
             svn.ra.reporter2_invoke_link_path(
                 self._reporter,
                 self._baton,
                 path,
                 url,
                 revision,
                 start_empty,
                 lock_token,
                 pool,
             )
 
         def finish_report(self, pool=None):
             svn.ra.reporter2_invoke_finish_report(
                 self._reporter, self._baton, pool
             )
 
         def abort_report(self, pool=None):
             svn.ra.reporter2_invoke_abort_report(
                 self._reporter, self._baton, pool
             )
 
     def do_update(self, revnum, path, *args, **kwargs):
         return self.Reporter(
             svn.ra.do_update(self.ra, revnum, path, *args, **kwargs)
         )

hgext/eol.py

@@ -1,479 +1,480
 """automatically manage newlines in repository files
 
 This extension allows you to manage the type of line endings (CRLF or
 LF) that are used in the repository and in the local working
 directory. That way you can get CRLF line endings on Windows and LF on
 Unix/Mac, thereby letting everybody use their OS native line endings.
 
 The extension reads its configuration from a versioned ``.hgeol``
 configuration file found in the root of the working directory. The
 ``.hgeol`` file use the same syntax as all other Mercurial
 configuration files. It uses two sections, ``[patterns]`` and
 ``[repository]``.
 
 The ``[patterns]`` section specifies how line endings should be
 converted between the working directory and the repository. The format is
 specified by a file pattern. The first match is used, so put more
 specific patterns first. The available line endings are ``LF``,
 ``CRLF``, and ``BIN``.
 
 Files with the declared format of ``CRLF`` or ``LF`` are always
 checked out and stored in the repository in that format and files
 declared to be binary (``BIN``) are left unchanged. Additionally,
 ``native`` is an alias for checking out in the platform's default line
 ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on
 Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's
 default behavior; it is only needed if you need to override a later,
 more general pattern.
 
 The optional ``[repository]`` section specifies the line endings to
 use for files stored in the repository. It has a single setting,
 ``native``, which determines the storage line endings for files
 declared as ``native`` in the ``[patterns]`` section. It can be set to
 ``LF`` or ``CRLF``. The default is ``LF``. For example, this means
 that on Windows, files configured as ``native`` (``CRLF`` by default)
 will be converted to ``LF`` when stored in the repository. Files
 declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section
 are always stored as-is in the repository.
 
 Example versioned ``.hgeol`` file::
 
   [patterns]
   **.py = native
   **.vcproj = CRLF
   **.txt = native
   Makefile = LF
   **.jpg = BIN
 
   [repository]
   native = LF
 
 .. note::
 
    The rules will first apply when files are touched in the working
    directory, e.g. by updating to null and back to tip to touch all files.
 
 The extension uses an optional ``[eol]`` section read from both the
 normal Mercurial configuration files and the ``.hgeol`` file, with the
 latter overriding the former. You can use that section to control the
 overall behavior. There are three settings:
 
 - ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or
   ``CRLF`` to override the default interpretation of ``native`` for
   checkout. This can be used with :hg:`archive` on Unix, say, to
   generate an archive where files have line endings for Windows.
 
 - ``eol.only-consistent`` (default True) can be set to False to make
   the extension convert files with inconsistent EOLs. Inconsistent
   means that there is both ``CRLF`` and ``LF`` present in the file.
   Such files are normally not touched under the assumption that they
   have mixed EOLs on purpose.
 
 - ``eol.fix-trailing-newline`` (default False) can be set to True to
   ensure that converted files end with a EOL character (either ``\\n``
   or ``\\r\\n`` as per the configured patterns).
 
 The extension provides ``cleverencode:`` and ``cleverdecode:`` filters
 like the deprecated win32text extension does. This means that you can
 disable win32text and enable eol and your filters will still work. You
 only need to these filters until you have prepared a ``.hgeol`` file.
 
 The ``win32text.forbid*`` hooks provided by the win32text extension
 have been unified into a single hook named ``eol.checkheadshook``. The
 hook will lookup the expected line endings from the ``.hgeol`` file,
 which means you must migrate to a ``.hgeol`` file first before using
 the hook. ``eol.checkheadshook`` only checks heads, intermediate
 invalid revisions will be pushed. To forbid them completely, use the
 ``eol.checkallhook`` hook. These hooks are best used as
 ``pretxnchangegroup`` hooks.
 
 See :hg:`help patterns` for more information about the glob patterns
 used.
 """
 
 
 import os
 import re
 from mercurial.i18n import _
 from mercurial import (
     config,
     error as errormod,
     extensions,
     match,
     pycompat,
     registrar,
     scmutil,
     util,
 )
 from mercurial.utils import stringutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'eol',
     b'fix-trailing-newline',
     default=False,
 )
 configitem(
     b'eol',
     b'native',
     default=pycompat.oslinesep,
 )
 configitem(
     b'eol',
     b'only-consistent',
     default=True,
 )
 
 # Matches a lone LF, i.e., one that is not part of CRLF.
 singlelf = re.compile(b'(^|[^\r])\n')
 
 
 def inconsistenteol(data):
     return b'\r\n' in data and singlelf.search(data)
 
 
 def tolf(s, params, ui, **kwargs):
     """Filter to convert to LF EOLs."""
     if stringutil.binary(s):
         return s
     if ui.configbool(b'eol', b'only-consistent') and inconsistenteol(s):
         return s
     if (
         ui.configbool(b'eol', b'fix-trailing-newline')
         and s
         and not s.endswith(b'\n')
     ):
         s = s + b'\n'
     return util.tolf(s)
 
 
 def tocrlf(s, params, ui, **kwargs):
     """Filter to convert to CRLF EOLs."""
     if stringutil.binary(s):
         return s
     if ui.configbool(b'eol', b'only-consistent') and inconsistenteol(s):
         return s
     if (
         ui.configbool(b'eol', b'fix-trailing-newline')
         and s
         and not s.endswith(b'\n')
     ):
         s = s + b'\n'
     return util.tocrlf(s)
 
 
 def isbinary(s, params, ui, **kwargs):
     """Filter to do nothing with the file."""
     return s
 
 
 filters = {
     b'to-lf': tolf,
     b'to-crlf': tocrlf,
     b'is-binary': isbinary,
     # The following provide backwards compatibility with win32text
     b'cleverencode:': tolf,
     b'cleverdecode:': tocrlf,
 }
 
 
 class eolfile:
     def __init__(self, ui, root, data):
         self._decode = {
             b'LF': b'to-lf',
             b'CRLF': b'to-crlf',
             b'BIN': b'is-binary',
         }
         self._encode = {
             b'LF': b'to-lf',
             b'CRLF': b'to-crlf',
             b'BIN': b'is-binary',
         }
 
         self.cfg = config.config()
         # Our files should not be touched. The pattern must be
         # inserted first override a '** = native' pattern.
         self.cfg.set(b'patterns', b'.hg*', b'BIN', b'eol')
         # We can then parse the user's patterns.
         self.cfg.parse(b'.hgeol', data)
 
         isrepolf = self.cfg.get(b'repository', b'native') != b'CRLF'
         self._encode[b'NATIVE'] = isrepolf and b'to-lf' or b'to-crlf'
         iswdlf = ui.config(b'eol', b'native') in (b'LF', b'\n')
         self._decode[b'NATIVE'] = iswdlf and b'to-lf' or b'to-crlf'
 
         include = []
         exclude = []
         self.patterns = []
         for pattern, style in self.cfg.items(b'patterns'):
             key = style.upper()
             if key == b'BIN':
                 exclude.append(pattern)
             else:
                 include.append(pattern)
             m = match.match(root, b'', [pattern])
             self.patterns.append((pattern, key, m))
         # This will match the files for which we need to care
         # about inconsistent newlines.
         self.match = match.match(root, b'', [], include, exclude)
 
     def copytoui(self, ui):
         newpatterns = {pattern for pattern, key, m in self.patterns}
         for section in (b'decode', b'encode'):
             for oldpattern, _filter in ui.configitems(section):
                 if oldpattern not in newpatterns:
                     if ui.configsource(section, oldpattern) == b'eol':
                         ui.setconfig(section, oldpattern, b'!', b'eol')
         for pattern, key, m in self.patterns:
             try:
                 ui.setconfig(b'decode', pattern, self._decode[key], b'eol')
                 ui.setconfig(b'encode', pattern, self._encode[key], b'eol')
             except KeyError:
                 ui.warn(
                     _(b"ignoring unknown EOL style '%s' from %s\n")
                     % (key, self.cfg.source(b'patterns', pattern))
                 )
         # eol.only-consistent can be specified in ~/.hgrc or .hgeol
         for k, v in self.cfg.items(b'eol'):
             ui.setconfig(b'eol', k, v, b'eol')
 
     def checkrev(self, repo, ctx, files):
         failed = []
         for f in files or ctx.files():
             if f not in ctx:
                 continue
             for pattern, key, m in self.patterns:
                 if not m(f):
                     continue
                 target = self._encode[key]
                 data = ctx[f].data()
                 if (
                     target == b"to-lf"
                     and b"\r\n" in data
                     or target == b"to-crlf"
                     and singlelf.search(data)
                 ):
                     failed.append((f, target, bytes(ctx)))
                 break
         return failed
 
 
 def parseeol(ui, repo, nodes):
     try:
         for node in nodes:
             try:
                 if node is None:
                     # Cannot use workingctx.data() since it would load
                     # and cache the filters before we configure them.
                     data = repo.wvfs(b'.hgeol').read()
                 else:
                     data = repo[node][b'.hgeol'].data()
                 return eolfile(ui, repo.root, data)
             except (IOError, LookupError):
                 pass
     except errormod.ConfigError as inst:
         ui.warn(
             _(
                 b"warning: ignoring .hgeol file due to parse error "
                 b"at %s: %s\n"
             )
             % (inst.location, inst.message)
         )
     return None
 
 
 def ensureenabled(ui):
     """make sure the extension is enabled when used as hook
 
     When eol is used through hooks, the extension is never formally loaded and
     enabled. This has some side effect, for example the config declaration is
     never loaded. This function ensure the extension is enabled when running
     hooks.
     """
     if b'eol' in ui._knownconfig:
         return
     ui.setconfig(b'extensions', b'eol', b'', source=b'internal')
     extensions.loadall(ui, [b'eol'])
 
 
 def _checkhook(ui, repo, node, headsonly):
     # Get revisions to check and touched files at the same time
     ensureenabled(ui)
     files = set()
     revs = set()
     for rev in range(repo[node].rev(), len(repo)):
         revs.add(rev)
         if headsonly:
             ctx = repo[rev]
             files.update(ctx.files())
             for pctx in ctx.parents():
                 revs.discard(pctx.rev())
     failed = []
     for rev in revs:
         ctx = repo[rev]
         eol = parseeol(ui, repo, [ctx.node()])
         if eol:
             failed.extend(eol.checkrev(repo, ctx, files))
 
     if failed:
         eols = {b'to-lf': b'CRLF', b'to-crlf': b'LF'}
         msgs = []
         for f, target, node in sorted(failed):
             msgs.append(
                 _(b"  %s in %s should not have %s line endings")
                 % (f, node, eols[target])
             )
         raise errormod.Abort(
             _(b"end-of-line check failed:\n") + b"\n".join(msgs)
         )
 
 
 def checkallhook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, False)
 
 
 def checkheadshook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, True)
 
 
 # "checkheadshook" used to be called "hook"
 hook = checkheadshook
 
 
 def preupdate(ui, repo, hooktype, parent1, parent2):
     p1node = scmutil.resolvehexnodeidprefix(repo, parent1)
     repo.loadeol([p1node])
     return False
 
 
 def uisetup(ui):
     ui.setconfig(b'hooks', b'preupdate.eol', preupdate, b'eol')
 
 
 def extsetup(ui):
     try:
         extensions.find(b'win32text')
         ui.warn(
             _(
                 b"the eol extension is incompatible with the "
                 b"win32text extension\n"
             )
         )
     except KeyError:
         pass
 
 
 def reposetup(ui, repo):
     uisetup(repo.ui)
 
     if not repo.local():
         return
     for name, fn in filters.items():
         repo.adddatafilter(name, fn)
 
     ui.setconfig(b'patch', b'eol', b'auto', b'eol')
 
     class eolrepo(repo.__class__):
         def loadeol(self, nodes):
             eol = parseeol(self.ui, self, nodes)
             if eol is None:
                 return None
             eol.copytoui(self.ui)
             return eol.match
 
         def _hgcleardirstate(self):
             self._eolmatch = self.loadeol([None])
             if not self._eolmatch:
                 self._eolmatch = util.never
                 return
 
             oldeol = None
             try:
                 cachemtime = os.path.getmtime(self.vfs.join(b"eol.cache"))
             except OSError:
                 cachemtime = 0
             else:
                 olddata = self.vfs.read(b"eol.cache")
                 if olddata:
                     oldeol = eolfile(self.ui, self.root, olddata)
 
             try:
                 eolmtime = os.path.getmtime(self.wjoin(b".hgeol"))
             except OSError:
                 eolmtime = 0
 
             if eolmtime >= cachemtime and eolmtime > 0:
                 self.ui.debug(b"eol: detected change in .hgeol\n")
 
                 hgeoldata = self.wvfs.read(b'.hgeol')
                 neweol = eolfile(self.ui, self.root, hgeoldata)
 
                 wlock = None
                 try:
                     wlock = self.wlock()
-                    for f in self.dirstate:
-                        if not self.dirstate.get_entry(f).maybe_clean:
-                            continue
-                        if oldeol is not None:
-                            if not oldeol.match(f) and not neweol.match(f):
+                    with self.dirstate.changing_files(self):
+                        for f in self.dirstate:
+                            if not self.dirstate.get_entry(f).maybe_clean:
                                 continue
-                            oldkey = None
-                            for pattern, key, m in oldeol.patterns:
-                                if m(f):
-                                    oldkey = key
-                                    break
-                            newkey = None
-                            for pattern, key, m in neweol.patterns:
-                                if m(f):
-                                    newkey = key
-                                    break
-                            if oldkey == newkey:
-                                continue
-                        # all normal files need to be looked at again since
-                        # the new .hgeol file specify a different filter
-                        self.dirstate.set_possibly_dirty(f)
-                    # Write the cache to update mtime and cache .hgeol
-                    with self.vfs(b"eol.cache", b"w") as f:
-                        f.write(hgeoldata)
+                            if oldeol is not None:
+                                if not oldeol.match(f) and not neweol.match(f):
+                                    continue
+                                oldkey = None
+                                for pattern, key, m in oldeol.patterns:
+                                    if m(f):
+                                        oldkey = key
+                                        break
+                                newkey = None
+                                for pattern, key, m in neweol.patterns:
+                                    if m(f):
+                                        newkey = key
+                                        break
+                                if oldkey == newkey:
+                                    continue
+                            # all normal files need to be looked at again since
+                            # the new .hgeol file specify a different filter
+                            self.dirstate.set_possibly_dirty(f)
+                        # Write the cache to update mtime and cache .hgeol
+                        with self.vfs(b"eol.cache", b"w") as f:
+                            f.write(hgeoldata)
                 except errormod.LockUnavailable:
                     # If we cannot lock the repository and clear the
                     # dirstate, then a commit might not see all files
                     # as modified. But if we cannot lock the
                     # repository, then we can also not make a commit,
                     # so ignore the error.
                     pass
                 finally:
                     if wlock is not None:
                         wlock.release()
 
         def commitctx(self, ctx, error=False, origctx=None):
             for f in sorted(ctx.added() + ctx.modified()):
                 if not self._eolmatch(f):
                     continue
                 fctx = ctx[f]
                 if fctx is None:
                     continue
                 data = fctx.data()
                 if stringutil.binary(data):
                     # We should not abort here, since the user should
                     # be able to say "** = native" to automatically
                     # have all non-binary files taken care of.
                     continue
                 if inconsistenteol(data):
                     raise errormod.Abort(
                         _(b"inconsistent newline style in %s\n") % f
                     )
             return super(eolrepo, self).commitctx(ctx, error, origctx)
 
     repo.__class__ = eolrepo
     repo._hgcleardirstate()

hgext/fastannotate/protocol.py

@@ -1,259 +1,262
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # protocol: logic for a server providing fastannotate support
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import contextlib
 import os
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     error,
     extensions,
     hg,
     util,
     wireprotov1peer,
     wireprotov1server,
 )
 from mercurial.utils import (
     urlutil,
 )
 from . import context
 
 # common
 
 
 def _getmaster(ui):
     """get the mainbranch, and enforce it is set"""
     master = ui.config(b'fastannotate', b'mainbranch')
     if not master:
         raise error.Abort(
             _(
                 b'fastannotate.mainbranch is required '
                 b'for both the client and the server'
             )
         )
     return master
 
 
 # server-side
 
 
 def _capabilities(orig, repo, proto):
     result = orig(repo, proto)
     result.append(b'getannotate')
     return result
 
 
 def _getannotate(repo, proto, path, lastnode):
     # output:
     #   FILE := vfspath + '\0' + str(size) + '\0' + content
     #   OUTPUT := '' | FILE + OUTPUT
     result = b''
     buildondemand = repo.ui.configbool(
         b'fastannotate', b'serverbuildondemand', True
     )
     with context.annotatecontext(repo, path) as actx:
         if buildondemand:
             # update before responding to the client
             master = _getmaster(repo.ui)
             try:
                 if not actx.isuptodate(master):
                     actx.annotate(master, master)
             except Exception:
                 # non-fast-forward move or corrupted. rebuild automically.
                 actx.rebuild()
                 try:
                     actx.annotate(master, master)
                 except Exception:
                     actx.rebuild()  # delete files
             finally:
                 # although the "with" context will also do a close/flush, we
                 # need to do it early so we can send the correct respond to
                 # client.
                 actx.close()
         # send back the full content of revmap and linelog, in the future we
         # may want to do some rsync-like fancy updating.
         # the lastnode check is not necessary if the client and the server
         # agree where the main branch is.
         if actx.lastnode != lastnode:
             for p in [actx.revmappath, actx.linelogpath]:
                 if not os.path.exists(p):
                     continue
                 with open(p, b'rb') as f:
                     content = f.read()
                 vfsbaselen = len(repo.vfs.base + b'/')
                 relpath = p[vfsbaselen:]
                 result += b'%s\0%d\0%s' % (relpath, len(content), content)
     return result
 
 
 def _registerwireprotocommand():
     if b'getannotate' in wireprotov1server.commands:
         return
     wireprotov1server.wireprotocommand(b'getannotate', b'path lastnode')(
         _getannotate
     )
 
 
 def serveruisetup(ui):
     _registerwireprotocommand()
     extensions.wrapfunction(wireprotov1server, b'_capabilities', _capabilities)
 
 
 # client-side
 
 
 def _parseresponse(payload):
     result = {}
     i = 0
     l = len(payload) - 1
     state = 0  # 0: vfspath, 1: size
     vfspath = size = b''
     while i < l:
         ch = payload[i : i + 1]
         if ch == b'\0':
             if state == 1:
                 result[vfspath] = payload[i + 1 : i + 1 + int(size)]
                 i += int(size)
                 state = 0
                 vfspath = size = b''
             elif state == 0:
                 state = 1
         else:
             if state == 1:
                 size += ch
             elif state == 0:
                 vfspath += ch
         i += 1
     return result
 
 
 def peersetup(ui, peer):
     class fastannotatepeer(peer.__class__):
         @wireprotov1peer.batchable
         def getannotate(self, path, lastnode=None):
             if not self.capable(b'getannotate'):
                 ui.warn(_(b'remote peer cannot provide annotate cache\n'))
                 return None, None
             else:
                 args = {b'path': path, b'lastnode': lastnode or b''}
                 return args, _parseresponse
 
     peer.__class__ = fastannotatepeer
 
 
 @contextlib.contextmanager
 def annotatepeer(repo):
     ui = repo.ui
 
     remotedest = ui.config(b'fastannotate', b'remotepath', b'default')
-    r = urlutil.get_unique_pull_path(b'fastannotate', repo, ui, remotedest)
-    remotepath = r[0]
+    remotepath = urlutil.get_unique_pull_path_obj(
+        b'fastannotate',
+        ui,
+        remotedest,
+    )
     peer = hg.peer(ui, {}, remotepath)
 
     try:
         yield peer
     finally:
         peer.close()
 
 
 def clientfetch(repo, paths, lastnodemap=None, peer=None):
     """download annotate cache from the server for paths"""
     if not paths:
         return
 
     if peer is None:
         with annotatepeer(repo) as peer:
             return clientfetch(repo, paths, lastnodemap, peer)
 
     if lastnodemap is None:
         lastnodemap = {}
 
     ui = repo.ui
     results = []
     with peer.commandexecutor() as batcher:
         ui.debug(b'fastannotate: requesting %d files\n' % len(paths))
         for p in paths:
             results.append(
                 batcher.callcommand(
                     b'getannotate',
                     {b'path': p, b'lastnode': lastnodemap.get(p)},
                 )
             )
 
         for result in results:
             r = result.result()
             # TODO: pconvert these paths on the server?
             r = {util.pconvert(p): v for p, v in r.items()}
             for path in sorted(r):
                 # ignore malicious paths
                 if not path.startswith(b'fastannotate/') or b'/../' in (
                     path + b'/'
                 ):
                     ui.debug(
                         b'fastannotate: ignored malicious path %s\n' % path
                     )
                     continue
                 content = r[path]
                 if ui.debugflag:
                     ui.debug(
                         b'fastannotate: writing %d bytes to %s\n'
                         % (len(content), path)
                     )
                 repo.vfs.makedirs(os.path.dirname(path))
                 with repo.vfs(path, b'wb') as f:
                     f.write(content)
 
 
 def _filterfetchpaths(repo, paths):
     """return a subset of paths whose history is long and need to fetch linelog
     from the server. works with remotefilelog and non-remotefilelog repos.
     """
     threshold = repo.ui.configint(b'fastannotate', b'clientfetchthreshold', 10)
     if threshold <= 0:
         return paths
 
     result = []
     for path in paths:
         try:
             if len(repo.file(path)) >= threshold:
                 result.append(path)
         except Exception:  # file not found etc.
             result.append(path)
 
     return result
 
 
 def localreposetup(ui, repo):
     class fastannotaterepo(repo.__class__):
         def prefetchfastannotate(self, paths, peer=None):
             master = _getmaster(self.ui)
             needupdatepaths = []
             lastnodemap = {}
             try:
                 for path in _filterfetchpaths(self, paths):
                     with context.annotatecontext(self, path) as actx:
                         if not actx.isuptodate(master, strict=False):
                             needupdatepaths.append(path)
                             lastnodemap[path] = actx.lastnode
                 if needupdatepaths:
                     clientfetch(self, needupdatepaths, lastnodemap, peer)
             except Exception as ex:
                 # could be directory not writable or so, not fatal
                 self.ui.debug(b'fastannotate: prefetch failed: %r\n' % ex)
 
     repo.__class__ = fastannotaterepo
 
 
 def clientreposetup(ui, repo):
     _registerwireprotocommand()
     if repo.local():
         localreposetup(ui, repo)
     # TODO: this mutates global state, but only if at least one repo
     # has the extension enabled. This is probably bad for hgweb.
     if peersetup not in hg.wirepeersetupfuncs:
         hg.wirepeersetupfuncs.append(peersetup)

hgext/fetch.py

@@ -1,198 +1,198
 # fetch.py - pull and merge remote changes
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''pull, update and merge in one command (DEPRECATED)'''
 
 
 from mercurial.i18n import _
 from mercurial.node import short
 from mercurial import (
     cmdutil,
     error,
     exchange,
     hg,
     lock,
     pycompat,
     registrar,
 )
 from mercurial.utils import (
     dateutil,
     urlutil,
 )
 
 release = lock.release
 cmdtable = {}
 command = registrar.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 
 @command(
     b'fetch',
     [
         (
             b'r',
             b'rev',
             [],
             _(b'a specific revision you would like to pull'),
             _(b'REV'),
         ),
         (b'', b'edit', None, _(b'invoke editor on commit messages')),
         (b'', b'force-editor', None, _(b'edit commit message (DEPRECATED)')),
         (b'', b'switch-parent', None, _(b'switch parents when merging')),
     ]
     + cmdutil.commitopts
     + cmdutil.commitopts2
     + cmdutil.remoteopts,
     _(b'hg fetch [SOURCE]'),
     helpcategory=command.CATEGORY_REMOTE_REPO_MANAGEMENT,
 )
 def fetch(ui, repo, source=b'default', **opts):
     """pull changes from a remote repository, merge new changes if needed.
 
     This finds all changes from the repository at the specified path
     or URL and adds them to the local repository.
 
     If the pulled changes add a new branch head, the head is
     automatically merged, and the result of the merge is committed.
     Otherwise, the working directory is updated to include the new
     changes.
 
     When a merge is needed, the working directory is first updated to
     the newly pulled changes. Local changes are then merged into the
     pulled changes. To switch the merge order, use --switch-parent.
 
     See :hg:`help dates` for a list of formats valid for -d/--date.
 
     Returns 0 on success.
     """
 
     opts = pycompat.byteskwargs(opts)
     date = opts.get(b'date')
     if date:
         opts[b'date'] = dateutil.parsedate(date)
 
     parent = repo.dirstate.p1()
     branch = repo.dirstate.branch()
     try:
         branchnode = repo.branchtip(branch)
     except error.RepoLookupError:
         branchnode = None
     if parent != branchnode:
         raise error.Abort(
             _(b'working directory not at branch tip'),
             hint=_(b"use 'hg update' to check out branch tip"),
         )
 
     wlock = lock = None
     try:
         wlock = repo.wlock()
         lock = repo.lock()
 
         cmdutil.bailifchanged(repo)
 
         bheads = repo.branchheads(branch)
         bheads = [head for head in bheads if len(repo[head].children()) == 0]
         if len(bheads) > 1:
             raise error.Abort(
                 _(
                     b'multiple heads in this branch '
                     b'(use "hg heads ." and "hg merge" to merge)'
                 )
             )
 
-        path = urlutil.get_unique_pull_path(b'fetch', repo, ui, source)[0]
+        path = urlutil.get_unique_pull_path_obj(b'fetch', ui, source)
         other = hg.peer(repo, opts, path)
-        ui.status(_(b'pulling from %s\n') % urlutil.hidepassword(path))
+        ui.status(_(b'pulling from %s\n') % urlutil.hidepassword(path.loc))
         revs = None
         if opts[b'rev']:
             try:
                 revs = [other.lookup(rev) for rev in opts[b'rev']]
             except error.CapabilityError:
                 err = _(
                     b"other repository doesn't support revision lookup, "
                     b"so a rev cannot be specified."
                 )
                 raise error.Abort(err)
 
         # Are there any changes at all?
         modheads = exchange.pull(repo, other, heads=revs).cgresult
         if modheads == 0:
             return 0
 
         # Is this a simple fast-forward along the current branch?
         newheads = repo.branchheads(branch)
         newchildren = repo.changelog.nodesbetween([parent], newheads)[2]
         if len(newheads) == 1 and len(newchildren):
             if newchildren[0] != parent:
                 return hg.update(repo, newchildren[0])
             else:
                 return 0
 
         # Are there more than one additional branch heads?
         newchildren = [n for n in newchildren if n != parent]
         newparent = parent
         if newchildren:
             newparent = newchildren[0]
             hg.clean(repo, newparent)
         newheads = [n for n in newheads if n != newparent]
         if len(newheads) > 1:
             ui.status(
                 _(
                     b'not merging with %d other new branch heads '
                     b'(use "hg heads ." and "hg merge" to merge them)\n'
                 )
                 % (len(newheads) - 1)
             )
             return 1
 
         if not newheads:
             return 0
 
         # Otherwise, let's merge.
         err = False
         if newheads:
             # By default, we consider the repository we're pulling
             # *from* as authoritative, so we merge our changes into
             # theirs.
             if opts[b'switch_parent']:
                 firstparent, secondparent = newparent, newheads[0]
             else:
                 firstparent, secondparent = newheads[0], newparent
                 ui.status(
                     _(b'updating to %d:%s\n')
                     % (repo.changelog.rev(firstparent), short(firstparent))
                 )
             hg.clean(repo, firstparent)
             p2ctx = repo[secondparent]
             ui.status(
                 _(b'merging with %d:%s\n') % (p2ctx.rev(), short(secondparent))
             )
             err = hg.merge(p2ctx, remind=False)
 
         if not err:
             # we don't translate commit messages
             message = cmdutil.logmessage(ui, opts) or (
                 b'Automated merge with %s' % urlutil.removeauth(other.url())
             )
             editopt = opts.get(b'edit') or opts.get(b'force_editor')
             editor = cmdutil.getcommiteditor(edit=editopt, editform=b'fetch')
             n = repo.commit(
                 message, opts[b'user'], opts[b'date'], editor=editor
             )
             ui.status(
                 _(b'new changeset %d:%s merges remote changes with local\n')
                 % (repo.changelog.rev(n), short(n))
             )
 
         return err
 
     finally:
         release(lock, wlock)

hgext/fix.py

@@ -1,958 +1,958
 # fix - rewrite file content in changesets and working copy
 #
 # Copyright 2018 Google LLC.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """rewrite file content in changesets or working copy (EXPERIMENTAL)
 
 Provides a command that runs configured tools on the contents of modified files,
 writing back any fixes to the working copy or replacing changesets.
 
 Here is an example configuration that causes :hg:`fix` to apply automatic
 formatting fixes to modified lines in C++ code::
 
   [fix]
   clang-format:command=clang-format --assume-filename={rootpath}
   clang-format:linerange=--lines={first}:{last}
   clang-format:pattern=set:**.cpp or **.hpp
 
 The :command suboption forms the first part of the shell command that will be
 used to fix a file. The content of the file is passed on standard input, and the
 fixed file content is expected on standard output. Any output on standard error
 will be displayed as a warning. If the exit status is not zero, the file will
 not be affected. A placeholder warning is displayed if there is a non-zero exit
 status but no standard error output. Some values may be substituted into the
 command::
 
   {rootpath}  The path of the file being fixed, relative to the repo root
   {basename}  The name of the file being fixed, without the directory path
 
 If the :linerange suboption is set, the tool will only be run if there are
 changed lines in a file. The value of this suboption is appended to the shell
 command once for every range of changed lines in the file. Some values may be
 substituted into the command::
 
   {first}   The 1-based line number of the first line in the modified range
   {last}    The 1-based line number of the last line in the modified range
 
 Deleted sections of a file will be ignored by :linerange, because there is no
 corresponding line range in the version being fixed.
 
 By default, tools that set :linerange will only be executed if there is at least
 one changed line range. This is meant to prevent accidents like running a code
 formatter in such a way that it unexpectedly reformats the whole file. If such a
 tool needs to operate on unchanged files, it should set the :skipclean suboption
 to false.
 
 The :pattern suboption determines which files will be passed through each
 configured tool. See :hg:`help patterns` for possible values. However, all
 patterns are relative to the repo root, even if that text says they are relative
 to the current working directory. If there are file arguments to :hg:`fix`, the
 intersection of these patterns is used.
 
 There is also a configurable limit for the maximum size of file that will be
 processed by :hg:`fix`::
 
   [fix]
   maxfilesize = 2MB
 
 Normally, execution of configured tools will continue after a failure (indicated
 by a non-zero exit status). It can also be configured to abort after the first
 such failure, so that no files will be affected if any tool fails. This abort
 will also cause :hg:`fix` to exit with a non-zero status::
 
   [fix]
   failure = abort
 
 When multiple tools are configured to affect a file, they execute in an order
 defined by the :priority suboption. The priority suboption has a default value
 of zero for each tool. Tools are executed in order of descending priority. The
 execution order of tools with equal priority is unspecified. For example, you
 could use the 'sort' and 'head' utilities to keep only the 10 smallest numbers
 in a text file by ensuring that 'sort' runs before 'head'::
 
   [fix]
   sort:command = sort -n
   head:command = head -n 10
   sort:pattern = numbers.txt
   head:pattern = numbers.txt
   sort:priority = 2
   head:priority = 1
 
 To account for changes made by each tool, the line numbers used for incremental
 formatting are recomputed before executing the next tool. So, each tool may see
 different values for the arguments added by the :linerange suboption.
 
 Each fixer tool is allowed to return some metadata in addition to the fixed file
 content. The metadata must be placed before the file content on stdout,
 separated from the file content by a zero byte. The metadata is parsed as a JSON
 value (so, it should be UTF-8 encoded and contain no zero bytes). A fixer tool
 is expected to produce this metadata encoding if and only if the :metadata
 suboption is true::
 
   [fix]
   tool:command = tool --prepend-json-metadata
   tool:metadata = true
 
 The metadata values are passed to hooks, which can be used to print summaries or
 perform other post-fixing work. The supported hooks are::
 
   "postfixfile"
     Run once for each file in each revision where any fixer tools made changes
     to the file content. Provides "$HG_REV" and "$HG_PATH" to identify the file,
     and "$HG_METADATA" with a map of fixer names to metadata values from fixer
     tools that affected the file. Fixer tools that didn't affect the file have a
     value of None. Only fixer tools that executed are present in the metadata.
 
   "postfix"
     Run once after all files and revisions have been handled. Provides
     "$HG_REPLACEMENTS" with information about what revisions were created and
     made obsolete. Provides a boolean "$HG_WDIRWRITTEN" to indicate whether any
     files in the working copy were updated. Provides a list "$HG_METADATA"
     mapping fixer tool names to lists of metadata values returned from
     executions that modified a file. This aggregates the same metadata
     previously passed to the "postfixfile" hook.
 
 Fixer tools are run in the repository's root directory. This allows them to read
 configuration files from the working copy, or even write to the working copy.
 The working copy is not updated to match the revision being fixed. In fact,
 several revisions may be fixed in parallel. Writes to the working copy are not
 amended into the revision being fixed; fixer tools should always write fixed
 file content back to stdout as documented above.
 """
 
 
 import collections
 import itertools
 import os
 import re
 import subprocess
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     nullrev,
     wdirrev,
 )
 
 from mercurial.utils import procutil
 
 from mercurial import (
     cmdutil,
     context,
     copies,
     error,
     logcmdutil,
     match as matchmod,
     mdiff,
     merge,
     mergestate as mergestatemod,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
     worker,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # Register the suboptions allowed for each configured fixer, and default values.
 FIXER_ATTRS = {
     b'command': None,
     b'linerange': None,
     b'pattern': None,
     b'priority': 0,
     b'metadata': False,
     b'skipclean': True,
     b'enabled': True,
 }
 
 for key, default in FIXER_ATTRS.items():
     configitem(b'fix', b'.*:%s$' % key, default=default, generic=True)
 
 # A good default size allows most source code files to be fixed, but avoids
 # letting fixer tools choke on huge inputs, which could be surprising to the
 # user.
 configitem(b'fix', b'maxfilesize', default=b'2MB')
 
 # Allow fix commands to exit non-zero if an executed fixer tool exits non-zero.
 # This helps users do shell scripts that stop when a fixer tool signals a
 # problem.
 configitem(b'fix', b'failure', default=b'continue')
 
 
 def checktoolfailureaction(ui, message, hint=None):
     """Abort with 'message' if fix.failure=abort"""
     action = ui.config(b'fix', b'failure')
     if action not in (b'continue', b'abort'):
         raise error.Abort(
             _(b'unknown fix.failure action: %s') % (action,),
             hint=_(b'use "continue" or "abort"'),
         )
     if action == b'abort':
         raise error.Abort(message, hint=hint)
 
 
 allopt = (b'', b'all', False, _(b'fix all non-public non-obsolete revisions'))
 baseopt = (
     b'',
     b'base',
     [],
     _(
         b'revisions to diff against (overrides automatic '
         b'selection, and applies to every revision being '
         b'fixed)'
     ),
     _(b'REV'),
 )
 revopt = (b'r', b'rev', [], _(b'revisions to fix (ADVANCED)'), _(b'REV'))
 sourceopt = (
     b's',
     b'source',
     [],
     _(b'fix the specified revisions and their descendants'),
     _(b'REV'),
 )
 wdiropt = (b'w', b'working-dir', False, _(b'fix the working directory'))
 wholeopt = (b'', b'whole', False, _(b'always fix every line of a file'))
 usage = _(b'[OPTION]... [FILE]...')
 
 
 @command(
     b'fix',
     [allopt, baseopt, revopt, sourceopt, wdiropt, wholeopt],
     usage,
     helpcategory=command.CATEGORY_FILE_CONTENTS,
 )
 def fix(ui, repo, *pats, **opts):
     """rewrite file content in changesets or working directory
 
     Runs any configured tools to fix the content of files. Only affects files
     with changes, unless file arguments are provided. Only affects changed lines
     of files, unless the --whole flag is used. Some tools may always affect the
     whole file regardless of --whole.
 
     If --working-dir is used, files with uncommitted changes in the working copy
     will be fixed. Note that no backup are made.
 
     If revisions are specified with --source, those revisions and their
     descendants will be checked, and they may be replaced with new revisions
     that have fixed file content. By automatically including the descendants,
     no merging, rebasing, or evolution will be required. If an ancestor of the
     working copy is included, then the working copy itself will also be fixed,
     and the working copy will be updated to the fixed parent.
 
     When determining what lines of each file to fix at each revision, the whole
     set of revisions being fixed is considered, so that fixes to earlier
     revisions are not forgotten in later ones. The --base flag can be used to
     override this default behavior, though it is not usually desirable to do so.
     """
     opts = pycompat.byteskwargs(opts)
     cmdutil.check_at_most_one_arg(opts, b'all', b'source', b'rev')
     cmdutil.check_incompatible_arguments(
         opts, b'working_dir', [b'all', b'source']
     )
 
     with repo.wlock(), repo.lock(), repo.transaction(b'fix'):
         revstofix = getrevstofix(ui, repo, opts)
         basectxs = getbasectxs(repo, opts, revstofix)
         workqueue, numitems = getworkqueue(
             ui, repo, pats, opts, revstofix, basectxs
         )
         basepaths = getbasepaths(repo, opts, workqueue, basectxs)
         fixers = getfixers(ui)
 
         # Rather than letting each worker independently fetch the files
         # (which also would add complications for shared/keepalive
         # connections), prefetch them all first.
         _prefetchfiles(repo, workqueue, basepaths)
 
         # There are no data dependencies between the workers fixing each file
         # revision, so we can use all available parallelism.
         def getfixes(items):
             for srcrev, path, dstrevs in items:
                 ctx = repo[srcrev]
                 olddata = ctx[path].data()
                 metadata, newdata = fixfile(
                     ui,
                     repo,
                     opts,
                     fixers,
                     ctx,
                     path,
                     basepaths,
                     basectxs[srcrev],
                 )
                 # We ungroup the work items now, because the code that consumes
                 # these results has to handle each dstrev separately, and in
                 # topological order. Because these are handled in topological
                 # order, it's important that we pass around references to
                 # "newdata" instead of copying it. Otherwise, we would be
                 # keeping more copies of file content in memory at a time than
                 # if we hadn't bothered to group/deduplicate the work items.
                 data = newdata if newdata != olddata else None
                 for dstrev in dstrevs:
                     yield (dstrev, path, metadata, data)
 
         results = worker.worker(
             ui, 1.0, getfixes, tuple(), workqueue, threadsafe=False
         )
 
         # We have to hold on to the data for each successor revision in memory
         # until all its parents are committed. We ensure this by committing and
         # freeing memory for the revisions in some topological order. This
         # leaves a little bit of memory efficiency on the table, but also makes
         # the tests deterministic. It might also be considered a feature since
         # it makes the results more easily reproducible.
         filedata = collections.defaultdict(dict)
         aggregatemetadata = collections.defaultdict(list)
         replacements = {}
         wdirwritten = False
         commitorder = sorted(revstofix, reverse=True)
         with ui.makeprogress(
             topic=_(b'fixing'), unit=_(b'files'), total=sum(numitems.values())
         ) as progress:
             for rev, path, filerevmetadata, newdata in results:
                 progress.increment(item=path)
                 for fixername, fixermetadata in filerevmetadata.items():
                     aggregatemetadata[fixername].append(fixermetadata)
                 if newdata is not None:
                     filedata[rev][path] = newdata
                     hookargs = {
                         b'rev': rev,
                         b'path': path,
                         b'metadata': filerevmetadata,
                     }
                     repo.hook(
                         b'postfixfile',
                         throw=False,
                         **pycompat.strkwargs(hookargs)
                     )
                 numitems[rev] -= 1
                 # Apply the fixes for this and any other revisions that are
                 # ready and sitting at the front of the queue. Using a loop here
                 # prevents the queue from being blocked by the first revision to
                 # be ready out of order.
                 while commitorder and not numitems[commitorder[-1]]:
                     rev = commitorder.pop()
                     ctx = repo[rev]
                     if rev == wdirrev:
                         writeworkingdir(repo, ctx, filedata[rev], replacements)
                         wdirwritten = bool(filedata[rev])
                     else:
                         replacerev(ui, repo, ctx, filedata[rev], replacements)
                     del filedata[rev]
 
         cleanup(repo, replacements, wdirwritten)
         hookargs = {
             b'replacements': replacements,
             b'wdirwritten': wdirwritten,
             b'metadata': aggregatemetadata,
         }
         repo.hook(b'postfix', throw=True, **pycompat.strkwargs(hookargs))
 
 
 def cleanup(repo, replacements, wdirwritten):
     """Calls scmutil.cleanupnodes() with the given replacements.
 
     "replacements" is a dict from nodeid to nodeid, with one key and one value
     for every revision that was affected by fixing. This is slightly different
     from cleanupnodes().
 
     "wdirwritten" is a bool which tells whether the working copy was affected by
     fixing, since it has no entry in "replacements".
 
     Useful as a hook point for extending "hg fix" with output summarizing the
     effects of the command, though we choose not to output anything here.
     """
     replacements = {prec: [succ] for prec, succ in replacements.items()}
     scmutil.cleanupnodes(repo, replacements, b'fix', fixphase=True)
 
 
 def getworkqueue(ui, repo, pats, opts, revstofix, basectxs):
     """Constructs a list of files to fix and which revisions each fix applies to
 
     To avoid duplicating work, there is usually only one work item for each file
     revision that might need to be fixed. There can be multiple work items per
     file revision if the same file needs to be fixed in multiple changesets with
     different baserevs. Each work item also contains a list of changesets where
     the file's data should be replaced with the fixed data. The work items for
     earlier changesets come earlier in the work queue, to improve pipelining by
     allowing the first changeset to be replaced while fixes are still being
     computed for later changesets.
 
     Also returned is a map from changesets to the count of work items that might
     affect each changeset. This is used later to count when all of a changeset's
     work items have been finished, without having to inspect the remaining work
     queue in each worker subprocess.
 
     The example work item (1, "foo/bar.txt", (1, 2, 3)) means that the data of
     bar.txt should be read from revision 1, then fixed, and written back to
     revisions 1, 2 and 3. Revision 1 is called the "srcrev" and the list of
     revisions is called the "dstrevs". In practice the srcrev is always one of
     the dstrevs, and we make that choice when constructing the work item so that
     the choice can't be made inconsistently later on. The dstrevs should all
     have the same file revision for the given path, so the choice of srcrev is
     arbitrary. The wdirrev can be a dstrev and a srcrev.
     """
     dstrevmap = collections.defaultdict(list)
     numitems = collections.defaultdict(int)
     maxfilesize = ui.configbytes(b'fix', b'maxfilesize')
     for rev in sorted(revstofix):
         fixctx = repo[rev]
         match = scmutil.match(fixctx, pats, opts)
         for path in sorted(
             pathstofix(ui, repo, pats, opts, match, basectxs[rev], fixctx)
         ):
             fctx = fixctx[path]
             if fctx.islink():
                 continue
             if fctx.size() > maxfilesize:
                 ui.warn(
                     _(b'ignoring file larger than %s: %s\n')
                     % (util.bytecount(maxfilesize), path)
                 )
                 continue
             baserevs = tuple(ctx.rev() for ctx in basectxs[rev])
             dstrevmap[(fctx.filerev(), baserevs, path)].append(rev)
             numitems[rev] += 1
     workqueue = [
         (min(dstrevs), path, dstrevs)
         for (_filerev, _baserevs, path), dstrevs in dstrevmap.items()
     ]
     # Move work items for earlier changesets to the front of the queue, so we
     # might be able to replace those changesets (in topological order) while
     # we're still processing later work items. Note the min() in the previous
     # expression, which means we don't need a custom comparator here. The path
     # is also important in the sort order to make the output order stable. There
     # are some situations where this doesn't help much, but some situations
     # where it lets us buffer O(1) files instead of O(n) files.
     workqueue.sort()
     return workqueue, numitems
 
 
 def getrevstofix(ui, repo, opts):
     """Returns the set of revision numbers that should be fixed"""
     if opts[b'all']:
         revs = repo.revs(b'(not public() and not obsolete()) or wdir()')
     elif opts[b'source']:
         source_revs = logcmdutil.revrange(repo, opts[b'source'])
         revs = set(repo.revs(b'(%ld::) - obsolete()', source_revs))
         if wdirrev in source_revs:
             # `wdir()::` is currently empty, so manually add wdir
             revs.add(wdirrev)
         if repo[b'.'].rev() in revs:
             revs.add(wdirrev)
     else:
         revs = set(logcmdutil.revrange(repo, opts[b'rev']))
         if opts.get(b'working_dir'):
             revs.add(wdirrev)
     # Allow fixing only wdir() even if there's an unfinished operation
     if not (len(revs) == 1 and wdirrev in revs):
         cmdutil.checkunfinished(repo)
         rewriteutil.precheck(repo, revs, b'fix')
     if (
         wdirrev in revs
         and mergestatemod.mergestate.read(repo).unresolvedcount()
     ):
         raise error.Abort(b'unresolved conflicts', hint=b"use 'hg resolve'")
     if not revs:
         raise error.Abort(
             b'no changesets specified', hint=b'use --source or --working-dir'
         )
     return revs
 
 
 def pathstofix(ui, repo, pats, opts, match, basectxs, fixctx):
     """Returns the set of files that should be fixed in a context
 
     The result depends on the base contexts; we include any file that has
     changed relative to any of the base contexts. Base contexts should be
     ancestors of the context being fixed.
     """
     files = set()
     for basectx in basectxs:
         stat = basectx.status(
             fixctx, match=match, listclean=bool(pats), listunknown=bool(pats)
         )
         files.update(
             set(
                 itertools.chain(
                     stat.added, stat.modified, stat.clean, stat.unknown
                 )
             )
         )
     return files
 
 
 def lineranges(opts, path, basepaths, basectxs, fixctx, content2):
     """Returns the set of line ranges that should be fixed in a file
 
     Of the form [(10, 20), (30, 40)].
 
     This depends on the given base contexts; we must consider lines that have
     changed versus any of the base contexts, and whether the file has been
     renamed versus any of them.
 
     Another way to understand this is that we exclude line ranges that are
     common to the file in all base contexts.
     """
     if opts.get(b'whole'):
         # Return a range containing all lines. Rely on the diff implementation's
         # idea of how many lines are in the file, instead of reimplementing it.
         return difflineranges(b'', content2)
 
     rangeslist = []
     for basectx in basectxs:
         basepath = basepaths.get((basectx.rev(), fixctx.rev(), path), path)
 
         if basepath in basectx:
             content1 = basectx[basepath].data()
         else:
             content1 = b''
         rangeslist.extend(difflineranges(content1, content2))
     return unionranges(rangeslist)
 
 
 def getbasepaths(repo, opts, workqueue, basectxs):
     if opts.get(b'whole'):
         # Base paths will never be fetched for line range determination.
         return {}
 
     basepaths = {}
     for srcrev, path, _dstrevs in workqueue:
         fixctx = repo[srcrev]
         for basectx in basectxs[srcrev]:
             basepath = copies.pathcopies(basectx, fixctx).get(path, path)
             if basepath in basectx:
                 basepaths[(basectx.rev(), fixctx.rev(), path)] = basepath
     return basepaths
 
 
 def unionranges(rangeslist):
     """Return the union of some closed intervals
 
     >>> unionranges([])
     []
     >>> unionranges([(1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (2, 100)])
     [(1, 100)]
     >>> unionranges([(1, 99), (1, 100)])
     [(1, 100)]
     >>> unionranges([(1, 100), (40, 60)])
     [(1, 100)]
     >>> unionranges([(1, 49), (50, 100)])
     [(1, 100)]
     >>> unionranges([(1, 48), (50, 100)])
     [(1, 48), (50, 100)]
     >>> unionranges([(1, 2), (3, 4), (5, 6)])
     [(1, 6)]
     """
     rangeslist = sorted(set(rangeslist))
     unioned = []
     if rangeslist:
         unioned, rangeslist = [rangeslist[0]], rangeslist[1:]
     for a, b in rangeslist:
         c, d = unioned[-1]
         if a > d + 1:
             unioned.append((a, b))
         else:
             unioned[-1] = (c, max(b, d))
     return unioned
 
 
 def difflineranges(content1, content2):
     """Return list of line number ranges in content2 that differ from content1.
 
     Line numbers are 1-based. The numbers are the first and last line contained
     in the range. Single-line ranges have the same line number for the first and
     last line. Excludes any empty ranges that result from lines that are only
     present in content1. Relies on mdiff's idea of where the line endings are in
     the string.
 
     >>> from mercurial import pycompat
     >>> lines = lambda s: b'\\n'.join([c for c in pycompat.iterbytestr(s)])
     >>> difflineranges2 = lambda a, b: difflineranges(lines(a), lines(b))
     >>> difflineranges2(b'', b'')
     []
     >>> difflineranges2(b'a', b'')
     []
     >>> difflineranges2(b'', b'A')
     [(1, 1)]
     >>> difflineranges2(b'a', b'a')
     []
     >>> difflineranges2(b'a', b'A')
     [(1, 1)]
     >>> difflineranges2(b'ab', b'')
     []
     >>> difflineranges2(b'', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abc', b'ac')
     []
     >>> difflineranges2(b'ab', b'aCb')
     [(2, 2)]
     >>> difflineranges2(b'abc', b'aBc')
     [(2, 2)]
     >>> difflineranges2(b'ab', b'AB')
     [(1, 2)]
     >>> difflineranges2(b'abcde', b'aBcDe')
     [(2, 2), (4, 4)]
     >>> difflineranges2(b'abcde', b'aBCDe')
     [(2, 4)]
     """
     ranges = []
     for lines, kind in mdiff.allblocks(content1, content2):
         firstline, lastline = lines[2:4]
         if kind == b'!' and firstline != lastline:
             ranges.append((firstline + 1, lastline))
     return ranges
 
 
 def getbasectxs(repo, opts, revstofix):
     """Returns a map of the base contexts for each revision
 
     The base contexts determine which lines are considered modified when we
     attempt to fix just the modified lines in a file. It also determines which
     files we attempt to fix, so it is important to compute this even when
     --whole is used.
     """
     # The --base flag overrides the usual logic, and we give every revision
     # exactly the set of baserevs that the user specified.
     if opts.get(b'base'):
         baserevs = set(logcmdutil.revrange(repo, opts.get(b'base')))
         if not baserevs:
             baserevs = {nullrev}
         basectxs = {repo[rev] for rev in baserevs}
         return {rev: basectxs for rev in revstofix}
 
     # Proceed in topological order so that we can easily determine each
     # revision's baserevs by looking at its parents and their baserevs.
     basectxs = collections.defaultdict(set)
     for rev in sorted(revstofix):
         ctx = repo[rev]
         for pctx in ctx.parents():
             if pctx.rev() in basectxs:
                 basectxs[rev].update(basectxs[pctx.rev()])
             else:
                 basectxs[rev].add(pctx)
     return basectxs
 
 
 def _prefetchfiles(repo, workqueue, basepaths):
     toprefetch = set()
 
     # Prefetch the files that will be fixed.
     for srcrev, path, _dstrevs in workqueue:
         if srcrev == wdirrev:
             continue
         toprefetch.add((srcrev, path))
 
     # Prefetch the base contents for lineranges().
     for (baserev, fixrev, path), basepath in basepaths.items():
         toprefetch.add((baserev, basepath))
 
     if toprefetch:
         scmutil.prefetchfiles(
             repo,
             [
                 (rev, scmutil.matchfiles(repo, [path]))
                 for rev, path in toprefetch
             ],
         )
 
 
 def fixfile(ui, repo, opts, fixers, fixctx, path, basepaths, basectxs):
     """Run any configured fixers that should affect the file in this context
 
     Returns the file content that results from applying the fixers in some order
     starting with the file's content in the fixctx. Fixers that support line
     ranges will affect lines that have changed relative to any of the basectxs
     (i.e. they will only avoid lines that are common to all basectxs).
 
     A fixer tool's stdout will become the file's new content if and only if it
     exits with code zero. The fixer tool's working directory is the repository's
     root.
     """
     metadata = {}
     newdata = fixctx[path].data()
     for fixername, fixer in fixers.items():
         if fixer.affects(opts, fixctx, path):
             ranges = lineranges(
                 opts, path, basepaths, basectxs, fixctx, newdata
             )
             command = fixer.command(ui, path, ranges)
             if command is None:
                 continue
             msg = b'fixing: %s - %s - %s\n'
             msg %= (fixctx, fixername, path)
             ui.debug(msg)
             ui.debug(b'subprocess: %s\n' % (command,))
             proc = subprocess.Popen(
                 procutil.tonativestr(command),
                 shell=True,
                 cwd=procutil.tonativestr(repo.root),
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
             )
             stdout, stderr = proc.communicate(newdata)
             if stderr:
                 showstderr(ui, fixctx.rev(), fixername, stderr)
             newerdata = stdout
             if fixer.shouldoutputmetadata():
                 try:
                     metadatajson, newerdata = stdout.split(b'\0', 1)
                     metadata[fixername] = pycompat.json_loads(metadatajson)
                 except ValueError:
                     ui.warn(
                         _(b'ignored invalid output from fixer tool: %s\n')
                         % (fixername,)
                     )
                     continue
             else:
                 metadata[fixername] = None
             if proc.returncode == 0:
                 newdata = newerdata
             else:
                 if not stderr:
                     message = _(b'exited with status %d\n') % (proc.returncode,)
                     showstderr(ui, fixctx.rev(), fixername, message)
                 checktoolfailureaction(
                     ui,
                     _(b'no fixes will be applied'),
                     hint=_(
                         b'use --config fix.failure=continue to apply any '
                         b'successful fixes anyway'
                     ),
                 )
     return metadata, newdata
 
 
 def showstderr(ui, rev, fixername, stderr):
     """Writes the lines of the stderr string as warnings on the ui
 
     Uses the revision number and fixername to give more context to each line of
     the error message. Doesn't include file names, since those take up a lot of
     space and would tend to be included in the error message if they were
     relevant.
     """
     for line in re.split(b'[\r\n]+', stderr):
         if line:
             ui.warn(b'[')
             if rev is None:
                 ui.warn(_(b'wdir'), label=b'evolve.rev')
             else:
                 ui.warn(b'%d' % rev, label=b'evolve.rev')
             ui.warn(b'] %s: %s\n' % (fixername, line))
 
 
 def writeworkingdir(repo, ctx, filedata, replacements):
     """Write new content to the working copy and check out the new p1 if any
 
     We check out a new revision if and only if we fixed something in both the
     working directory and its parent revision. This avoids the need for a full
     update/merge, and means that the working directory simply isn't affected
     unless the --working-dir flag is given.
 
     Directly updates the dirstate for the affected files.
     """
     for path, data in filedata.items():
         fctx = ctx[path]
         fctx.write(data, fctx.flags())
 
     oldp1 = repo.dirstate.p1()
     newp1 = replacements.get(oldp1, oldp1)
     if newp1 != oldp1:
         assert repo.dirstate.p2() == nullid
-        with repo.dirstate.parentchange():
+        with repo.dirstate.changing_parents(repo):
             scmutil.movedirstate(repo, repo[newp1])
 
 
 def replacerev(ui, repo, ctx, filedata, replacements):
     """Commit a new revision like the given one, but with file content changes
 
     "ctx" is the original revision to be replaced by a modified one.
 
     "filedata" is a dict that maps paths to their new file content. All other
     paths will be recreated from the original revision without changes.
     "filedata" may contain paths that didn't exist in the original revision;
     they will be added.
 
     "replacements" is a dict that maps a single node to a single node, and it is
     updated to indicate the original revision is replaced by the newly created
     one. No entry is added if the replacement's node already exists.
 
     The new revision has the same parents as the old one, unless those parents
     have already been replaced, in which case those replacements are the parents
     of this new revision. Thus, if revisions are replaced in topological order,
     there is no need to rebase them into the original topology later.
     """
 
     p1rev, p2rev = repo.changelog.parentrevs(ctx.rev())
     p1ctx, p2ctx = repo[p1rev], repo[p2rev]
     newp1node = replacements.get(p1ctx.node(), p1ctx.node())
     newp2node = replacements.get(p2ctx.node(), p2ctx.node())
 
     # We don't want to create a revision that has no changes from the original,
     # but we should if the original revision's parent has been replaced.
     # Otherwise, we would produce an orphan that needs no actual human
     # intervention to evolve. We can't rely on commit() to avoid creating the
     # un-needed revision because the extra field added below produces a new hash
     # regardless of file content changes.
     if (
         not filedata
         and p1ctx.node() not in replacements
         and p2ctx.node() not in replacements
     ):
         return
 
     extra = ctx.extra().copy()
     extra[b'fix_source'] = ctx.hex()
 
     wctx = context.overlayworkingctx(repo)
     wctx.setbase(repo[newp1node])
     merge.revert_to(ctx, wc=wctx)
     copies.graftcopies(wctx, ctx, ctx.p1())
 
     for path in filedata.keys():
         fctx = ctx[path]
         copysource = fctx.copysource()
         wctx.write(path, filedata[path], flags=fctx.flags())
         if copysource:
             wctx.markcopied(path, copysource)
 
     desc = rewriteutil.update_hash_refs(
         repo,
         ctx.description(),
         {oldnode: [newnode] for oldnode, newnode in replacements.items()},
     )
 
     memctx = wctx.tomemctx(
         text=desc,
         branch=ctx.branch(),
         extra=extra,
         date=ctx.date(),
         parents=(newp1node, newp2node),
         user=ctx.user(),
     )
 
     sucnode = memctx.commit()
     prenode = ctx.node()
     if prenode == sucnode:
         ui.debug(b'node %s already existed\n' % (ctx.hex()))
     else:
         replacements[ctx.node()] = sucnode
 
 
 def getfixers(ui):
     """Returns a map of configured fixer tools indexed by their names
 
     Each value is a Fixer object with methods that implement the behavior of the
     fixer's config suboptions. Does not validate the config values.
     """
     fixers = {}
     for name in fixernames(ui):
         enabled = ui.configbool(b'fix', name + b':enabled')
         command = ui.config(b'fix', name + b':command')
         pattern = ui.config(b'fix', name + b':pattern')
         linerange = ui.config(b'fix', name + b':linerange')
         priority = ui.configint(b'fix', name + b':priority')
         metadata = ui.configbool(b'fix', name + b':metadata')
         skipclean = ui.configbool(b'fix', name + b':skipclean')
         # Don't use a fixer if it has no pattern configured. It would be
         # dangerous to let it affect all files. It would be pointless to let it
         # affect no files. There is no reasonable subset of files to use as the
         # default.
         if command is None:
             ui.warn(
                 _(b'fixer tool has no command configuration: %s\n') % (name,)
             )
         elif pattern is None:
             ui.warn(
                 _(b'fixer tool has no pattern configuration: %s\n') % (name,)
             )
         elif not enabled:
             ui.debug(b'ignoring disabled fixer tool: %s\n' % (name,))
         else:
             fixers[name] = Fixer(
                 command, pattern, linerange, priority, metadata, skipclean
             )
     return collections.OrderedDict(
         sorted(fixers.items(), key=lambda item: item[1]._priority, reverse=True)
     )
 
 
 def fixernames(ui):
     """Returns the names of [fix] config options that have suboptions"""
     names = set()
     for k, v in ui.configitems(b'fix'):
         if b':' in k:
             names.add(k.split(b':', 1)[0])
     return names
 
 
 class Fixer:
     """Wraps the raw config values for a fixer with methods"""
 
     def __init__(
         self, command, pattern, linerange, priority, metadata, skipclean
     ):
         self._command = command
         self._pattern = pattern
         self._linerange = linerange
         self._priority = priority
         self._metadata = metadata
         self._skipclean = skipclean
 
     def affects(self, opts, fixctx, path):
         """Should this fixer run on the file at the given path and context?"""
         repo = fixctx.repo()
         matcher = matchmod.match(
             repo.root, repo.root, [self._pattern], ctx=fixctx
         )
         return matcher(path)
 
     def shouldoutputmetadata(self):
         """Should the stdout of this fixer start with JSON and a null byte?"""
         return self._metadata
 
     def command(self, ui, path, ranges):
         """A shell command to use to invoke this fixer on the given file/lines
 
         May return None if there is no appropriate command to run for the given
         parameters.
         """
         expand = cmdutil.rendercommandtemplate
         parts = [
             expand(
                 ui,
                 self._command,
                 {b'rootpath': path, b'basename': os.path.basename(path)},
             )
         ]
         if self._linerange:
             if self._skipclean and not ranges:
                 # No line ranges to fix, so don't run the fixer.
                 return None
             for first, last in ranges:
                 parts.append(
                     expand(
                         ui, self._linerange, {b'first': first, b'last': last}
                     )
                 )
         return b' '.join(parts)