merge with stable

contrib/packaging/docker/centos8

@@ -0,0 +1,15 b''
+FROM centos:centos8
+
+RUN groupadd -g %GID% build && \
+    useradd -u %UID% -g %GID% -s /bin/bash -d /build -m build
+
+RUN yum install -y \
+	gcc \
+	gettext \
+	make \
+	python3-devel \
+	python3-docutils \
+	rpm-build
+
+# For creating repo meta data
+RUN yum install -y createrepo

tests/unwrap-message-id.py

@@ -0,0 +1,6 b''
+from __future__ import absolute_import, print_function
+
+import re
+import sys
+
+print(re.sub(r"(?<=Message-Id:) \n ", " ", sys.stdin.read()), end="")

.hgsigs

@@ -185,3 +185,5 b' e386b5f4f8360dbb43a576dd9b1368e386fefa5b'
 e91930d712e8507d1bc1b2dffd96c83edc4cbed3 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl1DD/sQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91bvmD/4/QDZZGVe+WiMUxbT+grfFjwjX4nkg7Vt+6vQbjN68NC5XpSiCzW8uu0LRemX0KJKoOfQxqHk3YKkZZHIk10Fe6RSLWt8dqlfa2J9B2U8DwMEBykCOuxcLlDe7DGaaMXlXXRhNXebRheNPLeNe+r7beMAAjwchTIIJD5xcFnPRFR0nN7Vj7eRUdWIQ9H/s7TolPz1Mf7IWqapLjPtofiwSgtRoXfIAkuuabnE4eMVJ8rsLwcuMhxWP2zjEfEg68YkiGBAFmlnRk+3lJpiB9kVapB3cWcsWv2OBhz0D3NgGp82eWkjJCZZhZ+zHHrQ6L9zbiArzW9NVvPEAKLbl3XUhFUzFTUD+S38wsYLYL5RkzhlCI2/K1LJLOtj7r0Seen0v8X842p0cXmxTg/o1Vg3JOm04l9AwzCsnqwIqV7Ru//KPqH91MFFH6T6tbfjtLHRmjxRjMZmVt7ZQjS84opVCZwgUTZZJB2kd1goROjdowQVK6qsEonlzGjWb9zc3el5L9uzDeim3e5t2GNRVt8veQaLc+U2hHWniVsDJMvqp2Hr9IWUKp+bu/35B1nElvooS40gj2WhkfkCbbXSg9qnVLwGxxcGdF28Z0nhQcfKiJAc+8l9l19GNhdKxOi4zUXlp90opPWfT7wGQmysvTjQeFL2zX9ziuHUZZwlW1YbeMQ==
 a4e32fd539ab41489a51b2aa88bda9a73b839562 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl1xTxUQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91ZQgD/96mViQ6fEh84l4XyAlY6Dq3SgMqEXttsUpk/GPoW4ykDFKN6VoiOaPoyNODO/46V3yeAjYjy3vX7Ua4/MY1NlnNoliQcTYtRV3SlDdoueTPOLfO6YSV27LG+dX/HYvPc/htCVmIVItU1JL+KEpXnv+bT50Bk+m6OgzfJMDzdHQ5ICImT8gW7UXlH/mlNtWMOrJDk3cArGhGs/pTFVrfgRTfDfDGSA9xW0/QvsNI5iwZHgMYaqoPFDnw6d/NXWRlk77KNiXkBEOKHf6UEWecMKmiSCm8RePSiX9ezqdcBAHygOg4KUeiR2kPNl4QJtskyG4CwWxlmGlfgKx07s7rGafE+DWLEYC9Wa8qK6/LPiowm17m/UlAYxdFXaBCiN0wgEw7oNmjcx/791ez+CL1+h6pd0+iSVI4bO9/YZ8LPROYef18MFm+IFIDIOgZU4eUbpBrzBb3IM1a519xgnmWXAjtRtGWEZMuHaSoLJf2pDXvaUPX6YpJeqCBFO3q/swbiJsQsy6xRW0Dwtn7umU1PGdmMoTnskTRKy9Kgzv7lf/nsUuRbzzM4ut9m1TOo27AulObMrmQB4YvLi/LEnYaRNx18yaqOceMxb/mS0tHLgcZToy9rTV+vtC21vgwfzGia2neLLe50tnIsBPP/AdTOw9ZDMRfXMCajWM22hPxvnGcw==
 181e52f2b62f4768aa0d988936c929dc7c4a41a0 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl2UzlMQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91SDzD/0YZqtN+LK5AusJjWaTa61DRIPhJQoZD+HKg4kAzjL8zw8SxBGLxMZkGmve9QFMNzqIr5kkPk6yEKrEWYqyPtpwrv5Xh5D4d8AKfphdzwSr+BvMk4fBEvwnBhrUJtKDEiuYQdbh4+OQfQs1c3xhtinjXn30160uzFvLQY6/h4hxai2XWj4trgoNXqPHDHlQKc6kRfPpmNO2UZhG+2Xfsava2JpcP4xA2R0XkI10be5MDoGU4AFCMUcXZzIto0DYT+HOezowoNpdC1EWVHfa+bdrlzHHO7WPaTLzEPy44/IhXmNhbwFKOk5RZ/qBADQvs9BDfmIDczOoZKTC5+ESZM0PR2np5t7+JFMUeeRcINqBdSc4Aszw3iHjgNbJJ3viU72JZvGGGd9MglP590tA0proVGxQgvXDq3mtq3Se5yOLAjmRnktW5Tnt8/Z3ycuZz+QsTEMXR5uIZvgz63ibfsCGTXFYUz9h7McGgmhfKWvQw9+MH6kRbE9U8qaUumgf4zi4HNzmf8AyaMJo07DIMwWVgjlVUdWUlN/Eg61fU3wC79mV8mLVsi5/TZ986obz4csoYSYXyyez5ScRji+znSw8vUx0YhoiOQbDms/y2QZR/toyon554tHkDZsya2lhpwXs8T0IFZhERXsmz/XmT3fWnhSzyrUe6VjBMep1zn6lvQ==
+59338f9561099de77c684c00f76507f11e46ebe8 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl2ty1MQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91XBUD/wJqwW0cuMCUvuUODLIfWa7ZxNl1mV9eW3tFQEuLGry97s12KDwBe0Erdjj7DASl4/6Xpc4PYxelZwSw4xT1UQg7wd/C3daCq/cDXrAkl7ZNTAHu6iAnHh25mOpIBfhMbh4j3YD0A2OoI17QGScU6S7Uv0Gz1CY20lJmEqsMzuuDPm2zrdPnTWffRUuPgskAg3czaw45Na7nUBeaxN1On0O5WqMYZsCGyi14g5S0Z0LHMKRJzc/s48JUTDjTbbzJ6HBxrxWTW2v8gN2J6QDYykcLBB9kV6laal9jhWs9n/w0yWwHfBfJ+E4EiMXeRdZgGA55OCOuDxnmmONs1/Z0WwPo+vQlowEnjDMT0jPrPePZ5P4BDXZD3tGsmdXDHM7j+VfDyPh1FBFpcaej44t84X1OWtAnLZ3VMPLwobz9MOzz4wr9UuHq23hus0Fen+FJYOAlTx9qPAqBrCTpGl+h1DMKD62D7lF8Z1CxTlqg9PPBB7IZNCXoN7FZ4Wfhv1AarMVNNUgBx6m0r6OScCXrluuFklYDSIZrfgiwosXxsHW27RjxktrV4O+J1GT/chLBJFViTZg/gX/9UC3eLkzp1t6gC6T9SQ+lq0/I+1/rHQkxNaywLycBPOG1yb/59mibEwB9+Mu9anRYKFNHEktNoEmyw5G9UoZhD+1tHt4tkJCwA==
+ca3dca416f8d5863ca6f5a4a6a6bb835dcd5feeb 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl3BrQ4QHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91ZXjEACfBdZczf0a4bmeaaxRwxXAniSS4rVkF790g22fsvSZFvQEpmwqNtsvbTt3N1V2QSDSZyhBa+/qfpuZ689VXMlR3rcJOVjo/7193QLXHOPfRn7sDeeCxjsbtXXLbLa8UT56gtT5gUa4i0LC2kHBEi+UhV9EGgSaDTBxWUFJ9RY2sosy1XFiOUlkUoHUbqUF28J3/CxEXzULWkqTOPwh94JYsgXSSS69WNZEfsuEBSPCzn8Gd7z7lWudZ/VTZBTpTji7HQxpFtSZxNzpwmcmVOH9HlEKoA1K4JoR+1TMHqSytQXlz3FMF6c6Z1G+OPpwTGCjGTkB9ZAusP3gU8KIZTTEXthiEluRtnRq1yu4K2LTyY172JPJvANAWpVEvBvn4k5c9tDOEt9RCAPqCrgNGzDTrw02+gZyyNkjcS6hPn+cDJ6OQ1j2eCQtHlqfHLSc7FsRjUSTiKSEUTdWvHbNfOYe6Yth/tnQ7TnpnS9S0eiugFzZs2f8P85Gfa3uTFQIDm67Ud+8Yu1uOxa6bhECLaXEACnLofzz8sioLsJMiOoG2HmwhyPyfZUHXlb2zdsSP3LC+gKN39VvzSxhhjrIUJoM4ulP0GP1/lkMVzOady66iLaEwDvEn4FLmu395SubHwbre1Jx83hiCQpZfPkI0PhKnh4yVm+BRGUpX97rMTGjzw==

.hgtags

@@ -198,3 +198,5 b' e386b5f4f8360dbb43a576dd9b1368e386fefa5b'
 e91930d712e8507d1bc1b2dffd96c83edc4cbed3 5.1
 a4e32fd539ab41489a51b2aa88bda9a73b839562 5.1.1
 181e52f2b62f4768aa0d988936c929dc7c4a41a0 5.1.2
+59338f9561099de77c684c00f76507f11e46ebe8 5.2rc0
+ca3dca416f8d5863ca6f5a4a6a6bb835dcd5feeb 5.2

Makefile

@@ -183,16 +183,15 b' packaging_targets := \\'
   centos5 \
   centos6 \
   centos7 \
+  centos8 \
   deb \
   docker-centos5 \
   docker-centos6 \
   docker-centos7 \
+  docker-centos8 \
   docker-debian-jessie \
   docker-debian-stretch \
-  docker-fedora20 \
-  docker-fedora21 \
-  docker-fedora28 \
-  docker-fedora29 \
+  docker-fedora \
   docker-ubuntu-trusty \
   docker-ubuntu-trusty-ppa \
   docker-ubuntu-xenial \
@@ -201,10 +200,7 b' packaging_targets := \\'
   docker-ubuntu-artful-ppa \
   docker-ubuntu-bionic \
   docker-ubuntu-bionic-ppa \
-  fedora20 \
-  fedora21 \
-  fedora28 \
-  fedora29 \
+  fedora \
   linux-wheels \
   linux-wheels-x86_64 \
   linux-wheels-i686 \

black.toml

@@ -9,9 +9,7 b' build/'
 | \.mypy_cache/
 | \.venv/
 | mercurial/thirdparty/
-| hgext/fsmonitor/pywatchman/
 | contrib/python-zstandard/
-| contrib/grey.py
 '''
 skip-string-normalization = true
 quiet = true

contrib/automation/hgautomation/aws.py

@@ -191,6 +191,10 b" winrm set winrm/config/client/auth '@{Ba"
 $Setting = 'LocalAccountTokenFilterPolicy'
 Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force
 
+# Avoid long usernames in the temp directory path because the '~' causes extra quoting in ssh output
+[System.Environment]::SetEnvironmentVariable('TMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
+[System.Environment]::SetEnvironmentVariable('TEMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
+
 # Configure and restart the WinRM Service; Enable the required firewall exception
 Stop-Service -Name WinRM
 Set-Service -Name WinRM -StartupType Automatic

contrib/automation/hgautomation/linux.py

@@ -25,12 +25,12 b' DISTROS = {'
 }
 
 INSTALL_PYTHONS = r'''
-PYENV2_VERSIONS="2.7.16 pypy2.7-7.1.1"
-PYENV3_VERSIONS="3.5.7 3.6.9 3.7.4 3.8.0 pypy3.5-7.0.0 pypy3.6-7.1.1"
+PYENV2_VERSIONS="2.7.17 pypy2.7-7.2.0"
+PYENV3_VERSIONS="3.5.7 3.6.9 3.7.5 3.8.0 pypy3.5-7.0.0 pypy3.6-7.2.0"
 
 git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
 pushd /hgdev/pyenv
-git checkout d6d6bc8bb08bcdcbf4eb79509aa7061011ade1c4
+git checkout 0e7cfc3b3d4eca46ad83d632e1505f5932cd179b
 popd
 
 export PYENV_ROOT="/hgdev/pyenv"
@@ -266,6 +266,7 b' PACKAGES="\\'
     python3-fuzzywuzzy \
     python3-pygments \
     python3-vcr \
+    python3-venv \
     rsync \
     sqlite3 \
     subversion \

contrib/automation/linux-requirements-py3.txt

@@ -4,10 +4,25 b''
 #
 #    pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py3.txt contrib/automation/linux-requirements.txt.in
 #
+appdirs==1.4.3 \
+    --hash=sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92 \
+    --hash=sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e \
+    # via black
 astroid==2.2.5 \
     --hash=sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4 \
     --hash=sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4 \
     # via pylint
+attrs==19.3.0 \
+    --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
+    --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \
+    # via black
+black==19.10b0 ; python_version >= "3.6" and platform_python_implementation != "PyPy" \
+    --hash=sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b \
+    --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539
+click==7.0 \
+    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
+    # via black
 docutils==0.15.2 \
     --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
     --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
@@ -78,6 +93,9 b' multidict==4.5.2 \\'
     --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \
     --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b \
     # via yarl
+pathspec==0.6.0 \
+    --hash=sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c \
+    # via black
 pyflakes==2.1.1 \
     --hash=sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0 \
     --hash=sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2
@@ -104,10 +122,27 b' pyyaml==5.1.2 \\'
     --hash=sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41 \
     --hash=sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8 \
     # via vcrpy
+regex==2019.11.1 \
+    --hash=sha256:15454b37c5a278f46f7aa2d9339bda450c300617ca2fca6558d05d870245edc7 \
+    --hash=sha256:1ad40708c255943a227e778b022c6497c129ad614bb7a2a2f916e12e8a359ee7 \
+    --hash=sha256:5e00f65cc507d13ab4dfa92c1232d004fa202c1d43a32a13940ab8a5afe2fb96 \
+    --hash=sha256:604dc563a02a74d70ae1f55208ddc9bfb6d9f470f6d1a5054c4bd5ae58744ab1 \
+    --hash=sha256:720e34a539a76a1fedcebe4397290604cc2bdf6f81eca44adb9fb2ea071c0c69 \
+    --hash=sha256:7caf47e4a9ac6ef08cabd3442cc4ca3386db141fb3c8b2a7e202d0470028e910 \
+    --hash=sha256:c31eaf28c6fe75ea329add0022efeed249e37861c19681960f99bbc7db981fb2 \
+    --hash=sha256:c7393597191fc2043c744db021643549061e12abe0b3ff5c429d806de7b93b66 \
+    --hash=sha256:d2b302f8cdd82c8f48e9de749d1d17f85ce9a0f082880b9a4859f66b07037dc6 \
+    --hash=sha256:e3d8dd0ec0ea280cf89026b0898971f5750a7bd92cb62c51af5a52abd020054a \
+    --hash=sha256:ec032cbfed59bd5a4b8eab943c310acfaaa81394e14f44454ad5c9eba4f24a74 \
+    # via black
 six==1.12.0 \
     --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
     --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
     # via astroid, vcrpy
+toml==0.10.0 \
+    --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
+    --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \
+    # via black
 typed-ast==1.4.0 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \
     --hash=sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e \
     --hash=sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e \
@@ -146,4 +181,4 b' yarl==1.3.0 \\'
 
 # WARNING: The following packages were not pinned, but pip requires them to be
 # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
-# setuptools==41.0.1        # via python-levenshtein
+# setuptools==41.6.0        # via python-levenshtein

contrib/automation/linux-requirements.txt.in

@@ -1,3 +1,5 b''
+# black pulls in typed-ast, which doesn't install on PyPy.
+black ; python_version >= '3.6' and platform_python_implementation != 'PyPy'
 # Bazaar doesn't work with Python 3 nor PyPy.
 bzr ; python_version <= '2.7' and platform_python_implementation == 'CPython'
 docutils

contrib/byteify-strings.py

@@ -339,7 +339,7 b' def main():'
 
 
 if __name__ == '__main__':
-    if sys.version_info.major < 3:
-        print('This script must be run under Python 3.')
+    if sys.version_info[0:2] < (3, 7):
+        print('This script must be run under Python 3.7+')
         sys.exit(3)
     main()

contrib/examples/fix.hgrc

@@ -5,11 +5,5 b' clang-format:pattern = (**.c or **.cc or'
 rustfmt:command = rustfmt {rootpath}
 rustfmt:pattern = set:**.rs
 
-# We use black, but currently with
-# https://github.com/psf/black/pull/826 applied. For now
-# contrib/grey.py is our fork of black. You need to pip install
-# git+https://github.com/python/black/@d9e71a75ccfefa3d9156a64c03313a0d4ad981e5
-# to have the dependencies for grey.
-#
-# black:command = python3.7 contrib/grey.py --config=black.toml -
-# black:pattern = set:**.py - hgext/fsmonitor/pywatchman/** - mercurial/thirdparty/** - "contrib/python-zstandard/** - contrib/grey.py"
+black:command = black --config=black.toml -
+black:pattern = set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"

contrib/import-checker.py

@@ -4,6 +4,7 b' from __future__ import absolute_import, '
 
 import ast
 import collections
+import io
 import os
 import sys
 
@@ -754,7 +755,11 b' def sources(f, modname):'
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
-        with open(f, 'r') as src:
+        # Strictly speaking we should sniff for the magic header that denotes
+        # Python source file encoding. But in reality we don't use anything
+        # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
+        # simplicity is fine.
+        with io.open(f, 'r', encoding='utf-8') as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname.encode('utf8'), t, line
 

contrib/install-windows-dependencies.ps1

@@ -22,10 +22,10 b''
 $VC9_PYTHON_URL = "https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"
 $VC9_PYTHON_SHA256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf"
 
-$PYTHON27_x64_URL = "https://www.python.org/ftp/python/2.7.16/python-2.7.16.amd64.msi"
-$PYTHON27_x64_SHA256 = "7c0f45993019152d46041a7db4b947b919558fdb7a8f67bcd0535bc98d42b603"
-$PYTHON27_X86_URL = "https://www.python.org/ftp/python/2.7.16/python-2.7.16.msi"
-$PYTHON27_X86_SHA256 = "d57dc3e1ba490aee856c28b4915d09e3f49442461e46e481bc6b2d18207831d7"
+$PYTHON27_x64_URL = "https://www.python.org/ftp/python/2.7.17/python-2.7.17.amd64.msi"
+$PYTHON27_x64_SHA256 = "3b934447e3620e51d2daf5b2f258c9b617bcc686ca2f777a49aa3b47893abf1b"
+$PYTHON27_X86_URL = "https://www.python.org/ftp/python/2.7.17/python-2.7.17.msi"
+$PYTHON27_X86_SHA256 = "a4e3a321517c6b0c2693d6f712a0d18c82600b3d0c759c299b3d14384a17f863"
 
 $PYTHON35_x86_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4.exe"
 $PYTHON35_x86_SHA256 = "F27C2D67FD9688E4970F3BFF799BB9D722A0D6C2C13B04848E1F7D620B524B0E"
@@ -37,10 +37,10 b''
 $PYTHON36_x64_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe"
 $PYTHON36_x64_SHA256 = "96088A58B7C43BC83B84E6B67F15E8706C614023DD64F9A5A14E81FF824ADADC"
 
-$PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.4/python-3.7.4.exe"
-$PYTHON37_x86_SHA256 = "9a30ab5568ba37bfbcae5cdee19e9dc30765c42cf066f605221563ff8b20ee34"
-$PYTHON37_X64_URL = "https://www.python.org/ftp/python/3.7.4/python-3.7.4-amd64.exe"
-$PYTHON37_x64_SHA256 = "bab92f987320975c7826171a072bfd64f8f0941aaf2cdeba6924b7025c9968a3"
+$PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.5/python-3.7.5.exe"
+$PYTHON37_x86_SHA256 = "3c2ae8f72b48e6e0c2b482206e322bf5d0344ff91abc3b3c200cec9e275c7168"
+$PYTHON37_X64_URL = "https://www.python.org/ftp/python/3.7.5/python-3.7.5-amd64.exe"
+$PYTHON37_x64_SHA256 = "f3d60c127e7a92ed547efa3321bf70cd96b75c53bf4b903147015257c1314981"
 
 $PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.0/python-3.8.0.exe"
 $PYTHON38_x86_SHA256 = "b471908de5e10d8fb5c3351a5affb1172da7790c533e0c9ffbaeec9c11611b15"

contrib/packaging/Makefile

@@ -11,19 +11,17 b' UBUNTU_CODENAMES := \\'
   cosmic \
   disco
 
-FEDORA_RELEASES := \
-  20 \
-  21 \
-  28 \
-  29
+FEDORA_RELEASE := 31
 
 CENTOS_RELEASES := \
   5 \
   6 \
-  7
+  7 \
+  8
 
 # Build a Python for these CentOS releases.
 CENTOS_WITH_PYTHON_RELEASES := 5 6
+CENTOS_WITH_NONVERSIONED_PYTHON := 5 6 7
 
 help:
 	@echo 'Packaging Make Targets'
@@ -34,8 +32,8 b' help:'
 	@echo 'docker-debian-{$(strip $(DEBIAN_CODENAMES))}'
 	@echo '   Build Debian packages specific to a Debian distro using Docker.'
 	@echo ''
-	@echo 'docker-fedora{$(strip $(FEDORA_RELEASES))}'
-	@echo '   Build an RPM for a specific Fedora version using Docker.'
+	@echo 'docker-fedora'
+	@echo '   Build an RPM for a Fedora $(FEDORA_RELEASE) using Docker.'
 	@echo ''
 	@echo 'docker-ubuntu-{$(strip $(UBUNTU_CODENAMES))}'
 	@echo '   Build Debian package specific to an Ubuntu distro using Docker.'
@@ -59,8 +57,8 b' help:'
 	@echo 'centos{$(strip $(CENTOS_RELEASES))}'
 	@echo '   Build an RPM for a specific CentOS version locally'
 	@echo ''
-	@echo 'fedora{$(strip $(FEDORA_RELEASES))}'
-	@echo '   Build an RPM for a specific Fedora version locally'
+	@echo 'fedora'
+	@echo '   Build an RPM for Fedora $(FEDORA_RELEASE) locally'
 
 .PHONY: help
 
@@ -97,37 +95,30 b' endef'
 $(foreach codename,$(UBUNTU_CODENAMES),$(eval $(call ubuntu_targets,$(codename))))
 
 # Fedora targets.
-define fedora_targets
-.PHONY: fedora$(1)
-fedora$(1):
-	mkdir -p $$(HGROOT)/packages/fedora$(1)
+.PHONY: fedora
+fedora:
+	mkdir -p $(HGROOT)/packages/fedora$(FEDORA_RELEASE)
 	./buildrpm
-	cp $$(HGROOT)/contrib/packaging/rpmbuild/RPMS/*/* $$(HGROOT)/packages/fedora$(1)
-	cp $$(HGROOT)/contrib/packaging/rpmbuild/SRPMS/* $$(HGROOT)/packages/fedora$(1)
+	cp $(HGROOT)/contrib/packaging/rpmbuild/RPMS/*/* $(HGROOT)/packages/fedora$(FEDORA_RELEASE)
+	cp $(HGROOT)/contrib/packaging/rpmbuild/SRPMS/* $(HGROOT)/packages/fedora$(FEDORA_RELEASE)
 	rm -rf $(HGROOT)/rpmbuild
 
-.PHONY: docker-fedora$(1)
-docker-fedora$(1):
-	mkdir -p $$(HGROOT)/packages/fedora$(1)
-	./dockerrpm fedora$(1)
-
-endef
-
-$(foreach release,$(FEDORA_RELEASES),$(eval $(call fedora_targets,$(release))))
+.PHONY: docker-fedora
+docker-fedora:
+	./dockerrpm fedora$(FEDORA_RELEASE)
 
 # CentOS targets.
 define centos_targets
 .PHONY: centos$(1)
 centos$(1):
 	mkdir -p $$(HGROOT)/packages/centos$(1)
-	./buildrpm $$(if $$(filter $(1),$$(CENTOS_WITH_PYTHON_RELEASES)),--withpython)
+	./buildrpm $$(if $$(filter $(1),$$(CENTOS_WITH_PYTHON_RELEASES)),--withpython,$$(if $$(filter $(1),$$(CENTOS_WITH_NONVERSIONED_PYTHON)),--python python,))
 	cp $$(HGROOT)/contrib/packaging/rpmbuild/RPMS/*/* $$(HGROOT)/packages/centos$(1)
 	cp $$(HGROOT)/contrib/packaging/rpmbuild/SRPMS/* $$(HGROOT)/packages/centos$(1)
 
 .PHONY: docker-centos$(1)
 docker-centos$(1):
-	mkdir -p $$(HGROOT)/packages/centos$(1)
-	./dockerrpm centos$(1) $$(if $$(filter $(1),$$(CENTOS_WITH_PYTHON_RELEASES)),--withpython)
+	./dockerrpm centos$(1) $$(if $$(filter $(1),$$(CENTOS_WITH_PYTHON_RELEASES)),--withpython,$$(if $$(filter $(1),$$(CENTOS_WITH_NONVERSIONED_PYTHON)),--python python,))
 
 endef
 

contrib/packaging/builddeb

@@ -106,10 +106,10 b' if [ "$CLEANUP" ] ; then'
     echo
     OUTPUTDIR=${OUTPUTDIR:=packages/$DISTID-$CODENAME}
     mkdir -p "$OUTPUTDIR"
-    find ../mercurial*.deb ../mercurial_*.build ../mercurial_*.changes \
+    find ../mercurial*.deb ../mercurial_*.build* ../mercurial_*.changes \
           ../mercurial*.dsc ../mercurial*.gz \
           -type f -newer $control -print0 2>/dev/null | \
       xargs -Inarf -0 mv narf "$OUTPUTDIR"
     echo "Built packages for $debver:"
-    find "$PWD"/"$OUTPUTDIR" -type f -newer $control -name '*.deb'
+    find "$OUTPUTDIR" -type f -newer $control -name '*.deb'
 fi

contrib/packaging/buildrpm

@@ -1,16 +1,12 b''
 #!/bin/bash -e
 #
-# Build a Mercurial RPM from the current repo
-#
-# Tested on
-# - Fedora 20
-# - CentOS 5
-# - centOS 6
+# Build a Mercurial RPM from the current repo, mainly for Fedora/CentOS/RHEL
 
 . $(dirname $0)/packagelib.sh
 
 BUILD=1
 RPMBUILDDIR="$PWD/rpmbuild"
+PYTHONEXE=python3
 
 while [ "$1" ]; do
     case "$1" in
@@ -18,10 +14,16 b' while [ "$1" ]; do'
         shift
         BUILD=
         ;;
+    --python)
+        shift
+        PYTHONEXE=$1
+        shift
+        ;;
     --withpython | --with-python)
         shift
         PYTHONVER=2.7.16
         PYTHONMD5=f1a2ace631068444831d01485466ece0
+        PYTHONEXE=python
         ;;
     --rpmbuilddir )
         shift
@@ -51,7 +53,7 b' fi'
 gethgversion
 
 if [ -z "$type" ] ; then
-   release=1
+    release=1
 else
     release=0.9_$type
 fi
@@ -96,6 +98,7 b' rpmspec=$RPMBUILDDIR/SPECS/mercurial.spe'
 
 sed -e "s,^Version:.*,Version: $version," \
     -e "s,^Release:.*,Release: $release," \
+    -e "s/^%global pythonexe .*/%global pythonexe $PYTHONEXE/" \
     $specfile > $rpmspec
 
 echo >> $rpmspec
@@ -121,8 +124,8 b' for l in sorted(changelog, reverse=True)'
     if prevtitle != title:
         prevtitle = title
         print
-        print title
-    print "- %s" % l[3].strip()
+        print(title)
+    print("- %s" % l[3].strip())
 ' >> $rpmspec
 
 else
@@ -138,7 +141,7 b' def datestr(date, format):'
 for l in sys.stdin.readlines():
     tok = l.split("\t")
     hgdate = tuple(int(v) for v in tok[0].split())
-    print "* %s %s\n- %s" % (datestr(hgdate, "%a %b %d %Y"), tok[1], tok[2])
+    print("* %s %s\n- %s" % (datestr(hgdate, "%a %b %d %Y"), tok[1], tok[2]))
 ' >> $rpmspec
 
 fi

contrib/packaging/debian/control

@@ -7,21 +7,24 b' Build-Depends:'
  dh-python,
  less,
  netbase,
- python-all,
- python-all-dev,
- python-docutils,
+ python3-all,
+ python3-all-dev,
+ python3-docutils,
  unzip,
  zip
 Standards-Version: 3.9.4
-X-Python-Version: >= 2.7
+X-Python3-Version: >= 3.5
 
 Package: mercurial
 Depends:
- python,
+ sensible-utils,
  ${shlibs:Depends},
  ${misc:Depends},
- ${python:Depends},
- mercurial-common (= ${source:Version})
+ ${python3:Depends},
+Recommends: ca-certificates
+Suggests: wish
+Replaces: mercurial-common
+Breaks: mercurial-common
 Architecture: any
 Description: fast, easy to use, distributed revision control tool.
  Mercurial is a fast, lightweight Source Control Management system designed
@@ -36,19 +39,3 b' Description: fast, easy to use, distribu'
   * Easy-to-use command-line interface
   * Integrated stand-alone web interface
   * Small Python codebase
-
-Package: mercurial-common
-Architecture: all
-Depends:
- ${misc:Depends},
- ${python:Depends},
-Recommends: mercurial (= ${source:Version}), ca-certificates
-Suggests: wish
-Breaks: mercurial (<< ${source:Version})
-Replaces: mercurial (<< 2.6.3)
-Description: easy-to-use, scalable distributed version control system (common files)
- Mercurial is a fast, lightweight Source Control Management system designed
- for efficient handling of very large distributed projects.
- .
- This package contains the architecture independent components of Mercurial,
- and is generally useless without the mercurial package.

contrib/packaging/debian/rules

@@ -4,41 +4,41 b''
 
 CPUS=$(shell cat /proc/cpuinfo | grep -E ^processor | wc -l)
 
+export HGPYTHON3=1
+export PYTHON=python3
+
 %:
-	dh $@ --with python2
+	dh $@ --with python3
 
 override_dh_auto_test:
 	http_proxy='' dh_auto_test -- TESTFLAGS="-j$(CPUS)"
 
-override_dh_python2:
-	dh_python2
-	find debian/mercurial/usr/share -type d -empty -delete
+override_dh_python3:
+	dh_python3 --shebang=/usr/bin/python3
+
+override_dh_auto_clean:
+	$(MAKE) cleanbutpackages
+	$(MAKE) -C contrib/chg clean
 
-override_dh_install:
-	python$(PYVERS) setup.py install --root "$(CURDIR)"/debian/mercurial --install-layout=deb
+override_dh_auto_build:
+	$(MAKE) all
+	$(MAKE) -C contrib/chg all
+
+override_dh_auto_install:
+	python3 setup.py install --root "$(CURDIR)"/debian/mercurial --install-layout=deb
 	# chg
 	make -C contrib/chg \
 		DESTDIR="$(CURDIR)"/debian/mercurial \
 		PREFIX=/usr \
-		clean install
-	# remove arch-independent python stuff
-	find "$(CURDIR)"/debian/mercurial/usr/lib \
-		! -name '*.so' ! -type d -delete , \
-		-type d -empty -delete
-	python$(PYVERS) setup.py install --root "$(CURDIR)/debian/mercurial-common" --install-layout=deb
-	make install-doc PREFIX="$(CURDIR)"/debian/mercurial-common/usr
-	# remove arch-dependent python stuff
-	find "$(CURDIR)"/debian/mercurial-common/usr/lib \
-		-name '*.so' ! -type d -delete , \
-		-type d -empty -delete
-	cp contrib/hg-ssh "$(CURDIR)"/debian/mercurial-common/usr/bin
-	mkdir -p "$(CURDIR)"/debian/mercurial-common/usr/share/mercurial
-	cp contrib/hgk "$(CURDIR)"/debian/mercurial-common/usr/share/mercurial
-	mkdir -p "$(CURDIR)"/debian/mercurial-common/etc/mercurial/hgrc.d/
-	cp contrib/packaging/debian/*.rc "$(CURDIR)"/debian/mercurial-common/etc/mercurial/hgrc.d/
+		install
+	make install-doc PREFIX="$(CURDIR)"/debian/mercurial/usr
+	cp contrib/hg-ssh "$(CURDIR)"/debian/mercurial/usr/bin
+	mkdir -p "$(CURDIR)"/debian/mercurial/usr/share/mercurial
+	cp contrib/hgk "$(CURDIR)"/debian/mercurial/usr/share/mercurial
+	mkdir -p "$(CURDIR)"/debian/mercurial/etc/mercurial/hgrc.d/
+	cp contrib/packaging/debian/*.rc "$(CURDIR)"/debian/mercurial/etc/mercurial/hgrc.d/
 	# completions
-	mkdir -p "$(CURDIR)"/debian/mercurial-common/usr/share/bash-completion/completions
-	cp contrib/bash_completion "$(CURDIR)"/debian/mercurial-common/usr/share/bash-completion/completions/hg
-	mkdir -p "$(CURDIR)"/debian/mercurial-common/usr/share/zsh/vendor-completions
-	cp contrib/zsh_completion "$(CURDIR)"/debian/mercurial-common/usr/share/zsh/vendor-completions/_hg
-	rm "$(CURDIR)"/debian/mercurial-common/usr/bin/hg
+	mkdir -p "$(CURDIR)"/debian/mercurial/usr/share/bash-completion/completions
+	cp contrib/bash_completion "$(CURDIR)"/debian/mercurial/usr/share/bash-completion/completions/hg
+	mkdir -p "$(CURDIR)"/debian/mercurial/usr/share/zsh/vendor-completions
+	cp contrib/zsh_completion "$(CURDIR)"/debian/mercurial/usr/share/zsh/vendor-completions/_hg

contrib/packaging/docker/fedora.template

@@ -1,4 +1,4 b''
-FROM fedora:29
+FROM fedora:%OS_RELEASE%
 
 RUN groupadd -g 1000 build && \
     useradd -u 1000 -g 1000 -s /bin/bash -d /build -m build
@@ -7,8 +7,8 b' RUN dnf install -y \\'
 	gcc \
 	gettext \
 	make \
-	python-devel \
-	python-docutils \
+	python3-devel \
+	python3-docutils \
 	rpm-build
 
 # For creating repo meta data

contrib/packaging/dockerrpm

@@ -6,6 +6,14 b' export ROOTDIR=$(cd $BUILDDIR/../..; pwd'
 PLATFORM="$1"
 shift # extra params are passed to buildrpm
 
+DOCKERFILE="$PLATFORM"
+OS_RELEASE="${PLATFORM//[a-z]/}"
+case "$PLATFORM" in
+fedora*)
+    DOCKERFILE="${PLATFORM//[0-9]/}.template"
+    ;;
+esac
+
 DOCKER=$($BUILDDIR/hg-docker docker-path)
 
 CONTAINER=hg-docker-$PLATFORM
@@ -18,9 +26,14 b' else'
     DOCKERGID=$(id -g)
 fi
 
-$BUILDDIR/hg-docker build --build-arg UID=$DOCKERUID --build-arg GID=$DOCKERGID $BUILDDIR/docker/$PLATFORM $CONTAINER
+$BUILDDIR/hg-docker build \
+    --build-arg UID=$DOCKERUID \
+    --build-arg GID=$DOCKERGID \
+    --build-arg OS_RELEASE=${OS_RELEASE:-latest} \
+    $BUILDDIR/docker/$DOCKERFILE $CONTAINER
 
 RPMBUILDDIR=$ROOTDIR/packages/$PLATFORM
+mkdir -p $RPMBUILDDIR
 $ROOTDIR/contrib/packaging/buildrpm --rpmbuilddir $RPMBUILDDIR --prepare $*
 
 DSHARED=/mnt/shared

contrib/packaging/inno/requirements.txt

@@ -2,37 +2,119 b''
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes contrib/packaging/inno/requirements.txt.in -o contrib/packaging/inno/requirements.txt
+#    pip-compile --generate-hashes --output-file=contrib/packaging/inno/requirements.txt contrib/packaging/inno/requirements.txt.in
 #
-certifi==2018.11.29 \
-    --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \
-    --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \
+certifi==2019.9.11 \
+    --hash=sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50 \
+    --hash=sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef \
     # via dulwich
-configparser==3.7.3 \
-    --hash=sha256:27594cf4fc279f321974061ac69164aaebd2749af962ac8686b20503ac0bcf2d \
-    --hash=sha256:9d51fe0a382f05b6b117c5e601fc219fede4a8c71703324af3f7d883aef476a3 \
+cffi==1.13.1 \
+    --hash=sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa \
+    --hash=sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a \
+    --hash=sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400 \
+    --hash=sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365 \
+    --hash=sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98 \
+    --hash=sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526 \
+    --hash=sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14 \
+    --hash=sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5 \
+    --hash=sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e \
+    --hash=sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1 \
+    --hash=sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434 \
+    --hash=sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b \
+    --hash=sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730 \
+    --hash=sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43 \
+    --hash=sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4 \
+    --hash=sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331 \
+    --hash=sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36 \
+    --hash=sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599 \
+    --hash=sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8 \
+    --hash=sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8 \
+    --hash=sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa \
+    --hash=sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78 \
+    --hash=sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc \
+    --hash=sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e \
+    --hash=sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2 \
+    --hash=sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0 \
+    --hash=sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71 \
+    --hash=sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891 \
+    --hash=sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05 \
+    --hash=sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2 \
+    # via cryptography
+configparser==4.0.2 \
+    --hash=sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c \
+    --hash=sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df \
     # via entrypoints
-docutils==0.14 \
-    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
-    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
-    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
-dulwich==0.19.11 \
-    --hash=sha256:afbe070f6899357e33f63f3f3696e601731fef66c64a489dea1bc9f539f4a725
+cryptography==2.8 \
+    --hash=sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c \
+    --hash=sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595 \
+    --hash=sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad \
+    --hash=sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651 \
+    --hash=sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2 \
+    --hash=sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff \
+    --hash=sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d \
+    --hash=sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42 \
+    --hash=sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d \
+    --hash=sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e \
+    --hash=sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912 \
+    --hash=sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793 \
+    --hash=sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13 \
+    --hash=sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7 \
+    --hash=sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0 \
+    --hash=sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879 \
+    --hash=sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f \
+    --hash=sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9 \
+    --hash=sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2 \
+    --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \
+    --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 \
+    # via secretstorage
+docutils==0.15.2 \
+    --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
+    --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
+    --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99
+dulwich==0.19.13 \
+    --hash=sha256:0e442f6f96e6d97270a7cca4e75306b6b0228627bdf57dde3759e0e345a6b523 \
+    --hash=sha256:667f49536ccba09d3b90bac80d44048e45566f84b98a5e139cc8c70757a6ae60 \
+    --hash=sha256:82792a9d49b112fa2151fa0fb29b01667855a843ff99325b1c1578a4aec11b57 \
+    --hash=sha256:aa628449c5f594a9a282f4d9e5993fef65481ef5e3b9b6c52ff31200f8f5dc95 \
+    --hash=sha256:ab4668bc4e1996d12eb1910e123a09edcff8e166e7ec46db5aafb5c7e250b99f \
+    --hash=sha256:c35ed2cd5b263ce0d67758ffba590c0466ff13b048457ff060b7d2e6cb55a40e \
+    --hash=sha256:c8b48079a14850cbeb788b38e1061ae6db75061431c1c0f91382460be4c84bbe \
+    --hash=sha256:dfcd9943c69f963dd61a027f480d16f548ea5905c2485be8f4b8f130df2c32de \
+    --hash=sha256:e3693c3238c1a5fc1e4427281c4455d78549f4797f2a7107a5f4443b21efafb4
 entrypoints==0.3 \
     --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \
     --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \
     # via keyring
+enum34==1.1.6 \
+    --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
+    --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
+    --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
+    --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
+    # via cryptography
+ipaddress==1.0.23 \
+    --hash=sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc \
+    --hash=sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2 \
+    # via cryptography
 keyring==18.0.1 \
     --hash=sha256:67d6cc0132bd77922725fae9f18366bb314fd8f95ff4d323a4df41890a96a838 \
     --hash=sha256:7b29ebfcf8678c4da531b2478a912eea01e80007e5ddca9ee0c7038cb3489ec6
-pygments==2.3.1 \
-    --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \
-    --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d
+pycparser==2.19 \
+    --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
+    # via cffi
+pygments==2.4.2 \
+    --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \
+    --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297
 pywin32-ctypes==0.2.0 \
     --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
-    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98 \
+    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
+secretstorage==2.3.1 \
+    --hash=sha256:3af65c87765323e6f64c83575b05393f9e003431959c9395d1791d51497f29b6 \
     # via keyring
-urllib3==1.24.1 \
-    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
-    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
+    # via cryptography
+urllib3==1.25.6 \
+    --hash=sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398 \
+    --hash=sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86 \
     # via dulwich

contrib/packaging/inno/requirements.txt.in

@@ -2,3 +2,6 b' docutils'
 dulwich
 keyring
 pygments
+# Need to list explicitly so dependency gets pulled in when
+# not running on Windows.
+pywin32-ctypes

contrib/packaging/mercurial.spec

@@ -2,6 +2,8 b''
 
 %define withpython %{nil}
 
+%global pythonexe python3
+
 %if "%{?withpython}"
 
 %global pythonver %{withpython}
@@ -15,7 +17,7 b''
 
 %else
 
-%global pythonver %(python -c 'import sys;print ".".join(map(str, sys.version_info[:2]))')
+%global pythonver %(%{pythonexe} -c 'import sys;print(".".join(map(str, sys.version_info[:2])))')
 
 %endif
 
@@ -37,8 +39,8 b' BuildRequires: make, gcc, gettext'
 %if "%{?withpython}"
 BuildRequires: readline-devel, openssl-devel, ncurses-devel, zlib-devel, bzip2-devel
 %else
-BuildRequires: python >= 2.7, python-devel, python-docutils >= 0.5
-Requires: python >= 2.7
+BuildRequires: %{pythonexe} >= %{pythonver}, %{pythonexe}-devel, %{pythonexe}-docutils >= 0.5
+Requires: %{pythonexe} >= %{pythonver}
 %endif
 # The hgk extension uses the wish tcl interpreter, but we don't enforce it
 #Requires: tk
@@ -52,13 +54,15 b' for efficient handling of very large dis'
 %if "%{?withpython}"
 %setup -q -n mercurial-%{version}-%{release} -a1 -a2
 # despite the comments in cgi.py, we do this to prevent rpmdeps from picking /usr/local/bin/python up
-sed -i '1c#! /usr/bin/env python' %{pythonname}/Lib/cgi.py
+sed -i '1c#! /usr/bin/env %{pythonexe}' %{pythonname}/Lib/cgi.py
 %else
 %setup -q -n mercurial-%{version}-%{release}
 %endif
 
 %build
 
+export HGPYTHON3=1
+
 %if "%{?withpython}"
 
 PYPATH=$PWD/%{pythonname}
@@ -82,12 +86,16 b' export PYTHONPATH=$PWD/%{docutilsname}'
 
 %endif
 
-make all
+make all PYTHON=%{pythonexe}
 make -C contrib/chg
 
+sed -i -e '1s|#!/usr/bin/env python$|#!/usr/bin/env %{pythonexe}|' contrib/hg-ssh
+
 %install
 rm -rf $RPM_BUILD_ROOT
 
+export HGPYTHON3=1
+
 %if "%{?withpython}"
 
 PYPATH=$PWD/%{pythonname}
@@ -101,14 +109,14 b' cd %{docutilsname}'
 LD_LIBRARY_PATH=$PYPATH $PYPATH/python setup.py install --root="$RPM_BUILD_ROOT"
 cd -
 
-PATH=$PYPATH:$PATH LD_LIBRARY_PATH=$PYPATH make install DESTDIR=$RPM_BUILD_ROOT PREFIX=%{hgpyprefix} MANDIR=%{_mandir}
+PATH=$PYPATH:$PATH LD_LIBRARY_PATH=$PYPATH make install PYTHON=%{pythonexe} DESTDIR=$RPM_BUILD_ROOT PREFIX=%{hgpyprefix} MANDIR=%{_mandir}
 mkdir -p $RPM_BUILD_ROOT%{_bindir}
 ( cd $RPM_BUILD_ROOT%{_bindir}/ && ln -s ../..%{hgpyprefix}/bin/hg . )
 ( cd $RPM_BUILD_ROOT%{_bindir}/ && ln -s ../..%{hgpyprefix}/bin/python2.? %{pythonhg} )
 
 %else
 
-make install DESTDIR=$RPM_BUILD_ROOT PREFIX=%{_prefix} MANDIR=%{_mandir}
+make install PYTHON=%{pythonexe} DESTDIR=$RPM_BUILD_ROOT PREFIX=%{_prefix} MANDIR=%{_mandir}
 
 %endif
 
@@ -135,7 +143,7 b' rm -rf $RPM_BUILD_ROOT'
 
 %files
 %defattr(-,root,root,-)
-%doc CONTRIBUTORS COPYING doc/README doc/hg*.txt doc/hg*.html *.cgi contrib/*.fcgi
+%doc CONTRIBUTORS COPYING doc/README doc/hg*.txt doc/hg*.html *.cgi contrib/*.fcgi contrib/*.wsgi
 %doc %attr(644,root,root) %{_mandir}/man?/hg*
 %doc %attr(644,root,root) contrib/*.svg
 %dir %{_datadir}/zsh/

contrib/packaging/wix/requirements.txt

@@ -2,12 +2,12 b''
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes contrib/packaging/wix/requirements.txt.in -o contrib/packaging/wix/requirements.txt
+#    pip-compile --generate-hashes --output-file=contrib/packaging/wix/requirements.txt contrib/packaging/wix/requirements.txt.in
 #
-docutils==0.14 \
-    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
-    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
-    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
-pygments==2.3.1 \
-    --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \
-    --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d
+docutils==0.15.2 \
+    --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
+    --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
+    --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99
+pygments==2.4.2 \
+    --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \
+    --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297

contrib/simplemerge

@@ -18,6 +18,7 b' from mercurial import ('
 )
 from mercurial.utils import (
     procutil,
+    stringutil
 )
 
 options = [(b'L', b'label', [], _(b'labels to use on conflict markers')),
@@ -75,8 +76,7 b' try:'
                                      context.arbitraryfilectx(other),
                                      **pycompat.strkwargs(opts)))
 except ParseError as e:
-    if pycompat.ispy3:
-        e = str(e).encode('utf8')
+    e = stringutil.forcebytestr(e)
     pycompat.stdout.write(b"%s: %s\n" % (sys.argv[0].encode('utf8'), e))
     showhelp()
     sys.exit(1)

hgdemandimport/demandimportpy3.py

@@ -53,9 +53,13 b' class _lazyloaderex(importlib.util.LazyL'
 
 # This is 3.6+ because with Python 3.5 it isn't possible to lazily load
 # extensions. See the discussion in https://bugs.python.org/issue26186 for more.
-_extensions_loader = _lazyloaderex.factory(
-    importlib.machinery.ExtensionFileLoader
-)
+if sys.version_info[0:2] >= (3, 6):
+    _extensions_loader = _lazyloaderex.factory(
+        importlib.machinery.ExtensionFileLoader
+    )
+else:
+    _extensions_loader = importlib.machinery.ExtensionFileLoader
+
 _bytecode_loader = _lazyloaderex.factory(
     importlib.machinery.SourcelessFileLoader
 )

hgext/bugzilla.py

@@ -955,7 +955,7 b' class bzrestapi(bzaccess):'
     def _fetch(self, burl):
         try:
             resp = url.open(self.ui, burl)
-            return json.loads(resp.read())
+            return pycompat.json_loads(resp.read())
         except util.urlerr.httperror as inst:
             if inst.code == 401:
                 raise error.Abort(_(b'authorization failed'))
@@ -978,7 +978,7 b' class bzrestapi(bzaccess):'
         req = request_type(burl, data, {b'Content-Type': b'application/json'})
         try:
             resp = url.opener(self.ui).open(req)
-            return json.loads(resp.read())
+            return pycompat.json_loads(resp.read())
         except util.urlerr.httperror as inst:
             if inst.code == 401:
                 raise error.Abort(_(b'authorization failed'))

hgext/convert/gnuarch.py

@@ -7,7 +7,6 b''
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
-import email.parser as emailparser
 import os
 import shutil
 import stat
@@ -17,6 +16,7 b' from mercurial.i18n import _'
 from mercurial import (
     encoding,
     error,
+    mail,
     pycompat,
     util,
 )
@@ -69,7 +69,6 b' class gnuarch_source(common.converter_so'
         self.changes = {}
         self.parents = {}
         self.tags = {}
-        self.catlogparser = emailparser.Parser()
         self.encoding = encoding.encoding
         self.archives = []
 
@@ -299,26 +298,29 b' class gnuarch_source(common.converter_so'
 
     def _parsecatlog(self, data, rev):
         try:
-            catlog = self.catlogparser.parsestr(data)
+            catlog = mail.parsebytes(data)
 
             # Commit date
             self.changes[rev].date = dateutil.datestr(
-                dateutil.strdate(catlog[b'Standard-date'], b'%Y-%m-%d %H:%M:%S')
+                dateutil.strdate(catlog[r'Standard-date'], b'%Y-%m-%d %H:%M:%S')
             )
 
             # Commit author
-            self.changes[rev].author = self.recode(catlog[b'Creator'])
+            self.changes[rev].author = self.recode(catlog[r'Creator'])
 
             # Commit description
             self.changes[rev].summary = b'\n\n'.join(
-                (catlog[b'Summary'], catlog.get_payload())
+                (
+                    self.recode(catlog[r'Summary']),
+                    self.recode(catlog.get_payload()),
+                )
             )
             self.changes[rev].summary = self.recode(self.changes[rev].summary)
 
             # Commit revision origin when dealing with a branch or tag
-            if b'Continuation-of' in catlog:
+            if r'Continuation-of' in catlog:
                 self.changes[rev].continuationof = self.recode(
-                    catlog[b'Continuation-of']
+                    catlog[r'Continuation-of']
                 )
         except Exception:
             raise error.Abort(_(b'could not parse cat-log of %s') % rev)

hgext/fix.py

@@ -126,7 +126,6 b' from __future__ import absolute_import'
 
 import collections
 import itertools
-import json
 import os
 import re
 import subprocess
@@ -642,7 +641,7 b' def fixfile(ui, repo, opts, fixers, fixc'
             if fixer.shouldoutputmetadata():
                 try:
                     metadatajson, newerdata = stdout.split(b'\0', 1)
-                    metadata[fixername] = json.loads(metadatajson)
+                    metadata[fixername] = pycompat.json_loads(metadatajson)
                 except ValueError:
                     ui.warn(
                         _(b'ignored invalid output from fixer tool: %s\n')

hgext/fsmonitor/__init__.py

@@ -132,6 +132,7 b' from mercurial import ('
     util,
 )
 from mercurial import match as matchmod
+from mercurial.utils import stringutil
 
 from . import (
     pywatchman,
@@ -189,10 +190,10 b' def debuginstall(ui, fm):'
         fm.write(
             b"fsmonitor-watchman-version",
             _(b" watchman binary version %s\n"),
-            v[b"version"],
+            pycompat.bytestr(v["version"]),
         )
     except watchmanclient.Unavailable as e:
-        err = str(e)
+        err = stringutil.forcebytestr(e)
     fm.condwrite(
         err,
         b"fsmonitor-watchman-error",
@@ -207,15 +208,23 b' def _handleunavailable(ui, state, ex):'
     if isinstance(ex, watchmanclient.Unavailable):
         # experimental config: fsmonitor.verbose
         if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
-            if b'illegal_fstypes' not in str(ex):
-                ui.warn(str(ex) + b'\n')
+            if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
+                ui.warn(stringutil.forcebytestr(ex) + b'\n')
         if ex.invalidate:
             state.invalidate()
         # experimental config: fsmonitor.verbose
         if ui.configbool(b'fsmonitor', b'verbose'):
-            ui.log(b'fsmonitor', b'Watchman unavailable: %s\n', ex.msg)
+            ui.log(
+                b'fsmonitor',
+                b'Watchman unavailable: %s\n',
+                stringutil.forcebytestr(ex.msg),
+            )
     else:
-        ui.log(b'fsmonitor', b'Watchman exception: %s\n', ex)
+        ui.log(
+            b'fsmonitor',
+            b'Watchman exception: %s\n',
+            stringutil.forcebytestr(ex),
+        )
 
 
 def _hashignore(ignore):
@@ -227,8 +236,8 b' def _hashignore(ignore):'
 
     """
     sha1 = hashlib.sha1()
-    sha1.update(repr(ignore))
-    return sha1.hexdigest()
+    sha1.update(pycompat.byterepr(ignore))
+    return pycompat.sysbytes(sha1.hexdigest())
 
 
 _watchmanencoding = pywatchman.encoding.get_local_encoding()
@@ -245,12 +254,14 b' def _watchmantofsencoding(path):'
     try:
         decoded = path.decode(_watchmanencoding)
     except UnicodeDecodeError as e:
-        raise error.Abort(str(e), hint=b'watchman encoding error')
+        raise error.Abort(
+            stringutil.forcebytestr(e), hint=b'watchman encoding error'
+        )
 
     try:
         encoded = decoded.encode(_fsencoding, 'strict')
     except UnicodeEncodeError as e:
-        raise error.Abort(str(e))
+        raise error.Abort(stringutil.forcebytestr(e))
 
     return encoded
 
@@ -372,7 +383,7 b' def overridewalk(orig, self, match, subr'
     else:
         # We need to propagate the last observed clock up so that we
         # can use it for our next query
-        state.setlastclock(result[b'clock'])
+        state.setlastclock(pycompat.sysbytes(result[b'clock']))
         if result[b'is_fresh_instance']:
             if state.walk_on_invalidate:
                 state.invalidate()
@@ -396,8 +407,15 b' def overridewalk(orig, self, match, subr'
     # for name case changes.
     for entry in result[b'files']:
         fname = entry[b'name']
+
+        # Watchman always give us a str. Normalize to bytes on Python 3
+        # using Watchman's encoding, if needed.
+        if not isinstance(fname, bytes):
+            fname = fname.encode(_watchmanencoding)
+
         if _fixencoding:
             fname = _watchmantofsencoding(fname)
+
         if switch_slashes:
             fname = fname.replace(b'\\', b'/')
         if normalize:
@@ -486,9 +504,9 b' def overridewalk(orig, self, match, subr'
     for f in auditfail:
         results[f] = None
 
-    nf = iter(auditpass).next
+    nf = iter(auditpass)
     for st in util.statfiles([join(f) for f in auditpass]):
-        f = nf()
+        f = next(nf)
         if st or f in dmap:
             results[f] = st
 
@@ -916,7 +934,7 b' def reposetup(ui, repo):'
             return
 
         try:
-            client = watchmanclient.client(repo.ui, repo._root)
+            client = watchmanclient.client(repo.ui, repo.root)
         except Exception as ex:
             _handleunavailable(ui, fsmonitorstate, ex)
             return

hgext/fsmonitor/pywatchman/__init__.py

@@ -26,10 +26,8 b''
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
 
 import inspect
 import math
@@ -38,33 +36,22 b' import socket'
 import subprocess
 import time
 
+from . import capabilities, compat, encoding, load
+
+
 # Sometimes it's really hard to get Python extensions to compile,
 # so fall back to a pure Python implementation.
 try:
     from . import bser
+
     # Demandimport causes modules to be loaded lazily. Force the load now
     # so that we can fall back on pybser if bser doesn't exist
     bser.pdu_info
 except ImportError:
     from . import pybser as bser
 
-from mercurial.utils import (
-    procutil,
-)
 
-from mercurial import (
-    pycompat,
-)
-
-from . import (
-    capabilities,
-    compat,
-    encoding,
-    load,
-)
-
-
-if os.name == 'nt':
+if os.name == "nt":
     import ctypes
     import ctypes.wintypes
 
@@ -73,7 +60,7 b" if os.name == 'nt':"
     GENERIC_WRITE = 0x40000000
     FILE_FLAG_OVERLAPPED = 0x40000000
     OPEN_EXISTING = 3
-    INVALID_HANDLE_VALUE = -1
+    INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
     FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000
     FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100
     FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200
@@ -92,9 +79,11 b" if os.name == 'nt':"
 
     class OVERLAPPED(ctypes.Structure):
         _fields_ = [
-            ("Internal", ULONG_PTR), ("InternalHigh", ULONG_PTR),
-            ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD),
-            ("hEvent", wintypes.HANDLE)
+            ("Internal", ULONG_PTR),
+            ("InternalHigh", ULONG_PTR),
+            ("Offset", wintypes.DWORD),
+            ("OffsetHigh", wintypes.DWORD),
+            ("hEvent", wintypes.HANDLE),
         ]
 
         def __init__(self):
@@ -107,9 +96,15 b" if os.name == 'nt':"
     LPDWORD = ctypes.POINTER(wintypes.DWORD)
 
     CreateFile = ctypes.windll.kernel32.CreateFileA
-    CreateFile.argtypes = [wintypes.LPSTR, wintypes.DWORD, wintypes.DWORD,
-                           wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD,
-                           wintypes.HANDLE]
+    CreateFile.argtypes = [
+        wintypes.LPSTR,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.HANDLE,
+    ]
     CreateFile.restype = wintypes.HANDLE
 
     CloseHandle = ctypes.windll.kernel32.CloseHandle
@@ -117,13 +112,23 b" if os.name == 'nt':"
     CloseHandle.restype = wintypes.BOOL
 
     ReadFile = ctypes.windll.kernel32.ReadFile
-    ReadFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
-                         LPDWORD, ctypes.POINTER(OVERLAPPED)]
+    ReadFile.argtypes = [
+        wintypes.HANDLE,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        LPDWORD,
+        ctypes.POINTER(OVERLAPPED),
+    ]
     ReadFile.restype = wintypes.BOOL
 
     WriteFile = ctypes.windll.kernel32.WriteFile
-    WriteFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
-                          LPDWORD, ctypes.POINTER(OVERLAPPED)]
+    WriteFile.argtypes = [
+        wintypes.HANDLE,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        LPDWORD,
+        ctypes.POINTER(OVERLAPPED),
+    ]
     WriteFile.restype = wintypes.BOOL
 
     GetLastError = ctypes.windll.kernel32.GetLastError
@@ -135,34 +140,56 b" if os.name == 'nt':"
     SetLastError.restype = None
 
     FormatMessage = ctypes.windll.kernel32.FormatMessageA
-    FormatMessage.argtypes = [wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD,
-                              wintypes.DWORD, ctypes.POINTER(wintypes.LPSTR),
-                              wintypes.DWORD, wintypes.LPVOID]
+    FormatMessage.argtypes = [
+        wintypes.DWORD,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        ctypes.POINTER(wintypes.LPSTR),
+        wintypes.DWORD,
+        wintypes.LPVOID,
+    ]
     FormatMessage.restype = wintypes.DWORD
 
     LocalFree = ctypes.windll.kernel32.LocalFree
 
     GetOverlappedResult = ctypes.windll.kernel32.GetOverlappedResult
-    GetOverlappedResult.argtypes = [wintypes.HANDLE,
-                                    ctypes.POINTER(OVERLAPPED), LPDWORD,
-                                    wintypes.BOOL]
+    GetOverlappedResult.argtypes = [
+        wintypes.HANDLE,
+        ctypes.POINTER(OVERLAPPED),
+        LPDWORD,
+        wintypes.BOOL,
+    ]
     GetOverlappedResult.restype = wintypes.BOOL
 
-    GetOverlappedResultEx = getattr(ctypes.windll.kernel32,
-                                    'GetOverlappedResultEx', None)
+    GetOverlappedResultEx = getattr(
+        ctypes.windll.kernel32, "GetOverlappedResultEx", None
+    )
     if GetOverlappedResultEx is not None:
-        GetOverlappedResultEx.argtypes = [wintypes.HANDLE,
-                                          ctypes.POINTER(OVERLAPPED), LPDWORD,
-                                          wintypes.DWORD, wintypes.BOOL]
+        GetOverlappedResultEx.argtypes = [
+            wintypes.HANDLE,
+            ctypes.POINTER(OVERLAPPED),
+            LPDWORD,
+            wintypes.DWORD,
+            wintypes.BOOL,
+        ]
         GetOverlappedResultEx.restype = wintypes.BOOL
 
     WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx
-    WaitForSingleObjectEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL]
+    WaitForSingleObjectEx.argtypes = [
+        wintypes.HANDLE,
+        wintypes.DWORD,
+        wintypes.BOOL,
+    ]
     WaitForSingleObjectEx.restype = wintypes.DWORD
 
     CreateEvent = ctypes.windll.kernel32.CreateEventA
-    CreateEvent.argtypes = [LPDWORD, wintypes.BOOL, wintypes.BOOL,
-                            wintypes.LPSTR]
+    CreateEvent.argtypes = [
+        LPDWORD,
+        wintypes.BOOL,
+        wintypes.BOOL,
+        wintypes.LPSTR,
+    ]
     CreateEvent.restype = wintypes.HANDLE
 
     # Windows Vista is the minimum supported client for CancelIoEx.
@@ -178,9 +205,15 b' sniff_len = 13'
 if _debugging:
 
     def log(fmt, *args):
-        print('[%s] %s' %
-              (time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
-               fmt % args[:]))
+        print(
+            "[%s] %s"
+            % (
+                time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
+                fmt % args[:],
+            )
+        )
+
+
 else:
 
     def log(fmt, *args):
@@ -193,8 +226,16 b' def _win32_strerror(err):'
     # FormatMessage will allocate memory and assign it here
     buf = ctypes.c_char_p()
     FormatMessage(
-        FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER
-        | FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None)
+        FORMAT_MESSAGE_FROM_SYSTEM
+        | FORMAT_MESSAGE_ALLOCATE_BUFFER
+        | FORMAT_MESSAGE_IGNORE_INSERTS,
+        None,
+        err,
+        0,
+        buf,
+        0,
+        None,
+    )
     try:
         return buf.value
     finally:
@@ -211,21 +252,30 b' class WatchmanError(Exception):'
 
     def __str__(self):
         if self.cmd:
-            return '%s, while executing %s' % (self.msg, self.cmd)
+            return "%s, while executing %s" % (self.msg, self.cmd)
         return self.msg
 
 
+class BSERv1Unsupported(WatchmanError):
+    pass
+
+
+class UseAfterFork(WatchmanError):
+    pass
+
+
 class WatchmanEnvironmentError(WatchmanError):
     def __init__(self, msg, errno, errmsg, cmd=None):
         super(WatchmanEnvironmentError, self).__init__(
-            '{0}: errno={1} errmsg={2}'.format(msg, errno, errmsg),
-            cmd)
+            "{0}: errno={1} errmsg={2}".format(msg, errno, errmsg), cmd
+        )
 
 
 class SocketConnectError(WatchmanError):
     def __init__(self, sockpath, exc):
         super(SocketConnectError, self).__init__(
-            'unable to connect to %s: %s' % (sockpath, exc))
+            "unable to connect to %s: %s" % (sockpath, exc)
+        )
         self.sockpath = sockpath
         self.exc = exc
 
@@ -245,15 +295,16 b' class CommandError(WatchmanError):'
 
     self.msg is the message returned by watchman.
     """
+
     def __init__(self, msg, cmd=None):
         super(CommandError, self).__init__(
-            'watchman command error: %s' % (msg, ),
-            cmd,
+            "watchman command error: %s" % (msg,), cmd
         )
 
 
 class Transport(object):
     """ communication transport to the watchman server """
+
     buf = None
 
     def close(self):
@@ -289,7 +340,7 b' class Transport(object):'
         while True:
             b = self.readBytes(4096)
             if b"\n" in b:
-                result = b''.join(self.buf)
+                result = b"".join(self.buf)
                 (line, b) = b.split(b"\n", 1)
                 self.buf = [b]
                 return result + line
@@ -298,6 +349,7 b' class Transport(object):'
 
 class Codec(object):
     """ communication encoding for the watchman server """
+
     transport = None
 
     def __init__(self, transport):
@@ -315,9 +367,10 b' class Codec(object):'
 
 class UnixSocketTransport(Transport):
     """ local unix domain socket transport """
+
     sock = None
 
-    def __init__(self, sockpath, timeout, watchman_exe):
+    def __init__(self, sockpath, timeout):
         self.sockpath = sockpath
         self.timeout = timeout
 
@@ -331,8 +384,9 b' class UnixSocketTransport(Transport):'
             raise SocketConnectError(self.sockpath, e)
 
     def close(self):
-        self.sock.close()
-        self.sock = None
+        if self.sock:
+            self.sock.close()
+            self.sock = None
 
     def setTimeout(self, value):
         self.timeout = value
@@ -342,16 +396,16 b' class UnixSocketTransport(Transport):'
         try:
             buf = [self.sock.recv(size)]
             if not buf[0]:
-                raise WatchmanError('empty watchman response')
+                raise WatchmanError("empty watchman response")
             return buf[0]
         except socket.timeout:
-            raise SocketTimeout('timed out waiting for response')
+            raise SocketTimeout("timed out waiting for response")
 
     def write(self, data):
         try:
             self.sock.sendall(data)
         except socket.timeout:
-            raise SocketTimeout('timed out sending query command')
+            raise SocketTimeout("timed out sending query command")
 
 
 def _get_overlapped_result_ex_impl(pipe, olap, nbytes, millis, alertable):
@@ -364,7 +418,7 b' def _get_overlapped_result_ex_impl(pipe,'
     source code (see get_overlapped_result_ex_impl in stream_win.c). This
     way, maintenance should be simplified.
     """
-    log('Preparing to wait for maximum %dms', millis )
+    log("Preparing to wait for maximum %dms", millis)
     if millis != 0:
         waitReturnCode = WaitForSingleObjectEx(olap.hEvent, millis, alertable)
         if waitReturnCode == WAIT_OBJECT_0:
@@ -383,12 +437,12 b' def _get_overlapped_result_ex_impl(pipe,'
         elif waitReturnCode == WAIT_FAILED:
             # something went wrong calling WaitForSingleObjectEx
             err = GetLastError()
-            log('WaitForSingleObjectEx failed: %s', _win32_strerror(err))
+            log("WaitForSingleObjectEx failed: %s", _win32_strerror(err))
             return False
         else:
             # unexpected situation deserving investigation.
             err = GetLastError()
-            log('Unexpected error: %s', _win32_strerror(err))
+            log("Unexpected error: %s", _win32_strerror(err))
             return False
 
     return GetOverlappedResult(pipe, olap, nbytes, False)
@@ -397,36 +451,52 b' def _get_overlapped_result_ex_impl(pipe,'
 class WindowsNamedPipeTransport(Transport):
     """ connect to a named pipe """
 
-    def __init__(self, sockpath, timeout, watchman_exe):
+    def __init__(self, sockpath, timeout):
         self.sockpath = sockpath
         self.timeout = int(math.ceil(timeout * 1000))
         self._iobuf = None
 
-        self.pipe = CreateFile(sockpath, GENERIC_READ | GENERIC_WRITE, 0, None,
-                               OPEN_EXISTING, FILE_FLAG_OVERLAPPED, None)
+        if compat.PYTHON3:
+            sockpath = os.fsencode(sockpath)
+        self.pipe = CreateFile(
+            sockpath,
+            GENERIC_READ | GENERIC_WRITE,
+            0,
+            None,
+            OPEN_EXISTING,
+            FILE_FLAG_OVERLAPPED,
+            None,
+        )
 
-        if self.pipe == INVALID_HANDLE_VALUE:
+        err = GetLastError()
+        if self.pipe == INVALID_HANDLE_VALUE or self.pipe == 0:
             self.pipe = None
-            self._raise_win_err('failed to open pipe %s' % sockpath,
-                                GetLastError())
+            raise SocketConnectError(self.sockpath, self._make_win_err("", err))
 
         # event for the overlapped I/O operations
         self._waitable = CreateEvent(None, True, False, None)
+        err = GetLastError()
         if self._waitable is None:
-            self._raise_win_err('CreateEvent failed', GetLastError())
+            self._raise_win_err("CreateEvent failed", err)
 
         self._get_overlapped_result_ex = GetOverlappedResultEx
-        if (os.getenv('WATCHMAN_WIN7_COMPAT') == '1' or
-            self._get_overlapped_result_ex is None):
+        if (
+            os.getenv("WATCHMAN_WIN7_COMPAT") == "1"
+            or self._get_overlapped_result_ex is None
+        ):
             self._get_overlapped_result_ex = _get_overlapped_result_ex_impl
 
     def _raise_win_err(self, msg, err):
-        raise IOError('%s win32 error code: %d %s' %
-                      (msg, err, _win32_strerror(err)))
+        raise self._make_win_err(msg, err)
+
+    def _make_win_err(self, msg, err):
+        return IOError(
+            "%s win32 error code: %d %s" % (msg, err, _win32_strerror(err))
+        )
 
     def close(self):
         if self.pipe:
-            log('Closing pipe')
+            log("Closing pipe")
             CloseHandle(self.pipe)
         self.pipe = None
 
@@ -460,7 +530,7 b' class WindowsNamedPipeTransport(Transpor'
         olap = OVERLAPPED()
         olap.hEvent = self._waitable
 
-        log('made read buff of size %d', size)
+        log("made read buff of size %d", size)
 
         # ReadFile docs warn against sending in the nread parameter for async
         # operations, so we always collect it via GetOverlappedResultEx
@@ -469,23 +539,23 b' class WindowsNamedPipeTransport(Transpor'
         if not immediate:
             err = GetLastError()
             if err != ERROR_IO_PENDING:
-                self._raise_win_err('failed to read %d bytes' % size,
-                                    GetLastError())
+                self._raise_win_err("failed to read %d bytes" % size, err)
 
         nread = wintypes.DWORD()
-        if not self._get_overlapped_result_ex(self.pipe, olap, nread,
-                                              0 if immediate else self.timeout,
-                                              True):
+        if not self._get_overlapped_result_ex(
+            self.pipe, olap, nread, 0 if immediate else self.timeout, True
+        ):
             err = GetLastError()
             CancelIoEx(self.pipe, olap)
 
             if err == WAIT_TIMEOUT:
-                log('GetOverlappedResultEx timedout')
-                raise SocketTimeout('timed out after waiting %dms for read' %
-                                    self.timeout)
+                log("GetOverlappedResultEx timedout")
+                raise SocketTimeout(
+                    "timed out after waiting %dms for read" % self.timeout
+                )
 
-            log('GetOverlappedResultEx reports error %d', err)
-            self._raise_win_err('error while waiting for read', err)
+            log("GetOverlappedResultEx reports error %d", err)
+            self._raise_win_err("error while waiting for read", err)
 
         nread = nread.value
         if nread == 0:
@@ -494,7 +564,7 b' class WindowsNamedPipeTransport(Transpor'
             # other way this shows up is if the client has gotten in a weird
             # state, so let's bail out
             CancelIoEx(self.pipe, olap)
-            raise IOError('Async read yielded 0 bytes; unpossible!')
+            raise IOError("Async read yielded 0 bytes; unpossible!")
 
         # Holds precisely the bytes that we read from the prior request
         buf = buf[:nread]
@@ -511,21 +581,25 b' class WindowsNamedPipeTransport(Transpor'
         olap = OVERLAPPED()
         olap.hEvent = self._waitable
 
-        immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data),
-                              None, olap)
+        immediate = WriteFile(
+            self.pipe, ctypes.c_char_p(data), len(data), None, olap
+        )
 
         if not immediate:
             err = GetLastError()
             if err != ERROR_IO_PENDING:
-                self._raise_win_err('failed to write %d bytes' % len(data),
-                                    GetLastError())
+                self._raise_win_err(
+                    "failed to write %d bytes to handle %r"
+                    % (len(data), self.pipe),
+                    err,
+                )
 
         # Obtain results, waiting if needed
         nwrote = wintypes.DWORD()
-        if self._get_overlapped_result_ex(self.pipe, olap, nwrote,
-                                          0 if immediate else self.timeout,
-                                          True):
-            log('made write of %d bytes', nwrote.value)
+        if self._get_overlapped_result_ex(
+            self.pipe, olap, nwrote, 0 if immediate else self.timeout, True
+        ):
+            log("made write of %d bytes", nwrote.value)
             return nwrote.value
 
         err = GetLastError()
@@ -535,10 +609,21 b' class WindowsNamedPipeTransport(Transpor'
         CancelIoEx(self.pipe, olap)
 
         if err == WAIT_TIMEOUT:
-            raise SocketTimeout('timed out after waiting %dms for write' %
-                                self.timeout)
-        self._raise_win_err('error while waiting for write of %d bytes' %
-                            len(data), err)
+            raise SocketTimeout(
+                "timed out after waiting %dms for write" % self.timeout
+            )
+        self._raise_win_err(
+            "error while waiting for write of %d bytes" % len(data), err
+        )
+
+
+def _default_binpath(binpath=None):
+    if binpath:
+        return binpath
+    # The test harness sets WATCHMAN_BINARY to the binary under test,
+    # so we use that by default, otherwise, allow resolving watchman
+    # from the users PATH.
+    return os.environ.get("WATCHMAN_BINARY", "watchman")
 
 
 class CLIProcessTransport(Transport):
@@ -560,13 +645,14 b' class CLIProcessTransport(Transport):'
     It is the responsibility of the caller to set the send and
     receive codecs appropriately.
     """
+
     proc = None
     closed = True
 
-    def __init__(self, sockpath, timeout, watchman_exe):
+    def __init__(self, sockpath, timeout, binpath=None):
         self.sockpath = sockpath
         self.timeout = timeout
-        self.watchman_exe = watchman_exe
+        self.binpath = _default_binpath(binpath)
 
     def close(self):
         if self.proc:
@@ -574,32 +660,32 b' class CLIProcessTransport(Transport):'
                 self.proc.kill()
             self.proc.stdin.close()
             self.proc.stdout.close()
+            self.proc.wait()
             self.proc = None
 
     def _connect(self):
         if self.proc:
             return self.proc
         args = [
-            self.watchman_exe,
-            '--sockname={0}'.format(self.sockpath),
-            '--logfile=/BOGUS',
-            '--statefile=/BOGUS',
-            '--no-spawn',
-            '--no-local',
-            '--no-pretty',
-            '-j',
+            self.binpath,
+            "--sockname={0}".format(self.sockpath),
+            "--logfile=/BOGUS",
+            "--statefile=/BOGUS",
+            "--no-spawn",
+            "--no-local",
+            "--no-pretty",
+            "-j",
         ]
-        self.proc = subprocess.Popen(pycompat.rapply(procutil.tonativestr,
-                                                     args),
-                                     stdin=subprocess.PIPE,
-                                     stdout=subprocess.PIPE)
+        self.proc = subprocess.Popen(
+            args, stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        )
         return self.proc
 
     def readBytes(self, size):
         self._connect()
         res = self.proc.stdout.read(size)
-        if res == '':
-            raise WatchmanError('EOF on CLI process transport')
+        if not res:
+            raise WatchmanError("EOF on CLI process transport")
         return res
 
     def write(self, data):
@@ -616,13 +702,22 b' class CLIProcessTransport(Transport):'
 class BserCodec(Codec):
     """ use the BSER encoding.  This is the default, preferred codec """
 
+    def __init__(self, transport, value_encoding, value_errors):
+        super(BserCodec, self).__init__(transport)
+        self._value_encoding = value_encoding
+        self._value_errors = value_errors
+
     def _loads(self, response):
-        return bser.loads(response) # Defaults to BSER v1
+        return bser.loads(
+            response,
+            value_encoding=self._value_encoding,
+            value_errors=self._value_errors,
+        )
 
     def receive(self):
         buf = [self.transport.readBytes(sniff_len)]
         if not buf[0]:
-            raise WatchmanError('empty watchman response')
+            raise WatchmanError("empty watchman response")
 
         _1, _2, elen = bser.pdu_info(buf[0])
 
@@ -631,15 +726,15 b' class BserCodec(Codec):'
             buf.append(self.transport.readBytes(elen - rlen))
             rlen += len(buf[-1])
 
-        response = b''.join(buf)
+        response = b"".join(buf)
         try:
             res = self._loads(response)
             return res
         except ValueError as e:
-            raise WatchmanError('watchman response decode error: %s' % e)
+            raise WatchmanError("watchman response decode error: %s" % e)
 
     def send(self, *args):
-        cmd = bser.dumps(*args) # Defaults to BSER v1
+        cmd = bser.dumps(*args)  # Defaults to BSER v1
         self.transport.write(cmd)
 
 
@@ -648,74 +743,96 b' class ImmutableBserCodec(BserCodec):'
         immutable object support """
 
     def _loads(self, response):
-        return bser.loads(response, False) # Defaults to BSER v1
+        return bser.loads(
+            response,
+            False,
+            value_encoding=self._value_encoding,
+            value_errors=self._value_errors,
+        )
 
 
 class Bser2WithFallbackCodec(BserCodec):
     """ use BSER v2 encoding """
 
-    def __init__(self, transport):
-        super(Bser2WithFallbackCodec, self).__init__(transport)
-        # Once the server advertises support for bser-v2 we should switch this
-        # to 'required' on Python 3.
-        self.send(["version", {"optional": ["bser-v2"]}])
+    def __init__(self, transport, value_encoding, value_errors):
+        super(Bser2WithFallbackCodec, self).__init__(
+            transport, value_encoding, value_errors
+        )
+        if compat.PYTHON3:
+            bserv2_key = "required"
+        else:
+            bserv2_key = "optional"
+
+        self.send(["version", {bserv2_key: ["bser-v2"]}])
 
         capabilities = self.receive()
 
-        if 'error' in capabilities:
-          raise Exception('Unsupported BSER version')
+        if "error" in capabilities:
+            raise BSERv1Unsupported(
+                "The watchman server version does not support Python 3. Please "
+                "upgrade your watchman server."
+            )
 
-        if capabilities['capabilities']['bser-v2']:
+        if capabilities["capabilities"]["bser-v2"]:
             self.bser_version = 2
             self.bser_capabilities = 0
         else:
             self.bser_version = 1
             self.bser_capabilities = 0
 
-    def _loads(self, response):
-        return bser.loads(response)
-
     def receive(self):
         buf = [self.transport.readBytes(sniff_len)]
         if not buf[0]:
-            raise WatchmanError('empty watchman response')
+            raise WatchmanError("empty watchman response")
 
         recv_bser_version, recv_bser_capabilities, elen = bser.pdu_info(buf[0])
 
-        if hasattr(self, 'bser_version'):
-          # Readjust BSER version and capabilities if necessary
-          self.bser_version = max(self.bser_version, recv_bser_version)
-          self.capabilities = self.bser_capabilities & recv_bser_capabilities
+        if hasattr(self, "bser_version"):
+            # Readjust BSER version and capabilities if necessary
+            self.bser_version = max(self.bser_version, recv_bser_version)
+            self.capabilities = self.bser_capabilities & recv_bser_capabilities
 
         rlen = len(buf[0])
         while elen > rlen:
             buf.append(self.transport.readBytes(elen - rlen))
             rlen += len(buf[-1])
 
-        response = b''.join(buf)
+        response = b"".join(buf)
         try:
             res = self._loads(response)
             return res
         except ValueError as e:
-            raise WatchmanError('watchman response decode error: %s' % e)
+            raise WatchmanError("watchman response decode error: %s" % e)
 
     def send(self, *args):
-        if hasattr(self, 'bser_version'):
-            cmd = bser.dumps(*args, version=self.bser_version,
-                capabilities=self.bser_capabilities)
+        if hasattr(self, "bser_version"):
+            cmd = bser.dumps(
+                *args,
+                version=self.bser_version,
+                capabilities=self.bser_capabilities
+            )
         else:
             cmd = bser.dumps(*args)
         self.transport.write(cmd)
 
 
+class ImmutableBser2Codec(Bser2WithFallbackCodec, ImmutableBserCodec):
+    """ use the BSER encoding, decoding values using the newer
+        immutable object support """
+
+    pass
+
+
 class JsonCodec(Codec):
     """ Use json codec.  This is here primarily for testing purposes """
+
     json = None
 
     def __init__(self, transport):
         super(JsonCodec, self).__init__(transport)
         # optional dep on json, only if JsonCodec is used
         import json
+
         self.json = json
 
     def receive(self):
@@ -727,7 +844,7 b' class JsonCodec(Codec):'
             # but it's possible we might get non-ASCII bytes that are valid
             # UTF-8.
             if compat.PYTHON3:
-                line = line.decode('utf-8')
+                line = line.decode("utf-8")
             return self.json.loads(line)
         except Exception as e:
             print(e, line)
@@ -739,12 +856,13 b' class JsonCodec(Codec):'
         # containing Unicode strings to Unicode string. Even with (the default)
         # ensure_ascii=True, dumps returns a Unicode string.
         if compat.PYTHON3:
-            cmd = cmd.encode('ascii')
+            cmd = cmd.encode("ascii")
         self.transport.write(cmd + b"\n")
 
 
 class client(object):
     """ Handles the communication with the watchman service """
+
     sockpath = None
     transport = None
     sendCodec = None
@@ -754,60 +872,100 b' class client(object):'
     subs = {}  # Keyed by subscription name
     sub_by_root = {}  # Keyed by root, then by subscription name
     logs = []  # When log level is raised
-    unilateral = ['log', 'subscription']
+    unilateral = ["log", "subscription"]
     tport = None
     useImmutableBser = None
-    watchman_exe = None
+    pid = None
 
-    def __init__(self,
-                 sockpath=None,
-                 timeout=1.0,
-                 transport=None,
-                 sendEncoding=None,
-                 recvEncoding=None,
-                 useImmutableBser=False,
-                 watchman_exe=None):
+    def __init__(
+        self,
+        sockpath=None,
+        timeout=1.0,
+        transport=None,
+        sendEncoding=None,
+        recvEncoding=None,
+        useImmutableBser=False,
+        # use False for these two because None has a special
+        # meaning
+        valueEncoding=False,
+        valueErrors=False,
+        binpath=None,
+    ):
         self.sockpath = sockpath
         self.timeout = timeout
         self.useImmutableBser = useImmutableBser
-        self.watchman_exe = watchman_exe
+        self.binpath = _default_binpath(binpath)
 
         if inspect.isclass(transport) and issubclass(transport, Transport):
             self.transport = transport
         else:
-            transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local'
-            if transport == 'local' and os.name == 'nt':
+            transport = transport or os.getenv("WATCHMAN_TRANSPORT") or "local"
+            if transport == "local" and os.name == "nt":
                 self.transport = WindowsNamedPipeTransport
-            elif transport == 'local':
+            elif transport == "local":
                 self.transport = UnixSocketTransport
-            elif transport == 'cli':
+            elif transport == "cli":
                 self.transport = CLIProcessTransport
                 if sendEncoding is None:
-                    sendEncoding = 'json'
+                    sendEncoding = "json"
                 if recvEncoding is None:
                     recvEncoding = sendEncoding
             else:
-                raise WatchmanError('invalid transport %s' % transport)
+                raise WatchmanError("invalid transport %s" % transport)
 
-        sendEncoding = str(sendEncoding or os.getenv('WATCHMAN_ENCODING') or
-                           'bser')
-        recvEncoding = str(recvEncoding or os.getenv('WATCHMAN_ENCODING') or
-                           'bser')
+        sendEncoding = str(
+            sendEncoding or os.getenv("WATCHMAN_ENCODING") or "bser"
+        )
+        recvEncoding = str(
+            recvEncoding or os.getenv("WATCHMAN_ENCODING") or "bser"
+        )
 
         self.recvCodec = self._parseEncoding(recvEncoding)
         self.sendCodec = self._parseEncoding(sendEncoding)
 
+        # We want to act like the native OS methods as much as possible. This
+        # means returning bytestrings on Python 2 by default and Unicode
+        # strings on Python 3. However we take an optional argument that lets
+        # users override this.
+        if valueEncoding is False:
+            if compat.PYTHON3:
+                self.valueEncoding = encoding.get_local_encoding()
+                self.valueErrors = encoding.default_local_errors
+            else:
+                self.valueEncoding = None
+                self.valueErrors = None
+        else:
+            self.valueEncoding = valueEncoding
+            if valueErrors is False:
+                self.valueErrors = encoding.default_local_errors
+            else:
+                self.valueErrors = valueErrors
+
+    def _makeBSERCodec(self, codec):
+        def make_codec(transport):
+            return codec(transport, self.valueEncoding, self.valueErrors)
+
+        return make_codec
+
     def _parseEncoding(self, enc):
-        if enc == 'bser':
+        if enc == "bser":
             if self.useImmutableBser:
-                return ImmutableBserCodec
-            return BserCodec
-        elif enc == 'experimental-bser-v2':
-          return Bser2WithFallbackCodec
-        elif enc == 'json':
+                return self._makeBSERCodec(ImmutableBser2Codec)
+            return self._makeBSERCodec(Bser2WithFallbackCodec)
+        elif enc == "bser-v1":
+            if compat.PYTHON3:
+                raise BSERv1Unsupported(
+                    "Python 3 does not support the BSER v1 encoding: specify "
+                    '"bser" or omit the sendEncoding and recvEncoding '
+                    "arguments"
+                )
+            if self.useImmutableBser:
+                return self._makeBSERCodec(ImmutableBserCodec)
+            return self._makeBSERCodec(BserCodec)
+        elif enc == "json":
             return JsonCodec
         else:
-            raise WatchmanError('invalid encoding %s' % enc)
+            raise WatchmanError("invalid encoding %s" % enc)
 
     def _hasprop(self, result, name):
         if self.useImmutableBser:
@@ -817,26 +975,25 b' class client(object):'
     def _resolvesockname(self):
         # if invoked via a trigger, watchman will set this env var; we
         # should use it unless explicitly set otherwise
-        path = os.getenv('WATCHMAN_SOCK')
+        path = os.getenv("WATCHMAN_SOCK")
         if path:
             return path
 
-        cmd = [self.watchman_exe, '--output-encoding=bser', 'get-sockname']
+        cmd = [self.binpath, "--output-encoding=bser", "get-sockname"]
         try:
-            args = dict(stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                        close_fds=os.name != 'nt')
+            args = dict(
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )  # noqa: C408
 
-            if os.name == 'nt':
+            if os.name == "nt":
                 # if invoked via an application with graphical user interface,
                 # this call will cause a brief command window pop-up.
                 # Using the flag STARTF_USESHOWWINDOW to avoid this behavior.
                 startupinfo = subprocess.STARTUPINFO()
                 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
-                args['startupinfo'] = startupinfo
+                args["startupinfo"] = startupinfo
 
-            p = subprocess.Popen(pycompat.rapply(procutil.tonativestr, cmd),
-                                 **args)
+            p = subprocess.Popen(cmd, **args)
 
         except OSError as e:
             raise WatchmanError('"watchman" executable not in PATH (%s)' % e)
@@ -848,27 +1005,43 b' class client(object):'
             raise WatchmanError("watchman exited with code %d" % exitcode)
 
         result = bser.loads(stdout)
-        if b'error' in result:
-            raise WatchmanError('get-sockname error: %s' % result['error'])
+        if "error" in result:
+            raise WatchmanError("get-sockname error: %s" % result["error"])
 
-        return result[b'sockname']
+        return result["sockname"]
 
     def _connect(self):
         """ establish transport connection """
 
         if self.recvConn:
+            if self.pid != os.getpid():
+                raise UseAfterFork(
+                    "do not re-use a connection after fork; open a new client instead"
+                )
             return
 
         if self.sockpath is None:
             self.sockpath = self._resolvesockname()
 
-        self.tport = self.transport(self.sockpath, self.timeout, self.watchman_exe)
+        kwargs = {}
+        if self.transport == CLIProcessTransport:
+            kwargs["binpath"] = self.binpath
+
+        self.tport = self.transport(self.sockpath, self.timeout, **kwargs)
         self.sendConn = self.sendCodec(self.tport)
         self.recvConn = self.recvCodec(self.tport)
+        self.pid = os.getpid()
 
     def __del__(self):
         self.close()
 
+    def __enter__(self):
+        self._connect()
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self.close()
+
     def close(self):
         if self.tport:
             self.tport.close()
@@ -893,26 +1066,20 b' class client(object):'
 
         self._connect()
         result = self.recvConn.receive()
-        if self._hasprop(result, 'error'):
-            error = result['error']
-            if compat.PYTHON3 and isinstance(self.recvConn, BserCodec):
-                error = result['error'].decode('utf-8', 'surrogateescape')
-            raise CommandError(error)
+        if self._hasprop(result, "error"):
+            raise CommandError(result["error"])
 
-        if self._hasprop(result, 'log'):
-            log = result['log']
-            if compat.PYTHON3 and isinstance(self.recvConn, BserCodec):
-                log = log.decode('utf-8', 'surrogateescape')
-            self.logs.append(log)
+        if self._hasprop(result, "log"):
+            self.logs.append(result["log"])
 
-        if self._hasprop(result, 'subscription'):
-            sub = result['subscription']
+        if self._hasprop(result, "subscription"):
+            sub = result["subscription"]
             if not (sub in self.subs):
                 self.subs[sub] = []
             self.subs[sub].append(result)
 
             # also accumulate in {root,sub} keyed store
-            root = os.path.normcase(result['root'])
+            root = os.path.normpath(os.path.normcase(result["root"]))
             if not root in self.sub_by_root:
                 self.sub_by_root[root] = {}
             if not sub in self.sub_by_root[root]:
@@ -922,7 +1089,7 b' class client(object):'
         return result
 
     def isUnilateralResponse(self, res):
-        if 'unilateral' in res and res['unilateral']:
+        if "unilateral" in res and res["unilateral"]:
             return True
         # Fall back to checking for known unilateral responses
         for k in self.unilateral:
@@ -955,18 +1122,11 b' class client(object):'
         remove processing impacts both the unscoped and scoped stores
         for the subscription data.
         """
-        if compat.PYTHON3 and issubclass(self.recvCodec, BserCodec):
-            # People may pass in Unicode strings here -- but currently BSER only
-            # returns bytestrings. Deal with that.
-            if isinstance(root, str):
-                root = encoding.encode_local(root)
-            if isinstance(name, str):
-                name = name.encode('utf-8')
-
         if root is not None:
-            if not root in self.sub_by_root:
+            root = os.path.normpath(os.path.normcase(root))
+            if root not in self.sub_by_root:
                 return None
-            if not name in self.sub_by_root[root]:
+            if name not in self.sub_by_root[root]:
                 return None
             sub = self.sub_by_root[root][name]
             if remove:
@@ -976,7 +1136,7 b' class client(object):'
                     del self.subs[name]
             return sub
 
-        if not (name in self.subs):
+        if name not in self.subs:
             return None
         sub = self.subs[name]
         if remove:
@@ -992,7 +1152,7 b' class client(object):'
         and NOT returned via this method.
         """
 
-        log('calling client.query')
+        log("calling client.query")
         self._connect()
         try:
             self.sendConn.send(args)
@@ -1006,27 +1166,27 b' class client(object):'
             # When we can depend on Python 3, we can use PEP 3134
             # exception chaining here.
             raise WatchmanEnvironmentError(
-                'I/O error communicating with watchman daemon',
+                "I/O error communicating with watchman daemon",
                 ee.errno,
                 ee.strerror,
-                args)
+                args,
+            )
         except WatchmanError as ex:
             ex.setCommand(args)
             raise
 
     def capabilityCheck(self, optional=None, required=None):
         """ Perform a server capability check """
-        res = self.query('version', {
-            'optional': optional or [],
-            'required': required or []
-        })
+        res = self.query(
+            "version", {"optional": optional or [], "required": required or []}
+        )
 
-        if not self._hasprop(res, 'capabilities'):
+        if not self._hasprop(res, "capabilities"):
             # Server doesn't support capabilities, so we need to
             # synthesize the results based on the version
             capabilities.synthesize(res, optional)
-            if 'error' in res:
-                raise CommandError(res['error'])
+            if "error" in res:
+                raise CommandError(res["error"])
 
         return res
 

hgext/fsmonitor/pywatchman/bser.c

@@ -175,7 +175,22 b' static PyObject* bserobj_getattrro(PyObj'
     const char* item_name = NULL;
     PyObject* key = PyTuple_GET_ITEM(obj->keys, i);
 
-    item_name = PyBytes_AsString(key);
+    if (PyUnicode_Check(key)) {
+#if PY_MAJOR_VERSION >= 3
+      item_name = PyUnicode_AsUTF8(key);
+#else
+      PyObject* utf = PyUnicode_AsEncodedString(key, "utf-8", "ignore");
+      if (utf == NULL) {
+        goto bail;
+      }
+      item_name = PyBytes_AsString(utf);
+#endif
+    } else {
+      item_name = PyBytes_AsString(key);
+    }
+    if (item_name == NULL) {
+      goto bail;
+    }
     if (!strcmp(item_name, namestr)) {
       ret = PySequence_GetItem(obj->values, i);
       goto bail;
@@ -1147,11 +1162,15 b' static PyObject* bser_loads(PyObject* se'
 }
 
 static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) {
-  PyObject *load, *string;
+  PyObject* load;
+  PyObject* load_method;
+  PyObject* string;
+  PyObject* load_method_args;
+  PyObject* load_method_kwargs;
   PyObject* fp = NULL;
   PyObject* mutable_obj = NULL;
-  const char* value_encoding = NULL;
-  const char* value_errors = NULL;
+  PyObject* value_encoding = NULL;
+  PyObject* value_errors = NULL;
 
   static char* kw_list[] = {
       "fp", "mutable", "value_encoding", "value_errors", NULL};
@@ -1159,7 +1178,7 b' static PyObject* bser_load(PyObject* sel'
   if (!PyArg_ParseTupleAndKeywords(
           args,
           kw,
-          "OOzz:load",
+          "O|OOO:load",
           kw_list,
           &fp,
           &mutable_obj,
@@ -1172,8 +1191,33 b' static PyObject* bser_load(PyObject* sel'
   if (load == NULL) {
     return NULL;
   }
-  string = PyObject_CallMethod(
-      load, "load", "OOzz", fp, mutable_obj, value_encoding, value_errors);
+  load_method = PyObject_GetAttrString(load, "load");
+  if (load_method == NULL) {
+    return NULL;
+  }
+  // Mandatory method arguments
+  load_method_args = Py_BuildValue("(O)", fp);
+  if (load_method_args == NULL) {
+    return NULL;
+  }
+  // Optional method arguments
+  load_method_kwargs = PyDict_New();
+  if (load_method_kwargs == NULL) {
+    return NULL;
+  }
+  if (mutable_obj) {
+    PyDict_SetItemString(load_method_kwargs, "mutable", mutable_obj);
+  }
+  if (value_encoding) {
+    PyDict_SetItemString(load_method_kwargs, "value_encoding", value_encoding);
+  }
+  if (value_errors) {
+    PyDict_SetItemString(load_method_kwargs, "value_errors", value_errors);
+  }
+  string = PyObject_Call(load_method, load_method_args, load_method_kwargs);
+  Py_DECREF(load_method_kwargs);
+  Py_DECREF(load_method_args);
+  Py_DECREF(load_method);
   Py_DECREF(load);
   return string;
 }

hgext/fsmonitor/pywatchman/capabilities.py

@@ -26,20 +26,20 b''
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
 
 import re
 
+
 def parse_version(vstr):
     res = 0
-    for n in vstr.split('.'):
+    for n in vstr.split("."):
         res = res * 1000
         res = res + int(n)
     return res
 
+
 cap_versions = {
     "cmd-watch-del-all": "3.1.1",
     "cmd-watch-project": "3.1",
@@ -49,23 +49,29 b' cap_versions = {'
     "wildmatch": "3.7",
 }
 
+
 def check(version, name):
     if name in cap_versions:
         return version >= parse_version(cap_versions[name])
     return False
 
+
 def synthesize(vers, opts):
     """ Synthesize a capability enabled version response
         This is a very limited emulation for relatively recent feature sets
     """
-    parsed_version = parse_version(vers['version'])
-    vers['capabilities'] = {}
-    for name in opts['optional']:
-        vers['capabilities'][name] = check(parsed_version, name)
-    for name in opts['required']:
+    parsed_version = parse_version(vers["version"])
+    vers["capabilities"] = {}
+    for name in opts["optional"]:
+        vers["capabilities"][name] = check(parsed_version, name)
+    failed = False  # noqa: F841 T25377293 Grandfathered in
+    for name in opts["required"]:
         have = check(parsed_version, name)
-        vers['capabilities'][name] = have
+        vers["capabilities"][name] = have
         if not have:
-            vers['error'] = 'client required capability `' + name + \
-                            '` is not supported by this server'
+            vers["error"] = (
+                "client required capability `"
+                + name
+                + "` is not supported by this server"
+            )
     return vers

hgext/fsmonitor/pywatchman/compat.py

@@ -26,20 +26,22 b''
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
-
-'''Compatibility module across Python 2 and 3.'''
+from __future__ import absolute_import, division, print_function
 
 import sys
 
+
+"""Compatibility module across Python 2 and 3."""
+
+
+PYTHON2 = sys.version_info < (3, 0)
 PYTHON3 = sys.version_info >= (3, 0)
 
 # This is adapted from https://bitbucket.org/gutworth/six, and used under the
 # MIT license. See LICENSE for a full copyright notice.
 if PYTHON3:
+
     def reraise(tp, value, tb=None):
         try:
             if value is None:
@@ -50,16 +52,20 b' if PYTHON3:'
         finally:
             value = None
             tb = None
+
+
 else:
-    exec('''
+    exec(
+        """
 def reraise(tp, value, tb=None):
     try:
         raise tp, value, tb
     finally:
         tb = None
-'''.strip())
+""".strip()
+    )
 
 if PYTHON3:
     UNICODE = str
 else:
-    UNICODE = unicode
+    UNICODE = unicode  # noqa: F821 We handled versioning above

hgext/fsmonitor/pywatchman/encoding.py

@@ -26,48 +26,50 b''
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
-
-'''Module to deal with filename encoding on the local system, as returned by
-Watchman.'''
+from __future__ import absolute_import, division, print_function
 
 import sys
 
-from . import (
-    compat,
-)
+from . import compat
+
+
+"""Module to deal with filename encoding on the local system, as returned by
+Watchman."""
+
 
 if compat.PYTHON3:
-    default_local_errors = 'surrogateescape'
+    default_local_errors = "surrogateescape"
 
     def get_local_encoding():
-        if sys.platform == 'win32':
+        if sys.platform == "win32":
             # Watchman always returns UTF-8 encoded strings on Windows.
-            return 'utf-8'
+            return "utf-8"
         # On the Python 3 versions we support, sys.getfilesystemencoding never
         # returns None.
         return sys.getfilesystemencoding()
+
+
 else:
     # Python 2 doesn't support surrogateescape, so use 'strict' by
     # default. Users can register a custom surrogateescape error handler and use
     # that if they so desire.
-    default_local_errors = 'strict'
+    default_local_errors = "strict"
 
     def get_local_encoding():
-        if sys.platform == 'win32':
+        if sys.platform == "win32":
             # Watchman always returns UTF-8 encoded strings on Windows.
-            return 'utf-8'
+            return "utf-8"
         fsencoding = sys.getfilesystemencoding()
         if fsencoding is None:
             # This is very unlikely to happen, but if it does, just use UTF-8
-            fsencoding = 'utf-8'
+            fsencoding = "utf-8"
         return fsencoding
 
+
 def encode_local(s):
     return s.encode(get_local_encoding(), default_local_errors)
 
+
 def decode_local(bs):
     return bs.decode(get_local_encoding(), default_local_errors)

hgext/fsmonitor/pywatchman/load.py

@@ -26,17 +26,17 b''
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
+
+import ctypes
+
 
 try:
     from . import bser
 except ImportError:
     from . import pybser as bser
 
-import ctypes
 
 EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00"
 
@@ -95,13 +95,15 b' def load(fp, mutable=True, value_encodin'
         ctypes.resize(buf, total_len)
 
     body = (ctypes.c_char * (total_len - len(header))).from_buffer(
-        buf, len(header))
+        buf, len(header)
+    )
     read_len = _read_bytes(fp, body)
     if read_len < len(body):
-        raise RuntimeError('bser data ended early')
+        raise RuntimeError("bser data ended early")
 
     return bser.loads(
         (ctypes.c_char * total_len).from_buffer(buf, 0),
         mutable,
         value_encoding,
-        value_errors)
+        value_errors,
+    )

hgext/fsmonitor/pywatchman/pybser.py

@@ -26,10 +26,8 b''
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
 
 import binascii
 import collections
@@ -37,30 +35,31 b' import ctypes'
 import struct
 import sys
 
-from . import (
-    compat,
-)
+from . import compat
+
 
-BSER_ARRAY = b'\x00'
-BSER_OBJECT = b'\x01'
-BSER_BYTESTRING = b'\x02'
-BSER_INT8 = b'\x03'
-BSER_INT16 = b'\x04'
-BSER_INT32 = b'\x05'
-BSER_INT64 = b'\x06'
-BSER_REAL = b'\x07'
-BSER_TRUE = b'\x08'
-BSER_FALSE = b'\x09'
-BSER_NULL = b'\x0a'
-BSER_TEMPLATE = b'\x0b'
-BSER_SKIP = b'\x0c'
-BSER_UTF8STRING = b'\x0d'
+BSER_ARRAY = b"\x00"
+BSER_OBJECT = b"\x01"
+BSER_BYTESTRING = b"\x02"
+BSER_INT8 = b"\x03"
+BSER_INT16 = b"\x04"
+BSER_INT32 = b"\x05"
+BSER_INT64 = b"\x06"
+BSER_REAL = b"\x07"
+BSER_TRUE = b"\x08"
+BSER_FALSE = b"\x09"
+BSER_NULL = b"\x0a"
+BSER_TEMPLATE = b"\x0b"
+BSER_SKIP = b"\x0c"
+BSER_UTF8STRING = b"\x0d"
 
 if compat.PYTHON3:
     STRING_TYPES = (str, bytes)
     unicode = str
+
     def tobytes(i):
-        return str(i).encode('ascii')
+        return str(i).encode("ascii")
+
     long = int
 else:
     STRING_TYPES = (unicode, str)
@@ -72,6 +71,7 b' else:'
 EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00"
 EMPTY_HEADER_V2 = b"\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00"
 
+
 def _int_size(x):
     """Return the smallest size int that can store the value"""
     if -0x80 <= x <= 0x7F:
@@ -83,29 +83,34 b' def _int_size(x):'
     elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF):
         return 8
     else:
-        raise RuntimeError('Cannot represent value: ' + str(x))
+        raise RuntimeError("Cannot represent value: " + str(x))
+
 
 def _buf_pos(buf, pos):
     ret = buf[pos]
-    # In Python 2, buf is a str array so buf[pos] is a string. In Python 3, buf
-    # is a bytes array and buf[pos] is an integer.
-    if compat.PYTHON3:
+    # Normalize the return type to bytes
+    if compat.PYTHON3 and not isinstance(ret, bytes):
         ret = bytes((ret,))
     return ret
 
+
 class _bser_buffer(object):
-
     def __init__(self, version):
         self.bser_version = version
         self.buf = ctypes.create_string_buffer(8192)
         if self.bser_version == 1:
-            struct.pack_into(tobytes(len(EMPTY_HEADER)) + b's', self.buf, 0,
-                             EMPTY_HEADER)
+            struct.pack_into(
+                tobytes(len(EMPTY_HEADER)) + b"s", self.buf, 0, EMPTY_HEADER
+            )
             self.wpos = len(EMPTY_HEADER)
         else:
             assert self.bser_version == 2
-            struct.pack_into(tobytes(len(EMPTY_HEADER_V2)) + b's', self.buf, 0,
-                             EMPTY_HEADER_V2)
+            struct.pack_into(
+                tobytes(len(EMPTY_HEADER_V2)) + b"s",
+                self.buf,
+                0,
+                EMPTY_HEADER_V2,
+            )
             self.wpos = len(EMPTY_HEADER_V2)
 
     def ensure_size(self, size):
@@ -117,42 +122,68 b' class _bser_buffer(object):'
         to_write = size + 1
         self.ensure_size(to_write)
         if size == 1:
-            struct.pack_into(b'=cb', self.buf, self.wpos, BSER_INT8, val)
+            struct.pack_into(b"=cb", self.buf, self.wpos, BSER_INT8, val)
         elif size == 2:
-            struct.pack_into(b'=ch', self.buf, self.wpos, BSER_INT16, val)
+            struct.pack_into(b"=ch", self.buf, self.wpos, BSER_INT16, val)
         elif size == 4:
-            struct.pack_into(b'=ci', self.buf, self.wpos, BSER_INT32, val)
+            struct.pack_into(b"=ci", self.buf, self.wpos, BSER_INT32, val)
         elif size == 8:
-            struct.pack_into(b'=cq', self.buf, self.wpos, BSER_INT64, val)
+            struct.pack_into(b"=cq", self.buf, self.wpos, BSER_INT64, val)
         else:
-            raise RuntimeError('Cannot represent this long value')
+            raise RuntimeError("Cannot represent this long value")
         self.wpos += to_write
 
-
     def append_string(self, s):
         if isinstance(s, unicode):
-            s = s.encode('utf-8')
+            s = s.encode("utf-8")
         s_len = len(s)
         size = _int_size(s_len)
         to_write = 2 + size + s_len
         self.ensure_size(to_write)
         if size == 1:
-            struct.pack_into(b'=ccb' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT8, s_len, s)
+            struct.pack_into(
+                b"=ccb" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT8,
+                s_len,
+                s,
+            )
         elif size == 2:
-            struct.pack_into(b'=cch' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT16, s_len, s)
+            struct.pack_into(
+                b"=cch" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT16,
+                s_len,
+                s,
+            )
         elif size == 4:
-            struct.pack_into(b'=cci' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT32, s_len, s)
+            struct.pack_into(
+                b"=cci" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT32,
+                s_len,
+                s,
+            )
         elif size == 8:
-            struct.pack_into(b'=ccq' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT64, s_len, s)
+            struct.pack_into(
+                b"=ccq" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT64,
+                s_len,
+                s,
+            )
         else:
-            raise RuntimeError('Cannot represent this string value')
+            raise RuntimeError("Cannot represent this string value")
         self.wpos += to_write
 
-
     def append_recursive(self, val):
         if isinstance(val, bool):
             needed = 1
@@ -161,12 +192,12 b' class _bser_buffer(object):'
                 to_encode = BSER_TRUE
             else:
                 to_encode = BSER_FALSE
-            struct.pack_into(b'=c', self.buf, self.wpos, to_encode)
+            struct.pack_into(b"=c", self.buf, self.wpos, to_encode)
             self.wpos += needed
         elif val is None:
             needed = 1
             self.ensure_size(needed)
-            struct.pack_into(b'=c', self.buf, self.wpos, BSER_NULL)
+            struct.pack_into(b"=c", self.buf, self.wpos, BSER_NULL)
             self.wpos += needed
         elif isinstance(val, (int, long)):
             self.append_long(val)
@@ -175,61 +206,106 b' class _bser_buffer(object):'
         elif isinstance(val, float):
             needed = 9
             self.ensure_size(needed)
-            struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val)
+            struct.pack_into(b"=cd", self.buf, self.wpos, BSER_REAL, val)
             self.wpos += needed
-        elif isinstance(val, collections.Mapping) and \
-            isinstance(val, collections.Sized):
+        elif isinstance(val, collections.Mapping) and isinstance(
+            val, collections.Sized
+        ):
             val_len = len(val)
             size = _int_size(val_len)
             needed = 2 + size
             self.ensure_size(needed)
             if size == 1:
-                struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT8, val_len)
+                struct.pack_into(
+                    b"=ccb",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT8,
+                    val_len,
+                )
             elif size == 2:
-                struct.pack_into(b'=cch', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT16, val_len)
+                struct.pack_into(
+                    b"=cch",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT16,
+                    val_len,
+                )
             elif size == 4:
-                struct.pack_into(b'=cci', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT32, val_len)
+                struct.pack_into(
+                    b"=cci",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT32,
+                    val_len,
+                )
             elif size == 8:
-                struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT64, val_len)
+                struct.pack_into(
+                    b"=ccq",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT64,
+                    val_len,
+                )
             else:
-                raise RuntimeError('Cannot represent this mapping value')
+                raise RuntimeError("Cannot represent this mapping value")
             self.wpos += needed
             if compat.PYTHON3:
                 iteritems = val.items()
             else:
-                iteritems = val.iteritems()
+                iteritems = val.iteritems()  # noqa: B301 Checked version above
             for k, v in iteritems:
                 self.append_string(k)
                 self.append_recursive(v)
-        elif isinstance(val, collections.Iterable) and \
-            isinstance(val, collections.Sized):
+        elif isinstance(val, collections.Iterable) and isinstance(
+            val, collections.Sized
+        ):
             val_len = len(val)
             size = _int_size(val_len)
             needed = 2 + size
             self.ensure_size(needed)
             if size == 1:
-                struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT8, val_len)
+                struct.pack_into(
+                    b"=ccb", self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len
+                )
             elif size == 2:
-                struct.pack_into(b'=cch', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT16, val_len)
+                struct.pack_into(
+                    b"=cch",
+                    self.buf,
+                    self.wpos,
+                    BSER_ARRAY,
+                    BSER_INT16,
+                    val_len,
+                )
             elif size == 4:
-                struct.pack_into(b'=cci', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT32, val_len)
+                struct.pack_into(
+                    b"=cci",
+                    self.buf,
+                    self.wpos,
+                    BSER_ARRAY,
+                    BSER_INT32,
+                    val_len,
+                )
             elif size == 8:
-                struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT64, val_len)
+                struct.pack_into(
+                    b"=ccq",
+                    self.buf,
+                    self.wpos,
+                    BSER_ARRAY,
+                    BSER_INT64,
+                    val_len,
+                )
             else:
-                raise RuntimeError('Cannot represent this sequence value')
+                raise RuntimeError("Cannot represent this sequence value")
             self.wpos += needed
             for v in val:
                 self.append_recursive(v)
         else:
-            raise RuntimeError('Cannot represent unknown value type')
+            raise RuntimeError("Cannot represent unknown value type")
 
 
 def dumps(obj, version=1, capabilities=0):
@@ -238,18 +314,19 b' def dumps(obj, version=1, capabilities=0'
     # Now fill in the overall length
     if version == 1:
         obj_len = bser_buf.wpos - len(EMPTY_HEADER)
-        struct.pack_into(b'=i', bser_buf.buf, 3, obj_len)
+        struct.pack_into(b"=i", bser_buf.buf, 3, obj_len)
     else:
         obj_len = bser_buf.wpos - len(EMPTY_HEADER_V2)
-        struct.pack_into(b'=i', bser_buf.buf, 2, capabilities)
-        struct.pack_into(b'=i', bser_buf.buf, 7, obj_len)
-    return bser_buf.buf.raw[:bser_buf.wpos]
+        struct.pack_into(b"=i", bser_buf.buf, 2, capabilities)
+        struct.pack_into(b"=i", bser_buf.buf, 7, obj_len)
+    return bser_buf.buf.raw[: bser_buf.wpos]
+
 
 # This is a quack-alike with the bserObjectType in bser.c
 # It provides by getattr accessors and getitem for both index
 # and name.
 class _BunserDict(object):
-    __slots__ = ('_keys', '_values')
+    __slots__ = ("_keys", "_values")
 
     def __init__(self, keys, values):
         self._keys = keys
@@ -261,18 +338,19 b' class _BunserDict(object):'
     def __getitem__(self, key):
         if isinstance(key, (int, long)):
             return self._values[key]
-        elif key.startswith('st_'):
+        elif key.startswith("st_"):
             # hack^Wfeature to allow mercurial to use "st_size" to
             # reference "size"
             key = key[3:]
         try:
             return self._values[self._keys.index(key)]
         except ValueError:
-            raise KeyError('_BunserDict has no key %s' % key)
+            raise KeyError("_BunserDict has no key %s" % key)
 
     def __len__(self):
         return len(self._keys)
 
+
 class Bunser(object):
     def __init__(self, mutable=True, value_encoding=None, value_errors=None):
         self.mutable = mutable
@@ -281,7 +359,7 b' class Bunser(object):'
         if value_encoding is None:
             self.value_errors = None
         elif value_errors is None:
-            self.value_errors = 'strict'
+            self.value_errors = "strict"
         else:
             self.value_errors = value_errors
 
@@ -290,33 +368,35 b' class Bunser(object):'
         try:
             int_type = _buf_pos(buf, pos)
         except IndexError:
-            raise ValueError('Invalid bser int encoding, pos out of range')
+            raise ValueError("Invalid bser int encoding, pos out of range")
         if int_type == BSER_INT8:
             needed = 2
-            fmt = b'=b'
+            fmt = b"=b"
         elif int_type == BSER_INT16:
             needed = 3
-            fmt = b'=h'
+            fmt = b"=h"
         elif int_type == BSER_INT32:
             needed = 5
-            fmt = b'=i'
+            fmt = b"=i"
         elif int_type == BSER_INT64:
             needed = 9
-            fmt = b'=q'
+            fmt = b"=q"
         else:
-            raise ValueError('Invalid bser int encoding 0x%s' %
-                             binascii.hexlify(int_type).decode('ascii'))
+            raise ValueError(
+                "Invalid bser int encoding 0x%s at position %s"
+                % (binascii.hexlify(int_type).decode("ascii"), pos)
+            )
         int_val = struct.unpack_from(fmt, buf, pos + 1)[0]
         return (int_val, pos + needed)
 
     def unser_utf8_string(self, buf, pos):
         str_len, pos = self.unser_int(buf, pos + 1)
-        str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0]
-        return (str_val.decode('utf-8'), pos + str_len)
+        str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0]
+        return (str_val.decode("utf-8"), pos + str_len)
 
     def unser_bytestring(self, buf, pos):
         str_len, pos = self.unser_int(buf, pos + 1)
-        str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0]
+        str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0]
         if self.value_encoding is not None:
             str_val = str_val.decode(self.value_encoding, self.value_errors)
             # str_len stays the same because that's the length in bytes
@@ -325,12 +405,12 b' class Bunser(object):'
     def unser_array(self, buf, pos):
         arr_len, pos = self.unser_int(buf, pos + 1)
         arr = []
-        for i in range(arr_len):
+        for _ in range(arr_len):
             arr_item, pos = self.loads_recursive(buf, pos)
             arr.append(arr_item)
 
         if not self.mutable:
-          arr = tuple(arr)
+            arr = tuple(arr)
 
         return arr, pos
 
@@ -342,7 +422,7 b' class Bunser(object):'
             keys = []
             vals = []
 
-        for i in range(obj_len):
+        for _ in range(obj_len):
             key, pos = self.unser_utf8_string(buf, pos)
             val, pos = self.loads_recursive(buf, pos)
             if self.mutable:
@@ -359,13 +439,13 b' class Bunser(object):'
     def unser_template(self, buf, pos):
         val_type = _buf_pos(buf, pos + 1)
         if val_type != BSER_ARRAY:
-            raise RuntimeError('Expect ARRAY to follow TEMPLATE')
+            raise RuntimeError("Expect ARRAY to follow TEMPLATE")
         # force UTF-8 on keys
-        keys_bunser = Bunser(mutable=self.mutable, value_encoding='utf-8')
+        keys_bunser = Bunser(mutable=self.mutable, value_encoding="utf-8")
         keys, pos = keys_bunser.unser_array(buf, pos + 1)
         nitems, pos = self.unser_int(buf, pos)
         arr = []
-        for i in range(nitems):
+        for _ in range(nitems):
             if self.mutable:
                 obj = {}
             else:
@@ -392,11 +472,15 b' class Bunser(object):'
 
     def loads_recursive(self, buf, pos):
         val_type = _buf_pos(buf, pos)
-        if (val_type == BSER_INT8 or val_type == BSER_INT16 or
-            val_type == BSER_INT32 or val_type == BSER_INT64):
+        if (
+            val_type == BSER_INT8
+            or val_type == BSER_INT16
+            or val_type == BSER_INT32
+            or val_type == BSER_INT64
+        ):
             return self.unser_int(buf, pos)
         elif val_type == BSER_REAL:
-            val = struct.unpack_from(b'=d', buf, pos + 1)[0]
+            val = struct.unpack_from(b"=d", buf, pos + 1)[0]
             return (val, pos + 9)
         elif val_type == BSER_TRUE:
             return (True, pos + 1)
@@ -415,23 +499,26 b' class Bunser(object):'
         elif val_type == BSER_TEMPLATE:
             return self.unser_template(buf, pos)
         else:
-            raise ValueError('unhandled bser opcode 0x%s' %
-                             binascii.hexlify(val_type).decode('ascii'))
+            raise ValueError(
+                "unhandled bser opcode 0x%s"
+                % binascii.hexlify(val_type).decode("ascii")
+            )
 
 
 def _pdu_info_helper(buf):
+    bser_version = -1
     if buf[0:2] == EMPTY_HEADER[0:2]:
         bser_version = 1
         bser_capabilities = 0
         expected_len, pos2 = Bunser.unser_int(buf, 2)
     elif buf[0:2] == EMPTY_HEADER_V2[0:2]:
         if len(buf) < 8:
-            raise ValueError('Invalid BSER header')
+            raise ValueError("Invalid BSER header")
         bser_version = 2
         bser_capabilities = struct.unpack_from("I", buf, 2)[0]
         expected_len, pos2 = Bunser.unser_int(buf, 6)
     else:
-        raise ValueError('Invalid BSER header')
+        raise ValueError("Invalid BSER header")
 
     return bser_version, bser_capabilities, expected_len, pos2
 
@@ -470,14 +557,20 b' def loads(buf, mutable=True, value_encod'
     pos = info[3]
 
     if len(buf) != expected_len + pos:
-        raise ValueError('bser data len != header len')
+        raise ValueError(
+            "bser data len %d != header len %d" % (expected_len + pos, len(buf))
+        )
 
-    bunser = Bunser(mutable=mutable, value_encoding=value_encoding,
-                    value_errors=value_errors)
+    bunser = Bunser(
+        mutable=mutable,
+        value_encoding=value_encoding,
+        value_errors=value_errors,
+    )
 
     return bunser.loads_recursive(buf, pos)[0]
 
 
 def load(fp, mutable=True, value_encoding=None, value_errors=None):
     from . import load
+
     return load.load(fp, mutable, value_encoding, value_errors)

hgext/fsmonitor/state.py

@@ -14,6 +14,7 b' import struct'
 
 from mercurial.i18n import _
 from mercurial import (
+    encoding,
     pathutil,
     util,
 )
@@ -81,7 +82,7 b' class state(object):'
                 self.invalidate()
                 return None, None, None
             diskhostname = state[0]
-            hostname = socket.gethostname()
+            hostname = encoding.strtolocal(socket.gethostname())
             if diskhostname != hostname:
                 # file got moved to a different host
                 self._ui.log(
@@ -127,7 +128,7 b' class state(object):'
 
         with file:
             file.write(struct.pack(_versionformat, _version))
-            file.write(socket.gethostname() + b'\0')
+            file.write(encoding.strtolocal(socket.gethostname()) + b'\0')
             file.write(clock + b'\0')
             file.write(ignorehash + b'\0')
             if notefiles:

hgext/fsmonitor/watchmanclient.py

@@ -9,7 +9,14 b' from __future__ import absolute_import'
 
 import getpass
 
-from mercurial import util
+from mercurial import (
+    encoding,
+    util,
+)
+from mercurial.utils import (
+    procutil,
+    stringutil,
+)
 
 from . import pywatchman
 
@@ -22,12 +29,14 b' class Unavailable(Exception):'
             self.warn = False
         self.invalidate = invalidate
 
-    def __str__(self):
+    def __bytes__(self):
         if self.warn:
             return b'warning: Watchman unavailable: %s' % self.msg
         else:
             return b'Watchman unavailable: %s' % self.msg
 
+    __str__ = encoding.strmethod(__bytes__)
+
 
 class WatchmanNoRoot(Unavailable):
     def __init__(self, root, msg):
@@ -92,15 +101,17 b' class client(object):'
                 self._watchmanclient = pywatchman.client(
                     timeout=self._timeout,
                     useImmutableBser=True,
-                    watchman_exe=watchman_exe,
+                    binpath=procutil.tonativestr(watchman_exe),
                 )
             return self._watchmanclient.query(*watchmanargs)
         except pywatchman.CommandError as ex:
             if b'unable to resolve root' in ex.msg:
-                raise WatchmanNoRoot(self._root, ex.msg)
+                raise WatchmanNoRoot(
+                    self._root, stringutil.forcebytestr(ex.msg)
+                )
             raise Unavailable(ex.msg)
         except pywatchman.WatchmanError as ex:
-            raise Unavailable(str(ex))
+            raise Unavailable(stringutil.forcebytestr(ex))
 
     def command(self, *args):
         try:

hgext/gpg.py

@@ -76,10 +76,9 b' class gpg(object):'
             fp = os.fdopen(fd, r'wb')
             fp.write(data)
             fp.close()
-            gpgcmd = b"%s --logger-fd 1 --status-fd 1 --verify \"%s\" \"%s\"" % (
-                self.path,
-                sigfile,
-                datafile,
+            gpgcmd = (
+                b"%s --logger-fd 1 --status-fd 1 --verify \"%s\" \"%s\""
+                % (self.path, sigfile, datafile,)
             )
             ret = procutil.filter(b"", gpgcmd)
         finally:

hgext/histedit.py

@@ -217,6 +217,7 b' from mercurial import ('
     copies,
     destutil,
     discovery,
+    encoding,
     error,
     exchange,
     extensions,
@@ -1117,7 +1118,7 b' class histeditrule(object):'
         self.pos = pos
         self.conflicts = []
 
-    def __str__(self):
+    def __bytes__(self):
         # Some actions ('fold' and 'roll') combine a patch with a previous one.
         # Add a marker showing which patch they apply to, and also omit the
         # description for 'roll' (since it will get discarded). Example display:
@@ -1135,10 +1136,16 b' class histeditrule(object):'
         desc = self.ctx.description().splitlines()[0].strip()
         if self.action == b'roll':
             desc = b''
-        return b"#{0:<2} {1:<6} {2}:{3}   {4}".format(
-            self.origpos, action, r, h, desc
+        return b"#%s %s %d:%s   %s" % (
+            (b'%d' % self.origpos).ljust(2),
+            action.ljust(6),
+            r,
+            h,
+            desc,
         )
 
+    __str__ = encoding.strmethod(__bytes__)
+
     def checkconflicts(self, other):
         if other.pos > self.pos and other.origpos <= self.origpos:
             if set(other.ctx.files()) & set(self.ctx.files()) != set():
@@ -1315,7 +1322,7 b' def makecommands(rules):'
     our list of rules"""
     commands = []
     for rules in rules:
-        commands.append(b"{0} {1}\n".format(rules.action, rules.ctx))
+        commands.append(b'%s %s\n' % (rules.action, rules.ctx))
     return commands
 
 
@@ -1324,7 +1331,7 b' def addln(win, y, x, line, color=None):'
     whitespace characters, so that the color appears on the whole line"""
     maxy, maxx = win.getmaxyx()
     length = maxx - 1 - x
-    line = (b"{0:<%d}" % length).format(str(line).strip())[:length]
+    line = bytes(line).ljust(length)[:length]
     if y < 0:
         y = maxy + y
     if x < 0:
@@ -1395,17 +1402,17 b' def _chisteditmain(repo, rules, stdscr):'
         maxy, maxx = win.getmaxyx()
         length = maxx - 3
 
-        line = b"changeset: {0}:{1:<12}".format(ctx.rev(), ctx)
+        line = b"changeset: %d:%s" % (ctx.rev(), ctx.hex()[:12])
         win.addstr(1, 1, line[:length])
 
-        line = b"user:      {0}".format(ctx.user())
+        line = b"user:      %s" % ctx.user()
         win.addstr(2, 1, line[:length])
 
         bms = repo.nodebookmarks(ctx.node())
-        line = b"bookmark:  {0}".format(b' '.join(bms))
+        line = b"bookmark:  %s" % b' '.join(bms)
         win.addstr(3, 1, line[:length])
 
-        line = b"summary:   {0}".format(ctx.description().splitlines()[0])
+        line = b"summary:   %s" % (ctx.description().splitlines()[0])
         win.addstr(4, 1, line[:length])
 
         line = b"files:     "
@@ -1425,8 +1432,8 b' def _chisteditmain(repo, rules, stdscr):'
 
         conflicts = rule.conflicts
         if len(conflicts) > 0:
-            conflictstr = b','.join(map(lambda r: str(r.ctx), conflicts))
-            conflictstr = b"changed files overlap with {0}".format(conflictstr)
+            conflictstr = b','.join(map(lambda r: r.ctx.hex()[:12], conflicts))
+            conflictstr = b"changed files overlap with %s" % conflictstr
         else:
             conflictstr = b'no overlap'
 
@@ -1464,7 +1471,9 b' pgup/K: move patch up, pgdn/J: move patc'
 
         conflicts = [r.ctx for r in rules if r.conflicts]
         if len(conflicts) > 0:
-            line = b"potential conflict in %s" % b','.join(map(str, conflicts))
+            line = b"potential conflict in %s" % b','.join(
+                map(pycompat.bytestr, conflicts)
+            )
             addln(rulesscr, -1, 0, line, curses.color_pair(COLOR_WARN))
 
         for y, rule in enumerate(rules[start:]):
@@ -1601,7 +1610,7 b' pgup/K: move patch up, pgdn/J: move patc'
                 renderhelp(helpwin, state)
                 curses.doupdate()
                 # done rendering
-                ch = stdscr.getkey()
+                ch = encoding.strtolocal(stdscr.getkey())
         except curses.error:
             pass
 
@@ -1675,11 +1684,10 b' def _chistedit(ui, repo, *freeargs, **op'
         if type(rc) is list:
             ui.status(_(b"performing changes\n"))
             rules = makecommands(rc)
-            filename = repo.vfs.join(b'chistedit')
-            with open(filename, b'w+') as fp:
+            with repo.vfs(b'chistedit', b'w+') as fp:
                 for r in rules:
                     fp.write(r)
-            opts[b'commands'] = filename
+                opts['commands'] = fp.name
             return _texthistedit(ui, repo, *freeargs, **opts)
     except KeyboardInterrupt:
         pass

hgext/infinitepush/__init__.py

@@ -959,7 +959,7 b' def logservicecall(logger, service, **kw'
             service,
             eventtype=b'failure',
             elapsedms=(time.time() - start) * 1000,
-            errormsg=str(e),
+            errormsg=stringutil.forcebytestr(e),
             **kwargs
         )
         raise
@@ -1223,7 +1223,7 b' def storebundle(op, params, bundlefile):'
             scratchbranchparttype,
             eventtype=b'failure',
             elapsedms=(time.time() - parthandlerstart) * 1000,
-            errormsg=str(e),
+            errormsg=stringutil.forcebytestr(e),
         )
         raise
     finally:

hgext/lfs/blobstore.py

@@ -363,7 +363,7 b' class _gitlfsremote(object):'
                 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
             )
         try:
-            response = json.loads(rawjson)
+            response = pycompat.json_loads(rawjson)
         except ValueError:
             raise LfsRemoteError(
                 _(b'LFS server returns invalid JSON: %s')

hgext/lfs/wireprotolfsserver.py

@@ -133,7 +133,7 b' def _processbatchrequest(repo, req, res)'
         return True
 
     # XXX: specify an encoding?
-    lfsreq = json.loads(req.bodyfh.read())
+    lfsreq = pycompat.json_loads(req.bodyfh.read())
 
     # If no transfer handlers are explicitly requested, 'basic' is assumed.
     if r'basic' not in lfsreq.get(r'transfers', [r'basic']):

hgext/notify.py

@@ -148,7 +148,7 b' web.baseurl'
 from __future__ import absolute_import
 
 import email.errors as emailerrors
-import email.parser as emailparser
+import email.utils as emailutils
 import fnmatch
 import hashlib
 import socket
@@ -382,9 +382,8 b' class notifier(object):'
             )
             return
 
-        p = emailparser.Parser()
         try:
-            msg = p.parsestr(encoding.strfromlocal(data))
+            msg = mail.parsebytes(data)
         except emailerrors.MessageParseError as inst:
             raise error.Abort(inst)
 
@@ -392,16 +391,16 b' class notifier(object):'
         sender = msg[r'From']
         subject = msg[r'Subject']
         if sender is not None:
-            sender = encoding.strtolocal(sender)
+            sender = mail.headdecode(sender)
         if subject is not None:
-            subject = encoding.strtolocal(subject)
+            subject = mail.headdecode(subject)
         del msg[r'From'], msg[r'Subject']
 
         if not msg.is_multipart():
             # create fresh mime message from scratch
             # (multipart templates must take care of this themselves)
             headers = msg.items()
-            payload = msg.get_payload()
+            payload = msg.get_payload(decode=pycompat.ispy3)
             # for notification prefer readability over data precision
             msg = mail.mimeencode(self.ui, payload, self.charsets, self.test)
             # reinstate custom headers
@@ -440,7 +439,7 b' class notifier(object):'
             msg[r'Message-Id'] = messageid(ctx, self.domain, self.messageidseed)
         msg[r'To'] = encoding.strfromlocal(b', '.join(sorted(subs)))
 
-        msgtext = encoding.strtolocal(msg.as_string())
+        msgtext = msg.as_bytes() if pycompat.ispy3 else msg.as_string()
         if self.test:
             self.ui.write(msgtext)
             if not msgtext.endswith(b'\n'):
@@ -452,7 +451,7 b' class notifier(object):'
             )
             mail.sendmail(
                 self.ui,
-                stringutil.email(msg[r'From']),
+                emailutils.parseaddr(msg[r'From'])[1],
                 subs,
                 msgtext,
                 mbox=self.mbox,

hgext/patchbomb.py

@@ -960,7 +960,10 b' def email(ui, repo, *revs, **opts):'
                     hdr = pycompat.strurl(hdr)
                     change = True
                 if isinstance(val, bytes):
-                    val = pycompat.strurl(val)
+                    # header value should be ASCII since it's encoded by
+                    # mail.headencode(), but -n/--test disables it and raw
+                    # value of platform encoding is stored.
+                    val = encoding.strfromlocal(val)
                     if not change:
                         # prevent duplicate headers
                         del m[hdr]

hgext/phabricator.py

@@ -152,8 +152,8 b' def vcrcommand(name, flags, spec, helpca'
             value = r1params[key][0]
             # we want to compare json payloads without worrying about ordering
             if value.startswith(b'{') and value.endswith(b'}'):
-                r1json = json.loads(value)
-                r2json = json.loads(r2params[key][0])
+                r1json = pycompat.json_loads(value)
+                r2json = pycompat.json_loads(r2params[key][0])
                 if r1json != r2json:
                     return False
             elif r2params[key][0] != value:
@@ -307,7 +307,7 b' def callconduit(ui, name, params):'
         if isinstance(x, pycompat.unicode)
         else x,
         # json.loads only accepts bytes from py3.6+
-        json.loads(encoding.unifromlocal(body)),
+        pycompat.json_loads(encoding.unifromlocal(body)),
     )
     if parsed.get(b'error_code'):
         msg = _(b'Conduit Error (%s): %s') % (
@@ -332,7 +332,7 b' def debugcallconduit(ui, repo, name):'
         lambda x: encoding.unitolocal(x)
         if isinstance(x, pycompat.unicode)
         else x,
-        json.loads(rawparams),
+        pycompat.json_loads(rawparams),
     )
     # json.dumps only accepts unicode strings
     result = pycompat.rapply(
@@ -441,7 +441,7 b' def getoldnodedrevmap(repo, nodelist):'
                 )
                 unfi.ui.warn(
                     _(
-                        b'D%s: local tag removed - does not match '
+                        b'D%d: local tag removed - does not match '
                         b'Differential history\n'
                     )
                     % drev
@@ -1168,7 +1168,7 b' def phabsend(ui, repo, *revs, **opts):'
                         writediffproperties(unfi[newnode], diffmap[old.node()])
                     except util.urlerr.urlerror:
                         ui.warnnoi18n(
-                            b'Failed to update metadata for D%s\n' % drevid
+                            b'Failed to update metadata for D%d\n' % drevid
                         )
                 # Remove local tags since it's no longer necessary
                 tagname = b'D%d' % drevid
@@ -1208,7 +1208,7 b' def _confirmbeforesend(repo, revs, oldma'
         desc = ctx.description().splitlines()[0]
         oldnode, olddiff, drevid = oldmap.get(ctx.node(), (None, None, None))
         if drevid:
-            drevdesc = ui.label(b'D%s' % drevid, b'phabricator.drev')
+            drevdesc = ui.label(b'D%d' % drevid, b'phabricator.drev')
         else:
             drevdesc = ui.label(_(b'NEW'), b'phabricator.drev')
 
@@ -1613,7 +1613,7 b' def phabupdate(ui, repo, spec, **opts):'
 
     actions = []
     for f in flags:
-        actions.append({b'type': f, b'value': b'true'})
+        actions.append({b'type': f, b'value': True})
 
     drevs = querydrev(repo, spec)
     for i, drev in enumerate(drevs):

mercurial/commands.py

@@ -2267,7 +2267,13 b' def config(ui, repo, *values, **opts):'
             fm.write(b'value', b'%s\n', value)
         else:
             fm.write(b'name value', b'%s=%s\n', entryname, value)
-        fm.data(defaultvalue=defaultvalue)
+        if formatter.isprintable(defaultvalue):
+            fm.data(defaultvalue=defaultvalue)
+        elif isinstance(defaultvalue, list) and all(
+            formatter.isprintable(e) for e in defaultvalue
+        ):
+            fm.data(defaultvalue=fm.formatlist(defaultvalue, name=b'value'))
+        # TODO: no idea how to process unsupported defaultvalue types
         matched = True
     fm.end()
     if matched:

mercurial/config.py

@@ -212,11 +212,9 b' class config(object):'
     def read(self, path, fp=None, sections=None, remap=None):
         if not fp:
             fp = util.posixfile(path, b'rb')
-        assert (
-            getattr(fp, 'mode', r'rb') == r'rb'
-        ), b'config files must be opened in binary mode, got fp=%r mode=%r' % (
-            fp,
-            fp.mode,
+        assert getattr(fp, 'mode', r'rb') == r'rb', (
+            b'config files must be opened in binary mode, got fp=%r mode=%r'
+            % (fp, fp.mode,)
         )
         self.parse(
             path, fp.read(), sections=sections, remap=remap, include=self.read

mercurial/context.py

@@ -1187,10 +1187,9 b' class filectx(basefilectx):'
 
         assert (
             changeid is not None or fileid is not None or changectx is not None
-        ), b"bad args: changeid=%r, fileid=%r, changectx=%r" % (
-            changeid,
-            fileid,
-            changectx,
+        ), (
+            b"bad args: changeid=%r, fileid=%r, changectx=%r"
+            % (changeid, fileid, changectx,)
         )
 
         if filelog is not None:

mercurial/crecord.py

@@ -1816,7 +1816,7 b' are you sure you want to review/edit and'
             try:
                 patch = self.ui.edit(patch.getvalue(), b"", action=b"diff")
             except error.Abort as exc:
-                self.errorstr = str(exc)
+                self.errorstr = stringutil.forcebytestr(exc)
                 return None
             finally:
                 self.stdscr.clear()

mercurial/error.py

@@ -111,6 +111,14 b' class Abort(Hint, Exception):'
 
     __bytes__ = _tobytes
 
+    if pycompat.ispy3:
+
+        def __str__(self):
+            # the output would be unreadable if the message was translated,
+            # but do not replace it with encoding.strfromlocal(), which
+            # may raise another exception.
+            return pycompat.sysstr(self.__bytes__())
+
 
 class HookLoadError(Abort):
     """raised when loading a hook fails, aborting an operation

mercurial/formatter.py

@@ -136,6 +136,16 b' from .utils import ('
 pickle = util.pickle
 
 
+def isprintable(obj):
+    """Check if the given object can be directly passed in to formatter's
+    write() and data() functions
+
+    Returns False if the object is unsupported or must be pre-processed by
+    formatdate(), formatdict(), or formatlist().
+    """
+    return isinstance(obj, (type(None), bool, int, pycompat.long, float, bytes))
+
+
 class _nullconverter(object):
     '''convert non-primitive data types to be processed by formatter'''
 
@@ -505,6 +515,10 b' class templateformatter(baseformatter):'
         if part not in self._parts:
             return
         ref = self._parts[part]
+        # None can't be put in the mapping dict since it means <unset>
+        for k, v in item.items():
+            if v is None:
+                item[k] = templateutil.wrappedvalue(v)
         self._out.write(self._t.render(ref, item))
 
     @util.propertycache

mercurial/localrepo.py

@@ -1417,7 +1417,7 b' class localrepository(object):'
 
     def _refreshchangelog(self):
         """make sure the in memory changelog match the on-disk one"""
-        if b'changelog' in vars(self) and self.currenttransaction() is None:
+        if 'changelog' in vars(self) and self.currenttransaction() is None:
             del self.changelog
 
     @property

mercurial/mail.py

@@ -342,6 +342,7 b' def _encode(ui, s, charsets):'
         s.decode('ascii')
     except UnicodeDecodeError:
         for ics in (encoding.encoding, encoding.fallbackencoding):
+            ics = pycompat.sysstr(ics)
             try:
                 u = s.decode(ics)
             except UnicodeDecodeError:
@@ -362,13 +363,13 b' def headencode(ui, s, charsets=None, dis'
     if not display:
         # split into words?
         s, cs = _encode(ui, s, charsets)
-        return str(email.header.Header(s, cs))
+        return encoding.strtolocal(email.header.Header(s, cs).encode())
     return s
 
 
 def _addressencode(ui, name, addr, charsets=None):
     assert isinstance(addr, bytes)
-    name = headencode(ui, name, charsets)
+    name = encoding.strfromlocal(headencode(ui, name, charsets))
     try:
         acc, dom = addr.split(b'@')
         acc.decode('ascii')
@@ -440,6 +441,10 b' if pycompat.ispy3:'
         finally:
             fp.detach()
 
+    def parsebytes(data):
+        ep = email.parser.BytesParser()
+        return ep.parsebytes(data)
+
 
 else:
 
@@ -449,6 +454,10 b' else:'
         ep = email.parser.Parser()
         return ep.parse(fp)
 
+    def parsebytes(data):
+        ep = email.parser.Parser()
+        return ep.parsestr(data)
+
 
 def headdecode(s):
     '''Decodes RFC-2047 header'''
@@ -458,7 +467,7 b' def headdecode(s):'
             try:
                 uparts.append(part.decode(charset))
                 continue
-            except UnicodeDecodeError:
+            except (UnicodeDecodeError, LookupError):
                 pass
         # On Python 3, decode_header() may return either bytes or unicode
         # depending on whether the header has =?<charset>? or not

mercurial/manifest.py

@@ -867,12 +867,15 b' class treemanifest(object):'
         return not self._dirs or all(m._isempty() for m in self._dirs.values())
 
     def __repr__(self):
-        return b'<treemanifest dir=%s, node=%s, loaded=%s, dirty=%s at 0x%x>' % (
-            self._dir,
-            hex(self._node),
-            bool(self._loadfunc is _noop),
-            self._dirty,
-            id(self),
+        return (
+            b'<treemanifest dir=%s, node=%s, loaded=%s, dirty=%s at 0x%x>'
+            % (
+                self._dir,
+                hex(self._node),
+                bool(self._loadfunc is _noop),
+                self._dirty,
+                id(self),
+            )
         )
 
     def dir(self):

mercurial/pycompat.py

@@ -12,6 +12,7 b' from __future__ import absolute_import'
 
 import getopt
 import inspect
+import json
 import os
 import shlex
 import sys
@@ -88,6 +89,7 b' def rapply(f, xs):'
 
 if ispy3:
     import builtins
+    import codecs
     import functools
     import io
     import struct
@@ -347,6 +349,48 b' if ispy3:'
     iteritems = lambda x: x.items()
     itervalues = lambda x: x.values()
 
+    # Python 3.5's json.load and json.loads require str. We polyfill its
+    # code for detecting encoding from bytes.
+    if sys.version_info[0:2] < (3, 6):
+
+        def _detect_encoding(b):
+            bstartswith = b.startswith
+            if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
+                return 'utf-32'
+            if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
+                return 'utf-16'
+            if bstartswith(codecs.BOM_UTF8):
+                return 'utf-8-sig'
+
+            if len(b) >= 4:
+                if not b[0]:
+                    # 00 00 -- -- - utf-32-be
+                    # 00 XX -- -- - utf-16-be
+                    return 'utf-16-be' if b[1] else 'utf-32-be'
+                if not b[1]:
+                    # XX 00 00 00 - utf-32-le
+                    # XX 00 00 XX - utf-16-le
+                    # XX 00 XX -- - utf-16-le
+                    return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
+            elif len(b) == 2:
+                if not b[0]:
+                    # 00 XX - utf-16-be
+                    return 'utf-16-be'
+                if not b[1]:
+                    # XX 00 - utf-16-le
+                    return 'utf-16-le'
+            # default
+            return 'utf-8'
+
+        def json_loads(s, *args, **kwargs):
+            if isinstance(s, (bytes, bytearray)):
+                s = s.decode(_detect_encoding(s), 'surrogatepass')
+
+            return json.loads(s, *args, **kwargs)
+
+    else:
+        json_loads = json.loads
+
 else:
     import cStringIO
 
@@ -424,6 +468,7 b' else:'
     getargspec = inspect.getargspec
     iteritems = lambda x: x.iteritems()
     itervalues = lambda x: x.itervalues()
+    json_loads = json.loads
 
 isjython = sysplatform.startswith(b'java')
 

mercurial/scmutil.py

@@ -1854,7 +1854,7 b' class simplekeyvaluefile(object):'
                 raise error.CorruptedState(e % self.firstlinekey)
             d.update(updatedict)
         except ValueError as e:
-            raise error.CorruptedState(str(e))
+            raise error.CorruptedState(stringutil.forcebytestr(e))
         return d
 
     def write(self, data, firstline=None):

mercurial/templatefilters.py

@@ -31,9 +31,6 b' from .utils import ('
 urlerr = util.urlerr
 urlreq = util.urlreq
 
-if pycompat.ispy3:
-    long = int
-
 # filters are callables like:
 #   fn(obj)
 # with:
@@ -329,7 +326,7 b' def json(obj, paranoid=True):'
         return b'false'
     elif obj is True:
         return b'true'
-    elif isinstance(obj, (int, long, float)):
+    elif isinstance(obj, (int, pycompat.long, float)):
         return pycompat.bytestr(obj)
     elif isinstance(obj, bytes):
         return b'"%s"' % encoding.jsonescape(obj, paranoid=paranoid)

mercurial/ui.py

@@ -1564,6 +1564,7 b' class ui(object):'
         # - http://bugs.python.org/issue12833
         with self.timeblockedsection(b'stdio'):
             if usereadline:
+                self.flush()
                 prompt = encoding.strfromlocal(prompt)
                 line = encoding.strtolocal(pycompat.rawinput(prompt))
                 # When stdin is in binary mode on Windows, it can cause

setup.py

@@ -6,32 +6,27 b''
 
 import os
 
-supportedpy = '~= 2.7'
-if os.environ.get('HGALLOWPYTHON3', ''):
-    # Mercurial will never work on Python 3 before 3.5 due to a lack
-    # of % formatting on bytestrings, and can't work on 3.6.0 or 3.6.1
-    # due to a bug in % formatting in bytestrings.
-    # We cannot support Python 3.5.0, 3.5.1, 3.5.2 because of bug in
-    # codecs.escape_encode() where it raises SystemError on empty bytestring
-    # bug link: https://bugs.python.org/issue25270
-    #
-    # TODO: when we actually work on Python 3, use this string as the
-    # actual supportedpy string.
-    supportedpy = ','.join(
-        [
-            '>=2.7',
-            '!=3.0.*',
-            '!=3.1.*',
-            '!=3.2.*',
-            '!=3.3.*',
-            '!=3.4.*',
-            '!=3.5.0',
-            '!=3.5.1',
-            '!=3.5.2',
-            '!=3.6.0',
-            '!=3.6.1',
-        ]
-    )
+# Mercurial will never work on Python 3 before 3.5 due to a lack
+# of % formatting on bytestrings, and can't work on 3.6.0 or 3.6.1
+# due to a bug in % formatting in bytestrings.
+# We cannot support Python 3.5.0, 3.5.1, 3.5.2 because of bug in
+# codecs.escape_encode() where it raises SystemError on empty bytestring
+# bug link: https://bugs.python.org/issue25270
+supportedpy = ','.join(
+    [
+        '>=2.7',
+        '!=3.0.*',
+        '!=3.1.*',
+        '!=3.2.*',
+        '!=3.3.*',
+        '!=3.4.*',
+        '!=3.5.0',
+        '!=3.5.1',
+        '!=3.5.2',
+        '!=3.6.0',
+        '!=3.6.1',
+    ]
+)
 
 import sys, platform
 import sysconfig
@@ -89,39 +84,6 b' Python {py} detected.'
     printf(error, file=sys.stderr)
     sys.exit(1)
 
-# We don't yet officially support Python 3. But we want to allow developers to
-# hack on. Detect and disallow running on Python 3 by default. But provide a
-# backdoor to enable working on Python 3.
-if sys.version_info[0] != 2:
-    badpython = True
-
-    # Allow Python 3 from source checkouts.
-    if os.path.isdir('.hg') or 'HGPYTHON3' in os.environ:
-        badpython = False
-
-    if badpython:
-        error = """
-Python {py} detected.
-
-Mercurial currently has beta support for Python 3 and use of Python 2.7 is
-recommended for the best experience.
-
-Please re-run with Python 2.7 for a faster, less buggy experience.
-
-If you would like to beta test Mercurial with Python 3, this error can
-be suppressed by defining the HGPYTHON3 environment variable when invoking
-this command. No special environment variables or configuration changes are
-necessary to run `hg` with Python 3.
-
-See https://www.mercurial-scm.org/wiki/Python3 for more on Mercurial's
-Python 3 support.
-""".format(
-            py='.'.join('%d' % x for x in sys.version_info[0:2])
-        )
-
-        printf(error, file=sys.stderr)
-        sys.exit(1)
-
 if sys.version_info[0] >= 3:
     DYLIB_SUFFIX = sysconfig.get_config_vars()['EXT_SUFFIX']
 else:

tests/get-with-headers.py

@@ -98,7 +98,7 b' def request(host, path, show):'
         if formatjson:
             # json.dumps() will print trailing newlines. Eliminate them
             # to make tests easier to write.
-            data = json.loads(data)
+            data = pycompat.json_loads(data)
             lines = json.dumps(data, sort_keys=True, indent=2).splitlines()
             for line in lines:
                 bodyfh.write(pycompat.sysbytes(line.rstrip()))

tests/hghave.py

@@ -1,5 +1,6 b''
 from __future__ import absolute_import, print_function
 
+import distutils.version
 import os
 import re
 import socket
@@ -828,6 +829,17 b' def has_dev_full():'
     return os.path.exists('/dev/full')
 
 
+@check("ensurepip", "ensurepip module")
+def has_ensurepip():
+    try:
+        import ensurepip
+
+        ensurepip.bootstrap
+        return True
+    except ImportError:
+        return False
+
+
 @check("virtualenv", "Python virtualenv support")
 def has_virtualenv():
     try:
@@ -980,12 +992,10 b' def has_emacs():'
     return matchoutput('emacs --version', b'GNU Emacs 2(4.4|4.5|5|6|7|8|9)')
 
 
-# @check('black', 'the black formatter for python')
-@check('grey', 'grey, the fork of the black formatter for python')
+@check('black', 'the black formatter for python')
 def has_black():
-    # use that to actual black as soon as possible
-    # blackcmd = 'black --version'
-    blackcmd = 'python3 $RUNTESTDIR/../contrib/grey.py --version'
-    # version_regex = b'black, version \d'
-    version_regex = b'grey.py, version \d'
-    return matchoutput(blackcmd, version_regex)
+    blackcmd = 'black --version'
+    version_regex = b'black, version ([0-9a-b.]+)'
+    version = matchoutput(blackcmd, version_regex)
+    sv = distutils.version.StrictVersion
+    return version and sv(_strpath(version.group(1))) >= sv('19.10b0')

tests/run-tests.py

@@ -1504,7 +1504,7 b' class PythonTest(Test):'
         py3switch = self._py3warnings and b' -3' or b''
         # Quote the python(3) executable for Windows
         cmd = b'"%s"%s "%s"' % (PYTHON, py3switch, self.path)
-        vlog("# Running", cmd)
+        vlog("# Running", cmd.decode("utf-8"))
         normalizenewlines = os.name == 'nt'
         result = self._runcommand(cmd, env, normalizenewlines=normalizenewlines)
         if self._aborted:
@@ -1589,7 +1589,7 b' class TTest(Test):'
                 f.write(l)
 
         cmd = b'%s "%s"' % (self._shell, fname)
-        vlog("# Running", cmd)
+        vlog("# Running", cmd.decode("utf-8"))
 
         exitcode, output = self._runcommand(cmd, env)
 
@@ -1770,7 +1770,9 b' class TTest(Test):'
             if l.startswith(b'#require'):
                 lsplit = l.split()
                 if len(lsplit) < 2 or lsplit[0] != b'#require':
-                    after.setdefault(pos, []).append('  !!! invalid #require\n')
+                    after.setdefault(pos, []).append(
+                        b'  !!! invalid #require\n'
+                    )
                 if not skipping:
                     haveresult, message = self._hghave(lsplit[1:])
                     if not haveresult:
@@ -1780,19 +1782,19 b' class TTest(Test):'
             elif l.startswith(b'#if'):
                 lsplit = l.split()
                 if len(lsplit) < 2 or lsplit[0] != b'#if':
-                    after.setdefault(pos, []).append('  !!! invalid #if\n')
+                    after.setdefault(pos, []).append(b'  !!! invalid #if\n')
                 if skipping is not None:
-                    after.setdefault(pos, []).append('  !!! nested #if\n')
+                    after.setdefault(pos, []).append(b'  !!! nested #if\n')
                 skipping = not self._iftest(lsplit[1:])
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'#else'):
                 if skipping is None:
-                    after.setdefault(pos, []).append('  !!! missing #if\n')
+                    after.setdefault(pos, []).append(b'  !!! missing #if\n')
                 skipping = not skipping
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'#endif'):
                 if skipping is None:
-                    after.setdefault(pos, []).append('  !!! missing #if\n')
+                    after.setdefault(pos, []).append(b'  !!! missing #if\n')
                 skipping = None
                 after.setdefault(pos, []).append(l)
             elif skipping:
@@ -1841,7 +1843,7 b' class TTest(Test):'
         if inpython:
             script.append(b'EOF\n')
         if skipping is not None:
-            after.setdefault(pos, []).append('  !!! missing #endif\n')
+            after.setdefault(pos, []).append(b'  !!! missing #endif\n')
         addsalt(n + 1, False)
         # Need to end any current per-command trace
         if activetrace:
@@ -3111,12 +3113,14 b' class TestRunner(object):'
                 'extensions.logexceptions=%s' % logexceptions.decode('utf-8')
             )
 
-        vlog("# Using TESTDIR", self._testdir)
-        vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR'])
-        vlog("# Using HGTMP", self._hgtmp)
+        vlog("# Using TESTDIR", _strpath(self._testdir))
+        vlog("# Using RUNTESTDIR", _strpath(osenvironb[b'RUNTESTDIR']))
+        vlog("# Using HGTMP", _strpath(self._hgtmp))
         vlog("# Using PATH", os.environ["PATH"])
-        vlog("# Using", IMPL_PATH, osenvironb[IMPL_PATH])
-        vlog("# Writing to directory", self._outputdir)
+        vlog(
+            "# Using", _strpath(IMPL_PATH), _strpath(osenvironb[IMPL_PATH]),
+        )
+        vlog("# Writing to directory", _strpath(self._outputdir))
 
         try:
             return self._runtests(testdescs) or 0
@@ -3357,7 +3361,7 b' class TestRunner(object):'
         if self.options.keep_tmpdir:
             return
 
-        vlog("# Cleaning up HGTMP", self._hgtmp)
+        vlog("# Cleaning up HGTMP", _strpath(self._hgtmp))
         shutil.rmtree(self._hgtmp, True)
         for f in self._createdfiles:
             try:
@@ -3468,7 +3472,7 b' class TestRunner(object):'
         makedirs(self._pythondir)
         makedirs(self._bindir)
 
-        vlog("# Running", cmd)
+        vlog("# Running", cmd.decode("utf-8"))
         if subprocess.call(_strpath(cmd), shell=True) == 0:
             if not self.options.verbose:
                 try:
@@ -3643,13 +3647,11 b' class TestRunner(object):'
             if os.name == 'nt' and not p.endswith(b'.exe'):
                 p += b'.exe'
             found = self._findprogram(p)
+            p = p.decode("utf-8")
             if found:
-                vlog("# Found prerequisite", p, "at", found)
+                vlog("# Found prerequisite", p, "at", _strpath(found))
             else:
-                print(
-                    "WARNING: Did not find prerequisite tool: %s "
-                    % p.decode("utf-8")
-                )
+                print("WARNING: Did not find prerequisite tool: %s " % p)
 
 
 def aggregateexceptions(path):

tests/test-byteify-strings.t

@@ -1,4 +1,4 b''
-#require py3
+#require py37
 
   $ byteify_strings () {
   >   $PYTHON "$TESTDIR/../contrib/byteify-strings.py" "$@"

tests/test-check-code.t

@@ -21,7 +21,6 b' New errors are not allowed. Warnings are'
   Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
-  Skipping contrib/grey.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob)

tests/test-check-format.t

@@ -1,7 +1,5 b''
-#require grey
-
-(this should use the actual black as soon as possible)
+#require black
 
   $ cd $RUNTESTDIR/..
-  $ python3 contrib/grey.py --config=black.toml --check --diff `hg files 'set:**.py - hgext/fsmonitor/pywatchman/** - mercurial/thirdparty/** - "contrib/python-zstandard/**" - contrib/grey.py'`
+  $ black --config=black.toml --check --diff `hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"'`
 

tests/test-check-module-imports.t

@@ -20,7 +20,6 b' outputs, which should be fixed later.'
   > -X setup.py \
   > -X contrib/automation/ \
   > -X contrib/debugshell.py \
-  > -X contrib/grey.py \
   > -X contrib/hgweb.fcgi \
   > -X contrib/packaging/hg-docker \
   > -X contrib/packaging/hgpackaging/ \

tests/test-check-py3-compat.t

@@ -6,7 +6,6 b''
 #if no-py3
   $ testrepohg files 'set:(**.py)' \
   > -X contrib/automation/ \
-  > -X contrib/grey.py \
   > -X contrib/packaging/hgpackaging/ \
   > -X contrib/packaging/inno/ \
   > -X contrib/packaging/wix/ \

tests/test-config.t

@@ -87,6 +87,172 b' Test case sensitive configuration'
    }
   ]
 
+Test config default of various types:
+
+ {"defaultvalue": ""} for -T'json(defaultvalue)' looks weird, but that's
+ how the templater works. Unknown keywords are evaluated to "".
+
+ dynamicdefault
+
+  $ hg config --config alias.foo= alias -Tjson
+  [
+   {
+    "name": "alias.foo",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config alias.foo= alias -T'json(defaultvalue)'
+  [
+   {"defaultvalue": ""}
+  ]
+  $ hg config --config alias.foo= alias -T'{defaultvalue}\n'
+  
+
+ null
+
+  $ hg config --config auth.cookiefile= auth -Tjson
+  [
+   {
+    "defaultvalue": null,
+    "name": "auth.cookiefile",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config auth.cookiefile= auth -T'json(defaultvalue)'
+  [
+   {"defaultvalue": null}
+  ]
+  $ hg config --config auth.cookiefile= auth -T'{defaultvalue}\n'
+  
+
+ false
+
+  $ hg config --config commands.commit.post-status= commands -Tjson
+  [
+   {
+    "defaultvalue": false,
+    "name": "commands.commit.post-status",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config commands.commit.post-status= commands -T'json(defaultvalue)'
+  [
+   {"defaultvalue": false}
+  ]
+  $ hg config --config commands.commit.post-status= commands -T'{defaultvalue}\n'
+  False
+
+ true
+
+  $ hg config --config format.dotencode= format -Tjson
+  [
+   {
+    "defaultvalue": true,
+    "name": "format.dotencode",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config format.dotencode= format -T'json(defaultvalue)'
+  [
+   {"defaultvalue": true}
+  ]
+  $ hg config --config format.dotencode= format -T'{defaultvalue}\n'
+  True
+
+ bytes
+
+  $ hg config --config commands.resolve.mark-check= commands -Tjson
+  [
+   {
+    "defaultvalue": "none",
+    "name": "commands.resolve.mark-check",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config commands.resolve.mark-check= commands -T'json(defaultvalue)'
+  [
+   {"defaultvalue": "none"}
+  ]
+  $ hg config --config commands.resolve.mark-check= commands -T'{defaultvalue}\n'
+  none
+
+ empty list
+
+  $ hg config --config commands.show.aliasprefix= commands -Tjson
+  [
+   {
+    "defaultvalue": [],
+    "name": "commands.show.aliasprefix",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config commands.show.aliasprefix= commands -T'json(defaultvalue)'
+  [
+   {"defaultvalue": []}
+  ]
+  $ hg config --config commands.show.aliasprefix= commands -T'{defaultvalue}\n'
+  
+
+ nonempty list
+
+  $ hg config --config progress.format= progress -Tjson
+  [
+   {
+    "defaultvalue": ["topic", "bar", "number", "estimate"],
+    "name": "progress.format",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config progress.format= progress -T'json(defaultvalue)'
+  [
+   {"defaultvalue": ["topic", "bar", "number", "estimate"]}
+  ]
+  $ hg config --config progress.format= progress -T'{defaultvalue}\n'
+  topic bar number estimate
+
+ int
+
+  $ hg config --config profiling.freq= profiling -Tjson
+  [
+   {
+    "defaultvalue": 1000,
+    "name": "profiling.freq",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config profiling.freq= profiling -T'json(defaultvalue)'
+  [
+   {"defaultvalue": 1000}
+  ]
+  $ hg config --config profiling.freq= profiling -T'{defaultvalue}\n'
+  1000
+
+ float
+
+  $ hg config --config profiling.showmax= profiling -Tjson
+  [
+   {
+    "defaultvalue": 0.999,
+    "name": "profiling.showmax",
+    "source": "--config",
+    "value": ""
+   }
+  ]
+  $ hg config --config profiling.showmax= profiling -T'json(defaultvalue)'
+  [
+   {"defaultvalue": 0.999}
+  ]
+  $ hg config --config profiling.showmax= profiling -T'{defaultvalue}\n'
+  0.999
+
 Test empty config source:
 
   $ cat <<EOF > emptysource.py

tests/test-debian-packages.t

@@ -13,18 +13,16 b" Ensure debuild doesn't run the testsuite"
   $ make deb > $OUTPUTDIR/build.log 2>&1
   $ cd $OUTPUTDIR
   $ ls *.deb | grep -v 'dbg'
-  mercurial-common_*.deb (glob)
   mercurial_*.deb (glob)
-main deb should have .so but no .py
+should have .so and .py
   $ dpkg --contents mercurial_*.deb | egrep '(localrepo|parsers)'
-  * ./usr/lib/python2.7/dist-packages/mercurial/cext/parsers*.so (glob)
-mercurial-common should have py but no .so or pyc
-  $ dpkg --contents mercurial-common_*.deb | egrep '(localrepo|parsers.*so)'
-  * ./usr/lib/python2.7/dist-packages/mercurial/localrepo.py (glob)
-zsh completions should be in the common package
-  $ dpkg --contents mercurial-common_*.deb | egrep 'zsh.*[^/]$'
+  * ./usr/lib/python3/dist-packages/mercurial/cext/parsers*.so (glob)
+  * ./usr/lib/python3/dist-packages/mercurial/localrepo.py (glob)
+  * ./usr/lib/python3/dist-packages/mercurial/pure/parsers.py (glob)
+should have zsh completions
+  $ dpkg --contents mercurial_*.deb | egrep 'zsh.*[^/]$'
   * ./usr/share/zsh/vendor-completions/_hg (glob)
-chg should be installed alongside hg, in the 'mercurial' package
+should have chg
   $ dpkg --contents mercurial_*.deb | egrep 'chg$'
   * ./usr/bin/chg (glob)
 chg should come with a man page

tests/test-flagprocessor.t

@@ -209,12 +209,12 b' Ensure the data got to the server OK'
       insertflagprocessor(flag, processor, flagprocessors)
     File "*/mercurial/revlogutils/flagutil.py", line *, in insertflagprocessor (glob)
       raise error.Abort(msg)
-  mercurial.error.Abort: b"cannot register multiple processors on flag '0x8'." (py3 !)
+  mercurial.error.Abort: cannot register multiple processors on flag '0x8'. (py3 !)
   Abort: cannot register multiple processors on flag '0x8'. (no-py3 !)
   *** failed to set up extension duplicate: cannot register multiple processors on flag '0x8'.
   $ hg st 2>&1 | egrep 'cannot register multiple processors|flagprocessorext'
     File "*/tests/flagprocessorext.py", line *, in extsetup (glob)
-  mercurial.error.Abort: b"cannot register multiple processors on flag '0x8'." (py3 !)
+  mercurial.error.Abort: cannot register multiple processors on flag '0x8'. (py3 !)
   Abort: cannot register multiple processors on flag '0x8'. (no-py3 !)
   *** failed to set up extension duplicate: cannot register multiple processors on flag '0x8'.
     File "*/tests/flagprocessorext.py", line *, in b64decode (glob)

tests/test-highlight.t

@@ -966,7 +966,9 b' errors encountered'
   $ cd ..
   $ hg init eucjp
   $ cd eucjp
-  $ "$PYTHON" -c 'print("\265\376")' >> eucjp.txt  # Japanese kanji "Kyo"
+  >>> with open('eucjp.txt', 'wb') as fh:
+  ...     # Japanese kanji "Kyo"
+  ...     fh.write(u'\265\376'.encode('utf-8')) and None
   $ hg ci -Ama
   adding eucjp.txt
   $ hgserveget () {

tests/test-hook.t

@@ -988,7 +988,7 b' test python hooks'
   ModuleNotFoundError: No module named 'hgext_syntaxerror' (py36 !)
   Traceback (most recent call last): (py3 !)
   HookLoadError: preoutgoing.syntaxerror hook is invalid: import of "syntaxerror" failed (no-py3 !)
-  mercurial.error.HookLoadError: b'preoutgoing.syntaxerror hook is invalid: import of "syntaxerror" failed' (py3 !)
+  mercurial.error.HookLoadError: preoutgoing.syntaxerror hook is invalid: import of "syntaxerror" failed (py3 !)
   abort: preoutgoing.syntaxerror hook is invalid: import of "syntaxerror" failed
 
   $ echo '[hooks]' > ../a/.hg/hgrc
@@ -1161,7 +1161,7 b' make sure --traceback works on hook impo'
   ModuleNotFoundError: No module named 'hgext_importfail' (py36 !)
   Traceback (most recent call last):
   HookLoadError: precommit.importfail hook is invalid: import of "importfail" failed (no-py3 !)
-  mercurial.error.HookLoadError: b'precommit.importfail hook is invalid: import of "importfail" failed' (py3 !)
+  mercurial.error.HookLoadError: precommit.importfail hook is invalid: import of "importfail" failed (py3 !)
   abort: precommit.importfail hook is invalid: import of "importfail" failed
 
 Issue1827: Hooks Update & Commit not completely post operation

tests/test-install.t

@@ -236,25 +236,56 b' Verify the json works too:'
 
 #endif
 
-#if py3
-  $ HGALLOWPYTHON3=1
-  $ export HGALLOWPYTHON3
-#endif
-
-#if virtualenv
-
 Verify that Mercurial is installable with pip. Note that this MUST be
 the last test in this file, because we do some nasty things to the
 shell environment in order to make the virtualenv work reliably.
 
+On Python 3, we use the venv module, which is part of the standard library.
+But some Linux distros strip out this module's functionality involving pip,
+so we have to look for the ensurepip module, which these distros strip out
+completely.
+On Python 2, we use the 3rd party virtualenv module, if available.
+
   $ cd $TESTTMP
+  $ unset PYTHONPATH
+
+#if py3 ensurepip
+  $ "$PYTHON" -m venv installenv >> pip.log
+
+Note: we use this weird path to run pip and hg to avoid platform differences,
+since it's bin on most platforms but Scripts on Windows.
+  $ ./installenv/*/pip install --no-index $TESTDIR/.. >> pip.log
+  $ ./installenv/*/hg debuginstall || cat pip.log
+  checking encoding (ascii)...
+  checking Python executable (*) (glob)
+  checking Python version (3.*) (glob)
+  checking Python lib (*)... (glob)
+  checking Python security support (*) (glob)
+  checking Mercurial version (*) (glob)
+  checking Mercurial custom build (*) (glob)
+  checking module policy (*) (glob)
+  checking installed modules (*/mercurial)... (glob)
+  checking registered compression engines (*) (glob)
+  checking available compression engines (*) (glob)
+  checking available compression engines for wire protocol (*) (glob)
+  checking "re2" regexp engine \((available|missing)\) (re)
+  checking templates ($TESTTMP/installenv/*/site-packages/mercurial/templates)... (glob)
+  checking default template ($TESTTMP/installenv/*/site-packages/mercurial/templates/map-cmdline.default) (glob)
+  checking commit editor... (*) (glob)
+  checking username (test)
+  no problems detected
+#endif
+
+#if no-py3 virtualenv
+
 Note: --no-site-packages is deprecated, but some places have an
 ancient virtualenv from their linux distro or similar and it's not yet
 the default for them.
-  $ unset PYTHONPATH
+
   $ "$PYTHON" -m virtualenv --no-site-packages --never-download installenv >> pip.log
   DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. (?)
   DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support (?)
+
 Note: we use this weird path to run pip and hg to avoid platform differences,
 since it's bin on most platforms but Scripts on Windows.
   $ ./installenv/*/pip install --no-index $TESTDIR/.. >> pip.log
@@ -263,8 +294,7 b" since it's bin on most platforms but Scr"
   $ ./installenv/*/hg debuginstall || cat pip.log
   checking encoding (ascii)...
   checking Python executable (*) (glob)
-  checking Python version (2.*) (glob) (no-py3 !)
-  checking Python version (3.*) (glob) (py3 !)
+  checking Python version (2.*) (glob)
   checking Python lib (*)... (glob)
   checking Python security support (*) (glob)
     TLS 1.2 not supported by Python install; network connections lack modern security (?)

tests/test-keyword.t

@@ -256,7 +256,8 b' ie. if patch.diff wrapper acts as it sho'
 
 Pull from bundle and trigger notify
 
-  $ hg pull -u ../kw.hg
+  $ hg pull -u ../kw.hg | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py
   pulling from ../kw.hg
   requesting all changes
   adding changesets

tests/test-lfs-serve-access.t

@@ -354,7 +354,8 b' Test a checksum failure during the proce'
   $LOCALIP - - [$ERRDATE$] HG error:      localstore.download(oid, req.bodyfh) (glob)
   $LOCALIP - - [$ERRDATE$] HG error:      super(badstore, self).download(oid, src) (glob)
   $LOCALIP - - [$ERRDATE$] HG error:      _(b'corrupt remote lfs object: %s') % oid (glob)
-  $LOCALIP - - [$ERRDATE$] HG error:  LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (glob)
+  $LOCALIP - - [$ERRDATE$] HG error:  LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (no-py3 !)
+  $LOCALIP - - [$ERRDATE$] HG error:  hgext.lfs.blobstore.LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (py3 !)
   $LOCALIP - - [$ERRDATE$] HG error:   (glob)
   $LOCALIP - - [$ERRDATE$] Exception happened during processing request '/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d': (glob)
   Traceback (most recent call last):
@@ -376,7 +377,8 b' Test a checksum failure during the proce'
   $LOCALIP - - [$ERRDATE$] HG error:      blob = self._read(self.vfs, oid, verify) (glob)
   $LOCALIP - - [$ERRDATE$] HG error:      blobstore._verify(oid, b'dummy content') (glob)
   $LOCALIP - - [$ERRDATE$] HG error:      hint=_(b'run hg verify'), (glob)
-  $LOCALIP - - [$ERRDATE$] HG error:  LfsCorruptionError: detected corrupt lfs object: 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d (glob)
+  $LOCALIP - - [$ERRDATE$] HG error:  LfsCorruptionError: detected corrupt lfs object: 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d (no-py3 !)
+  $LOCALIP - - [$ERRDATE$] HG error:  hgext.lfs.blobstore.LfsCorruptionError: detected corrupt lfs object: 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d (py3 !)
   $LOCALIP - - [$ERRDATE$] HG error:   (glob)
 
 Basic Authorization headers are returned by the Batch API, and sent back with

tests/test-linerange.py

@@ -2,6 +2,7 b' from __future__ import absolute_import'
 
 import unittest
 from mercurial import error, mdiff
+from mercurial.utils import stringutil
 
 # for readability, line numbers are 0-origin
 text1 = b'''
@@ -228,7 +229,10 b' class blocksinrangetests(unittest.TestCa'
             try:
                 mdiff.blocksinrange(self.blocks, linerange2)
             except exctype as exc:
-                self.assertTrue('line range exceeds file size' in str(exc))
+                self.assertTrue(
+                    b'line range exceeds file size'
+                    in stringutil.forcebytestr(exc)
+                )
             else:
                 self.fail('%s not raised' % exctype.__name__)
 

tests/test-match.py

@@ -316,7 +316,7 b' class DifferenceMatcherTests(unittest.Te'
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
-    def testVisitdirIncludeIncludfe(self):
+    def testVisitdirIncludeInclude(self):
         m1 = matchmod.match(b'', b'', include=[b'path:dir/subdir'])
         m2 = matchmod.match(b'', b'', include=[b'rootfilesin:dir'])
         dm = matchmod.differencematcher(m1, m2)
@@ -430,7 +430,7 b' class IntersectionMatcherTests(unittest.'
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
-    def testVisitdirIncludeIncludfe(self):
+    def testVisitdirIncludeInclude(self):
         m1 = matchmod.match(b'', b'', include=[b'path:dir/subdir'])
         m2 = matchmod.match(b'', b'', include=[b'rootfilesin:dir'])
         im = matchmod.intersectmatchers(m1, m2)
@@ -644,7 +644,7 b' class UnionMatcherTests(unittest.TestCas'
 
     # We're using includematcher instead of patterns because it behaves slightly
     # better (giving narrower results) than patternmatcher.
-    def testVisitdirIncludeIncludfe(self):
+    def testVisitdirIncludeInclude(self):
         m1 = matchmod.match(b'', b'', include=[b'path:dir/subdir'])
         m2 = matchmod.match(b'', b'', include=[b'rootfilesin:dir'])
         um = matchmod.unionmatcher([m1, m2])

tests/test-notify-changegroup.t

@@ -39,6 +39,7 b' commit'
 push
 
   $ hg --traceback --cwd b push ../a 2>&1 |
+  >     "$PYTHON" $TESTDIR/unwrap-message-id.py | \
   >     "$PYTHON" -c 'from __future__ import print_function ; import sys,re; print(re.sub("\n\t", " ", sys.stdin.read()), end="")'
   pushing to ../a
   searching for changes
@@ -93,6 +94,7 b' unbundle with unrelated source'
 unbundle with correct source
 
   $ hg --config notify.sources=unbundle --cwd a unbundle ../test.hg 2>&1 |
+  >     "$PYTHON" $TESTDIR/unwrap-message-id.py | \
   >     "$PYTHON" -c 'from __future__ import print_function ; import sys,re; print(re.sub("\n\t", " ", sys.stdin.read()), end="")'
   adding changesets
   adding manifests
@@ -169,6 +171,7 b' merge as a different user'
 push
 
   $ hg --traceback --cwd b --config notify.fromauthor=True push ../a 2>&1 |
+  >     "$PYTHON" $TESTDIR/unwrap-message-id.py | \
   >     "$PYTHON" -c 'from __future__ import print_function ; import sys,re; print(re.sub("\n\t", " ", sys.stdin.read()), end="")'
   pushing to ../a
   searching for changes

tests/test-notify.t

@@ -196,7 +196,9 b' the python call below wraps continuation'
 of the very long subject line
 pull (minimal config)
 
-  $ hg --traceback --cwd b --config notify.domain=example.com --config notify.messageidseed=example pull ../a | "$PYTHON" $TESTTMP/filter.py
+  $ hg --traceback --cwd b --config notify.domain=example.com --config notify.messageidseed=example pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -255,7 +257,9 b' pull'
 
   $ hg --cwd b rollback
   repository tip rolled back to revision 0 (undo pull)
-  $ hg --traceback --cwd b pull ../a  | "$PYTHON" $TESTTMP/filter.py
+  $ hg --traceback --cwd b pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -303,7 +307,9 b' pull'
 
   $ hg --cwd b rollback
   repository tip rolled back to revision 0 (undo pull)
-  $ hg --traceback --config notify.maxdiffstat=1 --cwd b pull ../a | "$PYTHON" $TESTTMP/filter.py
+  $ hg --traceback --config notify.maxdiffstat=1 --cwd b pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -354,7 +360,9 b' test merge'
   (branch merge, don't forget to commit)
   $ hg ci -m merge -d '3 0'
   $ cd ..
-  $ hg --traceback --cwd b pull ../a | "$PYTHON" $TESTTMP/filter.py
+  $ hg --traceback --cwd b pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -418,8 +426,9 b' non-ascii content and truncation of mult'
   > EOF
   $ echo a >> a/a
   $ hg --cwd a --encoding utf-8 commit -A -d '0 0' \
-  >   -m `"$PYTHON" -c 'print("\xc3\xa0\xc3\xa1\xc3\xa2\xc3\xa3\xc3\xa4")'`
+  >   -m `"$PYTHON" -c 'import sys; getattr(sys.stdout, "buffer", sys.stdout).write(b"\xc3\xa0\xc3\xa1\xc3\xa2\xc3\xa3\xc3\xa4")'`
   $ hg --traceback --cwd b --encoding utf-8 pull ../a | \
+  >   "$PYTHON" $TESTDIR/unwrap-message-id.py | \
   >   "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
@@ -433,7 +442,8 b' non-ascii content and truncation of mult'
   Content-Transfer-Encoding: 8bit
   X-Test: foo
   Date: * (glob)
-  Subject: \xc3\xa0... (esc)
+  Subject: \xc3\xa0... (esc) (no-py3 !)
+  Subject: =?utf-8?b?w6AuLi4=?= (py3 !)
   From: [e-mail]
   X-Hg-Notification: changeset 0f25f9c22b4c
   Message-Id: <*> (glob)
@@ -473,7 +483,7 b' long lines'
   new changesets a846b5f6ebb7
   notify: sending 2 subscribers 1 changes
   (run 'hg update' to get a working copy)
-  $ "$PYTHON" $TESTTMP/filter.py < b/mbox
+  $ cat b/mbox | "$PYTHON" $TESTDIR/unwrap-message-id.py | "$PYTHON" $TESTTMP/filter.py
   From [e-mail] ... ... .. ..:..:.. .... (re)
   MIME-Version: 1.0
   Content-Type: text/plain; charset="*" (glob)
@@ -533,7 +543,9 b' long lines'
   (branches are permanent and global, did you want a bookmark?)
   $ echo a >> a/a
   $ hg --cwd a ci -m test -d '1 0'
-  $ hg --traceback --cwd b pull ../a | "$PYTHON" $TESTTMP/filter.py
+  $ hg --traceback --cwd b pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -563,7 +575,9 b' from different branch'
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo a >> a/a
   $ hg --cwd a ci -m test -d '1 0'
-  $ hg --traceback --cwd b pull ../a | "$PYTHON" $TESTTMP/filter.py
+  $ hg --traceback --cwd b pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -592,7 +606,9 b' default template:'
   $ mv "$HGRCPATH.new" $HGRCPATH
   $ echo a >> a/a
   $ hg --cwd a commit -m 'default template'
-  $ hg --cwd b pull ../a -q | "$PYTHON" $TESTTMP/filter.py
+  $ hg --cwd b pull ../a -q | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   MIME-Version: 1.0
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: 7bit
@@ -621,7 +637,9 b' with style:'
   > EOF
   $ echo a >> a/a
   $ hg --cwd a commit -m 'with style'
-  $ hg --cwd b pull ../a -q | "$PYTHON" $TESTTMP/filter.py
+  $ hg --cwd b pull ../a -q | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   MIME-Version: 1.0
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: 7bit
@@ -644,7 +662,9 b' with template (overrides style):'
   > EOF
   $ echo a >> a/a
   $ hg --cwd a commit -m 'with template'
-  $ hg --cwd b pull ../a -q | "$PYTHON" $TESTTMP/filter.py
+  $ hg --cwd b pull ../a -q | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py | \
+  >  "$PYTHON" $TESTTMP/filter.py
   MIME-Version: 1.0
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: 7bit
@@ -675,7 +695,8 b' showfunc diff'
   > EOF
   $ hg commit -Am addfunction
   adding f1
-  $ hg --cwd ../b pull ../a
+  $ hg --cwd ../b pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py
   pulling from ../a
   searching for changes
   adding changesets
@@ -718,7 +739,8 b' showfunc diff'
   > }
   > EOF
   $ hg commit -m changefunction
-  $ hg --cwd ../b --config notify.showfunc=True pull ../a
+  $ hg --cwd ../b --config notify.showfunc=True pull ../a | \
+  >  "$PYTHON" $TESTDIR/unwrap-message-id.py
   pulling from ../a
   searching for changes
   adding changesets

tests/test-patchbomb.t

@@ -445,7 +445,9 b' with a specific bundle type'
 
 utf-8 patch:
   $ "$PYTHON" -c 'fp = open("utf", "wb"); fp.write(b"h\xC3\xB6mma!\n"); fp.close();'
-  $ hg commit -A -d '4 0' -m 'utf-8 content'
+  $ hg commit -A -d '4 0' \
+  >   --encoding "utf-8" \
+  >   -m `"$PYTHON" -c 'import sys; getattr(sys.stdout, "buffer", sys.stdout).write(b"\xc3\xa7a")'`
   adding description
   adding utf
 
@@ -454,16 +456,16 b' no mime encoding for email --test:'
   this patch series consists of 1 patches.
   
   
-  displaying [PATCH] utf-8 content ...
+  displaying [PATCH] ?a ...
   MIME-Version: 1.0
   Content-Type: text/plain; charset="iso-8859-1"
   Content-Transfer-Encoding: quoted-printable
-  Subject: [PATCH] utf-8 content
-  X-Mercurial-Node: 909a00e13e9d78b575aeee23dddbada46d5a143f
+  Subject: [PATCH] ?a
+  X-Mercurial-Node: f81ef97829467e868fc405fccbcfa66217e4d3e6
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <909a00e13e9d78b575ae.240@test-hostname>
-  X-Mercurial-Series-Id: <909a00e13e9d78b575ae.240@test-hostname>
+  Message-Id: <f81ef97829467e868fc4.240@test-hostname>
+  X-Mercurial-Series-Id: <f81ef97829467e868fc4.240@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:04:00 +0000
   From: quux
@@ -474,18 +476,18 b' no mime encoding for email --test:'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID 909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID f81ef97829467e868fc405fccbcfa66217e4d3e6
   # Parent  ff2c9fa2018b15fa74b33363bda9527323e2a99f
-  utf-8 content
-  
-  diff -r ff2c9fa2018b -r 909a00e13e9d description
+  ?a
+  
+  diff -r ff2c9fa2018b -r f81ef9782946 description
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/description	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,3 @@
   +a multiline
   +
   +description
-  diff -r ff2c9fa2018b -r 909a00e13e9d utf
+  diff -r ff2c9fa2018b -r f81ef9782946 utf
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/utf	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,1 @@
@@ -497,35 +499,36 b' mime encoded mbox (base64):'
   this patch series consists of 1 patches.
   
   
-  sending [PATCH] utf-8 content ...
+  sending [PATCH] ?a ...
 
   $ cat mbox
   From quux ... ... .. ..:..:.. .... (re)
   MIME-Version: 1.0
   Content-Type: text/plain; charset="utf-8"
   Content-Transfer-Encoding: base64
-  Subject: [PATCH] utf-8 content
-  X-Mercurial-Node: 909a00e13e9d78b575aeee23dddbada46d5a143f
+  Subject: [PATCH] ?a
+  X-Mercurial-Node: f81ef97829467e868fc405fccbcfa66217e4d3e6
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <909a00e13e9d78b575ae.240@test-hostname>
-  X-Mercurial-Series-Id: <909a00e13e9d78b575ae.240@test-hostname>
+  Message-Id: <f81ef97829467e868fc4.240@test-hostname>
+  X-Mercurial-Series-Id: <f81ef97829467e868fc4.240@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:04:00 +0000
-  From: Q <quux>
+  From: Q <quux> (no-py3 !)
+  From: =?iso-8859-1?q?Q?= <quux> (py3 !)
   To: foo
   Cc: bar
   
   IyBIRyBjaGFuZ2VzZXQgcGF0Y2gKIyBVc2VyIHRlc3QKIyBEYXRlIDQgMAojICAgICAgVGh1IEph
-  biAwMSAwMDowMDowNCAxOTcwICswMDAwCiMgTm9kZSBJRCA5MDlhMDBlMTNlOWQ3OGI1NzVhZWVl
-  MjNkZGRiYWRhNDZkNWExNDNmCiMgUGFyZW50ICBmZjJjOWZhMjAxOGIxNWZhNzRiMzMzNjNiZGE5
-  NTI3MzIzZTJhOTlmCnV0Zi04IGNvbnRlbnQKCmRpZmYgLXIgZmYyYzlmYTIwMThiIC1yIDkwOWEw
-  MGUxM2U5ZCBkZXNjcmlwdGlvbgotLS0gL2Rldi9udWxsCVRodSBKYW4gMDEgMDA6MDA6MDAgMTk3
-  MCArMDAwMAorKysgYi9kZXNjcmlwdGlvbglUaHUgSmFuIDAxIDAwOjAwOjA0IDE5NzAgKzAwMDAK
-  QEAgLTAsMCArMSwzIEBACithIG11bHRpbGluZQorCitkZXNjcmlwdGlvbgpkaWZmIC1yIGZmMmM5
-  ZmEyMDE4YiAtciA5MDlhMDBlMTNlOWQgdXRmCi0tLSAvZGV2L251bGwJVGh1IEphbiAwMSAwMDow
-  MDowMCAxOTcwICswMDAwCisrKyBiL3V0ZglUaHUgSmFuIDAxIDAwOjAwOjA0IDE5NzAgKzAwMDAK
-  QEAgLTAsMCArMSwxIEBACitow7ZtbWEhCg==
+  biAwMSAwMDowMDowNCAxOTcwICswMDAwCiMgTm9kZSBJRCBmODFlZjk3ODI5NDY3ZTg2OGZjNDA1
+  ZmNjYmNmYTY2MjE3ZTRkM2U2CiMgUGFyZW50ICBmZjJjOWZhMjAxOGIxNWZhNzRiMzMzNjNiZGE5
+  NTI3MzIzZTJhOTlmCj9hCgpkaWZmIC1yIGZmMmM5ZmEyMDE4YiAtciBmODFlZjk3ODI5NDYgZGVz
+  Y3JpcHRpb24KLS0tIC9kZXYvbnVsbAlUaHUgSmFuIDAxIDAwOjAwOjAwIDE5NzAgKzAwMDAKKysr
+  IGIvZGVzY3JpcHRpb24JVGh1IEphbiAwMSAwMDowMDowNCAxOTcwICswMDAwCkBAIC0wLDAgKzEs
+  MyBAQAorYSBtdWx0aWxpbmUKKworZGVzY3JpcHRpb24KZGlmZiAtciBmZjJjOWZhMjAxOGIgLXIg
+  ZjgxZWY5NzgyOTQ2IHV0ZgotLS0gL2Rldi9udWxsCVRodSBKYW4gMDEgMDA6MDA6MDAgMTk3MCAr
+  MDAwMAorKysgYi91dGYJVGh1IEphbiAwMSAwMDowMDowNCAxOTcwICswMDAwCkBAIC0wLDAgKzEs
+  MSBAQAoraMO2bW1hIQo=
   
   
   >>> import base64
@@ -540,18 +543,18 b' mime encoded mbox (base64):'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID 909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID f81ef97829467e868fc405fccbcfa66217e4d3e6
   # Parent  ff2c9fa2018b15fa74b33363bda9527323e2a99f
-  utf-8 content
-  
-  diff -r ff2c9fa2018b -r 909a00e13e9d description
+  ?a
+  
+  diff -r ff2c9fa2018b -r f81ef9782946 description
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/description	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,3 @@
   +a multiline
   +
   +description
-  diff -r ff2c9fa2018b -r 909a00e13e9d utf
+  diff -r ff2c9fa2018b -r f81ef9782946 utf
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/utf	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,1 @@
@@ -574,11 +577,11 b' no mime encoding for email --test:'
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: quoted-printable
   Subject: [PATCH] long line
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <a2ea8fc83dd8b93cfd86.240@test-hostname>
-  X-Mercurial-Series-Id: <a2ea8fc83dd8b93cfd86.240@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.240@test-hostname>
+  X-Mercurial-Series-Id: <0c7b871cb86b61a1c07e.240@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:04:00 +0000
   From: quux
@@ -589,11 +592,11 b' no mime encoding for email --test:'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -628,11 +631,11 b' mime encoded mbox (quoted-printable):'
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: quoted-printable
   Subject: [PATCH] long line
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <a2ea8fc83dd8b93cfd86.240@test-hostname>
-  X-Mercurial-Series-Id: <a2ea8fc83dd8b93cfd86.240@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.240@test-hostname>
+  X-Mercurial-Series-Id: <0c7b871cb86b61a1c07e.240@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:04:00 +0000
   From: quux
@@ -643,11 +646,11 b' mime encoded mbox (quoted-printable):'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -690,11 +693,11 b' iso-8859-1 mbox:'
   Content-Type: text/plain; charset="iso-8859-1"
   Content-Transfer-Encoding: quoted-printable
   Subject: [PATCH] isolatin 8-bit encoding
-  X-Mercurial-Node: 240fb913fc1b7ff15ddb9f33e73d82bf5277c720
+  X-Mercurial-Node: 4d6f44f466c96d89f2e7e865a70ff41d8b6eee37
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <240fb913fc1b7ff15ddb.300@test-hostname>
-  X-Mercurial-Series-Id: <240fb913fc1b7ff15ddb.300@test-hostname>
+  Message-Id: <4d6f44f466c96d89f2e7.300@test-hostname>
+  X-Mercurial-Series-Id: <4d6f44f466c96d89f2e7.300@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:05:00 +0000
   From: quux
@@ -705,11 +708,11 b' iso-8859-1 mbox:'
   # User test
   # Date 5 0
   #      Thu Jan 01 00:00:05 1970 +0000
-  # Node ID 240fb913fc1b7ff15ddb9f33e73d82bf5277c720
-  # Parent  a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  # Node ID 4d6f44f466c96d89f2e7e865a70ff41d8b6eee37
+  # Parent  0c7b871cb86b61a1c07e244393603c361e4a178d
   isolatin 8-bit encoding
   
-  diff -r a2ea8fc83dd8 -r 240fb913fc1b isolatin
+  diff -r 0c7b871cb86b -r 4d6f44f466c9 isolatin
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/isolatin	Thu Jan 01 00:00:05 1970 +0000
   @@ -0,0 +1,1 @@
@@ -937,11 +940,11 b' test inline for single patch (quoted-pri'
   Content-Type: multipart/mixed; boundary="===*==" (glob)
   MIME-Version: 1.0
   Subject: [PATCH] test
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <a2ea8fc83dd8b93cfd86.60@test-hostname>
-  X-Mercurial-Series-Id: <a2ea8fc83dd8b93cfd86.60@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.60@test-hostname>
+  X-Mercurial-Series-Id: <0c7b871cb86b61a1c07e.60@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:01:00 +0000
   From: quux
@@ -958,11 +961,11 b' test inline for single patch (quoted-pri'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -1088,10 +1091,10 b' test inline for multiple patches:'
   Content-Type: multipart/mixed; boundary="===*==" (glob)
   MIME-Version: 1.0
   Subject: [PATCH 3 of 3] long line
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 3
   X-Mercurial-Series-Total: 3
-  Message-Id: <a2ea8fc83dd8b93cfd86.63@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.63@test-hostname>
   X-Mercurial-Series-Id: <8580ff50825a50c8f716.61@test-hostname>
   In-Reply-To: <patchbomb.60@test-hostname>
   References: <patchbomb.60@test-hostname>
@@ -1111,11 +1114,11 b' test inline for multiple patches:'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -1199,11 +1202,11 b' test attach for single patch (quoted-pri'
   Content-Type: multipart/mixed; boundary="===*==" (glob)
   MIME-Version: 1.0
   Subject: [PATCH] test
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <a2ea8fc83dd8b93cfd86.60@test-hostname>
-  X-Mercurial-Series-Id: <a2ea8fc83dd8b93cfd86.60@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.60@test-hostname>
+  X-Mercurial-Series-Id: <0c7b871cb86b61a1c07e.60@test-hostname>
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Thu, 01 Jan 1970 00:01:00 +0000
   From: quux
@@ -1229,11 +1232,11 b' test attach for single patch (quoted-pri'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -1438,10 +1441,10 b' test attach for multiple patches:'
   Content-Type: multipart/mixed; boundary="===*==" (glob)
   MIME-Version: 1.0
   Subject: [PATCH 3 of 3] long line
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 3
   X-Mercurial-Series-Total: 3
-  Message-Id: <a2ea8fc83dd8b93cfd86.63@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.63@test-hostname>
   X-Mercurial-Series-Id: <8580ff50825a50c8f716.61@test-hostname>
   In-Reply-To: <patchbomb.60@test-hostname>
   References: <patchbomb.60@test-hostname>
@@ -1470,11 +1473,11 b' test attach for multiple patches:'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -1929,11 +1932,11 b' test inreplyto:'
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: 7bit
   Subject: [PATCH] Added tag two, two.diff for changeset ff2c9fa2018b
-  X-Mercurial-Node: 7aead2484924c445ad8ce2613df91f52f9e502ed
+  X-Mercurial-Node: 9cea7492c36bdda2c72e7dd5f35f7fc367adeb2c
   X-Mercurial-Series-Index: 1
   X-Mercurial-Series-Total: 1
-  Message-Id: <7aead2484924c445ad8c.60@test-hostname>
-  X-Mercurial-Series-Id: <7aead2484924c445ad8c.60@test-hostname>
+  Message-Id: <9cea7492c36bdda2c72e.60@test-hostname>
+  X-Mercurial-Series-Id: <9cea7492c36bdda2c72e.60@test-hostname>
   In-Reply-To: <baz>
   References: <baz>
   User-Agent: Mercurial-patchbomb/* (glob)
@@ -1946,11 +1949,11 b' test inreplyto:'
   # User test
   # Date 0 0
   #      Thu Jan 01 00:00:00 1970 +0000
-  # Node ID 7aead2484924c445ad8ce2613df91f52f9e502ed
-  # Parent  045ca29b1ea20e4940411e695e20e521f2f0f98e
+  # Node ID 9cea7492c36bdda2c72e7dd5f35f7fc367adeb2c
+  # Parent  3b775b32716d9b54291ccddf0a36ceea45449bfb
   Added tag two, two.diff for changeset ff2c9fa2018b
   
-  diff -r 045ca29b1ea2 -r 7aead2484924 .hgtags
+  diff -r 3b775b32716d -r 9cea7492c36b .hgtags
   --- a/.hgtags	Thu Jan 01 00:00:00 1970 +0000
   +++ b/.hgtags	Thu Jan 01 00:00:00 1970 +0000
   @@ -2,3 +2,5 @@
@@ -2397,9 +2400,12 b' test multi-address parsing:'
   User-Agent: Mercurial-patchbomb/* (glob)
   Date: Tue, 01 Jan 1980 00:01:00 +0000
   From: quux
-  To: spam <spam>, eggs, toast
-  Cc: foo, [e-mail], "A, B <>" <[e-mail]>
-  Bcc: "Quux, A." <quux>
+  To: spam <spam>, eggs, toast (no-py3 !)
+  Cc: foo, [e-mail], "A, B <>" <[e-mail]> (no-py3 !)
+  Bcc: "Quux, A." <quux> (no-py3 !)
+  To: =?iso-8859-1?q?spam?= <spam>, eggs, toast (py3 !)
+  Cc: foo, [e-mail], =?iso-8859-1?q?A=2C_B_=3C=3E?= <[e-mail]> (py3 !)
+  Bcc: =?iso-8859-1?q?Quux=2C_A=2E?= <quux> (py3 !)
   
   # HG changeset patch
   # User test
@@ -2601,17 +2607,17 b' test outgoing:'
   |
   o  9:2f9fa9b998c5 d
   |
-  | o  8:7aead2484924 Added tag two, two.diff for changeset ff2c9fa2018b
+  | o  8:9cea7492c36b Added tag two, two.diff for changeset ff2c9fa2018b
   | |
-  | o  7:045ca29b1ea2 Added tag one, one.patch for changeset 97d72e5f12c7
+  | o  7:3b775b32716d Added tag one, one.patch for changeset 97d72e5f12c7
   | |
-  | o  6:5d5ef15dfe5e Added tag zero, zero.foo for changeset 8580ff50825a
+  | o  6:c41d7353114c Added tag zero, zero.foo for changeset 8580ff50825a
   | |
-  | o  5:240fb913fc1b isolatin 8-bit encoding
+  | o  5:4d6f44f466c9 isolatin 8-bit encoding
   | |
-  | o  4:a2ea8fc83dd8 long line
+  | o  4:0c7b871cb86b long line
   | |
-  | o  3:909a00e13e9d utf-8 content
+  | o  3:f81ef9782946 \xe7a (esc)
   | |
   | o  2:ff2c9fa2018b c
   |/
@@ -2673,15 +2679,16 b' test outgoing:'
   @@ -0,0 +1,1 @@
   +c
   
-  displaying [PATCH 2 of 6] utf-8 content ...
+  displaying [PATCH 2 of 6] \xe7a ... (esc)
   MIME-Version: 1.0
   Content-Type: text/plain; charset="iso-8859-1"
   Content-Transfer-Encoding: quoted-printable
-  Subject: [PATCH 2 of 6] utf-8 content
-  X-Mercurial-Node: 909a00e13e9d78b575aeee23dddbada46d5a143f
+  Subject: [PATCH 2 of 6] \xe7a (esc) (no-py3 !)
+  Subject: =?utf-8?b?W1BBVENIIDIgb2YgNl0gw6dh?= (py3 !)
+  X-Mercurial-Node: f81ef97829467e868fc405fccbcfa66217e4d3e6
   X-Mercurial-Series-Index: 2
   X-Mercurial-Series-Total: 6
-  Message-Id: <909a00e13e9d78b575ae.315532862@test-hostname>
+  Message-Id: <f81ef97829467e868fc4.315532862@test-hostname>
   X-Mercurial-Series-Id: <ff2c9fa2018b15fa74b3.315532861@test-hostname>
   In-Reply-To: <patchbomb.315532860@test-hostname>
   References: <patchbomb.315532860@test-hostname>
@@ -2694,18 +2701,18 b' test outgoing:'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID 909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID f81ef97829467e868fc405fccbcfa66217e4d3e6
   # Parent  ff2c9fa2018b15fa74b33363bda9527323e2a99f
-  utf-8 content
-  
-  diff -r ff2c9fa2018b -r 909a00e13e9d description
+  =E7a
+  
+  diff -r ff2c9fa2018b -r f81ef9782946 description
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/description	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,3 @@
   +a multiline
   +
   +description
-  diff -r ff2c9fa2018b -r 909a00e13e9d utf
+  diff -r ff2c9fa2018b -r f81ef9782946 utf
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/utf	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,1 @@
@@ -2716,10 +2723,10 b' test outgoing:'
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: quoted-printable
   Subject: [PATCH 3 of 6] long line
-  X-Mercurial-Node: a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  X-Mercurial-Node: 0c7b871cb86b61a1c07e244393603c361e4a178d
   X-Mercurial-Series-Index: 3
   X-Mercurial-Series-Total: 6
-  Message-Id: <a2ea8fc83dd8b93cfd86.315532863@test-hostname>
+  Message-Id: <0c7b871cb86b61a1c07e.315532863@test-hostname>
   X-Mercurial-Series-Id: <ff2c9fa2018b15fa74b3.315532861@test-hostname>
   In-Reply-To: <patchbomb.315532860@test-hostname>
   References: <patchbomb.315532860@test-hostname>
@@ -2732,11 +2739,11 b' test outgoing:'
   # User test
   # Date 4 0
   #      Thu Jan 01 00:00:04 1970 +0000
-  # Node ID a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
-  # Parent  909a00e13e9d78b575aeee23dddbada46d5a143f
+  # Node ID 0c7b871cb86b61a1c07e244393603c361e4a178d
+  # Parent  f81ef97829467e868fc405fccbcfa66217e4d3e6
   long line
   
-  diff -r 909a00e13e9d -r a2ea8fc83dd8 long
+  diff -r f81ef9782946 -r 0c7b871cb86b long
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/long	Thu Jan 01 00:00:04 1970 +0000
   @@ -0,0 +1,4 @@
@@ -2763,10 +2770,10 b' test outgoing:'
   Content-Type: text/plain; charset="iso-8859-1"
   Content-Transfer-Encoding: quoted-printable
   Subject: [PATCH 4 of 6] isolatin 8-bit encoding
-  X-Mercurial-Node: 240fb913fc1b7ff15ddb9f33e73d82bf5277c720
+  X-Mercurial-Node: 4d6f44f466c96d89f2e7e865a70ff41d8b6eee37
   X-Mercurial-Series-Index: 4
   X-Mercurial-Series-Total: 6
-  Message-Id: <240fb913fc1b7ff15ddb.315532864@test-hostname>
+  Message-Id: <4d6f44f466c96d89f2e7.315532864@test-hostname>
   X-Mercurial-Series-Id: <ff2c9fa2018b15fa74b3.315532861@test-hostname>
   In-Reply-To: <patchbomb.315532860@test-hostname>
   References: <patchbomb.315532860@test-hostname>
@@ -2779,11 +2786,11 b' test outgoing:'
   # User test
   # Date 5 0
   #      Thu Jan 01 00:00:05 1970 +0000
-  # Node ID 240fb913fc1b7ff15ddb9f33e73d82bf5277c720
-  # Parent  a2ea8fc83dd8b93cfd86ac97b28287204ab806e1
+  # Node ID 4d6f44f466c96d89f2e7e865a70ff41d8b6eee37
+  # Parent  0c7b871cb86b61a1c07e244393603c361e4a178d
   isolatin 8-bit encoding
   
-  diff -r a2ea8fc83dd8 -r 240fb913fc1b isolatin
+  diff -r 0c7b871cb86b -r 4d6f44f466c9 isolatin
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/isolatin	Thu Jan 01 00:00:05 1970 +0000
   @@ -0,0 +1,1 @@
@@ -2794,10 +2801,10 b' test outgoing:'
   Content-Type: text/plain; charset="us-ascii"
   Content-Transfer-Encoding: 7bit
   Subject: [PATCH 5 of 6] Added tag zero, zero.foo for changeset 8580ff50825a
-  X-Mercurial-Node: 5d5ef15dfe5e7bd3a4ee154b5fff76c7945ec433
+  X-Mercurial-Node: c41d7353114ccb07a50a822ad5ddf47051c88ec2
   X-Mercurial-Series-Index: 5
   X-Mercurial-Series-Total: 6
-  Message-Id: <5d5ef15dfe5e7bd3a4ee.315532865@test-hostname>
+  Message-Id: <c41d7353114ccb07a50a.315532865@test-hostname>
   X-Mercurial-Series-Id: <ff2c9fa2018b15fa74b3.315532861@test-hostname>
   In-Reply-To: <patchbomb.315532860@test-hostname>
   References: <patchbomb.315532860@test-hostname>
@@ -2810,11 +2817,11 b' test outgoing:'
   # User test
   # Date 0 0
   #      Thu Jan 01 00:00:00 1970 +0000
-  # Node ID 5d5ef15dfe5e7bd3a4ee154b5fff76c7945ec433
-  # Parent  240fb913fc1b7ff15ddb9f33e73d82bf5277c720
+  # Node ID c41d7353114ccb07a50a822ad5ddf47051c88ec2
+  # Parent  4d6f44f466c96d89f2e7e865a70ff41d8b6eee37
   Added tag zero, zero.foo for changeset 8580ff50825a
   
-  diff -r 240fb913fc1b -r 5d5ef15dfe5e .hgtags
+  diff -r 4d6f44f466c9 -r c41d7353114c .hgtags
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/.hgtags	Thu Jan 01 00:00:00 1970 +0000
   @@ -0,0 +1,2 @@

tests/test-worker.t

@@ -85,7 +85,7 b' Known exception should be caught, but pr'
   $ hg --config "extensions.t=$abspath" --config 'worker.numcpus=8' \
   > test 100000.0 abort --traceback 2>&1 | egrep '(SystemExit|Abort)'
       raise error.Abort(b'known exception')
-  mercurial.error.Abort: b'known exception' (py3 !)
+  mercurial.error.Abort: known exception (py3 !)
   Abort: known exception (no-py3 !)
   SystemExit: 255